diff options
Diffstat (limited to 'dpdk/dpdk-2.2.0_patches')
26 files changed, 0 insertions, 4589 deletions
diff --git a/dpdk/dpdk-2.2.0_patches/0001-e1000-Set-VLAN-Rx-Offload-tag-correctly.patch b/dpdk/dpdk-2.2.0_patches/0001-e1000-Set-VLAN-Rx-Offload-tag-correctly.patch deleted file mode 100644 index 6ed2fc61..00000000 --- a/dpdk/dpdk-2.2.0_patches/0001-e1000-Set-VLAN-Rx-Offload-tag-correctly.patch +++ /dev/null @@ -1,75 +0,0 @@ -From 4a599535445d16a1c55fac0bd71edc443c6c23f2 Mon Sep 17 00:00:00 2001 -From: Damjan Marion <damarion@cisco.com> -Date: Wed, 16 Dec 2015 03:21:21 +0100 -Subject: [PATCH 1/4] e1000: Set VLAN Rx Offload tag correctly - ---- - drivers/net/e1000/igb_rxtx.c | 30 ++++++++++++++++++++++++++++++ - lib/librte_ether/rte_ether.h | 3 +++ - 2 files changed, 33 insertions(+) - -diff --git a/drivers/net/e1000/igb_rxtx.c b/drivers/net/e1000/igb_rxtx.c -index 996e7da..cbe80a1 100644 ---- a/drivers/net/e1000/igb_rxtx.c -+++ b/drivers/net/e1000/igb_rxtx.c -@@ -910,6 +910,21 @@ eth_igb_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(rxq, hlen_type_rss); - pkt_flags = pkt_flags | rx_desc_status_to_pkt_flags(staterr); - pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr); -+ { -+ /* -+ * Check packet for VLAN ethernet types and set -+ * RX Offload flag PKT_RX_VLAN_PKT accordingly. -+ */ -+ struct ether_hdr *eth_hdr = -+ rte_pktmbuf_mtod(rxm, struct ether_hdr *); -+ u16 eth_type = rte_be_to_cpu_16(eth_hdr->ether_type); -+ -+ if ((eth_type == ETHER_TYPE_VLAN) || -+ (eth_type == ETHER_TYPE_VLAN_AD) || -+ (eth_type == ETHER_TYPE_VLAN_9100) || -+ (eth_type == ETHER_TYPE_VLAN_9200)) -+ pkt_flags |= PKT_RX_VLAN_PKT; -+ } - rxm->ol_flags = pkt_flags; - rxm->packet_type = igb_rxd_pkt_info_to_pkt_type(rxd.wb.lower. - lo_dword.hs_rss.pkt_info); -@@ -1146,6 +1161,21 @@ eth_igb_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(rxq, hlen_type_rss); - pkt_flags = pkt_flags | rx_desc_status_to_pkt_flags(staterr); - pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr); -+ { -+ /* -+ * Check packet for VLAN ethernet types and set -+ * RX Offload flag PKT_RX_VLAN_PKT accordingly. -+ */ -+ struct ether_hdr *eth_hdr = -+ rte_pktmbuf_mtod(rxm, struct ether_hdr *); -+ u16 eth_type = rte_be_to_cpu_16(eth_hdr->ether_type); -+ -+ if ((eth_type == ETHER_TYPE_VLAN) || -+ (eth_type == ETHER_TYPE_VLAN_AD) || -+ (eth_type == ETHER_TYPE_VLAN_9100) || -+ (eth_type == ETHER_TYPE_VLAN_9200)) -+ pkt_flags |= PKT_RX_VLAN_PKT; -+ } - first_seg->ol_flags = pkt_flags; - first_seg->packet_type = igb_rxd_pkt_info_to_pkt_type(rxd.wb. - lower.lo_dword.hs_rss.pkt_info); -diff --git a/lib/librte_ether/rte_ether.h b/lib/librte_ether/rte_ether.h -index 07c17d7..fd646ec 100644 ---- a/lib/librte_ether/rte_ether.h -+++ b/lib/librte_ether/rte_ether.h -@@ -332,6 +332,9 @@ struct vxlan_hdr { - #define ETHER_TYPE_1588 0x88F7 /**< IEEE 802.1AS 1588 Precise Time Protocol. */ - #define ETHER_TYPE_SLOW 0x8809 /**< Slow protocols (LACP and Marker). */ - #define ETHER_TYPE_TEB 0x6558 /**< Transparent Ethernet Bridging. */ -+#define ETHER_TYPE_VLAN_AD 0x88a8 /**< IEEE 802.1AD VLAN tagging. */ -+#define ETHER_TYPE_VLAN_9100 0x9100 /**< VLAN 0x9100 tagging. */ -+#define ETHER_TYPE_VLAN_9200 0x9200 /**< VLAN 0x9200 tagging. */ - - #define ETHER_VXLAN_HLEN (sizeof(struct udp_hdr) + sizeof(struct vxlan_hdr)) - /**< VXLAN tunnel header length. */ --- -2.5.0 - diff --git a/dpdk/dpdk-2.2.0_patches/0002-ixgbe-Wait-a-bit-longer-for-autonegotiation-to-leave.patch b/dpdk/dpdk-2.2.0_patches/0002-ixgbe-Wait-a-bit-longer-for-autonegotiation-to-leave.patch deleted file mode 100644 index b7a50298..00000000 --- a/dpdk/dpdk-2.2.0_patches/0002-ixgbe-Wait-a-bit-longer-for-autonegotiation-to-leave.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 009cd67e5b1ed0592de0fb6ae2fa662ffc172dde Mon Sep 17 00:00:00 2001 -From: Damjan Marion <damarion@cisco.com> -Date: Wed, 16 Dec 2015 03:22:11 +0100 -Subject: [PATCH 2/4] ixgbe: Wait a bit longer for autonegotiation to leave - ---- - drivers/net/ixgbe/base/ixgbe_82599.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/drivers/net/ixgbe/base/ixgbe_82599.c b/drivers/net/ixgbe/base/ixgbe_82599.c -index f0deb59..ae66380 100644 ---- a/drivers/net/ixgbe/base/ixgbe_82599.c -+++ b/drivers/net/ixgbe/base/ixgbe_82599.c -@@ -2442,7 +2442,7 @@ s32 ixgbe_reset_pipeline_82599(struct ixgbe_hw *hw) - IXGBE_WRITE_REG(hw, IXGBE_AUTOC, - autoc_reg ^ (0x4 << IXGBE_AUTOC_LMS_SHIFT)); - /* Wait for AN to leave state 0 */ -- for (i = 0; i < 10; i++) { -+ for (i = 0; i < 50; i++) { - msec_delay(4); - anlp1_reg = IXGBE_READ_REG(hw, IXGBE_ANLP1); - if (anlp1_reg & IXGBE_ANLP1_AN_STATE_MASK) --- -2.5.0 - diff --git a/dpdk/dpdk-2.2.0_patches/0003-virtio-Cleanup-virtio-pmd-debug-log-output-reset.patch b/dpdk/dpdk-2.2.0_patches/0003-virtio-Cleanup-virtio-pmd-debug-log-output-reset.patch deleted file mode 100644 index 874f666b..00000000 --- a/dpdk/dpdk-2.2.0_patches/0003-virtio-Cleanup-virtio-pmd-debug-log-output-reset.patch +++ /dev/null @@ -1,76 +0,0 @@ -From e2592eb622c33791d8ae51153360bd8249bdd056 Mon Sep 17 00:00:00 2001 -From: Damjan Marion <damarion@cisco.com> -Date: Wed, 16 Dec 2015 03:29:22 +0100 -Subject: [PATCH 3/4] virtio: Cleanup virtio pmd debug log output, reset - ---- - drivers/net/virtio/virtio_ethdev.c | 10 +++++----- - drivers/net/virtio/virtio_rxtx.c | 4 +++- - 2 files changed, 8 insertions(+), 6 deletions(-) - -diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c -index d928339..2fa1587 100644 ---- a/drivers/net/virtio/virtio_ethdev.c -+++ b/drivers/net/virtio/virtio_ethdev.c -@@ -1635,24 +1635,24 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet - link.link_speed = SPEED_10G; - - if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { -- PMD_INIT_LOG(DEBUG, "Get link status from hw"); - vtpci_read_dev_config(hw, - offsetof(struct virtio_net_config, status), - &status, sizeof(status)); - if ((status & VIRTIO_NET_S_LINK_UP) == 0) { - link.link_status = 0; -- PMD_INIT_LOG(DEBUG, "Port %d is down", -- dev->data->port_id); - } else { - link.link_status = 1; -- PMD_INIT_LOG(DEBUG, "Port %d is up", -- dev->data->port_id); - } - } else { - link.link_status = 1; /* Link up */ - } - virtio_dev_atomic_write_link_status(dev, &link); - -+ /* This message is far too noisy for normal use */ -+ if (0) -+ PMD_INIT_LOG(DEBUG, "Port %d is %s\n", dev->data->port_id, -+ link.link_status ? "up" : "down"); -+ - return (old.link_status == link.link_status) ? -1 : 0; - } - -diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c -index 74b39ef..2512bce 100644 ---- a/drivers/net/virtio/virtio_rxtx.c -+++ b/drivers/net/virtio/virtio_rxtx.c -@@ -618,6 +618,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) - rxm->next = NULL; - rxm->pkt_len = (uint32_t)(len[i] - hdr_size); - rxm->data_len = (uint16_t)(len[i] - hdr_size); -+ rxm->ol_flags = 0; - - if (hw->vlan_strip) - rte_vlan_strip(rxm); -@@ -737,6 +738,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, - rxm->vlan_tci = 0; - rxm->pkt_len = (uint32_t)(len[0] - hdr_size); - rxm->data_len = (uint16_t)(len[0] - hdr_size); -+ rxm->ol_flags = 0; - - rxm->port = rxvq->port_id; - rx_pkts[nb_rx] = rxm; -@@ -838,7 +840,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) - if (unlikely(nb_pkts < 1)) - return nb_pkts; - -- PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); -+ PMD_TX_LOG(DEBUG, "%d packets to xmit\n", nb_pkts); - nb_used = VIRTQUEUE_NUSED(txvq); - - virtio_rmb(); --- -2.5.0 - diff --git a/dpdk/dpdk-2.2.0_patches/0004-mbuf-rearrange-rte_mbuf-metadata-to-suit-vpp.patch b/dpdk/dpdk-2.2.0_patches/0004-mbuf-rearrange-rte_mbuf-metadata-to-suit-vpp.patch deleted file mode 100644 index bee64dff..00000000 --- a/dpdk/dpdk-2.2.0_patches/0004-mbuf-rearrange-rte_mbuf-metadata-to-suit-vpp.patch +++ /dev/null @@ -1,83 +0,0 @@ -From b8b575a3398c480f6e02525a0933e5e057139b78 Mon Sep 17 00:00:00 2001 -From: Damjan Marion <damarion@cisco.com> -Date: Wed, 16 Dec 2015 04:25:23 +0100 -Subject: [PATCH 4/4] mbuf: rearrange rte_mbuf metadata to suit vpp - ---- - .../linuxapp/eal/include/exec-env/rte_kni_common.h | 5 +++-- - lib/librte_mbuf/rte_mbuf.h | 20 ++++++++++++-------- - 2 files changed, 15 insertions(+), 10 deletions(-) - -diff --git a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h -index bd1cc09..a68a949 100644 ---- a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h -+++ b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h -@@ -120,11 +120,12 @@ struct rte_kni_mbuf { - char pad2[4]; - uint32_t pkt_len; /**< Total pkt len: sum of all segment data_len. */ - uint16_t data_len; /**< Amount of data in segment buffer. */ -+ char pad3[8]; -+ void *next; - - /* fields on second cache line */ -- char pad3[8] __attribute__((__aligned__(RTE_CACHE_LINE_SIZE))); -+ char pad4[16] __attribute__((__aligned__(RTE_CACHE_LINE_SIZE))); - void *pool; -- void *next; - }; - - /* -diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h -index f234ac9..a1b4405 100644 ---- a/lib/librte_mbuf/rte_mbuf.h -+++ b/lib/librte_mbuf/rte_mbuf.h -@@ -734,6 +734,12 @@ struct rte_mbuf_offload; - /** - * The generic rte_mbuf, containing a packet mbuf. - */ -+/* -+ * offload in the second cache line, next in the first. Better for vpp -+ * at least as of right now. -+ * If you change this structure, you must change the user-mode -+ * version in rte_mbuf.h -+ */ - struct rte_mbuf { - MARKER cacheline0; - -@@ -786,6 +792,12 @@ struct rte_mbuf { - uint32_t pkt_len; /**< Total pkt len: sum of all segments. */ - uint16_t data_len; /**< Amount of data in segment buffer. */ - uint16_t vlan_tci; /**< VLAN Tag Control Identifier (CPU order) */ -+ uint32_t seqn; /**< Sequence number. See also rte_reorder_insert() */ -+ uint16_t vlan_tci_outer; /**< Outer VLAN Tag Control Identifier (CPU order) */ -+ struct rte_mbuf *next; /**< Next segment of scattered packet. */ -+ -+ /* second cache line - fields only used in slow path or on TX */ -+ MARKER cacheline1 __rte_cache_aligned; - - union { - uint32_t rss; /**< RSS hash result if RSS enabled */ -@@ -809,20 +821,12 @@ struct rte_mbuf { - uint32_t usr; /**< User defined tags. See rte_distributor_process() */ - } hash; /**< hash information */ - -- uint32_t seqn; /**< Sequence number. See also rte_reorder_insert() */ -- -- uint16_t vlan_tci_outer; /**< Outer VLAN Tag Control Identifier (CPU order) */ -- -- /* second cache line - fields only used in slow path or on TX */ -- MARKER cacheline1 __rte_cache_aligned; -- - union { - void *userdata; /**< Can be used for external metadata */ - uint64_t udata64; /**< Allow 8-byte userdata on 32-bit */ - }; - - struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */ -- struct rte_mbuf *next; /**< Next segment of scattered packet. */ - - /* fields to support TX offloads */ - union { --- -2.5.0 - diff --git a/dpdk/dpdk-2.2.0_patches/0005-missing-include.patch b/dpdk/dpdk-2.2.0_patches/0005-missing-include.patch deleted file mode 100644 index c6211cd4..00000000 --- a/dpdk/dpdk-2.2.0_patches/0005-missing-include.patch +++ /dev/null @@ -1,24 +0,0 @@ -From a8767269f3ee545141e83e5a5f62ff24c29248a9 Mon Sep 17 00:00:00 2001 -From: Damjan Marion <damarion@cisco.com> -Date: Wed, 16 Dec 2015 04:43:40 +0100 -Subject: [PATCH 5/5] missing include - ---- - lib/librte_eal/linuxapp/eal/eal_timer.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/lib/librte_eal/linuxapp/eal/eal_timer.c b/lib/librte_eal/linuxapp/eal/eal_timer.c -index 9ceff33..d0792be 100644 ---- a/lib/librte_eal/linuxapp/eal/eal_timer.c -+++ b/lib/librte_eal/linuxapp/eal/eal_timer.c -@@ -51,6 +51,7 @@ - #include <rte_memzone.h> - #include <rte_eal.h> - #include <rte_debug.h> -+#include <rte_lcore.h> - - #include "eal_private.h" - #include "eal_internal_cfg.h" --- -2.5.0 - diff --git a/dpdk/dpdk-2.2.0_patches/0006-Fix-a-crash-in-igb_uio-driver-when-the-device-is-rem.patch b/dpdk/dpdk-2.2.0_patches/0006-Fix-a-crash-in-igb_uio-driver-when-the-device-is-rem.patch deleted file mode 100644 index f15e4aab..00000000 --- a/dpdk/dpdk-2.2.0_patches/0006-Fix-a-crash-in-igb_uio-driver-when-the-device-is-rem.patch +++ /dev/null @@ -1,33 +0,0 @@ -From 68d23609ec0c42773043383ff2939a30830e8069 Mon Sep 17 00:00:00 2001 -From: Bud Grise <griseb@cisco.com> -Date: Tue, 2 Feb 2016 12:45:44 -0800 -Subject: [PATCH 6/8] Fix a crash in igb_uio driver when the device is removed. - -This crash happens because the device still has MSI configured, -the fix is to free the IRQ. - -Signed-off-by: Todd Foggoa (tfoggoa) <tfoggoa@cisco.com> ---- - lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c -index f5617d2..23a5cfa 100644 ---- a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c -+++ b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c -@@ -571,6 +571,12 @@ igbuio_pci_remove(struct pci_dev *dev) - udev = info->priv; - - sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp); -+ -+ if (info->irq && (info->irq != UIO_IRQ_CUSTOM)){ -+ free_irq(info->irq, info->uio_dev); -+ info->irq = UIO_IRQ_NONE; -+ } -+ - uio_unregister_device(info); - igbuio_pci_release_iomem(info); - if (udev->mode == RTE_INTR_MODE_MSIX) --- -2.2.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0007-Allow-applications-to-override-rte_delay_us.patch b/dpdk/dpdk-2.2.0_patches/0007-Allow-applications-to-override-rte_delay_us.patch deleted file mode 100644 index 4a1494ef..00000000 --- a/dpdk/dpdk-2.2.0_patches/0007-Allow-applications-to-override-rte_delay_us.patch +++ /dev/null @@ -1,43 +0,0 @@ -From 5d03f3ca8ddc7313de59e54d83912b1f3c049170 Mon Sep 17 00:00:00 2001 -From: "Todd Foggoa (tfoggoa)" <tfoggoa@cisco.com> -Date: Wed, 3 Feb 2016 08:35:27 -0800 -Subject: [PATCH 7/8] Allow applications to override rte_delay_us() - -Some applications may wish to define their own implentation of -usec delay other than the existing blocking one. The default -behavior remains unchanged. - -Signed-off-by: Todd Foggoa (tfoggoa) <tfoggoa@cisco.com> ---- - lib/librte_eal/common/eal_common_timer.c | 12 ++++++++++++ - 1 file changed, 12 insertions(+) - -diff --git a/lib/librte_eal/common/eal_common_timer.c b/lib/librte_eal/common/eal_common_timer.c -index 72371b8..5189fa5 100644 ---- a/lib/librte_eal/common/eal_common_timer.c -+++ b/lib/librte_eal/common/eal_common_timer.c -@@ -47,9 +47,21 @@ - /* The frequency of the RDTSC timer resolution */ - static uint64_t eal_tsc_resolution_hz; - -+/* Allow an override of the rte_delay_us function */ -+int rte_delay_us_override (unsigned us) __attribute__((weak)); -+ -+int -+rte_delay_us_override(__attribute__((unused)) unsigned us) -+{ -+ return 0; -+} -+ - void - rte_delay_us(unsigned us) - { -+ if (rte_delay_us_override(us)) -+ return; -+ - const uint64_t start = rte_get_timer_cycles(); - const uint64_t ticks = (uint64_t)us * rte_get_timer_hz() / 1E6; - while ((rte_get_timer_cycles() - start) < ticks) --- -2.2.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0008-Add-missing-init-of-packet_type-field.patch b/dpdk/dpdk-2.2.0_patches/0008-Add-missing-init-of-packet_type-field.patch deleted file mode 100644 index 04c1fb39..00000000 --- a/dpdk/dpdk-2.2.0_patches/0008-Add-missing-init-of-packet_type-field.patch +++ /dev/null @@ -1,70 +0,0 @@ -From 666ceb1d0c11e2ca69baacc272b7d5690d3f11a3 Mon Sep 17 00:00:00 2001 -From: Bud Grise <griseb@cisco.com> -Date: Mon, 1 Feb 2016 14:28:01 -0500 -Subject: [PATCH 8/8] Add missing init of packet_type field. - -This can cause packets to be mishandled in systems with more than -one type of driver in use. - -Signed-off-by: Todd Foggoa (tfoggoa) <tfoggoa@cisco.com> ---- - drivers/net/e1000/em_rxtx.c | 2 ++ - drivers/net/virtio/virtio_rxtx.c | 2 ++ - drivers/net/vmxnet3/vmxnet3_rxtx.c | 1 + - 3 files changed, 5 insertions(+) - -diff --git a/drivers/net/e1000/em_rxtx.c b/drivers/net/e1000/em_rxtx.c -index d8fb252..8796c8a 100644 ---- a/drivers/net/e1000/em_rxtx.c -+++ b/drivers/net/e1000/em_rxtx.c -@@ -799,6 +799,7 @@ eth_em_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - rxm->ol_flags = rx_desc_status_to_pkt_flags(status); - rxm->ol_flags = rxm->ol_flags | - rx_desc_error_to_pkt_flags(rxd.errors); -+ rxm->packet_type = RTE_PTYPE_UNKNOWN; - - /* Only valid if PKT_RX_VLAN_PKT set in pkt_flags */ - rxm->vlan_tci = rte_le_to_cpu_16(rxd.special); -@@ -1025,6 +1026,7 @@ eth_em_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - first_seg->ol_flags = rx_desc_status_to_pkt_flags(status); - first_seg->ol_flags = first_seg->ol_flags | - rx_desc_error_to_pkt_flags(rxd.errors); -+ first_seg->packet_type = RTE_PTYPE_UNKNOWN; - - /* Only valid if PKT_RX_VLAN_PKT set in pkt_flags */ - rxm->vlan_tci = rte_le_to_cpu_16(rxd.special); -diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c -index 2512bce..a74c816 100644 ---- a/drivers/net/virtio/virtio_rxtx.c -+++ b/drivers/net/virtio/virtio_rxtx.c -@@ -619,6 +619,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) - rxm->pkt_len = (uint32_t)(len[i] - hdr_size); - rxm->data_len = (uint16_t)(len[i] - hdr_size); - rxm->ol_flags = 0; -+ rxm->packet_type = RTE_PTYPE_UNKNOWN; - - if (hw->vlan_strip) - rte_vlan_strip(rxm); -@@ -739,6 +740,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, - rxm->pkt_len = (uint32_t)(len[0] - hdr_size); - rxm->data_len = (uint16_t)(len[0] - hdr_size); - rxm->ol_flags = 0; -+ rxm->packet_type = RTE_PTYPE_UNKNOWN; - - rxm->port = rxvq->port_id; - rx_pkts[nb_rx] = rxm; -diff --git a/drivers/net/vmxnet3/vmxnet3_rxtx.c b/drivers/net/vmxnet3/vmxnet3_rxtx.c -index 4de5d89..c76b230 100644 ---- a/drivers/net/vmxnet3/vmxnet3_rxtx.c -+++ b/drivers/net/vmxnet3/vmxnet3_rxtx.c -@@ -640,6 +640,7 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) - rxm->data_off = RTE_PKTMBUF_HEADROOM; - rxm->ol_flags = 0; - rxm->vlan_tci = 0; -+ rxm->packet_type = RTE_PTYPE_UNKNOWN; - - vmxnet3_rx_offload(rcd, rxm); - --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0009-Temporarily-disable-unthrottled-log-message.patch b/dpdk/dpdk-2.2.0_patches/0009-Temporarily-disable-unthrottled-log-message.patch deleted file mode 100644 index 62132b13..00000000 --- a/dpdk/dpdk-2.2.0_patches/0009-Temporarily-disable-unthrottled-log-message.patch +++ /dev/null @@ -1,26 +0,0 @@ -From da141a8f16224a97a1a4093a3293f9bb6b15fa90 Mon Sep 17 00:00:00 2001 -From: Dave Barach <dave@barachs.net> -Date: Tue, 9 Feb 2016 10:22:39 -0500 -Subject: [PATCH] Temporarily disable unthrottled log message. - -Signed-off-by: Dave Barach <dave@barachs.net> ---- - lib/librte_eal/linuxapp/eal/eal_interrupts.c | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c -index 06b26a9..8d918a4 100644 ---- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c -+++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c -@@ -711,6 +711,8 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds) - if (errno == EINTR || errno == EWOULDBLOCK) - continue; - -+ /* $$$ disable to avoid filling /var/log */ -+ if (0) - RTE_LOG(ERR, EAL, "Error reading from file " - "descriptor %d: %s\n", - events[n].data.fd, --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0010-enic-fix-dma-addr-of-outgoing-packets.patch b/dpdk/dpdk-2.2.0_patches/0010-enic-fix-dma-addr-of-outgoing-packets.patch deleted file mode 100644 index a524007c..00000000 --- a/dpdk/dpdk-2.2.0_patches/0010-enic-fix-dma-addr-of-outgoing-packets.patch +++ /dev/null @@ -1,28 +0,0 @@ -From c68ded695938b43682d4bd7dfaf40e5b267dfe3b Mon Sep 17 00:00:00 2001 -From: Yoann Desmouceaux <ydesmouc@cisco.com> -Date: Fri, 19 Feb 2016 12:49:29 +0100 -Subject: [PATCH] enic: fix dma addr of outgoing packets - -The enic PMD driver send function uses a constant offset instead of relying on the data_off in the mbuf to find the start of the packet. - -Signed-off-by: Yoann Desmouceaux <ydesmouc@cisco.com> ---- - drivers/net/enic/enic_main.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index 07a9810..f818c32 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -166,7 +166,7 @@ void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, - uint16_t mss = 0; - uint8_t vlan_tag_insert = 0; - uint64_t bus_addr = (dma_addr_t) -- (tx_pkt->buf_physaddr + RTE_PKTMBUF_HEADROOM); -+ (tx_pkt->buf_physaddr + tx_pkt->data_off); - - if (sop) { - if (ol_flags & PKT_TX_VLAN_PKT) --- -2.1.4 - diff --git a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch b/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch deleted file mode 100644 index 2aa4840d..00000000 --- a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch +++ /dev/null @@ -1,1349 +0,0 @@ -From 057358356e7d05f07ab2df37c12b1cce37a3cca9 Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Fri, 4 Mar 2016 13:09:00 -0800 -Subject: [PATCH 11/22] enic: improve Rx performance - - This is a wholesale replacement of the Enic PMD receive path in order - to improve performance and code clarity. The changes are: - - Simplify and reduce code path length of receive function. - - Put most of the fast-path receive functions in one file. - - Reduce the number of posted_index updates (pay attention to - rx_free_thresh) - - Remove the unneeded container structure around the RQ mbuf ring - - Prefetch next Mbuf and descriptors while processing the current one - - Use a lookup table for converting CQ flags to mbuf flags. - - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/Makefile | 1 + - drivers/net/enic/base/vnic_rq.c | 99 ++--------- - drivers/net/enic/base/vnic_rq.h | 147 +--------------- - drivers/net/enic/enic.h | 16 +- - drivers/net/enic/enic_ethdev.c | 27 ++- - drivers/net/enic/enic_main.c | 321 ++++++++++------------------------ - drivers/net/enic/enic_res.h | 16 +- - drivers/net/enic/enic_rx.c | 370 ++++++++++++++++++++++++++++++++++++++++ - 8 files changed, 511 insertions(+), 486 deletions(-) - create mode 100644 drivers/net/enic/enic_rx.c - -diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile -index f0ee093..f316274 100644 ---- a/drivers/net/enic/Makefile -+++ b/drivers/net/enic/Makefile -@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src - # - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c -+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c -diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c -index 1441604..cb62c5e 100644 ---- a/drivers/net/enic/base/vnic_rq.c -+++ b/drivers/net/enic/base/vnic_rq.c -@@ -35,77 +35,21 @@ - #include "vnic_dev.h" - #include "vnic_rq.h" - --static int vnic_rq_alloc_bufs(struct vnic_rq *rq) --{ -- struct vnic_rq_buf *buf; -- unsigned int i, j, count = rq->ring.desc_count; -- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); -- -- for (i = 0; i < blks; i++) { -- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC); -- if (!rq->bufs[i]) -- return -ENOMEM; -- } -- -- for (i = 0; i < blks; i++) { -- buf = rq->bufs[i]; -- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { -- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; -- buf->desc = (u8 *)rq->ring.descs + -- rq->ring.desc_size * buf->index; -- if (buf->index + 1 == count) { -- buf->next = rq->bufs[0]; -- break; -- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { -- buf->next = rq->bufs[i + 1]; -- } else { -- buf->next = buf + 1; -- buf++; -- } -- } -- } -- -- rq->to_use = rq->to_clean = rq->bufs[0]; -- -- return 0; --} -- --int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, -- unsigned int desc_size) --{ -- int mem_size = 0; -- -- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size); -- -- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) * -- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count); -- -- return mem_size; --} -- - void vnic_rq_free(struct vnic_rq *rq) - { - struct vnic_dev *vdev; -- unsigned int i; - - vdev = rq->vdev; - - vnic_dev_free_desc_ring(vdev, &rq->ring); - -- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { -- if (rq->bufs[i]) { -- kfree(rq->bufs[i]); -- rq->bufs[i] = NULL; -- } -- } -- - rq->ctrl = NULL; - } - - int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) - { -- int err; -+ int rc; - char res_name[NAME_MAX]; - static int instance; - -@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, - vnic_rq_disable(rq); - - snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index); -- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, -+ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, - rq->socket_id, res_name); -- if (err) -- return err; -- -- err = vnic_rq_alloc_bufs(rq); -- if (err) { -- vnic_rq_free(rq); -- return err; -- } -- -- return 0; -+ return rc; - } - - void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, -@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, - iowrite32(fetch_index, &rq->ctrl->fetch_index); - iowrite32(posted_index, &rq->ctrl->posted_index); - -- rq->to_use = rq->to_clean = -- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; - } - - void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, -@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, - fetch_index, fetch_index, - error_interrupt_enable, - error_interrupt_offset); -+ rq->rxst_idx = 0; -+ rq->tot_pkts = 0; - } - - void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) -@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq) - } - - void vnic_rq_clean(struct vnic_rq *rq, -- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) -+ void (*buf_clean)(struct rte_mbuf **buf)) - { -- struct vnic_rq_buf *buf; -- u32 fetch_index; -+ struct rte_mbuf **buf; -+ u32 fetch_index, i; - unsigned int count = rq->ring.desc_count; - -- buf = rq->to_clean; -- -- while (vnic_rq_desc_used(rq) > 0) { -+ buf = &rq->mbuf_ring[0]; - -- (*buf_clean)(rq, buf); -- -- buf = rq->to_clean = buf->next; -- rq->ring.desc_avail++; -+ for (i = 0; i < count; i++) { -+ (*buf_clean)(buf); -+ buf++; - } -+ rq->ring.desc_avail = count - 1; -+ rq->rx_nb_hold = 0; - - /* Use current fetch_index as the ring starting point */ - fetch_index = ioread32(&rq->ctrl->fetch_index); -@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq, - /* Hardware surprise removal: reset fetch_index */ - fetch_index = 0; - } -- rq->to_use = rq->to_clean = -- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; -+ - iowrite32(fetch_index, &rq->ctrl->posted_index); - - vnic_dev_clear_desc_ring(&rq->ring); -diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h -index 0f5c3c1..e083ccc 100644 ---- a/drivers/net/enic/base/vnic_rq.h -+++ b/drivers/net/enic/base/vnic_rq.h -@@ -66,42 +66,22 @@ struct vnic_rq_ctrl { - u32 pad10; - }; - --/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ --#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 --#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 --#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ -- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ -- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) --#define VNIC_RQ_BUF_BLK_SZ(entries) \ -- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) --#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ -- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) --#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) -- --struct vnic_rq_buf { -- struct vnic_rq_buf *next; -- dma_addr_t dma_addr; -- void *os_buf; -- unsigned int os_buf_index; -- unsigned int len; -- unsigned int index; -- void *desc; -- uint64_t wr_id; --}; -- - struct vnic_rq { - unsigned int index; -+ unsigned int posted_index; - struct vnic_dev *vdev; -- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ -+ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ - struct vnic_dev_ring ring; -- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; -- struct vnic_rq_buf *to_use; -- struct vnic_rq_buf *to_clean; -+ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ -+ unsigned int mbuf_next_idx; /* next mb to consume */ - void *os_buf_head; - unsigned int pkts_outstanding; -- -+ uint16_t rx_nb_hold; -+ uint16_t rx_free_thresh; - unsigned int socket_id; - struct rte_mempool *mp; -+ uint16_t rxst_idx; -+ uint32_t tot_pkts; - }; - - static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) -@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) - return rq->ring.desc_count - rq->ring.desc_avail - 1; - } - --static inline void *vnic_rq_next_desc(struct vnic_rq *rq) --{ -- return rq->to_use->desc; --} -- --static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) --{ -- return rq->to_use->index; --} -- --static inline void vnic_rq_post(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len, -- uint64_t wrid) --{ -- struct vnic_rq_buf *buf = rq->to_use; -- -- buf->os_buf = os_buf; -- buf->os_buf_index = os_buf_index; -- buf->dma_addr = dma_addr; -- buf->len = len; -- buf->wr_id = wrid; -- -- buf = buf->next; -- rq->to_use = buf; -- rq->ring.desc_avail--; -- -- /* Move the posted_index every nth descriptor -- */ -- --#ifndef VNIC_RQ_RETURN_RATE --#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ --#endif -- -- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &rq->ctrl->posted_index); -- } --} -- --static inline void vnic_rq_post_commit(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len) --{ -- struct vnic_rq_buf *buf = rq->to_use; -- -- buf->os_buf = os_buf; -- buf->os_buf_index = os_buf_index; -- buf->dma_addr = dma_addr; -- buf->len = len; -- -- buf = buf->next; -- rq->to_use = buf; -- rq->ring.desc_avail--; -- -- /* Move the posted_index every descriptor -- */ -- -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &rq->ctrl->posted_index); --} - --static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) --{ -- rq->ring.desc_avail += count; --} - - enum desc_return_options { - VNIC_RQ_RETURN_DESC, - VNIC_RQ_DEFER_RETURN_DESC, - }; - --static inline int vnic_rq_service(struct vnic_rq *rq, -- struct cq_desc *cq_desc, u16 completed_index, -- int desc_return, int (*buf_service)(struct vnic_rq *rq, -- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, -- int skipped, void *opaque), void *opaque) --{ -- struct vnic_rq_buf *buf; -- int skipped; -- int eop = 0; -- -- buf = rq->to_clean; -- while (1) { -- -- skipped = (buf->index != completed_index); -- -- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque)) -- eop++; -- -- if (desc_return == VNIC_RQ_RETURN_DESC) -- rq->ring.desc_avail++; -- -- rq->to_clean = buf->next; -- -- if (!skipped) -- break; -- -- buf = rq->to_clean; -- } -- return eop; --} -- - static inline int vnic_rq_fill(struct vnic_rq *rq, - int (*buf_fill)(struct vnic_rq *rq)) - { -@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq); - void vnic_rq_enable(struct vnic_rq *rq); - int vnic_rq_disable(struct vnic_rq *rq); - void vnic_rq_clean(struct vnic_rq *rq, -- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); --int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, -- unsigned int desc_size); -- -+ void (*buf_clean)(struct rte_mbuf **buf)); - #endif /* _VNIC_RQ_H_ */ -diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h -index 9e78305..8c914f5 100644 ---- a/drivers/net/enic/enic.h -+++ b/drivers/net/enic/enic.h -@@ -45,6 +45,7 @@ - #include "vnic_nic.h" - #include "vnic_rss.h" - #include "enic_res.h" -+#include "cq_enet_desc.h" - - #define DRV_NAME "enic_pmd" - #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" -@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) - return (struct enic *)eth_dev->data->dev_private; - } - -+#define RTE_LIBRTE_ENIC_ASSERT_ENABLE -+#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE -+#define ASSERT(x) do { \ -+ if (!(x)) \ -+ rte_panic("ENIC: x"); \ -+} while (0) -+#else -+#define ASSERT(x) -+#endif -+ - extern void enic_fdir_stats_get(struct enic *enic, - struct rte_eth_fdir_stats *stats); - extern int enic_fdir_add_fltr(struct enic *enic, -@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, - uint16_t ol_flags, uint16_t vlan_tag); - - extern void enic_post_wq_index(struct vnic_wq *wq); --extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, -- unsigned int budget, unsigned int *work_done); - extern int enic_probe(struct enic *enic); - extern int enic_clsf_init(struct enic *enic); - extern void enic_clsf_destroy(struct enic *enic); -+uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts); -+ - #endif /* _ENIC_H_ */ -diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c -index 2a88043..6f2ada5 100644 ---- a/drivers/net/enic/enic_ethdev.c -+++ b/drivers/net/enic/enic_ethdev.c -@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, - uint16_t queue_idx, - uint16_t nb_desc, - unsigned int socket_id, -- __rte_unused const struct rte_eth_rxconf *rx_conf, -+ const struct rte_eth_rxconf *rx_conf, - struct rte_mempool *mp) - { - int ret; -@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, - return ret; - } - -+ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh; -+ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, -+ enic->rq[queue_idx].rx_free_thresh); -+ - return enicpmd_dev_setup_intr(enic); - } - -@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, - DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM; -+ device_info->default_rxconf = (struct rte_eth_rxconf) { -+ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH -+ }; - } - - static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) -@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - return index; - } - --static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -- uint16_t nb_pkts) --{ -- struct vnic_rq *rq = (struct vnic_rq *)rx_queue; -- unsigned int work_done; -- -- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done)) -- dev_err(enic, "error in enicpmd poll\n"); -- -- return work_done; --} -- - static const struct eth_dev_ops enicpmd_eth_dev_ops = { - .dev_configure = enicpmd_dev_configure, - .dev_start = enicpmd_dev_start, -@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) - enic->port_id = eth_dev->data->port_id; - enic->rte_dev = eth_dev; - eth_dev->dev_ops = &enicpmd_eth_dev_ops; -- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts; -+ eth_dev->rx_pkt_burst = &enic_recv_pkts; - eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; - - pdev = eth_dev->pci_dev; -@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = { - * Register as the [Poll Mode] Driver of Cisco ENIC device. - */ - static int --rte_enic_pmd_init(const char *name __rte_unused, -- const char *params __rte_unused) -+rte_enic_pmd_init(__rte_unused const char *name, -+ __rte_unused const char *params) - { - ENICPMD_FUNC_TRACE(); - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index f818c32..9fff020 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -60,6 +60,17 @@ - #include "vnic_nic.h" - #include "enic_vnic_wq.h" - -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+ - static inline int enic_is_sriov_vf(struct enic *enic) - { - return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; -@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr) - return !is_mcast_addr(addr) && !is_zero_addr(addr); - } - --static inline struct rte_mbuf * --enic_rxmbuf_alloc(struct rte_mempool *mp) -+static void -+enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) - { -- struct rte_mbuf *m; -+ uint16_t i; - -- m = __rte_mbuf_raw_alloc(mp); -- __rte_mbuf_sanity_check_raw(m, 0); -- return m; -+ if (!rq || !rq->mbuf_ring) { -+ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); -+ return; -+ } -+ -+ for (i = 0; i < enic->config.rq_desc_count; i++) { -+ if (rq->mbuf_ring[i]) { -+ rte_pktmbuf_free_seg(rq->mbuf_ring[i]); -+ rq->mbuf_ring[i] = NULL; -+ } -+ } - } - -+ - void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) - { - vnic_set_hdr_split_size(enic->vdev, split_hdr_size); -@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) - } - - static void --enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf) -+enic_free_rq_buf(struct rte_mbuf **mbuf) - { -- if (!buf->os_buf) -+ if (*mbuf == NULL) - return; - -- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf); -- buf->os_buf = NULL; -+ rte_pktmbuf_free(*mbuf); -+ mbuf = NULL; - } - - void enic_init_vnic_resources(struct enic *enic) -@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic) - } - - --static int enic_rq_alloc_buf(struct vnic_rq *rq) -+static int -+enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - { -- struct enic *enic = vnic_dev_priv(rq->vdev); -+ struct rte_mbuf *mb; -+ struct rq_enet_desc *rqd = rq->ring.descs; -+ unsigned i; - dma_addr_t dma_addr; -- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); -- uint8_t type = RQ_ENET_TYPE_ONLY_SOP; -- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev); -- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp); -- struct rte_mbuf *hdr_mbuf = NULL; -- -- if (!mbuf) { -- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n"); -- return -1; -- } -- -- if (unlikely(split_hdr_size)) { -- if (vnic_rq_desc_avail(rq) < 2) { -- rte_mempool_put(mbuf->pool, mbuf); -- return -1; -- } -- hdr_mbuf = enic_rxmbuf_alloc(rq->mp); -- if (!hdr_mbuf) { -- rte_mempool_put(mbuf->pool, mbuf); -- dev_err(enic, -- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n"); -- return -1; -- } -- -- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM; -- -- hdr_mbuf->nb_segs = 2; -- hdr_mbuf->port = enic->port_id; -- hdr_mbuf->next = mbuf; -- -- dma_addr = (dma_addr_t) -- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off); -- -- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size); - -- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr, -- (unsigned int)split_hdr_size, 0 /*wrid*/); -+ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, -+ rq->ring.desc_count); - -- desc = vnic_rq_next_desc(rq); -- type = RQ_ENET_TYPE_NOT_SOP; -- } else { -- mbuf->nb_segs = 1; -- mbuf->port = enic->port_id; -- } -- -- mbuf->data_off = RTE_PKTMBUF_HEADROOM; -- mbuf->next = NULL; -- -- dma_addr = (dma_addr_t) -- (mbuf->buf_physaddr + mbuf->data_off); -- -- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len); -- -- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr, -- (unsigned int)mbuf->buf_len, 0 /*wrid*/); -- -- return 0; --} -- --static int enic_rq_indicate_buf(struct vnic_rq *rq, -- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, -- int skipped, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(rq->vdev); -- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque; -- struct rte_mbuf *rx_pkt = NULL; -- struct rte_mbuf *hdr_rx_pkt = NULL; -- -- u8 type, color, eop, sop, ingress_port, vlan_stripped; -- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; -- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; -- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; -- u8 packet_error; -- u16 q_number, completed_index, bytes_written, vlan_tci, checksum; -- u32 rss_hash; -- -- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, -- &type, &color, &q_number, &completed_index, -- &ingress_port, &fcoe, &eop, &sop, &rss_type, -- &csum_not_calc, &rss_hash, &bytes_written, -- &packet_error, &vlan_stripped, &vlan_tci, &checksum, -- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, -- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, -- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, -- &fcs_ok); -- -- rx_pkt = (struct rte_mbuf *)buf->os_buf; -- buf->os_buf = NULL; -- -- if (unlikely(packet_error)) { -- dev_err(enic, "packet error\n"); -- rx_pkt->data_len = 0; -- return 0; -- } -- -- if (unlikely(skipped)) { -- rx_pkt->data_len = 0; -- return 0; -- } -- -- if (likely(!vnic_get_hdr_split_size(enic->vdev))) { -- /* No header split configured */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- -- if (ipv4) { -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && (!tcp_udp_csum_ok)) -- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; -- } else { -- /* Header split */ -- if (sop && !eop) { -- /* This piece is header */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- } else { -- if (sop && eop) { -- /* The packet is smaller than split_hdr_size */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- if (ipv4) { -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- rx_pkt->ol_flags |= -- PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && -- (!tcp_udp_csum_ok)) -- rx_pkt->ol_flags |= -- PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; -- } else { -- /* Payload */ -- hdr_rx_pkt = *rx_pkt_bucket; -- hdr_rx_pkt->pkt_len += bytes_written; -- if (ipv4) { -- hdr_rx_pkt->packet_type = -- RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- hdr_rx_pkt->ol_flags |= -- PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && -- (!tcp_udp_csum_ok)) -- hdr_rx_pkt->ol_flags |= -- PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- hdr_rx_pkt->packet_type = -- RTE_PTYPE_L3_IPV6; -- } -+ for (i = 0; i < rq->ring.desc_count; i++, rqd++) { -+ mb = rte_rxmbuf_alloc(rq->mp); -+ if (mb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed queue_id=%u", -+ (unsigned)rq->index); -+ return -ENOMEM; - } -- } - -- rx_pkt->data_len = bytes_written; -+ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); - -- if (rss_hash) { -- rx_pkt->ol_flags |= PKT_RX_RSS_HASH; -- rx_pkt->hash.rss = rss_hash; -+ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, -+ mb->buf_len); -+ rq->mbuf_ring[i] = mb; - } - -- if (vlan_tci) { -- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT; -- rx_pkt->vlan_tci = vlan_tci; -- } -+ /* make sure all prior writes are complete before doing the PIO write */ -+ rte_rmb(); - -- return eop; --} -+ /* Post all but the last 2 cache lines' worth of descriptors */ -+ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE -+ / sizeof(struct rq_enet_desc)); -+ rq->rx_nb_hold = 0; - --static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, -- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(vdev); -- -- return vnic_rq_service(&enic->rq[q_number], cq_desc, -- completed_index, VNIC_RQ_RETURN_DESC, -- enic_rq_indicate_buf, opaque); -- --} -+ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", -+ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ rte_rmb(); - --int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, -- unsigned int budget, unsigned int *work_done) --{ -- struct enic *enic = vnic_dev_priv(rq->vdev); -- unsigned int cq = enic_cq_rq(enic, rq->index); -- int err = 0; -- -- *work_done = vnic_cq_service(&enic->cq[cq], -- budget, enic_rq_service, (void *)rx_pkts); -- -- if (*work_done) { -- vnic_rq_fill(rq, enic_rq_alloc_buf); -+ return 0; - -- /* Need at least one buffer on ring to get going */ -- if (vnic_rq_desc_used(rq) == 0) { -- dev_err(enic, "Unable to alloc receive buffers\n"); -- err = -1; -- } -- } -- return err; - } - - static void * -@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle, - int enic_enable(struct enic *enic) - { - unsigned int index; -+ int err; - struct rte_eth_dev *eth_dev = enic->rte_dev; - - eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); -@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic) - dev_warning(enic, "Init of hash table for clsf failed."\ - "Flow director feature will not work\n"); - -- /* Fill RQ bufs */ - for (index = 0; index < enic->rq_count; index++) { -- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf); -- -- /* Need at least one buffer on ring to get going -- */ -- if (vnic_rq_desc_used(&enic->rq[index]) == 0) { -- dev_err(enic, "Unable to alloc receive buffers\n"); -- return -1; -+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); -+ if (err) { -+ dev_err(enic, "Failed to alloc RX queue mbufs\n"); -+ return err; - } - } - -@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq) - struct vnic_rq *rq = (struct vnic_rq *)rxq; - struct enic *enic = vnic_dev_priv(rq->vdev); - -+ enic_rxmbuf_queue_release(enic, rq); -+ rte_free(rq->mbuf_ring); -+ rq->mbuf_ring = NULL; - vnic_rq_free(rq); - vnic_cq_free(&enic->cq[rq->index]); - } -@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - unsigned int socket_id, struct rte_mempool *mp, - uint16_t nb_desc) - { -- int err; -+ int rc; - struct vnic_rq *rq = &enic->rq[queue_idx]; - - rq->socket_id = socket_id; -@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - } - - /* Allocate queue resources */ -- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx, -- enic->config.rq_desc_count, -- sizeof(struct rq_enet_desc)); -- if (err) { -+ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, -+ enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); -+ if (rc) { - dev_err(enic, "error in allocation of rq\n"); -- return err; -+ goto err_exit; - } - -- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, -+ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, - socket_id, enic->config.rq_desc_count, - sizeof(struct cq_enet_rq_desc)); -- if (err) { -- vnic_rq_free(rq); -+ if (rc) { - dev_err(enic, "error in allocation of cq for rq\n"); -+ goto err_free_rq_exit; - } - -- return err; -+ /* Allocate the mbuf ring */ -+ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", -+ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, -+ RTE_CACHE_LINE_SIZE, rq->socket_id); -+ -+ if (rq->mbuf_ring != NULL) -+ return 0; -+ -+ /* cleanup on error */ -+ vnic_cq_free(&enic->cq[queue_idx]); -+err_free_rq_exit: -+ vnic_rq_free(rq); -+err_exit: -+ return -ENOMEM; - } - - void enic_free_wq(void *txq) -@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic) - - for (i = 0; i < enic->wq_count; i++) - vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); -+ - for (i = 0; i < enic->rq_count; i++) - vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); - for (i = 0; i < enic->cq_count; i++) -@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic) - - /* Set ingress vlan rewrite mode before vnic initialization */ - err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, -- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN); -+ IG_VLAN_REWRITE_MODE_PASS_THRU); - if (err) { - dev_err(enic, - "Failed to set ingress vlan rewrite mode, aborting.\n"); -diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h -index 49f7e22..33f2e84 100644 ---- a/drivers/net/enic/enic_res.h -+++ b/drivers/net/enic/enic_res.h -@@ -52,6 +52,7 @@ - #define ENIC_UNICAST_PERFECT_FILTERS 32 - - #define ENIC_NON_TSO_MAX_DESC 16 -+#define ENIC_DEFAULT_RX_FREE_THRESH 32 - - #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) - -@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, - WQ_ENET_OFFLOAD_MODE_TSO, - eop, 1 /* SOP */, eop, loopback); - } --static inline void enic_queue_rq_desc(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len) --{ -- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); -- u64 wrid = 0; -- u8 type = os_buf_index ? -- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP; -- -- rq_enet_desc_enc(desc, -- (u64)dma_addr | VNIC_PADDR_TARGET, -- type, (u16)len); -- -- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid); --} - - struct enic; - -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -new file mode 100644 -index 0000000..945a60f ---- /dev/null -+++ b/drivers/net/enic/enic_rx.c -@@ -0,0 +1,370 @@ -+/* -+ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. -+ * Copyright 2007 Nuova Systems, Inc. All rights reserved. -+ * -+ * Copyright (c) 2014, Cisco Systems, Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ * -+ */ -+ -+#include <rte_mbuf.h> -+#include <rte_ethdev.h> -+#include <rte_prefetch.h> -+ -+#include "enic_compat.h" -+#include "rq_enet_desc.h" -+#include "enic.h" -+ -+#define RTE_PMD_USE_PREFETCH -+ -+#ifdef RTE_PMD_USE_PREFETCH -+/* -+ * Prefetch a cache line into all cache levels. -+ */ -+#define rte_enic_prefetch(p) rte_prefetch0(p) -+#else -+#define rte_enic_prefetch(p) do {} while (0) -+#endif -+ -+#ifdef RTE_PMD_PACKET_PREFETCH -+#define rte_packet_prefetch(p) rte_prefetch1(p) -+#else -+#define rte_packet_prefetch(p) do {} while (0) -+#endif -+ -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) -+{ -+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) -+{ -+ return(le16_to_cpu(crd->bytes_written_flags) & -+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_packet_error(uint16_t bwflags) -+{ -+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == -+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_eop(uint16_t ciflags) -+{ -+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) -+ == CQ_ENET_RQ_DESC_FLAGS_EOP; -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) -+{ -+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> -+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); -+} -+ -+static inline uint32_t -+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) -+{ -+ return le32_to_cpu(cqrd->rss_hash); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_FCS_OK); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) -+{ -+ return le16_to_cpu(cqrd->vlan); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ return le16_to_cpu(cqrd->bytes_written_flags) & -+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; -+} -+ -+static inline uint64_t -+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t bwflags; -+ uint64_t pkt_err_flags = 0; -+ -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ /* Check for packet error. Can't be more specific than MAC error */ -+ if (enic_cq_rx_desc_packet_error(bwflags)) { -+ pkt_err_flags |= PKT_RX_MAC_ERR; -+ } -+ -+ /* Check for bad FCS. MAC error isn't quite, but no other choice */ -+ if (!enic_cq_rx_desc_fcs_ok(cqrd)) { -+ pkt_err_flags |= PKT_RX_MAC_ERR; -+ } -+ return pkt_err_flags; -+} -+ -+/* -+ * Lookup table to translate RX CQ flags to mbuf flags. -+ */ -+static inline uint32_t -+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint8_t cqrd_flags = cqrd->flags; -+ static const uint32_t cq_type_table[128] __rte_cache_aligned = { -+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, -+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_UDP, -+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_TCP, -+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_FRAG, -+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, -+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_UDP, -+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_TCP, -+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_FRAG, -+ /* All others reserved */ -+ }; -+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT -+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 -+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; -+ return cq_type_table[cqrd_flags]; -+} -+ -+static inline void -+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t ciflags, bwflags, pkt_flags = 0; -+ ciflags = enic_cq_rx_desc_ciflags(cqrd); -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ ASSERT(mbuf->ol_flags == 0); -+ -+ /* flags are meaningless if !EOP */ -+ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -+ goto mbuf_flags_done; -+ -+ /* VLAN stripping */ -+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { -+ pkt_flags |= PKT_RX_VLAN_PKT; -+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); -+ } else { -+ mbuf->vlan_tci = 0; -+ } -+ -+ /* RSS flag */ -+ if (enic_cq_rx_desc_rss_type(cqrd)) { -+ pkt_flags |= PKT_RX_RSS_HASH; -+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); -+ } -+ -+ /* checksum flags */ -+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && -+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { -+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_IP_CKSUM_BAD; -+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { -+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_L4_CKSUM_BAD; -+ } -+ } -+ -+ mbuf_flags_done: -+ mbuf->ol_flags = pkt_flags; -+} -+ -+static inline uint32_t -+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -+{ -+ uint32_t d = i0 + i1; -+ ASSERT(i0 < n_descriptors); -+ ASSERT(i1 < n_descriptors); -+ d -= (d >= n_descriptors) ? n_descriptors : 0; -+ return d; -+} -+ -+ -+uint16_t -+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts) -+{ -+ struct vnic_rq *rq = rx_queue; -+ struct enic *enic = vnic_dev_priv(rq->vdev); -+ unsigned int rx_id; -+ struct rte_mbuf *nmb, *rxmb; -+ uint16_t nb_rx = 0; -+ uint16_t nb_hold; -+ struct vnic_cq *cq; -+ volatile struct cq_desc *cqd_ptr; -+ uint8_t color; -+ -+ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ -+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -+ -+ while (nb_rx < nb_pkts) { -+ uint16_t rx_pkt_len; -+ volatile struct rq_enet_desc *rqd_ptr; -+ dma_addr_t dma_addr; -+ struct cq_desc cqd; -+ uint64_t ol_err_flags; -+ -+ /* Check for pkts available */ -+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -+ & CQ_DESC_COLOR_MASK; -+ if (color == cq->last_color) -+ break; -+ -+ /* Get the cq descriptor and rq pointer */ -+ cqd = *cqd_ptr; -+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -+ -+ /* allocate a new mbuf */ -+ nmb = rte_rxmbuf_alloc(rq->mp); -+ if (nmb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", -+ enic->port_id, (unsigned)rq->index); -+ rte_eth_devices[enic->port_id]. -+ data->rx_mbuf_alloc_failed++; -+ break; -+ } -+ -+ /* Check for FCS or packet errors */ -+ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); -+ if (ol_err_flags == 0) -+ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -+ else -+ rx_pkt_len = 0; -+ -+ /* Get the mbuf to return and replace with one just allocated */ -+ rxmb = rq->mbuf_ring[rx_id]; -+ rq->mbuf_ring[rx_id] = nmb; -+ -+ /* Increment cqd, rqd, mbuf_table index */ -+ rx_id++; -+ if (unlikely(rx_id == rq->ring.desc_count)) { -+ rx_id = 0; -+ cq->last_color = cq->last_color ? 0 : 1; -+ } -+ -+ /* Prefetch next mbuf & desc while processing current one */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ rte_enic_prefetch(cqd_ptr); -+ rte_enic_prefetch(rq->mbuf_ring[rx_id]); -+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -+ + rx_id); -+ -+ /* Push descriptor for newly allocated mbuf */ -+ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); -+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); -+ -+ /* Fill in the rest of the mbuf */ -+ rxmb->data_off = RTE_PKTMBUF_HEADROOM; -+ rxmb->nb_segs = 1; -+ rxmb->next = NULL; -+ rxmb->pkt_len = rx_pkt_len; -+ rxmb->data_len = rx_pkt_len; -+ rxmb->port = enic->port_id; -+ rxmb->ol_flags = ol_err_flags; -+ if (!ol_err_flags) -+ enic_cq_rx_to_pkt_flags(&cqd, rxmb); -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -+ -+ /* prefetch mbuf data for caller */ -+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -+ RTE_PKTMBUF_HEADROOM)); -+ -+ /* store the mbuf address into the next entry of the array */ -+ rx_pkts[nb_rx++] = rxmb; -+ } -+ -+ nb_hold += nb_rx; -+ cq->to_clean = rx_id; -+ -+ if (nb_hold > rq->rx_free_thresh) { -+ rq->posted_index = enic_ring_add(rq->ring.desc_count, -+ rq->posted_index, nb_hold); -+ nb_hold = 0; -+ rte_mb(); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ } -+ -+ rq->rx_nb_hold = nb_hold; -+ -+ return nb_rx; -+} --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0012-enic-fix-last-packet-not-being-sent.patch b/dpdk/dpdk-2.2.0_patches/0012-enic-fix-last-packet-not-being-sent.patch deleted file mode 100644 index 218a42f6..00000000 --- a/dpdk/dpdk-2.2.0_patches/0012-enic-fix-last-packet-not-being-sent.patch +++ /dev/null @@ -1,39 +0,0 @@ -From a31a1dbdf5e1ff46d04f50fea02e83453b84652c Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Tue, 8 Mar 2016 10:49:07 -0800 -Subject: [PATCH 12/22] enic: fix last packet not being sent - - The last packet of the tx burst function array was not being - emitted until the subsequent call. The nic descriptor index - was being set to the current tx descriptor instead of one past - the descriptor as required by the nic. - - Fixes: d739ba4c6abf ("enic: improve Tx packet rate") - - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/base/enic_vnic_wq.h | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h -index e3ea574..b019109 100644 ---- a/drivers/net/enic/base/enic_vnic_wq.h -+++ b/drivers/net/enic/base/enic_vnic_wq.h -@@ -69,11 +69,11 @@ static inline void enic_vnic_post_wq(struct vnic_wq *wq, - buf->wr_id = wrid; - - buf = buf->next; -- if (cq_entry) -- enic_vnic_post_wq_index(wq); -+ wq->ring.desc_avail -= desc_skip_cnt; - wq->to_use = buf; - -- wq->ring.desc_avail -= desc_skip_cnt; -+ if (cq_entry) -+ enic_vnic_post_wq_index(wq); - } - - #endif /* _ENIC_VNIC_WQ_H_ */ --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0013-enic-add-missing-newline-to-print-statements.patch b/dpdk/dpdk-2.2.0_patches/0013-enic-add-missing-newline-to-print-statements.patch deleted file mode 100644 index 97a424f1..00000000 --- a/dpdk/dpdk-2.2.0_patches/0013-enic-add-missing-newline-to-print-statements.patch +++ /dev/null @@ -1,47 +0,0 @@ -From a1ed99bc24f88f061d75eed0db84dc6355855dd2 Mon Sep 17 00:00:00 2001 -From: Nelson Escobar <neescoba@cisco.com> -Date: Thu, 17 Mar 2016 15:48:13 -0700 -Subject: [PATCH 13/22] enic: add missing newline to print statements - - Add the missing '\n' character to the end of a few print statements. - - Fixes: fefed3d1e62c ("enic: new driver") - - Signed-off-by: Nelson Escobar <neescoba@cisco.com> - Acked-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_main.c | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index 9fff020..e30672c 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -342,13 +342,13 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - unsigned i; - dma_addr_t dma_addr; - -- dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, -+ dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index, - rq->ring.desc_count); - - for (i = 0; i < rq->ring.desc_count; i++, rqd++) { - mb = rte_rxmbuf_alloc(rq->mp); - if (mb == NULL) { -- dev_err(enic, "RX mbuf alloc failed queue_id=%u", -+ dev_err(enic, "RX mbuf alloc failed queue_id=%u\n", - (unsigned)rq->index); - return -ENOMEM; - } -@@ -388,7 +388,7 @@ enic_alloc_consistent(__rte_unused void *priv, size_t size, - rz = rte_memzone_reserve_aligned((const char *)name, - size, SOCKET_ID_ANY, 0, ENIC_ALIGN); - if (!rz) { -- pr_err("%s : Failed to allocate memory requested for %s", -+ pr_err("%s : Failed to allocate memory requested for %s\n", - __func__, name); - return NULL; - } --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0014-vmxnet3-support-jumbo-frames.patch b/dpdk/dpdk-2.2.0_patches/0014-vmxnet3-support-jumbo-frames.patch deleted file mode 100644 index cae055eb..00000000 --- a/dpdk/dpdk-2.2.0_patches/0014-vmxnet3-support-jumbo-frames.patch +++ /dev/null @@ -1,171 +0,0 @@ -From fef2b892245d5a2f3c68d2e03a6c5f2a40205cf7 Mon Sep 17 00:00:00 2001 -From: Steve Shin <jonshin@cisco.com> -Date: Wed, 23 Mar 2016 09:54:54 -0700 -Subject: [PATCH 14/22] vmxnet3: support jumbo frames - ---- - drivers/net/vmxnet3/vmxnet3_ethdev.c | 3 +- - drivers/net/vmxnet3/vmxnet3_ring.h | 2 + - drivers/net/vmxnet3/vmxnet3_rxtx.c | 77 ++++++++++++++++++++++-------------- - 3 files changed, 52 insertions(+), 30 deletions(-) - -diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c -index c363bf6..b78acd4 100644 ---- a/drivers/net/vmxnet3/vmxnet3_ethdev.c -+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c -@@ -425,6 +425,7 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) - { - struct rte_eth_conf port_conf = dev->data->dev_conf; - struct vmxnet3_hw *hw = dev->data->dev_private; -+ uint32_t mtu = dev->data->mtu; - Vmxnet3_DriverShared *shared = hw->shared; - Vmxnet3_DSDevRead *devRead = &shared->devRead; - uint32_t *mac_ptr; -@@ -442,7 +443,7 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) - devRead->misc.driverInfo.vmxnet3RevSpt = 1; - devRead->misc.driverInfo.uptVerSpt = 1; - -- devRead->misc.mtu = rte_le_to_cpu_32(dev->data->mtu); -+ devRead->misc.mtu = rte_le_to_cpu_32(mtu); - devRead->misc.queueDescPA = hw->queueDescPA; - devRead->misc.queueDescLen = hw->queue_desc_len; - devRead->misc.numTxQueues = hw->num_tx_queues; -diff --git a/drivers/net/vmxnet3/vmxnet3_ring.h b/drivers/net/vmxnet3/vmxnet3_ring.h -index 612487e..b1582f8 100644 ---- a/drivers/net/vmxnet3/vmxnet3_ring.h -+++ b/drivers/net/vmxnet3/vmxnet3_ring.h -@@ -171,6 +171,8 @@ typedef struct vmxnet3_rx_queue { - uint32_t qid1; - uint32_t qid2; - Vmxnet3_RxQueueDesc *shared; -+ struct rte_mbuf *start_seg; -+ struct rte_mbuf *last_seg; - struct vmxnet3_rxq_stats stats; - bool stopped; - uint16_t queue_id; /**< Device RX queue index. */ -diff --git a/drivers/net/vmxnet3/vmxnet3_rxtx.c b/drivers/net/vmxnet3/vmxnet3_rxtx.c -index c76b230..59b6db8 100644 ---- a/drivers/net/vmxnet3/vmxnet3_rxtx.c -+++ b/drivers/net/vmxnet3/vmxnet3_rxtx.c -@@ -547,7 +547,6 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) - vmxnet3_rx_queue_t *rxq; - Vmxnet3_RxCompDesc *rcd; - vmxnet3_buf_info_t *rbi; -- Vmxnet3_RxDesc *rxd; - struct rte_mbuf *rxm = NULL; - struct vmxnet3_hw *hw; - -@@ -572,37 +571,16 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) - - idx = rcd->rxdIdx; - ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1); -- rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx; - rbi = rxq->cmd_ring[ring_idx].buf_info + idx; - -- if (unlikely(rcd->sop != 1 || rcd->eop != 1)) { -- rte_pktmbuf_free_seg(rbi->m); -- PMD_RX_LOG(DEBUG, "Packet spread across multiple buffers\n)"); -- goto rcd_done; -- } -- - PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx); - -+ #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER -+ Vmxnet3_RxDesc *rxd -+ = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx; - VMXNET3_ASSERT(rcd->len <= rxd->len); - VMXNET3_ASSERT(rbi->m); -- -- if (unlikely(rcd->len == 0)) { -- PMD_RX_LOG(DEBUG, "Rx buf was skipped. rxring[%d][%d]\n)", -- ring_idx, idx); -- VMXNET3_ASSERT(rcd->sop && rcd->eop); -- rte_pktmbuf_free_seg(rbi->m); -- goto rcd_done; -- } -- -- /* Assuming a packet is coming in a single packet buffer */ -- if (unlikely(rxd->btype != VMXNET3_RXD_BTYPE_HEAD)) { -- PMD_RX_LOG(DEBUG, -- "Alert : Misbehaving device, incorrect " -- " buffer type used. iPacket dropped."); -- rte_pktmbuf_free_seg(rbi->m); -- goto rcd_done; -- } -- VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD); -+ #endif - - /* Get the packet buffer pointer from buf_info */ - rxm = rbi->m; -@@ -615,7 +593,7 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) - rxq->cmd_ring[ring_idx].next2comp = idx; - - /* For RCD with EOP set, check if there is frame error */ -- if (unlikely(rcd->err)) { -+ if (unlikely(rcd->eop && rcd->err)) { - rxq->stats.drop_total++; - rxq->stats.drop_err++; - -@@ -642,9 +620,49 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) - rxm->vlan_tci = 0; - rxm->packet_type = RTE_PTYPE_UNKNOWN; - -- vmxnet3_rx_offload(rcd, rxm); -+ /* -+ * If this is the first buffer of the received packet, -+ * set the pointer to the first mbuf of the packet -+ * Otherwise, update the total length and the number of segments -+ * of the current scattered packet, and update the pointer to -+ * the last mbuf of the current packet. -+ */ -+ if (rcd->sop) { -+#ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER -+ VMXNET3_ASSERT(!rxq->start_seg); -+ VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD); -+#endif -+ -+ if (unlikely(rcd->len == 0)) { -+ PMD_RX_LOG(DEBUG, -+ "Rx buf was skipped. rxring[%d][%d])", -+ ring_idx, idx); -+ rte_pktmbuf_free_seg(rbi->m); -+ goto rcd_done; -+ } -+ -+ rxq->start_seg = rxm; -+ vmxnet3_rx_offload(rcd, rxm); -+ } else { -+ struct rte_mbuf *start = rxq->start_seg; -+ -+#ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER -+ VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY); -+ VMXNET3_ASSERT(start != NULL); -+#endif -+ -+ start->pkt_len += rxm->data_len; -+ start->nb_segs++; -+ -+ rxq->last_seg->next = rxm; -+ } -+ rxq->last_seg = rxm; -+ -+ if (rcd->eop) { -+ rx_pkts[nb_rx++] = rxq->start_seg; -+ rxq->start_seg = NULL; -+ } - -- rx_pkts[nb_rx++] = rxm; - rcd_done: - rxq->cmd_ring[ring_idx].next2comp = idx; - VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size); -@@ -945,6 +963,7 @@ vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev) - } - } - rxq->stopped = FALSE; -+ rxq->start_seg = NULL; - } - - for (i = 0; i < dev->data->nb_tx_queues; i++) { --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0015-enic-fix-crash-when-allocating-too-many-queues.patch b/dpdk/dpdk-2.2.0_patches/0015-enic-fix-crash-when-allocating-too-many-queues.patch deleted file mode 100644 index 2c0e65d3..00000000 --- a/dpdk/dpdk-2.2.0_patches/0015-enic-fix-crash-when-allocating-too-many-queues.patch +++ /dev/null @@ -1,51 +0,0 @@ -From 7a7fa2891df4ec4af0c34f3bbd203e1376e83951 Mon Sep 17 00:00:00 2001 -From: Nelson Escobar <neescoba@cisco.com> -Date: Thu, 17 Mar 2016 15:49:58 -0700 -Subject: [PATCH 15/22] enic: fix crash when allocating too many queues - - Add checks to make sure we don't try to allocate more tx or rx queues - than we support. - - Fixes: fefed3d1e62c ("enic: new driver") - - Signed-off-by: Nelson Escobar <neescoba@cisco.com> - Reviewed-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_ethdev.c | 14 ++++++++++++++ - 1 file changed, 14 insertions(+) - -diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c -index 6f2ada5..6c3c734 100644 ---- a/drivers/net/enic/enic_ethdev.c -+++ b/drivers/net/enic/enic_ethdev.c -@@ -174,6 +174,13 @@ static int enicpmd_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, - struct enic *enic = pmd_priv(eth_dev); - - ENICPMD_FUNC_TRACE(); -+ if (queue_idx >= ENIC_WQ_MAX) { -+ dev_err(enic, -+ "Max number of TX queues exceeded. Max is %d\n", -+ ENIC_WQ_MAX); -+ return -EINVAL; -+ } -+ - eth_dev->data->tx_queues[queue_idx] = (void *)&enic->wq[queue_idx]; - - ret = enic_alloc_wq(enic, queue_idx, socket_id, nb_desc); -@@ -262,6 +269,13 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, - struct enic *enic = pmd_priv(eth_dev); - - ENICPMD_FUNC_TRACE(); -+ if (queue_idx >= ENIC_RQ_MAX) { -+ dev_err(enic, -+ "Max number of RX queues exceeded. Max is %d\n", -+ ENIC_RQ_MAX); -+ return -EINVAL; -+ } -+ - eth_dev->data->rx_queues[queue_idx] = (void *)&enic->rq[queue_idx]; - - ret = enic_alloc_rq(enic, queue_idx, socket_id, mp, nb_desc); --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0016-enic-fix-mbuf-flags-on-Rx.patch b/dpdk/dpdk-2.2.0_patches/0016-enic-fix-mbuf-flags-on-Rx.patch deleted file mode 100644 index 895d5715..00000000 --- a/dpdk/dpdk-2.2.0_patches/0016-enic-fix-mbuf-flags-on-Rx.patch +++ /dev/null @@ -1,43 +0,0 @@ -From 3ffb9431d6ba34dbcffab5cff4c060d5dca167e1 Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Thu, 17 Mar 2016 15:57:05 -0700 -Subject: [PATCH 16/22] enic: fix mbuf flags on Rx - - In the receive path, the function to set mbuf ol_flags used the - mbuf packet_type before it was set. - - Fixes: 947d860c821f ("enic: improve Rx performance") - - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_rx.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -index 945a60f..59ebaa4 100644 ---- a/drivers/net/enic/enic_rx.c -+++ b/drivers/net/enic/enic_rx.c -@@ -210,7 +210,7 @@ enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) - ciflags = enic_cq_rx_desc_ciflags(cqrd); - bwflags = enic_cq_rx_desc_bwflags(cqrd); - -- ASSERT(mbuf->ol_flags == 0); -+ mbuf->ol_flags = 0; - - /* flags are meaningless if !EOP */ - if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -@@ -340,10 +340,10 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - rxmb->pkt_len = rx_pkt_len; - rxmb->data_len = rx_pkt_len; - rxmb->port = enic->port_id; -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); - rxmb->ol_flags = ol_err_flags; - if (!ol_err_flags) - enic_cq_rx_to_pkt_flags(&cqd, rxmb); -- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); - - /* prefetch mbuf data for caller */ - rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0017-enic-fix-error-packets-handling.patch b/dpdk/dpdk-2.2.0_patches/0017-enic-fix-error-packets-handling.patch deleted file mode 100644 index 28c05e88..00000000 --- a/dpdk/dpdk-2.2.0_patches/0017-enic-fix-error-packets-handling.patch +++ /dev/null @@ -1,117 +0,0 @@ -From 678e5952cf49bb66c2d697581a70dc8c7d703e8f Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Thu, 17 Mar 2016 15:57:06 -0700 -Subject: [PATCH 17/22] enic: fix error packets handling - - If the packet_error bit in the completion descriptor is set, the - remainder of the descriptor and data are invalid. PKT_RX_MAC_ERR - was set in the mbuf->ol_flags if packet_error was set and used - later to indicate an error packet. But since PKT_RX_MAC_ERR is - defined as 0, mbuf flags and packet types and length were being - misinterpreted. - - Make the function enic_cq_rx_to_pkt_err_flags() return true for error - packets and use the return value instead of mbuf->ol_flags to indicate - error packets. Also remove warning for error packets and rely on - rx_error stats. - - Fixes: 947d860c821f ("enic: improve Rx performance") - - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_rx.c | 43 ++++++++++++++++++------------------------- - 1 file changed, 18 insertions(+), 25 deletions(-) - -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -index 59ebaa4..817a891 100644 ---- a/drivers/net/enic/enic_rx.c -+++ b/drivers/net/enic/enic_rx.c -@@ -129,13 +129,6 @@ enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) - return le32_to_cpu(cqrd->rss_hash); - } - --static inline uint8_t --enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) --{ -- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == -- CQ_ENET_RQ_DESC_FLAGS_FCS_OK); --} -- - static inline uint16_t - enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) - { -@@ -150,25 +143,21 @@ enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) - CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; - } - --static inline uint64_t --enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) -+static inline uint8_t -+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) - { - struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; - uint16_t bwflags; -+ int ret = 0; - uint64_t pkt_err_flags = 0; - - bwflags = enic_cq_rx_desc_bwflags(cqrd); -- -- /* Check for packet error. Can't be more specific than MAC error */ -- if (enic_cq_rx_desc_packet_error(bwflags)) { -- pkt_err_flags |= PKT_RX_MAC_ERR; -- } -- -- /* Check for bad FCS. MAC error isn't quite, but no other choice */ -- if (!enic_cq_rx_desc_fcs_ok(cqrd)) { -- pkt_err_flags |= PKT_RX_MAC_ERR; -+ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { -+ pkt_err_flags = PKT_RX_MAC_ERR; -+ ret = 1; - } -- return pkt_err_flags; -+ *pkt_err_flags_out = pkt_err_flags; -+ return ret; - } - - /* -@@ -282,6 +271,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - dma_addr_t dma_addr; - struct cq_desc cqd; - uint64_t ol_err_flags; -+ uint8_t packet_error; - - /* Check for pkts available */ - color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -@@ -303,9 +293,9 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - break; - } - -- /* Check for FCS or packet errors */ -- ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); -- if (ol_err_flags == 0) -+ /* A packet error means descriptor and data are untrusted */ -+ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); -+ if (!packet_error) - rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); - else - rx_pkt_len = 0; -@@ -340,10 +330,13 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - rxmb->pkt_len = rx_pkt_len; - rxmb->data_len = rx_pkt_len; - rxmb->port = enic->port_id; -- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -- rxmb->ol_flags = ol_err_flags; -- if (!ol_err_flags) -+ if (!packet_error) { -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); - enic_cq_rx_to_pkt_flags(&cqd, rxmb); -+ } else { -+ rxmb->packet_type = 0; -+ rxmb->ol_flags = 0; -+ } - - /* prefetch mbuf data for caller */ - rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0018-enic-remove-packet-error-conditional.patch b/dpdk/dpdk-2.2.0_patches/0018-enic-remove-packet-error-conditional.patch deleted file mode 100644 index 3f29f6ea..00000000 --- a/dpdk/dpdk-2.2.0_patches/0018-enic-remove-packet-error-conditional.patch +++ /dev/null @@ -1,58 +0,0 @@ -From 2fa6a45ff9f9fb3108b09403e32393416bd0a732 Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Thu, 17 Mar 2016 15:57:07 -0700 -Subject: [PATCH 18/22] enic: remove packet error conditional - - small cleanup to remove conditional. - - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_rx.c | 10 +++------- - 1 file changed, 3 insertions(+), 7 deletions(-) - -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -index 817a891..232987a 100644 ---- a/drivers/net/enic/enic_rx.c -+++ b/drivers/net/enic/enic_rx.c -@@ -266,7 +266,6 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - nb_hold = rq->rx_nb_hold; /* mbufs held by software */ - - while (nb_rx < nb_pkts) { -- uint16_t rx_pkt_len; - volatile struct rq_enet_desc *rqd_ptr; - dma_addr_t dma_addr; - struct cq_desc cqd; -@@ -295,10 +294,6 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - - /* A packet error means descriptor and data are untrusted */ - packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); -- if (!packet_error) -- rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -- else -- rx_pkt_len = 0; - - /* Get the mbuf to return and replace with one just allocated */ - rxmb = rq->mbuf_ring[rx_id]; -@@ -327,16 +322,17 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - rxmb->data_off = RTE_PKTMBUF_HEADROOM; - rxmb->nb_segs = 1; - rxmb->next = NULL; -- rxmb->pkt_len = rx_pkt_len; -- rxmb->data_len = rx_pkt_len; - rxmb->port = enic->port_id; - if (!packet_error) { -+ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); - rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); - enic_cq_rx_to_pkt_flags(&cqd, rxmb); - } else { -+ rxmb->pkt_len = 0; - rxmb->packet_type = 0; - rxmb->ol_flags = 0; - } -+ rxmb->data_len = rxmb->pkt_len; - - /* prefetch mbuf data for caller */ - rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0019-enic-update-maintainers.patch b/dpdk/dpdk-2.2.0_patches/0019-enic-update-maintainers.patch deleted file mode 100644 index 74151772..00000000 --- a/dpdk/dpdk-2.2.0_patches/0019-enic-update-maintainers.patch +++ /dev/null @@ -1,42 +0,0 @@ -From 8ad252ab40b8f95db8413220146d54bf8a7d7be8 Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Fri, 18 Mar 2016 11:27:07 -0700 -Subject: [PATCH 19/22] enic: update maintainers - - Change maintainers for ENIC PMD and fix pointer to enic - documentation in MAINTAINERS. - - Signed-off-by: John Daley <johndale@cisco.com> ---- - MAINTAINERS | 3 ++- - doc/guides/nics/enic.rst | 2 +- - 2 files changed, 3 insertions(+), 2 deletions(-) - -diff --git a/MAINTAINERS b/MAINTAINERS -index b90aeea..f5b8bb4 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -264,8 +264,9 @@ F: doc/guides/nics/cxgbe.rst - - Cisco enic - M: John Daley <johndale@cisco.com> --M: Sujith Sankar <ssujith@cisco.com> -+M: Nelson Escobar <neescoba@cisco.com> - F: drivers/net/enic/ -+F: doc/guides/nics/enic.rst - - Combo szedata2 - M: Matej Vido <matejvido@gmail.com> -diff --git a/doc/guides/nics/enic.rst b/doc/guides/nics/enic.rst -index 2a228fd..e67c3db 100644 ---- a/doc/guides/nics/enic.rst -+++ b/doc/guides/nics/enic.rst -@@ -218,4 +218,4 @@ Any questions or bugs should be reported to DPDK community and to the ENIC PMD - maintainers: - - - John Daley <johndale@cisco.com> --- Sujith Sankar <ssujith@cisco.com> -+- Nelson Escobar <neescoba@cisco.com> --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0020-enic-fix-Rx-descriptor-limit.patch b/dpdk/dpdk-2.2.0_patches/0020-enic-fix-Rx-descriptor-limit.patch deleted file mode 100644 index db2ac646..00000000 --- a/dpdk/dpdk-2.2.0_patches/0020-enic-fix-Rx-descriptor-limit.patch +++ /dev/null @@ -1,66 +0,0 @@ -From ce6badc60736f5e78a295f30fe84c3e40ad0c330 Mon Sep 17 00:00:00 2001 -From: Nelson Escobar <neescoba@cisco.com> -Date: Fri, 18 Mar 2016 11:33:34 -0700 -Subject: [PATCH 20/22] enic: fix Rx descriptor limit - - On initialization, the rq descriptor count was set to the limit - of the vic. When the requested number of rx descriptors was - less than this count, enic_alloc_rq() was incorrectly setting - the count to the lower value. This results in later calls to - enic_alloc_rq() incorrectly using the lower value as the adapter - limit. - - Fixes: fefed3d1e62c ("enic: new driver") - - Signed-off-by: Nelson Escobar <neescoba@cisco.com> - Reviewed-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_main.c | 14 ++++++-------- - 1 file changed, 6 insertions(+), 8 deletions(-) - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index e30672c..2f79cf0 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -524,24 +524,22 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - "policy. Applying the value in the adapter "\ - "policy (%d).\n", - queue_idx, nb_desc, enic->config.rq_desc_count); -- } else if (nb_desc != enic->config.rq_desc_count) { -- enic->config.rq_desc_count = nb_desc; -- dev_info(enic, -- "RX Queues - effective number of descs:%d\n", -- nb_desc); -+ nb_desc = enic->config.rq_desc_count; - } -+ dev_info(enic, "RX Queues - effective number of descs:%d\n", -+ nb_desc); - } - - /* Allocate queue resources */ - rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, -- enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); -+ nb_desc, sizeof(struct rq_enet_desc)); - if (rc) { - dev_err(enic, "error in allocation of rq\n"); - goto err_exit; - } - - rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, -- socket_id, enic->config.rq_desc_count, -+ socket_id, nb_desc, - sizeof(struct cq_enet_rq_desc)); - if (rc) { - dev_err(enic, "error in allocation of cq for rq\n"); -@@ -550,7 +548,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - - /* Allocate the mbuf ring */ - rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", -- sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, -+ sizeof(struct rte_mbuf *) * nb_desc, - RTE_CACHE_LINE_SIZE, rq->socket_id); - - if (rq->mbuf_ring != NULL) --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0021-enic-fix-TX-hang-when-number-of-packets-queue-size.patch b/dpdk/dpdk-2.2.0_patches/0021-enic-fix-TX-hang-when-number-of-packets-queue-size.patch deleted file mode 100644 index 154e6f1e..00000000 --- a/dpdk/dpdk-2.2.0_patches/0021-enic-fix-TX-hang-when-number-of-packets-queue-size.patch +++ /dev/null @@ -1,89 +0,0 @@ -From e89ea2a038987102d9eb0a7ea217d7a301b484cb Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Thu, 24 Mar 2016 14:00:39 -0700 -Subject: [PATCH 21/22] enic: fix TX hang when number of packets > queue - size - - If the nb_pkts parameter to rte_eth_tx_burst() was greater than - the TX descriptor count, a completion was not being requested - from the NIC, so descriptors would not be released back to the - host causing a lock-up. - - Introduce a limit of how many TX descriptors can be used in a single - call to the enic PMD burst TX function before requesting a completion. - - Fixes: d739ba4c6abf ("enic: improve Tx packet rate") - - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_ethdev.c | 20 ++++++++++++++++---- - drivers/net/enic/enic_res.h | 1 + - 2 files changed, 17 insertions(+), 4 deletions(-) - -diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c -index 6c3c734..61bb83c 100644 ---- a/drivers/net/enic/enic_ethdev.c -+++ b/drivers/net/enic/enic_ethdev.c -@@ -510,7 +510,7 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui - static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) - { -- unsigned int index; -+ uint16_t index; - unsigned int frags; - unsigned int pkt_len; - unsigned int seg_len; -@@ -522,6 +522,7 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - unsigned short vlan_id; - unsigned short ol_flags; - uint8_t last_seg, eop; -+ unsigned int host_tx_descs = 0; - - for (index = 0; index < nb_pkts; index++) { - tx_pkt = *tx_pkts++; -@@ -537,6 +538,7 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - return index; - } - } -+ - pkt_len = tx_pkt->pkt_len; - vlan_id = tx_pkt->vlan_tci; - ol_flags = tx_pkt->ol_flags; -@@ -546,9 +548,19 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - next_tx_pkt = tx_pkt->next; - seg_len = tx_pkt->data_len; - inc_len += seg_len; -- eop = (pkt_len == inc_len) || (!next_tx_pkt); -- last_seg = eop && -- (index == ((unsigned int)nb_pkts - 1)); -+ -+ host_tx_descs++; -+ last_seg = 0; -+ eop = 0; -+ if ((pkt_len == inc_len) || !next_tx_pkt) { -+ eop = 1; -+ /* post if last packet in batch or > thresh */ -+ if ((index == (nb_pkts - 1)) || -+ (host_tx_descs > ENIC_TX_POST_THRESH)) { -+ last_seg = 1; -+ host_tx_descs = 0; -+ } -+ } - enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len, - !frags, eop, last_seg, ol_flags, vlan_id); - tx_pkt = next_tx_pkt; -diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h -index 33f2e84..00fa71d 100644 ---- a/drivers/net/enic/enic_res.h -+++ b/drivers/net/enic/enic_res.h -@@ -53,6 +53,7 @@ - - #define ENIC_NON_TSO_MAX_DESC 16 - #define ENIC_DEFAULT_RX_FREE_THRESH 32 -+#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2) - - #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) - --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0022-bonding-fix-bond-link-detect-in-non-interrupt-mode.patch b/dpdk/dpdk-2.2.0_patches/0022-bonding-fix-bond-link-detect-in-non-interrupt-mode.patch deleted file mode 100644 index 3e03c896..00000000 --- a/dpdk/dpdk-2.2.0_patches/0022-bonding-fix-bond-link-detect-in-non-interrupt-mode.patch +++ /dev/null @@ -1,76 +0,0 @@ -From a2f08a919c72af29c56b937e6c92eb104037fed5 Mon Sep 17 00:00:00 2001 -From: Nelson Escobar <neescoba@cisco.com> -Date: Tue, 22 Mar 2016 13:42:08 -0700 -Subject: [PATCH 22/22] bonding: fix bond link detect in non-interrupt mode - - Stopping then re-starting a bond interface containing slaves that - used polling for link detection caused the bond to think all slave - links were down and inactive. - - Move the start of the polling for link from slave_add() to - bond_ethdev_start() and in bond_ethdev_stop() make sure we clear - the last_link_status of the slaves. - - Signed-off-by: Nelson Escobar <neescoba@cisco.com> - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/bonding/rte_eth_bond_pmd.c | 27 +++++++++++++++++---------- - 1 file changed, 17 insertions(+), 10 deletions(-) - -diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c -index b1373c6..d32c6f5 100644 ---- a/drivers/net/bonding/rte_eth_bond_pmd.c -+++ b/drivers/net/bonding/rte_eth_bond_pmd.c -@@ -1447,18 +1447,11 @@ slave_add(struct bond_dev_private *internals, - slave_details->port_id = slave_eth_dev->data->port_id; - slave_details->last_link_status = 0; - -- /* If slave device doesn't support interrupts then we need to enabled -- * polling to monitor link status */ -+ /* Mark slave devices that don't support interrupts so we can -+ * compensate when we start the bond -+ */ - if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) { - slave_details->link_status_poll_enabled = 1; -- -- if (!internals->link_status_polling_enabled) { -- internals->link_status_polling_enabled = 1; -- -- rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000, -- bond_ethdev_slave_link_status_change_monitor, -- (void *)&rte_eth_devices[internals->port_id]); -- } - } - - slave_details->link_status_wait_to_complete = 0; -@@ -1543,6 +1536,18 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) - eth_dev->data->port_id, internals->slaves[i].port_id); - return -1; - } -+ /* We will need to poll for link status if any slave doesn't -+ * support interrupts -+ */ -+ if (internals->slaves[i].link_status_poll_enabled) -+ internals->link_status_polling_enabled = 1; -+ } -+ /* start polling if needed */ -+ if (internals->link_status_polling_enabled) { -+ rte_eal_alarm_set( -+ internals->link_status_polling_interval_ms * 1000, -+ bond_ethdev_slave_link_status_change_monitor, -+ (void *)&rte_eth_devices[internals->port_id]); - } - - if (internals->user_defined_primary_port) -@@ -1615,6 +1620,8 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) - - internals->active_slave_count = 0; - internals->link_status_polling_enabled = 0; -+ for (i = 0; i < internals->slave_count; i++) -+ internals->slaves[i].last_link_status = 0; - - eth_dev->data->dev_link.link_status = 0; - eth_dev->data->dev_started = 0; --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0023-enic-expose-RX-missed-packets-counter.patch b/dpdk/dpdk-2.2.0_patches/0023-enic-expose-RX-missed-packets-counter.patch deleted file mode 100644 index 53f3eaa7..00000000 --- a/dpdk/dpdk-2.2.0_patches/0023-enic-expose-RX-missed-packets-counter.patch +++ /dev/null @@ -1,27 +0,0 @@ -commit 7182d3e7d17722d088322695fc09f0d3bb7f1eab -Author: John Daley <johndale@cisco.com> -Date: Wed Mar 30 11:07:31 2016 -0700 - - enic: expose Rx missed packets counter - - Update the 'imissed' counter with the number of packets dropped - by the NIC. - - Fixes: fefed3d1e62c ("enic: new driver") - - Signed-off-by: John Daley <johndale@cisco.com> - Reviewed-by: Nelson Escobar <neescoba@cisco.com> - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index 2f79cf0..e3da51d 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -246,6 +246,8 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) - r_stats->ierrors = stats->rx.rx_errors; - r_stats->oerrors = stats->tx.tx_errors; - -+ r_stats->imissed = stats->rx.rx_drop; -+ - r_stats->imcasts = stats->rx.rx_multicast_frames_ok; - r_stats->rx_nombuf = stats->rx.rx_no_bufs; - } diff --git a/dpdk/dpdk-2.2.0_patches/0024-enic-fix-imissed-rx-counter.patch b/dpdk/dpdk-2.2.0_patches/0024-enic-fix-imissed-rx-counter.patch deleted file mode 100644 index 81e7bf3d..00000000 --- a/dpdk/dpdk-2.2.0_patches/0024-enic-fix-imissed-rx-counter.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 3433c7828ec909fccb768636ee21867030da14c9 Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Tue, 26 Apr 2016 13:30:50 -0700 -Subject: [PATCH 1/3] enic: fix 'imissed' to count drops due to lack of RX - buffers - -Fixes: 7182d3e7d177 ("enic: expose Rx missed packets counter") -Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_main.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index 60fe765..be4e9e5 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -243,10 +243,10 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) - r_stats->ibytes = stats->rx.rx_bytes_ok; - r_stats->obytes = stats->tx.tx_bytes_ok; - -- r_stats->ierrors = stats->rx.rx_errors; -+ r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop; - r_stats->oerrors = stats->tx.tx_errors; - -- r_stats->imissed = stats->rx.rx_drop; -+ r_stats->imissed = stats->rx.rx_no_bufs; - - r_stats->rx_nombuf = stats->rx.rx_no_bufs; - } --- -2.7.0 - diff --git a/dpdk/dpdk-2.2.0_patches/0025-enic-fix-misalignment-of-Rx-mbuf-data.patch b/dpdk/dpdk-2.2.0_patches/0025-enic-fix-misalignment-of-Rx-mbuf-data.patch deleted file mode 100644 index 69ca3f31..00000000 --- a/dpdk/dpdk-2.2.0_patches/0025-enic-fix-misalignment-of-Rx-mbuf-data.patch +++ /dev/null @@ -1,55 +0,0 @@ -From 454eb71eca1912e32a509c738a99a340cc2488cf Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Mon, 25 Apr 2016 16:24:53 -0700 -Subject: [PATCH 2/3] enic: fix misalignment of Rx mbuf data - -Data DMA used m->data_off of uninitialized mbufs instead of -RTE_PKTMBUF_HEADROOM, potentially causing Rx data to be -placed at the wrong alignment in the mbuf. - -Fixes: 947d860c821f ("enic: improve Rx performance") -Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/enic_main.c | 5 +++-- - drivers/net/enic/enic_rx.c | 6 ++++-- - 2 files changed, 7 insertions(+), 4 deletions(-) - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index be4e9e5..646d87f 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -354,10 +354,11 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - return -ENOMEM; - } - -- dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); -+ dma_addr = (dma_addr_t)(mb->buf_physaddr -+ + RTE_PKTMBUF_HEADROOM); - - rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, -- mb->buf_len); -+ mb->buf_len - RTE_PKTMBUF_HEADROOM); - rq->mbuf_ring[i] = mb; - } - -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -index 232987a..39bb55c 100644 ---- a/drivers/net/enic/enic_rx.c -+++ b/drivers/net/enic/enic_rx.c -@@ -314,9 +314,11 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - + rx_id); - - /* Push descriptor for newly allocated mbuf */ -- dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); -+ dma_addr = (dma_addr_t)(nmb->buf_physaddr -+ + RTE_PKTMBUF_HEADROOM); - rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); -+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len -+ - RTE_PKTMBUF_HEADROOM); - - /* Fill in the rest of the mbuf */ - rxmb->data_off = RTE_PKTMBUF_HEADROOM; --- -2.7.0 - diff --git a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch b/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch deleted file mode 100644 index 4858b8f5..00000000 --- a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch +++ /dev/null @@ -1,1844 +0,0 @@ -From ca6bbb723880e91d006de6cc485259da988859aa Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Tue, 5 Apr 2016 15:19:06 -0700 -Subject: [PATCH 3/3] enic: Optimization of Tx path to reduce Host CPU - overhead, cleanup - -Optimizations and cleanup: -- flatten packet send path -- flatten mbuf free path -- disable CQ entry writing and use CQ messages instead -- use rte_mempool_put_bulk() to bulk return freed mbufs -- remove unnecessary fields vnic_bufs struct, use contiguous array of cache - aligned divisible elements. No next pointers. -- use local variables inside per packet loop instead of fields in structs. -- factor book keeping out of the per packet tx loop where possible - (removed several conditionals) -- put Tx and Rx code in 1 file (enic_rxtx.c) - -Reviewed-by: Nelson Escobar <neescoba@cisco.com> -Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/Makefile | 2 +- - drivers/net/enic/base/enic_vnic_wq.h | 79 ------ - drivers/net/enic/base/vnic_cq.h | 37 +-- - drivers/net/enic/base/vnic_rq.h | 2 +- - drivers/net/enic/base/vnic_wq.c | 89 +++--- - drivers/net/enic/base/vnic_wq.h | 113 +------- - drivers/net/enic/enic.h | 27 +- - drivers/net/enic/enic_ethdev.c | 67 +---- - drivers/net/enic/enic_main.c | 132 +++------ - drivers/net/enic/enic_res.h | 81 +----- - drivers/net/enic/enic_rx.c | 361 ------------------------- - drivers/net/enic/enic_rxtx.c | 505 +++++++++++++++++++++++++++++++++++ - 12 files changed, 635 insertions(+), 860 deletions(-) - delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h - delete mode 100644 drivers/net/enic/enic_rx.c - create mode 100644 drivers/net/enic/enic_rxtx.c - -diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile -index f316274..3926b79 100644 ---- a/drivers/net/enic/Makefile -+++ b/drivers/net/enic/Makefile -@@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src - # - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c --SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c -+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c -diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h -deleted file mode 100644 -index b019109..0000000 ---- a/drivers/net/enic/base/enic_vnic_wq.h -+++ /dev/null -@@ -1,79 +0,0 @@ --/* -- * Copyright 2008-2015 Cisco Systems, Inc. All rights reserved. -- * Copyright 2007 Nuova Systems, Inc. All rights reserved. -- * -- * Copyright (c) 2015, Cisco Systems, Inc. -- * All rights reserved. -- * -- * Redistribution and use in source and binary forms, with or without -- * modification, are permitted provided that the following conditions -- * are met: -- * -- * 1. Redistributions of source code must retain the above copyright -- * notice, this list of conditions and the following disclaimer. -- * -- * 2. Redistributions in binary form must reproduce the above copyright -- * notice, this list of conditions and the following disclaimer in -- * the documentation and/or other materials provided with the -- * distribution. -- * -- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -- * POSSIBILITY OF SUCH DAMAGE. -- * -- */ -- --#ifndef _ENIC_VNIC_WQ_H_ --#define _ENIC_VNIC_WQ_H_ -- --#include "vnic_dev.h" --#include "vnic_cq.h" -- --static inline void enic_vnic_post_wq_index(struct vnic_wq *wq) --{ -- struct vnic_wq_buf *buf = wq->to_use; -- -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &wq->ctrl->posted_index); --} -- --static inline void enic_vnic_post_wq(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, -- unsigned int len, int sop, -- uint8_t desc_skip_cnt, uint8_t cq_entry, -- uint8_t compressed_send, uint64_t wrid) --{ -- struct vnic_wq_buf *buf = wq->to_use; -- -- buf->sop = sop; -- buf->cq_entry = cq_entry; -- buf->compressed_send = compressed_send; -- buf->desc_skip_cnt = desc_skip_cnt; -- buf->os_buf = os_buf; -- buf->dma_addr = dma_addr; -- buf->len = len; -- buf->wr_id = wrid; -- -- buf = buf->next; -- wq->ring.desc_avail -= desc_skip_cnt; -- wq->to_use = buf; -- -- if (cq_entry) -- enic_vnic_post_wq_index(wq); --} -- --#endif /* _ENIC_VNIC_WQ_H_ */ -diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h -index 922391b..ffc1aaa 100644 ---- a/drivers/net/enic/base/vnic_cq.h -+++ b/drivers/net/enic/base/vnic_cq.h -@@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq *cq, - u8 type, u16 q_number, u16 completed_index, void *opaque), - void *opaque) - { -- struct cq_desc *cq_desc; -+ struct cq_desc *cq_desc, *cq_desc_last; - unsigned int work_done = 0; - u16 q_number, completed_index; -- u8 type, color; -- struct rte_mbuf **rx_pkts = opaque; -- unsigned int ret; -+ u8 type, color, type_color; - - cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + - cq->ring.desc_size * cq->to_clean); -- cq_desc_dec(cq_desc, &type, &color, -- &q_number, &completed_index); -+ -+ type_color = cq_desc->type_color; -+ color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; -+ if (color == cq->last_color) -+ return 0; - - while (color != cq->last_color) { -- if (opaque) -- opaque = (void *)&(rx_pkts[work_done]); -+ cq_desc_last = cq_desc; - -- ret = (*q_service)(cq->vdev, cq_desc, type, -- q_number, completed_index, opaque); - cq->to_clean++; - if (cq->to_clean == cq->ring.desc_count) { - cq->to_clean = 0; - cq->last_color = cq->last_color ? 0 : 1; - } - -+ work_done++; -+ if (work_done >= work_to_do) -+ break; -+ - cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + - cq->ring.desc_size * cq->to_clean); -- cq_desc_dec(cq_desc, &type, &color, -- &q_number, &completed_index); - -- if (ret) -- work_done++; -- if (work_done >= work_to_do) -- break; -+ type_color = cq_desc->type_color; -+ color = (type_color >> CQ_DESC_COLOR_SHIFT) -+ & CQ_DESC_COLOR_MASK; -+ - } - -+ cq_desc_dec(cq_desc_last, &type, &color, -+ &q_number, &completed_index); -+ -+ (*q_service)(cq->vdev, cq_desc, type, -+ q_number, completed_index, opaque); - return work_done; - } - -diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h -index e083ccc..424415c 100644 ---- a/drivers/net/enic/base/vnic_rq.h -+++ b/drivers/net/enic/base/vnic_rq.h -@@ -74,7 +74,7 @@ struct vnic_rq { - struct vnic_dev_ring ring; - struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ - unsigned int mbuf_next_idx; /* next mb to consume */ -- void *os_buf_head; -+ void *mb_head; - unsigned int pkts_outstanding; - uint16_t rx_nb_hold; - uint16_t rx_free_thresh; -diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c -index a3ef417..ccbbd61 100644 ---- a/drivers/net/enic/base/vnic_wq.c -+++ b/drivers/net/enic/base/vnic_wq.c -@@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq, - - static int vnic_wq_alloc_bufs(struct vnic_wq *wq) - { -- struct vnic_wq_buf *buf; -- unsigned int i, j, count = wq->ring.desc_count; -- unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count); -- -- for (i = 0; i < blks; i++) { -- wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC); -- if (!wq->bufs[i]) -- return -ENOMEM; -- } -- -- for (i = 0; i < blks; i++) { -- buf = wq->bufs[i]; -- for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) { -- buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j; -- buf->desc = (u8 *)wq->ring.descs + -- wq->ring.desc_size * buf->index; -- if (buf->index + 1 == count) { -- buf->next = wq->bufs[0]; -- break; -- } else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) { -- buf->next = wq->bufs[i + 1]; -- } else { -- buf->next = buf + 1; -- buf++; -- } -- } -- } -- -- wq->to_use = wq->to_clean = wq->bufs[0]; -- -+ unsigned int count = wq->ring.desc_count; -+ /* Allocate the mbuf ring */ -+ wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs", -+ sizeof(struct vnic_wq_buf) * count, -+ RTE_CACHE_LINE_SIZE, wq->socket_id); -+ wq->head_idx = 0; -+ wq->tail_idx = 0; -+ if (wq->bufs == NULL) -+ return -ENOMEM; - return 0; - } - - void vnic_wq_free(struct vnic_wq *wq) - { - struct vnic_dev *vdev; -- unsigned int i; - - vdev = wq->vdev; - - vnic_dev_free_desc_ring(vdev, &wq->ring); - -- for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) { -- if (wq->bufs[i]) { -- kfree(wq->bufs[i]); -- wq->bufs[i] = NULL; -- } -- } -- -+ rte_free(wq->bufs); - wq->ctrl = NULL; - } - --int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, -- unsigned int desc_size) --{ -- int mem_size = 0; -- -- mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size); -- -- mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) * -- VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count); -- -- return mem_size; --} -- - - int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) -@@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index, - iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset); - iowrite32(0, &wq->ctrl->error_status); - -- wq->to_use = wq->to_clean = -- &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)]; -+ wq->head_idx = fetch_index; -+ wq->tail_idx = wq->head_idx; - } - - void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, -@@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, - vnic_wq_init_start(wq, cq_index, 0, 0, - error_interrupt_enable, - error_interrupt_offset); -+ wq->last_completed_index = 0; - } - - void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error) -@@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq) - return -ETIMEDOUT; - } - -+static inline uint32_t -+buf_idx_incr(uint32_t n_descriptors, uint32_t idx) -+{ -+ idx++; -+ if (unlikely(idx == n_descriptors)) -+ idx = 0; -+ return idx; -+} -+ - void vnic_wq_clean(struct vnic_wq *wq, -- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)) -+ void (*buf_clean)(struct vnic_wq_buf *buf)) - { - struct vnic_wq_buf *buf; -+ unsigned int to_clean = wq->tail_idx; - -- buf = wq->to_clean; -+ buf = &wq->bufs[to_clean]; - - while (vnic_wq_desc_used(wq) > 0) { - -- (*buf_clean)(wq, buf); -+ (*buf_clean)(buf); -+ to_clean = buf_idx_incr(wq->ring.desc_count, to_clean); - -- buf = wq->to_clean = buf->next; -+ buf = &wq->bufs[to_clean]; - wq->ring.desc_avail++; - } - -- wq->to_use = wq->to_clean = wq->bufs[0]; -+ wq->head_idx = 0; -+ wq->tail_idx = 0; - - iowrite32(0, &wq->ctrl->fetch_index); - iowrite32(0, &wq->ctrl->posted_index); -diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h -index c23de62..37c3ff9 100644 ---- a/drivers/net/enic/base/vnic_wq.h -+++ b/drivers/net/enic/base/vnic_wq.h -@@ -64,42 +64,23 @@ struct vnic_wq_ctrl { - u32 pad9; - }; - -+/* 16 bytes */ - struct vnic_wq_buf { -- struct vnic_wq_buf *next; -- dma_addr_t dma_addr; -- void *os_buf; -- unsigned int len; -- unsigned int index; -- int sop; -- void *desc; -- uint64_t wr_id; /* Cookie */ -- uint8_t cq_entry; /* Gets completion event from hw */ -- uint8_t desc_skip_cnt; /* Num descs to occupy */ -- uint8_t compressed_send; /* Both hdr and payload in one desc */ -+ struct rte_mempool *pool; -+ void *mb; - }; - --/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */ --#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32 --#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64 --#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \ -- ((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \ -- VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES)) --#define VNIC_WQ_BUF_BLK_SZ(entries) \ -- (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf)) --#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \ -- DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries)) --#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096) -- - struct vnic_wq { - unsigned int index; - struct vnic_dev *vdev; - struct vnic_wq_ctrl __iomem *ctrl; /* memory-mapped */ - struct vnic_dev_ring ring; -- struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX]; -- struct vnic_wq_buf *to_use; -- struct vnic_wq_buf *to_clean; -- unsigned int pkts_outstanding; -+ struct vnic_wq_buf *bufs; -+ unsigned int head_idx; -+ unsigned int tail_idx; - unsigned int socket_id; -+ const struct rte_memzone *cqmsg_rz; -+ uint16_t last_completed_index; - }; - - static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq) -@@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq *wq) - return wq->ring.desc_count - wq->ring.desc_avail - 1; - } - --static inline void *vnic_wq_next_desc(struct vnic_wq *wq) --{ -- return wq->to_use->desc; --} -- - #define PI_LOG2_CACHE_LINE_SIZE 5 - #define PI_INDEX_BITS 12 - #define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1) -@@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t addr, unsigned int len, - PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF); - } - --static inline void vnic_wq_post(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, -- unsigned int len, int sop, int eop, -- uint8_t desc_skip_cnt, uint8_t cq_entry, -- uint8_t compressed_send, uint64_t wrid) --{ -- struct vnic_wq_buf *buf = wq->to_use; -- -- buf->sop = sop; -- buf->cq_entry = cq_entry; -- buf->compressed_send = compressed_send; -- buf->desc_skip_cnt = desc_skip_cnt; -- buf->os_buf = os_buf; -- buf->dma_addr = dma_addr; -- buf->len = len; -- buf->wr_id = wrid; -- -- buf = buf->next; -- if (eop) { --#ifdef DO_PREFETCH -- uint64_t wr = vnic_cached_posted_index(dma_addr, len, -- buf->index); --#endif -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); --#ifdef DO_PREFETCH -- /* Intel chipsets seem to limit the rate of PIOs that we can -- * push on the bus. Thus, it is very important to do a single -- * 64 bit write here. With two 32-bit writes, my maximum -- * pkt/sec rate was cut almost in half. -AJF -- */ -- iowrite64((uint64_t)wr, &wq->ctrl->posted_index); --#else -- iowrite32(buf->index, &wq->ctrl->posted_index); --#endif -- } -- wq->to_use = buf; -- -- wq->ring.desc_avail -= desc_skip_cnt; --} -- --static inline void vnic_wq_service(struct vnic_wq *wq, -- struct cq_desc *cq_desc, u16 completed_index, -- void (*buf_service)(struct vnic_wq *wq, -- struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque), -- void *opaque) --{ -- struct vnic_wq_buf *buf; -- -- buf = wq->to_clean; -- while (1) { -- -- (*buf_service)(wq, cq_desc, buf, opaque); -- -- wq->ring.desc_avail++; -- -- wq->to_clean = buf->next; -- -- if (buf->index == completed_index) -- break; -- -- buf = wq->to_clean; -- } --} -- - void vnic_wq_free(struct vnic_wq *wq); - int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, - unsigned int desc_count, unsigned int desc_size); -@@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq); - void vnic_wq_enable(struct vnic_wq *wq); - int vnic_wq_disable(struct vnic_wq *wq); - void vnic_wq_clean(struct vnic_wq *wq, -- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)); --int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, -- unsigned int desc_size); -+ void (*buf_clean)(struct vnic_wq_buf *buf)); - - #endif /* _VNIC_WQ_H_ */ -diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h -index 8c914f5..43b82a6 100644 ---- a/drivers/net/enic/enic.h -+++ b/drivers/net/enic/enic.h -@@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) - return (struct enic *)eth_dev->data->dev_private; - } - -+static inline uint32_t -+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -+{ -+ uint32_t d = i0 + i1; -+ d -= (d >= n_descriptors) ? n_descriptors : 0; -+ return d; -+} -+ -+static inline uint32_t -+enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -+{ -+ int32_t d = i1 - i0; -+ return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d); -+} -+ -+static inline uint32_t -+enic_ring_incr(uint32_t n_descriptors, uint32_t idx) -+{ -+ idx++; -+ if (unlikely(idx == n_descriptors)) -+ idx = 0; -+ return idx; -+} -+ - #define RTE_LIBRTE_ENIC_ASSERT_ENABLE - #ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE - #define ASSERT(x) do { \ -@@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic); - extern void enic_clsf_destroy(struct enic *enic); - uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts); -- -+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, -+ uint16_t nb_pkts); - #endif /* _ENIC_H_ */ -diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c -index 6bea940..697ff82 100644 ---- a/drivers/net/enic/enic_ethdev.c -+++ b/drivers/net/enic/enic_ethdev.c -@@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui - enic_del_mac_address(enic); - } - -- --static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, -- uint16_t nb_pkts) --{ -- uint16_t index; -- unsigned int frags; -- unsigned int pkt_len; -- unsigned int seg_len; -- unsigned int inc_len; -- unsigned int nb_segs; -- struct rte_mbuf *tx_pkt, *next_tx_pkt; -- struct vnic_wq *wq = (struct vnic_wq *)tx_queue; -- struct enic *enic = vnic_dev_priv(wq->vdev); -- unsigned short vlan_id; -- unsigned short ol_flags; -- uint8_t last_seg, eop; -- unsigned int host_tx_descs = 0; -- -- for (index = 0; index < nb_pkts; index++) { -- tx_pkt = *tx_pkts++; -- inc_len = 0; -- nb_segs = tx_pkt->nb_segs; -- if (nb_segs > vnic_wq_desc_avail(wq)) { -- if (index > 0) -- enic_post_wq_index(wq); -- -- /* wq cleanup and try again */ -- if (!enic_cleanup_wq(enic, wq) || -- (nb_segs > vnic_wq_desc_avail(wq))) { -- return index; -- } -- } -- -- pkt_len = tx_pkt->pkt_len; -- vlan_id = tx_pkt->vlan_tci; -- ol_flags = tx_pkt->ol_flags; -- for (frags = 0; inc_len < pkt_len; frags++) { -- if (!tx_pkt) -- break; -- next_tx_pkt = tx_pkt->next; -- seg_len = tx_pkt->data_len; -- inc_len += seg_len; -- -- host_tx_descs++; -- last_seg = 0; -- eop = 0; -- if ((pkt_len == inc_len) || !next_tx_pkt) { -- eop = 1; -- /* post if last packet in batch or > thresh */ -- if ((index == (nb_pkts - 1)) || -- (host_tx_descs > ENIC_TX_POST_THRESH)) { -- last_seg = 1; -- host_tx_descs = 0; -- } -- } -- enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len, -- !frags, eop, last_seg, ol_flags, vlan_id); -- tx_pkt = next_tx_pkt; -- } -- } -- -- enic_cleanup_wq(enic, wq); -- return index; --} -- - static const struct eth_dev_ops enicpmd_eth_dev_ops = { - .dev_configure = enicpmd_dev_configure, - .dev_start = enicpmd_dev_start, -@@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) - enic->rte_dev = eth_dev; - eth_dev->dev_ops = &enicpmd_eth_dev_ops; - eth_dev->rx_pkt_burst = &enic_recv_pkts; -- eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; -+ eth_dev->tx_pkt_burst = &enic_xmit_pkts; - - pdev = eth_dev->pci_dev; - rte_eth_copy_pci_info(eth_dev, pdev); -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index 646d87f..ba73604 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -40,11 +40,11 @@ - #include <libgen.h> - - #include <rte_pci.h> --#include <rte_memzone.h> - #include <rte_malloc.h> - #include <rte_mbuf.h> - #include <rte_string_fns.h> - #include <rte_ethdev.h> -+#include <rte_memzone.h> - - #include "enic_compat.h" - #include "enic.h" -@@ -58,7 +58,6 @@ - #include "vnic_cq.h" - #include "vnic_intr.h" - #include "vnic_nic.h" --#include "enic_vnic_wq.h" - - static inline struct rte_mbuf * - rte_rxmbuf_alloc(struct rte_mempool *mp) -@@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) - } - } - -- - void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) - { - vnic_set_hdr_split_size(enic->vdev, split_hdr_size); - } - --static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf) -+static void enic_free_wq_buf(struct vnic_wq_buf *buf) - { -- struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf; -+ struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb; - - rte_mempool_put(mbuf->pool, mbuf); -- buf->os_buf = NULL; --} -- --static void enic_wq_free_buf(struct vnic_wq *wq, -- __rte_unused struct cq_desc *cq_desc, -- struct vnic_wq_buf *buf, -- __rte_unused void *opaque) --{ -- enic_free_wq_buf(wq, buf); --} -- --static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, -- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(vdev); -- -- vnic_wq_service(&enic->wq[q_number], cq_desc, -- completed_index, enic_wq_free_buf, -- opaque); -- -- return 0; -+ buf->mb = NULL; - } - - static void enic_log_q_error(struct enic *enic) -@@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic) - } - } - --unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq) --{ -- unsigned int cq = enic_cq_wq(enic, wq->index); -- -- /* Return the work done */ -- return vnic_cq_service(&enic->cq[cq], -- -1 /*wq_work_to_do*/, enic_wq_service, NULL); --} -- --void enic_post_wq_index(struct vnic_wq *wq) --{ -- enic_vnic_post_wq_index(wq); --} -- --void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, -- struct rte_mbuf *tx_pkt, unsigned short len, -- uint8_t sop, uint8_t eop, uint8_t cq_entry, -- uint16_t ol_flags, uint16_t vlan_tag) --{ -- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); -- uint16_t mss = 0; -- uint8_t vlan_tag_insert = 0; -- uint64_t bus_addr = (dma_addr_t) -- (tx_pkt->buf_physaddr + tx_pkt->data_off); -- -- if (sop) { -- if (ol_flags & PKT_TX_VLAN_PKT) -- vlan_tag_insert = 1; -- -- if (enic->hw_ip_checksum) { -- if (ol_flags & PKT_TX_IP_CKSUM) -- mss |= ENIC_CALC_IP_CKSUM; -- -- if (ol_flags & PKT_TX_TCP_UDP_CKSUM) -- mss |= ENIC_CALC_TCP_UDP_CKSUM; -- } -- } -- -- wq_enet_desc_enc(desc, -- bus_addr, -- len, -- mss, -- 0 /* header_length */, -- 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */, -- eop, -- cq_entry, -- 0 /* fcoe_encap */, -- vlan_tag_insert, -- vlan_tag, -- 0 /* loopback */); -- -- enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len, -- sop, -- 1 /*desc_skip_cnt*/, -- cq_entry, -- 0 /*compressed send*/, -- 0 /*wrid*/); --} - - void enic_dev_stats_clear(struct enic *enic) - { -@@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic) - unsigned int error_interrupt_enable = 1; - unsigned int error_interrupt_offset = 0; - unsigned int index = 0; -+ unsigned int cq_idx; -+ -+ vnic_dev_stats_clear(enic->vdev); - - for (index = 0; index < enic->rq_count; index++) { - vnic_rq_init(&enic->rq[index], - enic_cq_rq(enic, index), - error_interrupt_enable, - error_interrupt_offset); -+ -+ cq_idx = enic_cq_rq(enic, index); -+ vnic_cq_init(&enic->cq[cq_idx], -+ 0 /* flow_control_enable */, -+ 1 /* color_enable */, -+ 0 /* cq_head */, -+ 0 /* cq_tail */, -+ 1 /* cq_tail_color */, -+ 0 /* interrupt_enable */, -+ 1 /* cq_entry_enable */, -+ 0 /* cq_message_enable */, -+ 0 /* interrupt offset */, -+ 0 /* cq_message_addr */); - } - - for (index = 0; index < enic->wq_count; index++) { -@@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic) - enic_cq_wq(enic, index), - error_interrupt_enable, - error_interrupt_offset); -- } -- -- vnic_dev_stats_clear(enic->vdev); - -- for (index = 0; index < enic->cq_count; index++) { -- vnic_cq_init(&enic->cq[index], -+ cq_idx = enic_cq_wq(enic, index); -+ vnic_cq_init(&enic->cq[cq_idx], - 0 /* flow_control_enable */, - 1 /* color_enable */, - 0 /* cq_head */, - 0 /* cq_tail */, - 1 /* cq_tail_color */, - 0 /* interrupt_enable */, -- 1 /* cq_entry_enable */, -- 0 /* cq_message_enable */, -+ 0 /* cq_entry_enable */, -+ 1 /* cq_message_enable */, - 0 /* interrupt offset */, -- 0 /* cq_message_addr */); -+ (u64)enic->wq[index].cqmsg_rz->phys_addr); - } - - vnic_intr_init(&enic->intr, -@@ -569,6 +502,7 @@ void enic_free_wq(void *txq) - struct vnic_wq *wq = (struct vnic_wq *)txq; - struct enic *enic = vnic_dev_priv(wq->vdev); - -+ rte_memzone_free(wq->cqmsg_rz); - vnic_wq_free(wq); - vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); - } -@@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, - int err; - struct vnic_wq *wq = &enic->wq[queue_idx]; - unsigned int cq_index = enic_cq_wq(enic, queue_idx); -+ char name[NAME_MAX]; -+ static int instance; - - wq->socket_id = socket_id; - if (nb_desc) { -@@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, - dev_err(enic, "error in allocation of cq for wq\n"); - } - -+ /* setup up CQ message */ -+ snprintf((char *)name, sizeof(name), -+ "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx, -+ instance++); -+ -+ wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name, -+ sizeof(uint32_t), -+ SOCKET_ID_ANY, 0, -+ ENIC_ALIGN); -+ if (!wq->cqmsg_rz) -+ return -ENOMEM; -+ - return err; - } - -diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h -index 00fa71d..3e1bdf5 100644 ---- a/drivers/net/enic/enic_res.h -+++ b/drivers/net/enic/enic_res.h -@@ -53,89 +53,10 @@ - - #define ENIC_NON_TSO_MAX_DESC 16 - #define ENIC_DEFAULT_RX_FREE_THRESH 32 --#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2) -+#define ENIC_TX_XMIT_MAX 64 - - #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) - --static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- unsigned int mss_or_csum_offset, unsigned int hdr_len, -- int vlan_tag_insert, unsigned int vlan_tag, -- int offload_mode, int cq_entry, int sop, int eop, int loopback) --{ -- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); -- u8 desc_skip_cnt = 1; -- u8 compressed_send = 0; -- u64 wrid = 0; -- -- wq_enet_desc_enc(desc, -- (u64)dma_addr | VNIC_PADDR_TARGET, -- (u16)len, -- (u16)mss_or_csum_offset, -- (u16)hdr_len, (u8)offload_mode, -- (u8)eop, (u8)cq_entry, -- 0, /* fcoe_encap */ -- (u8)vlan_tag_insert, -- (u16)vlan_tag, -- (u8)loopback); -- -- vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt, -- (u8)cq_entry, compressed_send, wrid); --} -- --static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- 0, 0, 0, 0, 0, -- eop, 0 /* !SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf, -- dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert, -- unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- 0, 0, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_CSUM, -- eop, 1 /* SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- int ip_csum, int tcpudp_csum, int vlan_tag_insert, -- unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- (ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0), -- 0, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_CSUM, -- eop, 1 /* SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- unsigned int csum_offset, unsigned int hdr_len, -- int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- csum_offset, hdr_len, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_CSUM_L4, -- eop, 1 /* SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- unsigned int mss, unsigned int hdr_len, int vlan_tag_insert, -- unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- mss, hdr_len, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_TSO, -- eop, 1 /* SOP */, eop, loopback); --} -- - struct enic; - - int enic_get_vnic_config(struct enic *); -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -deleted file mode 100644 -index 39bb55c..0000000 ---- a/drivers/net/enic/enic_rx.c -+++ /dev/null -@@ -1,361 +0,0 @@ --/* -- * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. -- * Copyright 2007 Nuova Systems, Inc. All rights reserved. -- * -- * Copyright (c) 2014, Cisco Systems, Inc. -- * All rights reserved. -- * -- * Redistribution and use in source and binary forms, with or without -- * modification, are permitted provided that the following conditions -- * are met: -- * -- * 1. Redistributions of source code must retain the above copyright -- * notice, this list of conditions and the following disclaimer. -- * -- * 2. Redistributions in binary form must reproduce the above copyright -- * notice, this list of conditions and the following disclaimer in -- * the documentation and/or other materials provided with the -- * distribution. -- * -- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -- * POSSIBILITY OF SUCH DAMAGE. -- * -- */ -- --#include <rte_mbuf.h> --#include <rte_ethdev.h> --#include <rte_prefetch.h> -- --#include "enic_compat.h" --#include "rq_enet_desc.h" --#include "enic.h" -- --#define RTE_PMD_USE_PREFETCH -- --#ifdef RTE_PMD_USE_PREFETCH --/* -- * Prefetch a cache line into all cache levels. -- */ --#define rte_enic_prefetch(p) rte_prefetch0(p) --#else --#define rte_enic_prefetch(p) do {} while (0) --#endif -- --#ifdef RTE_PMD_PACKET_PREFETCH --#define rte_packet_prefetch(p) rte_prefetch1(p) --#else --#define rte_packet_prefetch(p) do {} while (0) --#endif -- --static inline struct rte_mbuf * --rte_rxmbuf_alloc(struct rte_mempool *mp) --{ -- struct rte_mbuf *m; -- -- m = __rte_mbuf_raw_alloc(mp); -- __rte_mbuf_sanity_check_raw(m, 0); -- return m; --} -- --static inline uint16_t --enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) --{ -- return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; --} -- --static inline uint16_t --enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) --{ -- return(le16_to_cpu(crd->bytes_written_flags) & -- ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); --} -- --static inline uint8_t --enic_cq_rx_desc_packet_error(uint16_t bwflags) --{ -- return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == -- CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); --} -- --static inline uint8_t --enic_cq_rx_desc_eop(uint16_t ciflags) --{ -- return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) -- == CQ_ENET_RQ_DESC_FLAGS_EOP; --} -- --static inline uint8_t --enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) --{ -- return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & -- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == -- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); --} -- --static inline uint8_t --enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) --{ -- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == -- CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); --} -- --static inline uint8_t --enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) --{ -- return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == -- CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); --} -- --static inline uint8_t --enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) --{ -- return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> -- CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); --} -- --static inline uint32_t --enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) --{ -- return le32_to_cpu(cqrd->rss_hash); --} -- --static inline uint16_t --enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) --{ -- return le16_to_cpu(cqrd->vlan); --} -- --static inline uint16_t --enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- return le16_to_cpu(cqrd->bytes_written_flags) & -- CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; --} -- --static inline uint8_t --enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- uint16_t bwflags; -- int ret = 0; -- uint64_t pkt_err_flags = 0; -- -- bwflags = enic_cq_rx_desc_bwflags(cqrd); -- if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { -- pkt_err_flags = PKT_RX_MAC_ERR; -- ret = 1; -- } -- *pkt_err_flags_out = pkt_err_flags; -- return ret; --} -- --/* -- * Lookup table to translate RX CQ flags to mbuf flags. -- */ --static inline uint32_t --enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- uint8_t cqrd_flags = cqrd->flags; -- static const uint32_t cq_type_table[128] __rte_cache_aligned = { -- [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, -- [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -- | RTE_PTYPE_L4_UDP, -- [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -- | RTE_PTYPE_L4_TCP, -- [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -- | RTE_PTYPE_L4_FRAG, -- [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, -- [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -- | RTE_PTYPE_L4_UDP, -- [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -- | RTE_PTYPE_L4_TCP, -- [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -- | RTE_PTYPE_L4_FRAG, -- /* All others reserved */ -- }; -- cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT -- | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 -- | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; -- return cq_type_table[cqrd_flags]; --} -- --static inline void --enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- uint16_t ciflags, bwflags, pkt_flags = 0; -- ciflags = enic_cq_rx_desc_ciflags(cqrd); -- bwflags = enic_cq_rx_desc_bwflags(cqrd); -- -- mbuf->ol_flags = 0; -- -- /* flags are meaningless if !EOP */ -- if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -- goto mbuf_flags_done; -- -- /* VLAN stripping */ -- if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { -- pkt_flags |= PKT_RX_VLAN_PKT; -- mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); -- } else { -- mbuf->vlan_tci = 0; -- } -- -- /* RSS flag */ -- if (enic_cq_rx_desc_rss_type(cqrd)) { -- pkt_flags |= PKT_RX_RSS_HASH; -- mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); -- } -- -- /* checksum flags */ -- if (!enic_cq_rx_desc_csum_not_calc(cqrd) && -- (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { -- if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) -- pkt_flags |= PKT_RX_IP_CKSUM_BAD; -- if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { -- if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) -- pkt_flags |= PKT_RX_L4_CKSUM_BAD; -- } -- } -- -- mbuf_flags_done: -- mbuf->ol_flags = pkt_flags; --} -- --static inline uint32_t --enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) --{ -- uint32_t d = i0 + i1; -- ASSERT(i0 < n_descriptors); -- ASSERT(i1 < n_descriptors); -- d -= (d >= n_descriptors) ? n_descriptors : 0; -- return d; --} -- -- --uint16_t --enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -- uint16_t nb_pkts) --{ -- struct vnic_rq *rq = rx_queue; -- struct enic *enic = vnic_dev_priv(rq->vdev); -- unsigned int rx_id; -- struct rte_mbuf *nmb, *rxmb; -- uint16_t nb_rx = 0; -- uint16_t nb_hold; -- struct vnic_cq *cq; -- volatile struct cq_desc *cqd_ptr; -- uint8_t color; -- -- cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -- -- nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -- -- while (nb_rx < nb_pkts) { -- volatile struct rq_enet_desc *rqd_ptr; -- dma_addr_t dma_addr; -- struct cq_desc cqd; -- uint64_t ol_err_flags; -- uint8_t packet_error; -- -- /* Check for pkts available */ -- color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -- & CQ_DESC_COLOR_MASK; -- if (color == cq->last_color) -- break; -- -- /* Get the cq descriptor and rq pointer */ -- cqd = *cqd_ptr; -- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -- -- /* allocate a new mbuf */ -- nmb = rte_rxmbuf_alloc(rq->mp); -- if (nmb == NULL) { -- dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", -- enic->port_id, (unsigned)rq->index); -- rte_eth_devices[enic->port_id]. -- data->rx_mbuf_alloc_failed++; -- break; -- } -- -- /* A packet error means descriptor and data are untrusted */ -- packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); -- -- /* Get the mbuf to return and replace with one just allocated */ -- rxmb = rq->mbuf_ring[rx_id]; -- rq->mbuf_ring[rx_id] = nmb; -- -- /* Increment cqd, rqd, mbuf_table index */ -- rx_id++; -- if (unlikely(rx_id == rq->ring.desc_count)) { -- rx_id = 0; -- cq->last_color = cq->last_color ? 0 : 1; -- } -- -- /* Prefetch next mbuf & desc while processing current one */ -- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -- rte_enic_prefetch(cqd_ptr); -- rte_enic_prefetch(rq->mbuf_ring[rx_id]); -- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -- + rx_id); -- -- /* Push descriptor for newly allocated mbuf */ -- dma_addr = (dma_addr_t)(nmb->buf_physaddr -- + RTE_PKTMBUF_HEADROOM); -- rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len -- - RTE_PKTMBUF_HEADROOM); -- -- /* Fill in the rest of the mbuf */ -- rxmb->data_off = RTE_PKTMBUF_HEADROOM; -- rxmb->nb_segs = 1; -- rxmb->next = NULL; -- rxmb->port = enic->port_id; -- if (!packet_error) { -- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -- enic_cq_rx_to_pkt_flags(&cqd, rxmb); -- } else { -- rxmb->pkt_len = 0; -- rxmb->packet_type = 0; -- rxmb->ol_flags = 0; -- } -- rxmb->data_len = rxmb->pkt_len; -- -- /* prefetch mbuf data for caller */ -- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -- RTE_PKTMBUF_HEADROOM)); -- -- /* store the mbuf address into the next entry of the array */ -- rx_pkts[nb_rx++] = rxmb; -- } -- -- nb_hold += nb_rx; -- cq->to_clean = rx_id; -- -- if (nb_hold > rq->rx_free_thresh) { -- rq->posted_index = enic_ring_add(rq->ring.desc_count, -- rq->posted_index, nb_hold); -- nb_hold = 0; -- rte_mb(); -- iowrite32(rq->posted_index, &rq->ctrl->posted_index); -- } -- -- rq->rx_nb_hold = nb_hold; -- -- return nb_rx; --} -diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c -new file mode 100644 -index 0000000..71ca34e ---- /dev/null -+++ b/drivers/net/enic/enic_rxtx.c -@@ -0,0 +1,505 @@ -+/* -+ * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved. -+ * Copyright 2007 Nuova Systems, Inc. All rights reserved. -+ * -+ * Copyright (c) 2016, Cisco Systems, Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ * -+ */ -+ -+#include <rte_mbuf.h> -+#include <rte_ethdev.h> -+#include <rte_prefetch.h> -+#include <rte_memzone.h> -+ -+#include "enic_compat.h" -+#include "rq_enet_desc.h" -+#include "enic.h" -+ -+#define RTE_PMD_USE_PREFETCH -+ -+#ifdef RTE_PMD_USE_PREFETCH -+/* -+ * Prefetch a cache line into all cache levels. -+ */ -+#define rte_enic_prefetch(p) rte_prefetch0(p) -+#else -+#define rte_enic_prefetch(p) do {} while (0) -+#endif -+ -+#ifdef RTE_PMD_PACKET_PREFETCH -+#define rte_packet_prefetch(p) rte_prefetch1(p) -+#else -+#define rte_packet_prefetch(p) do {} while (0) -+#endif -+ -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) -+{ -+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) -+{ -+ return(le16_to_cpu(crd->bytes_written_flags) & -+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_packet_error(uint16_t bwflags) -+{ -+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == -+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_eop(uint16_t ciflags) -+{ -+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) -+ == CQ_ENET_RQ_DESC_FLAGS_EOP; -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) -+{ -+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> -+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); -+} -+ -+static inline uint32_t -+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) -+{ -+ return le32_to_cpu(cqrd->rss_hash); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) -+{ -+ return le16_to_cpu(cqrd->vlan); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ return le16_to_cpu(cqrd->bytes_written_flags) & -+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; -+} -+ -+static inline uint8_t -+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t bwflags; -+ int ret = 0; -+ uint64_t pkt_err_flags = 0; -+ -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { -+ pkt_err_flags = PKT_RX_MAC_ERR; -+ ret = 1; -+ } -+ *pkt_err_flags_out = pkt_err_flags; -+ return ret; -+} -+ -+/* -+ * Lookup table to translate RX CQ flags to mbuf flags. -+ */ -+static inline uint32_t -+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint8_t cqrd_flags = cqrd->flags; -+ static const uint32_t cq_type_table[128] __rte_cache_aligned = { -+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, -+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_UDP, -+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_TCP, -+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_FRAG, -+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, -+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_UDP, -+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_TCP, -+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_FRAG, -+ /* All others reserved */ -+ }; -+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT -+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 -+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; -+ return cq_type_table[cqrd_flags]; -+} -+ -+static inline void -+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t ciflags, bwflags, pkt_flags = 0; -+ ciflags = enic_cq_rx_desc_ciflags(cqrd); -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ mbuf->ol_flags = 0; -+ -+ /* flags are meaningless if !EOP */ -+ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -+ goto mbuf_flags_done; -+ -+ /* VLAN stripping */ -+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { -+ pkt_flags |= PKT_RX_VLAN_PKT; -+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); -+ } else { -+ mbuf->vlan_tci = 0; -+ } -+ -+ /* RSS flag */ -+ if (enic_cq_rx_desc_rss_type(cqrd)) { -+ pkt_flags |= PKT_RX_RSS_HASH; -+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); -+ } -+ -+ /* checksum flags */ -+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && -+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { -+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_IP_CKSUM_BAD; -+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { -+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_L4_CKSUM_BAD; -+ } -+ } -+ -+ mbuf_flags_done: -+ mbuf->ol_flags = pkt_flags; -+} -+ -+uint16_t -+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts) -+{ -+ struct vnic_rq *rq = rx_queue; -+ struct enic *enic = vnic_dev_priv(rq->vdev); -+ unsigned int rx_id; -+ struct rte_mbuf *nmb, *rxmb; -+ uint16_t nb_rx = 0; -+ uint16_t nb_hold; -+ struct vnic_cq *cq; -+ volatile struct cq_desc *cqd_ptr; -+ uint8_t color; -+ -+ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ -+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -+ -+ while (nb_rx < nb_pkts) { -+ volatile struct rq_enet_desc *rqd_ptr; -+ dma_addr_t dma_addr; -+ struct cq_desc cqd; -+ uint64_t ol_err_flags; -+ uint8_t packet_error; -+ -+ /* Check for pkts available */ -+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -+ & CQ_DESC_COLOR_MASK; -+ if (color == cq->last_color) -+ break; -+ -+ /* Get the cq descriptor and rq pointer */ -+ cqd = *cqd_ptr; -+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -+ -+ /* allocate a new mbuf */ -+ nmb = rte_rxmbuf_alloc(rq->mp); -+ if (nmb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", -+ enic->port_id, (unsigned)rq->index); -+ rte_eth_devices[enic->port_id]. -+ data->rx_mbuf_alloc_failed++; -+ break; -+ } -+ -+ /* A packet error means descriptor and data are untrusted */ -+ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); -+ -+ /* Get the mbuf to return and replace with one just allocated */ -+ rxmb = rq->mbuf_ring[rx_id]; -+ rq->mbuf_ring[rx_id] = nmb; -+ -+ /* Increment cqd, rqd, mbuf_table index */ -+ rx_id++; -+ if (unlikely(rx_id == rq->ring.desc_count)) { -+ rx_id = 0; -+ cq->last_color = cq->last_color ? 0 : 1; -+ } -+ -+ /* Prefetch next mbuf & desc while processing current one */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ rte_enic_prefetch(cqd_ptr); -+ rte_enic_prefetch(rq->mbuf_ring[rx_id]); -+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -+ + rx_id); -+ -+ /* Push descriptor for newly allocated mbuf */ -+ dma_addr = (dma_addr_t)(nmb->buf_physaddr -+ + RTE_PKTMBUF_HEADROOM); -+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len -+ - RTE_PKTMBUF_HEADROOM); -+ -+ /* Fill in the rest of the mbuf */ -+ rxmb->data_off = RTE_PKTMBUF_HEADROOM; -+ rxmb->nb_segs = 1; -+ rxmb->next = NULL; -+ rxmb->port = enic->port_id; -+ if (!packet_error) { -+ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -+ enic_cq_rx_to_pkt_flags(&cqd, rxmb); -+ } else { -+ rxmb->pkt_len = 0; -+ rxmb->packet_type = 0; -+ rxmb->ol_flags = 0; -+ } -+ rxmb->data_len = rxmb->pkt_len; -+ -+ /* prefetch mbuf data for caller */ -+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -+ RTE_PKTMBUF_HEADROOM)); -+ -+ /* store the mbuf address into the next entry of the array */ -+ rx_pkts[nb_rx++] = rxmb; -+ } -+ -+ nb_hold += nb_rx; -+ cq->to_clean = rx_id; -+ -+ if (nb_hold > rq->rx_free_thresh) { -+ rq->posted_index = enic_ring_add(rq->ring.desc_count, -+ rq->posted_index, nb_hold); -+ nb_hold = 0; -+ rte_mb(); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ } -+ -+ rq->rx_nb_hold = nb_hold; -+ -+ return nb_rx; -+} -+ -+static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) -+{ -+ struct vnic_wq_buf *buf; -+ struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS]; -+ unsigned int nb_to_free, nb_free = 0, i; -+ struct rte_mempool *pool; -+ unsigned int tail_idx; -+ unsigned int desc_count = wq->ring.desc_count; -+ -+ nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index) -+ + 1; -+ tail_idx = wq->tail_idx; -+ buf = &wq->bufs[tail_idx]; -+ pool = ((struct rte_mbuf *)buf->mb)->pool; -+ for (i = 0; i < nb_to_free; i++) { -+ buf = &wq->bufs[tail_idx]; -+ m = (struct rte_mbuf *)(buf->mb); -+ if (likely(m->pool == pool)) { -+ ASSERT(nb_free < ENIC_MAX_WQ_DESCS); -+ free[nb_free++] = m; -+ } else { -+ rte_mempool_put_bulk(pool, (void *)free, nb_free); -+ free[0] = m; -+ nb_free = 1; -+ pool = m->pool; -+ } -+ tail_idx = enic_ring_incr(desc_count, tail_idx); -+ buf->mb = NULL; -+ } -+ -+ rte_mempool_put_bulk(pool, (void **)free, nb_free); -+ -+ wq->tail_idx = tail_idx; -+ wq->ring.desc_avail += nb_to_free; -+} -+ -+unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq) -+{ -+ u16 completed_index; -+ -+ completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff; -+ -+ if (wq->last_completed_index != completed_index) { -+ enic_free_wq_bufs(wq, completed_index); -+ wq->last_completed_index = completed_index; -+ } -+ return 0; -+} -+ -+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, -+ uint16_t nb_pkts) -+{ -+ uint16_t index; -+ unsigned int pkt_len, data_len; -+ unsigned int nb_segs; -+ struct rte_mbuf *tx_pkt; -+ struct vnic_wq *wq = (struct vnic_wq *)tx_queue; -+ struct enic *enic = vnic_dev_priv(wq->vdev); -+ unsigned short vlan_id; -+ unsigned short ol_flags; -+ unsigned int wq_desc_avail; -+ int head_idx; -+ struct vnic_wq_buf *buf; -+ unsigned int hw_ip_cksum_enabled; -+ unsigned int desc_count; -+ struct wq_enet_desc *descs, *desc_p, desc_tmp; -+ uint16_t mss; -+ uint8_t vlan_tag_insert; -+ uint8_t eop; -+ uint64_t bus_addr; -+ -+ enic_cleanup_wq(enic, wq); -+ wq_desc_avail = vnic_wq_desc_avail(wq); -+ head_idx = wq->head_idx; -+ desc_count = wq->ring.desc_count; -+ -+ nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX); -+ -+ hw_ip_cksum_enabled = enic->hw_ip_checksum; -+ for (index = 0; index < nb_pkts; index++) { -+ tx_pkt = *tx_pkts++; -+ nb_segs = tx_pkt->nb_segs; -+ if (nb_segs > wq_desc_avail) { -+ if (index > 0) -+ goto post; -+ goto done; -+ } -+ -+ pkt_len = tx_pkt->pkt_len; -+ data_len = tx_pkt->data_len; -+ vlan_id = tx_pkt->vlan_tci; -+ ol_flags = tx_pkt->ol_flags; -+ -+ mss = 0; -+ vlan_tag_insert = 0; -+ bus_addr = (dma_addr_t) -+ (tx_pkt->buf_physaddr + tx_pkt->data_off); -+ -+ descs = (struct wq_enet_desc *)wq->ring.descs; -+ desc_p = descs + head_idx; -+ -+ eop = (data_len == pkt_len); -+ -+ if (ol_flags & PKT_TX_VLAN_PKT) -+ vlan_tag_insert = 1; -+ -+ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM)) -+ mss |= ENIC_CALC_IP_CKSUM; -+ -+ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM)) -+ mss |= ENIC_CALC_TCP_UDP_CKSUM; -+ -+ wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop, -+ eop, 0, vlan_tag_insert, vlan_id, 0); -+ -+ *desc_p = desc_tmp; -+ buf = &wq->bufs[head_idx]; -+ buf->mb = (void *)tx_pkt; -+ head_idx = enic_ring_incr(desc_count, head_idx); -+ wq_desc_avail--; -+ -+ if (!eop) { -+ for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt = -+ tx_pkt->next) { -+ data_len = tx_pkt->data_len; -+ -+ if (tx_pkt->next == NULL) -+ eop = 1; -+ desc_p = descs + head_idx; -+ bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr -+ + tx_pkt->data_off); -+ wq_enet_desc_enc((struct wq_enet_desc *) -+ &desc_tmp, bus_addr, data_len, -+ mss, 0, 0, eop, eop, 0, -+ vlan_tag_insert, vlan_id, 0); -+ -+ *desc_p = desc_tmp; -+ buf = &wq->bufs[head_idx]; -+ buf->mb = (void *)tx_pkt; -+ head_idx = enic_ring_incr(desc_count, head_idx); -+ wq_desc_avail--; -+ } -+ } -+ } -+ post: -+ rte_wmb(); -+ iowrite32(head_idx, &wq->ctrl->posted_index); -+ done: -+ wq->ring.desc_avail = wq_desc_avail; -+ wq->head_idx = head_idx; -+ -+ return index; -+} --- -2.7.0 - |