Add support of Ethernet link bonding utilizing DPDK link bonding

poll mode driver library. The bonded interfaces to be created on
VPP startup is specified in the dpdk section of startup.conf
or qn.conf, using DPDK EAL command. Following is an example of a
dpdk section white listing PCI addressses of 4 ethernet interfacess
to be under VPP control plus two bonded interface and the PCI addresses
of the slaves in each:
dpdk { socket-mem 1024,1024
dev 0000:0f:00.0 dev 0000:10:00.0 dev 0000:11:00.0 dev 0000:12:00.0
vdev eth_bond0,mode=2,slave=0000:0f:00.0,slave=0000:11:00.0,xmit_policy=l34
vdev  eth_bond1,mode=2,slave=0000:10:00.0,slave=0000:12:00.0,xmit_policy=l34
}
Note that only balance XOR (mode 2) is supported and "xmit_policy=l34"
specifies to use layer 3 SIP/DIP and layer 4 Sport/Dport for load
balance. Using "xmit_policy=l2" for SMAC/DMAC or "xmit_policy=l23" for
SMAC/DMAC and SIP/DIP should also work.

Change-Id: Iaf6438686fa20cce893cb5a823b76e2886b4360b
Signed-off-by: John Lo <loj@cisco.com>

4 files changed, 63 insertions, 4 deletions

diff --git a/dpdk/Makefile b/dpdk/Makefile
index d7d46713..307faf06 100644
--- a/dpdk/Makefile
+++ b/dpdk/Makefile
@@ -101,14 +101,14 @@ $(B)/custom-config: $(B)/.patch.ok Makefile
 	$(call set,RTE_LIBRTE_E1000_DEBUG_INIT,$(DPDK_DEBUG))
 	$(call set,RTE_LIBRTE_VIRTIO_DEBUG_INIT,$(DPDK_DEBUG))
 	$(call set,RTE_LIBRTE_VMXNET3_DEBUG_INIT,$(DPDK_DEBUG))
+	$(call set,RTE_LIBRTE_PMD_BOND,y)
+	$(call set,RTE_LIBRTE_IP_FRAG,y)
 	@# not needed
-	$(call set,RTE_LIBRTE_PMD_BOND,n)
 	$(call set,RTE_LIBRTE_TIMER,n)
 	$(call set,RTE_LIBRTE_CFGFILE,n)
 	$(call set,RTE_LIBRTE_LPM,n)
 	$(call set,RTE_LIBRTE_ACL,n)
 	$(call set,RTE_LIBRTE_POWER,n)
-	$(call set,RTE_LIBRTE_IP_FRAG,n)
 	$(call set,RTE_LIBRTE_DISTRIBUTOR,n)
 	$(call set,RTE_LIBRTE_REORDER,n)
 	$(call set,RTE_LIBRTE_PORT,n)
diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c
index 327f9dec..08fe27e5 100644
--- a/vnet/vnet/devices/dpdk/device.c
+++ b/vnet/vnet/devices/dpdk/device.c
@@ -812,6 +812,9 @@ static u8 * format_dpdk_device_name (u8 * s, va_list * args)
       device_name = "FortyGigabitEthernet";
       break;
 
+    case VNET_DPDK_PORT_TYPE_ETH_BOND:
+      return format(s, "BondEthernet%d", dm->devices[i].device_index);
+
     case VNET_DPDK_PORT_TYPE_ETH_SWITCH:
       device_name = "EthernetSwitch";
       break;
@@ -926,8 +929,12 @@ static u8 * format_dpdk_device_type (u8 * s, va_list * args)
 #endif
 
     case VNET_DPDK_PMD_AF_PACKET:
-  dev_type = "af_packet";
-  break;
+	dev_type = "af_packet";
+	break;
+ 
+    case VNET_DPDK_PMD_BOND:
+	dev_type = "Ethernet Bonding";
+	break;
 
     default:
     case VNET_DPDK_PMD_UNKNOWN:
diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h
index da4e381b..f17c53c7 100644
--- a/vnet/vnet/devices/dpdk/dpdk.h
+++ b/vnet/vnet/devices/dpdk/dpdk.h
@@ -51,6 +51,7 @@
 #include <rte_virtio_net.h>
 #include <rte_pci_dev_ids.h>
 #include <rte_version.h>
+#include <rte_eth_bond.h>
 
 #include <vnet/unix/pcap.h>
 #include <vnet/devices/virtio/vhost-user.h>
@@ -90,6 +91,7 @@ typedef enum {
   _ ("rte_enic_pmd", ENIC)        \
   _ ("rte_vmxnet3_pmd", VMXNET3)  \
   _ ("AF_PACKET PMD", AF_PACKET)  \
+  _ ("rte_bond_pmd", BOND)        \
   _ ("rte_pmd_fm10k", FM10K)      \
   _ ("rte_cxgbe_pmd", CXGBE)
 
@@ -108,6 +110,7 @@ typedef enum {
   VNET_DPDK_PORT_TYPE_ETH_1G,
   VNET_DPDK_PORT_TYPE_ETH_10G,
   VNET_DPDK_PORT_TYPE_ETH_40G,
+  VNET_DPDK_PORT_TYPE_ETH_BOND,
   VNET_DPDK_PORT_TYPE_ETH_SWITCH,
 #ifdef NETMAP
   VNET_DPDK_PORT_TYPE_NETMAP,
diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c
index c9187d5e..f958f81e 100644
--- a/vnet/vnet/devices/dpdk/init.c
+++ b/vnet/vnet/devices/dpdk/init.c
@@ -465,6 +465,10 @@ dpdk_lib_init (dpdk_main_t * dm)
             xd->af_packet_port_id = af_packet_port_id++;
             break;
 
+          case VNET_DPDK_PMD_BOND:
+            xd->port_type = VNET_DPDK_PORT_TYPE_ETH_BOND;
+            break;
+
           default:
             xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
         }
@@ -1589,7 +1593,9 @@ dpdk_process (vlib_main_t * vm,
               vlib_frame_t * f)
 {
   clib_error_t * error;
+  vnet_main_t * vnm = vnet_get_main();
   dpdk_main_t * dm = &dpdk_main;
+  ethernet_main_t * em = &ethernet_main;
   dpdk_device_t * xd;
   vlib_thread_main_t * tm = vlib_get_thread_main();
   void *vu_state;
@@ -1630,6 +1636,45 @@ dpdk_process (vlib_main_t * vm,
       dpdk_update_link_state (xd, now);
     }
 
+{ // Setup MACs for bond interfaces and their links which was initialized in
+  // dpdk_port_setup() but needs to be done again here to take effect.
+  int nports = rte_eth_dev_count();
+  if (nports > 0) {
+      for (i = 0; i < nports; i++) {
+	  struct rte_eth_dev_info dev_info;
+	  rte_eth_dev_info_get(i, &dev_info);
+	  if (!dev_info.driver_name)
+	      dev_info.driver_name = dev_info.pci_dev->driver->name;
+	  ASSERT(dev_info.driver_name);
+	  if (strncmp(dev_info.driver_name, "rte_bond_pmd", 12) == 0) {
+	      u8  addr[6]; 
+	      u8  slink[16];
+	      int nlink = rte_eth_bond_slaves_get(i, slink, 16);
+	      if (nlink > 0) {
+		  vnet_hw_interface_t * hi;
+		  ethernet_interface_t * ei;
+		  /* Get MAC of 1st slave link */
+		  rte_eth_macaddr_get(slink[0], (struct ether_addr *)addr);
+		  /* Set MAC of bounded interface to that of 1st slave link */
+		  rte_eth_bond_mac_address_set(i, (struct ether_addr *)addr);
+		  /* Populate MAC of bonded interface in VPP hw tables */
+		  hi = vnet_get_hw_interface (
+		      vnm, dm->devices[i].vlib_hw_if_index);
+		  ei = pool_elt_at_index (em->interfaces, hi->hw_instance);
+		  memcpy (hi->hw_address, addr, 6);
+		  memcpy (ei->address, addr, 6);
+		  /* Add MAC to other slave links */
+		  while (nlink > 1) {
+		      nlink--;
+		      rte_eth_dev_mac_addr_add(
+			  slink[nlink], (struct ether_addr *)addr, 0);
+		  }
+	      }
+	  }
+      }
+  }
+}
+
   while (1)
     {
       vlib_process_wait_for_event_or_clock (vm, 5.0);
@@ -1733,6 +1778,10 @@ do {                                                  \
   _(rte_cxgbe_driver)
 #endif
 
+#ifdef RTE_LIBRTE_PMD_BOND
+  _(bond_drv)
+#endif
+
 #undef _
 
 /*