From d9bf9abbabac7ea637a25461757303a92e321f7e Mon Sep 17 00:00:00 2001 From: John Lo Date: Thu, 25 Feb 2016 11:17:55 -0500 Subject: Add support of Ethernet link bonding utilizing DPDK link bonding poll mode driver library. The bonded interfaces to be created on VPP startup is specified in the dpdk section of startup.conf or qn.conf, using DPDK EAL command. Following is an example of a dpdk section white listing PCI addressses of 4 ethernet interfacess to be under VPP control plus two bonded interface and the PCI addresses of the slaves in each: dpdk { socket-mem 1024,1024 dev 0000:0f:00.0 dev 0000:10:00.0 dev 0000:11:00.0 dev 0000:12:00.0 vdev eth_bond0,mode=2,slave=0000:0f:00.0,slave=0000:11:00.0,xmit_policy=l34 vdev eth_bond1,mode=2,slave=0000:10:00.0,slave=0000:12:00.0,xmit_policy=l34 } Note that only balance XOR (mode 2) is supported and "xmit_policy=l34" specifies to use layer 3 SIP/DIP and layer 4 Sport/Dport for load balance. Using "xmit_policy=l2" for SMAC/DMAC or "xmit_policy=l23" for SMAC/DMAC and SIP/DIP should also work. Change-Id: Iaf6438686fa20cce893cb5a823b76e2886b4360b Signed-off-by: John Lo --- dpdk/Makefile | 4 ++-- vnet/vnet/devices/dpdk/device.c | 11 +++++++-- vnet/vnet/devices/dpdk/dpdk.h | 3 +++ vnet/vnet/devices/dpdk/init.c | 49 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/dpdk/Makefile b/dpdk/Makefile index d7d4671322c..307faf067de 100644 --- a/dpdk/Makefile +++ b/dpdk/Makefile @@ -101,14 +101,14 @@ $(B)/custom-config: $(B)/.patch.ok Makefile $(call set,RTE_LIBRTE_E1000_DEBUG_INIT,$(DPDK_DEBUG)) $(call set,RTE_LIBRTE_VIRTIO_DEBUG_INIT,$(DPDK_DEBUG)) $(call set,RTE_LIBRTE_VMXNET3_DEBUG_INIT,$(DPDK_DEBUG)) + $(call set,RTE_LIBRTE_PMD_BOND,y) + $(call set,RTE_LIBRTE_IP_FRAG,y) @# not needed - $(call set,RTE_LIBRTE_PMD_BOND,n) $(call set,RTE_LIBRTE_TIMER,n) $(call set,RTE_LIBRTE_CFGFILE,n) $(call set,RTE_LIBRTE_LPM,n) $(call set,RTE_LIBRTE_ACL,n) $(call set,RTE_LIBRTE_POWER,n) - $(call set,RTE_LIBRTE_IP_FRAG,n) $(call set,RTE_LIBRTE_DISTRIBUTOR,n) $(call set,RTE_LIBRTE_REORDER,n) $(call set,RTE_LIBRTE_PORT,n) diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c index 327f9dec5a3..08fe27e5c16 100644 --- a/vnet/vnet/devices/dpdk/device.c +++ b/vnet/vnet/devices/dpdk/device.c @@ -812,6 +812,9 @@ static u8 * format_dpdk_device_name (u8 * s, va_list * args) device_name = "FortyGigabitEthernet"; break; + case VNET_DPDK_PORT_TYPE_ETH_BOND: + return format(s, "BondEthernet%d", dm->devices[i].device_index); + case VNET_DPDK_PORT_TYPE_ETH_SWITCH: device_name = "EthernetSwitch"; break; @@ -926,8 +929,12 @@ static u8 * format_dpdk_device_type (u8 * s, va_list * args) #endif case VNET_DPDK_PMD_AF_PACKET: - dev_type = "af_packet"; - break; + dev_type = "af_packet"; + break; + + case VNET_DPDK_PMD_BOND: + dev_type = "Ethernet Bonding"; + break; default: case VNET_DPDK_PMD_UNKNOWN: diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h index da4e381b3c1..f17c53c7a3b 100644 --- a/vnet/vnet/devices/dpdk/dpdk.h +++ b/vnet/vnet/devices/dpdk/dpdk.h @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -90,6 +91,7 @@ typedef enum { _ ("rte_enic_pmd", ENIC) \ _ ("rte_vmxnet3_pmd", VMXNET3) \ _ ("AF_PACKET PMD", AF_PACKET) \ + _ ("rte_bond_pmd", BOND) \ _ ("rte_pmd_fm10k", FM10K) \ _ ("rte_cxgbe_pmd", CXGBE) @@ -108,6 +110,7 @@ typedef enum { VNET_DPDK_PORT_TYPE_ETH_1G, VNET_DPDK_PORT_TYPE_ETH_10G, VNET_DPDK_PORT_TYPE_ETH_40G, + VNET_DPDK_PORT_TYPE_ETH_BOND, VNET_DPDK_PORT_TYPE_ETH_SWITCH, #ifdef NETMAP VNET_DPDK_PORT_TYPE_NETMAP, diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c index c9187d5eeb5..f958f81e35f 100644 --- a/vnet/vnet/devices/dpdk/init.c +++ b/vnet/vnet/devices/dpdk/init.c @@ -465,6 +465,10 @@ dpdk_lib_init (dpdk_main_t * dm) xd->af_packet_port_id = af_packet_port_id++; break; + case VNET_DPDK_PMD_BOND: + xd->port_type = VNET_DPDK_PORT_TYPE_ETH_BOND; + break; + default: xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN; } @@ -1589,7 +1593,9 @@ dpdk_process (vlib_main_t * vm, vlib_frame_t * f) { clib_error_t * error; + vnet_main_t * vnm = vnet_get_main(); dpdk_main_t * dm = &dpdk_main; + ethernet_main_t * em = ðernet_main; dpdk_device_t * xd; vlib_thread_main_t * tm = vlib_get_thread_main(); void *vu_state; @@ -1630,6 +1636,45 @@ dpdk_process (vlib_main_t * vm, dpdk_update_link_state (xd, now); } +{ // Setup MACs for bond interfaces and their links which was initialized in + // dpdk_port_setup() but needs to be done again here to take effect. + int nports = rte_eth_dev_count(); + if (nports > 0) { + for (i = 0; i < nports; i++) { + struct rte_eth_dev_info dev_info; + rte_eth_dev_info_get(i, &dev_info); + if (!dev_info.driver_name) + dev_info.driver_name = dev_info.pci_dev->driver->name; + ASSERT(dev_info.driver_name); + if (strncmp(dev_info.driver_name, "rte_bond_pmd", 12) == 0) { + u8 addr[6]; + u8 slink[16]; + int nlink = rte_eth_bond_slaves_get(i, slink, 16); + if (nlink > 0) { + vnet_hw_interface_t * hi; + ethernet_interface_t * ei; + /* Get MAC of 1st slave link */ + rte_eth_macaddr_get(slink[0], (struct ether_addr *)addr); + /* Set MAC of bounded interface to that of 1st slave link */ + rte_eth_bond_mac_address_set(i, (struct ether_addr *)addr); + /* Populate MAC of bonded interface in VPP hw tables */ + hi = vnet_get_hw_interface ( + vnm, dm->devices[i].vlib_hw_if_index); + ei = pool_elt_at_index (em->interfaces, hi->hw_instance); + memcpy (hi->hw_address, addr, 6); + memcpy (ei->address, addr, 6); + /* Add MAC to other slave links */ + while (nlink > 1) { + nlink--; + rte_eth_dev_mac_addr_add( + slink[nlink], (struct ether_addr *)addr, 0); + } + } + } + } + } +} + while (1) { vlib_process_wait_for_event_or_clock (vm, 5.0); @@ -1733,6 +1778,10 @@ do { \ _(rte_cxgbe_driver) #endif +#ifdef RTE_LIBRTE_PMD_BOND + _(bond_drv) +#endif + #undef _ /* -- cgit 1.2.3-korg