diff options
Diffstat (limited to 'dpdk')
18 files changed, 2411 insertions, 0 deletions
diff --git a/dpdk/Makefile b/dpdk/Makefile new file mode 100644 index 00000000..179e3565 --- /dev/null +++ b/dpdk/Makefile @@ -0,0 +1,453 @@ +# Copyright (c) 2015 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Scripts require non-POSIX parts of bash +SHELL := /bin/bash + +DPDK_BUILD_DIR ?= $(CURDIR)/_build +DPDK_INSTALL_DIR ?= $(CURDIR)/_install +DPDK_PKTMBUF_HEADROOM ?= 128 +DPDK_DOWNLOAD_DIR ?= $(HOME)/Downloads +DPDK_DEBUG ?= n +DPDK_MLX4_PMD ?= n +DPDK_MLX5_PMD ?= n + +B := $(DPDK_BUILD_DIR) +I := $(DPDK_INSTALL_DIR) +DPDK_VERSION ?= 17.08 +PKG_SUFFIX ?= vpp2 +DPDK_BASE_URL ?= http://fast.dpdk.org/rel +DPDK_TARBALL := dpdk-$(DPDK_VERSION).tar.xz +DPDK_TAR_URL := $(DPDK_BASE_URL)/$(DPDK_TARBALL) +DPDK_17.05_TARBALL_MD5_CKSUM := 0a68c31cd6a6cabeed0a4331073e4c05 +DPDK_17.08_TARBALL_MD5_CKSUM := 0641f59ea8ea98afefa7cfa2699f6241 +DPDK_SOURCE := $(B)/dpdk-$(DPDK_VERSION) +MACHINE=$(shell uname -m) + +ISA_L_CRYPTO_LIB := n + +IPSEC_MB_VER ?= 0.46 +ISA_L_CRYPTO_VER := 2.18.0 + +ifeq ($(MACHINE),$(filter $(MACHINE),x86_64)) + AESNI ?= y +# DPDK pre 17.08 depends on ISA-L Crypto library for GCM PMD + ifneq ($(firstword $(sort $(DPDK_VERSION), 17.08)), 17.08) + ISA_L_CRYPTO_LIB = y + IPSEC_MB_VER = 0.45 + $(info Building ISA-L Crypto $(ISA_L_CRYPTO_VER) library) + endif +$(info Building IPSec-MB $(IPSEC_MB_VER) library) +else + AESNI ?= N +endif + +AESNIMB_LIB_TARBALL := v$(IPSEC_MB_VER).tar.gz +AESNIMB_LIB_TARBALL_URL := http://github.com/01org/intel-ipsec-mb/archive/$(AESNIMB_LIB_TARBALL) +AESNIMB_LIB_SOURCE := $(B)/intel-ipsec-mb-$(IPSEC_MB_VER) + +ISA_L_CRYPTO_LIB_TARBALL := v$(ISA_L_CRYPTO_VER).tar.gz +ISA_L_CRYPTO_LIB_TARBALL_URL := http://github.com/01org/isa-l_crypto/archive/$(ISA_L_CRYPTO_LIB_TARBALL) +ISA_L_CRYPTO_LIB_SOURCE := $(B)/isa-l_crypto-$(ISA_L_CRYPTO_VER) +ISA_L_CRYPTO_INSTALL_DIR := $(ISA_L_CRYPTO_LIB_SOURCE)/install + +ifneq (,$(findstring clang,$(CC))) +DPDK_CC=clang +else ifneq (,$(findstring icc,$(CC))) +DPDK_CC=icc +else +DPDK_CC=gcc +endif + +############################################################################## +# Intel x86 +############################################################################## +ifeq ($(MACHINE),$(filter $(MACHINE),x86_64 i686)) +DPDK_TARGET ?= $(MACHINE)-native-linuxapp-$(DPDK_CC) +DPDK_MACHINE ?= nhm +DPDK_TUNE ?= core-avx2 + +############################################################################## +# ARM64 +############################################################################## +else ifeq ($(MACHINE),aarch64) +export CROSS="" +DPDK_TARGET ?= arm64-armv8a-linuxapp-$(DPDK_CC) +DPDK_MACHINE ?= armv8a +DPDK_TUNE ?= generic + +CPU_IMP_ARM = 0x41 +CPU_IMP_CAVIUM = 0x43 + +CPU_PART_ARM_CORTEX_A53 = 0xd03 +CPU_PART_ARM_CORTEX_A57 = 0xd07 +CPU_PART_ARM_CORTEX_A72 = 0xd08 +CPU_PART_ARM_CORTEX_A73 = 0xd09 + +CPU_PART_CAVIUM_THUNDERX = 0x0a1 +CPU_PART_CAVIUM_THUNDERX_81XX = 0x0a2 +CPU_PART_CAVIUM_THUNDERX_83XX = 0x0a3 + +MIDR_IMPLEMENTER=$(shell awk '/implementer/ {print $$4;exit}' /proc/cpuinfo) +MIDR_PARTNUM=$(shell awk '/part/ {print $$4;exit}' /proc/cpuinfo) + +ifeq ($(MIDR_IMPLEMENTER),$(CPU_IMP_ARM)) +############################################################################## +# Arm Cortex +############################################################################## +CPU_PART_ARM_TUNE := $(CPU_PART_ARM_CORTEX_A53)/cortex-a53 \ + $(CPU_PART_ARM_CORTEX_A57)/cortex-a57 \ + $(CPU_PART_ARM_CORTEX_A72)/cortex-a72 \ + $(CPU_PART_ARM_CORTEX_A73)/cortex-a73 +CPU_TUNE = $(notdir $(filter $(MIDR_PARTNUM)/%,$(CPU_PART_ARM_TUNE))) +ifneq ($(CPU_TUNE),) +DPDK_TUNE = $(CPU_TUNE) +else +$(warning Unknown Arm CPU) +endif + +else ifeq ($(MIDR_IMPLEMENTER),$(CPU_IMP_CAVIUM)) +############################################################################## +# Cavium ThunderX +############################################################################## +ifneq (,$(findstring $(MIDR_PARTNUM),$(CPU_PART_CAVIUM_THUNDERX) \ + $(CPU_PART_CAVIUM_THUNDERX_81XX) $(CPU_PART_CAVIUM_THUNDERX_83XX))) +DPDK_TARGET = arm64-thunderx-linuxapp-$(DPDK_CC) +DPDK_MACHINE = thunderx +else +$(warning Unknown Cavium CPU) +endif +endif + +############################################################################## +# Unknown platform +############################################################################## +else +$(error Unknown platform) +endif + +# /proc/cpuinfo does not exist on platforms without a /proc and on some +# platforms, notably inside containers, it has no content. In those cases +# we assume there's 1 processor; we use 2*ncpu for the -j option. +# NB: GNU Make 4.2 will let us use '$(file </proc/cpuinfo)' to both test +# for file presence and content; for now this will have to do. +JOBS := $(if $(shell [ -f /proc/cpuinfo ] && head /proc/cpuinfo),\ + $(shell expr 2 '*' $$(grep -c ^processor /proc/cpuinfo)), 2) + +# compiler/linker custom arguments +DPDK_CPU_CFLAGS := -pie -fPIC +DPDK_EXTRA_LDFLAGS := -g + +ifeq ($(DPDK_DEBUG),n) +DPDK_EXTRA_CFLAGS := -g -mtune=$(DPDK_TUNE) +else +DPDK_EXTRA_CFLAGS := -g -O0 +endif + +ifeq ($(ISA_L_CRYPTO_LIB),y) +DPDK_EXTRA_CFLAGS += -I$(ISA_L_CRYPTO_INSTALL_DIR)/include -Wl,-z,muldefs +DPDK_EXTRA_LDFLAGS += -L$(I)/lib +endif +DPDK_MAKE_EXTRA_ARGS += AESNI_MULTI_BUFFER_LIB_PATH=$(AESNIMB_LIB_SOURCE) + +# assemble DPDK make arguments +DPDK_MAKE_ARGS := -C $(DPDK_SOURCE) -j $(JOBS) \ + T=$(DPDK_TARGET) \ + RTE_CONFIG_TEMPLATE=../custom-config \ + EXTRA_CFLAGS="$(DPDK_EXTRA_CFLAGS)" \ + EXTRA_LDFLAGS="$(DPDK_EXTRA_LDFLAGS)" \ + CPU_CFLAGS="$(DPDK_CPU_CFLAGS)" \ + DESTDIR=$(I) \ + $(DPDK_MAKE_EXTRA_ARGS) + +define set +@if grep -q CONFIG_$1 $@ ; \ + then sed -i -e 's/.*\(CONFIG_$1=\).*/\1$2/' $@ ; \ + else echo CONFIG_$1=$2 >> $@ ; \ +fi +endef + +all: build + +$(B)/custom-config: $(B)/.patch.ok Makefile + @echo --- generating custom config from $(DPDK_SOURCE)/config/defconfig_$(DPDK_TARGET) --- + @cpp -undef -ffreestanding -x assembler-with-cpp $(DPDK_SOURCE)/config/defconfig_$(DPDK_TARGET) $@ + $(call set,RTE_MACHINE,$(DPDK_MACHINE)) + @# modify options + $(call set,RTE_MAX_LCORE,256) + $(call set,RTE_PKTMBUF_HEADROOM,$(DPDK_PKTMBUF_HEADROOM)) + $(call set,RTE_LIBEAL_USE_HPET,y) + $(call set,RTE_BUILD_COMBINE_LIBS,y) + $(call set,RTE_PCI_CONFIG,y) + $(call set,RTE_PCI_EXTENDED_TAG,"on") + $(call set,RTE_PCI_MAX_READ_REQUEST_SIZE,4096) + @# enable debug init for device drivers + $(call set,RTE_LIBRTE_I40E_DEBUG_INIT,$(DPDK_DEBUG)) + $(call set,RTE_LIBRTE_IXGBE_DEBUG_INIT,$(DPDK_DEBUG)) + $(call set,RTE_LIBRTE_E1000_DEBUG_INIT,$(DPDK_DEBUG)) + $(call set,RTE_LIBRTE_VIRTIO_DEBUG_INIT,$(DPDK_DEBUG)) + $(call set,RTE_LIBRTE_VMXNET3_DEBUG_INIT,$(DPDK_DEBUG)) + $(call set,RTE_LIBRTE_PMD_BOND,y) + $(call set,RTE_LIBRTE_IP_FRAG,y) + $(call set,RTE_LIBRTE_PMD_QAT,y) + $(call set,RTE_LIBRTE_PMD_AESNI_MB,$(AESNI)) + $(call set,RTE_LIBRTE_PMD_AESNI_GCM,$(AESNI)) + $(call set,RTE_LIBRTE_MLX4_PMD,$(DPDK_MLX4_PMD)) + $(call set,RTE_LIBRTE_MLX5_PMD,$(DPDK_MLX5_PMD)) + @# not needed + $(call set,RTE_LIBRTE_PMD_TAP,n) + $(call set,RTE_LIBRTE_TIMER,n) + $(call set,RTE_LIBRTE_CFGFILE,n) + $(call set,RTE_LIBRTE_LPM,n) + $(call set,RTE_LIBRTE_ACL,n) + $(call set,RTE_LIBRTE_POWER,n) + $(call set,RTE_LIBRTE_DISTRIBUTOR,n) + $(call set,RTE_LIBRTE_PORT,n) + $(call set,RTE_LIBRTE_TABLE,n) + $(call set,RTE_LIBRTE_PIPELINE,n) + $(call set,RTE_KNI_KMOD,n) + $(call set,RTE_EAL_IGB_UIO,n) + @rm -f .config.ok + +$(CURDIR)/$(DPDK_TARBALL): + @if [ -e $(DPDK_DOWNLOAD_DIR)/$(DPDK_TARBALL) ] ; \ + then cp $(DPDK_DOWNLOAD_DIR)/$(DPDK_TARBALL) $(CURDIR) ; \ + else curl -o $(CURDIR)/$(DPDK_TARBALL) -LO $(DPDK_TAR_URL) ; \ + fi + @rm -f $(B)/.download.ok + +$(CURDIR)/$(AESNIMB_LIB_TARBALL): + @if [ -e $(DPDK_DOWNLOAD_DIR)/$(AESNIMB_LIB_TARBALL) ] ; \ + then cp $(DPDK_DOWNLOAD_DIR)/$(AESNIMB_LIB_TARBALL) $(CURDIR) ; \ + else curl -o $@ -LO $(AESNIMB_LIB_TARBALL_URL) ; \ + fi + +$(CURDIR)/$(ISA_L_CRYPTO_LIB_TARBALL): + @if [ -e $(DPDK_DOWNLOAD_DIR)/$(ISA_L_CRYPTO_LIB_TARBALL) ] ; \ + then cp $(DPDK_DOWNLOAD_DIR)/$(ISA_L_CRYPTO_LIB_TARBALL) $(CURDIR) ; \ + else curl -o $@ -LO $(ISA_L_CRYPTO_LIB_TARBALL_URL) ; \ + fi + +DPDK_DOWNLOADS = $(CURDIR)/$(DPDK_TARBALL) +ifeq ($(AESNI),y) +DPDK_DOWNLOADS += $(CURDIR)/$(AESNIMB_LIB_TARBALL) +endif +ifeq ($(ISA_L_CRYPTO_LIB),y) +DPDK_DOWNLOADS += $(CURDIR)/$(ISA_L_CRYPTO_LIB_TARBALL) +endif + +$(B)/.download.ok: $(DPDK_DOWNLOADS) + @mkdir -p $(B) + @openssl md5 $< | cut -f 2 -d " " - > $(B)/$(DPDK_TARBALL).md5sum + @([ "$$(<$(B)/$(DPDK_TARBALL).md5sum)" = "$(DPDK_$(DPDK_VERSION)_TARBALL_MD5_CKSUM)" ] || \ + ( echo "Bad Checksum! Please remove $< and retry" && \ + rm $(B)/$(DPDK_TARBALL).md5sum && false )) + @touch $@ + +.PHONY: download +download: $(B)/.download.ok + +$(B)/.extract.ok: $(B)/.download.ok + @echo --- extracting $(DPDK_TARBALL) --- + @tar --directory $(B) --extract --file $(CURDIR)/$(DPDK_TARBALL) +ifeq ($(AESNI),y) + @echo --- extracting $(AESNIMB_LIB_TARBALL) --- + @tar --directory $(B) --extract --file $(CURDIR)/$(AESNIMB_LIB_TARBALL) +endif +ifeq ($(ISA_L_CRYPTO_LIB),y) + @echo --- extracting $(ISA_L_CRYPTO_LIB_TARBALL) --- + @tar --directory $(B) --extract --file $(CURDIR)/$(ISA_L_CRYPTO_LIB_TARBALL) +endif + @touch $@ + +.PHONY: extract +extract: $(B)/.extract.ok + +$(B)/.patch.ok: $(B)/.extract.ok +ifneq ($(wildcard $(CURDIR)/dpdk-$(DPDK_VERSION)_patches/*.patch),) + @echo --- patching --- + @for f in $(CURDIR)/dpdk-$(DPDK_VERSION)_patches/*.patch ; do \ + echo Applying patch: $$(basename $$f) ; \ + patch -p1 -d $(DPDK_SOURCE) < $$f ; \ + done +endif + @touch $@ + +.PHONY: patch +patch: $(B)/.patch.ok + +$(B)/.config.ok: $(B)/.patch.ok $(B)/custom-config + @make $(DPDK_MAKE_ARGS) config + @touch $@ + +.PHONY: config +config: $(B)/.config.ok + +.PHONY: build-ipsec-mb +build-ipsec-mb: + mkdir -p $(I)/lib/ + # Do not build GCM stuff if we are building ISA_L + make -C $(AESNIMB_LIB_SOURCE) -j NO_GCM=$(ISA_L_CRYPTO_LIB) + cp $(AESNIMB_LIB_SOURCE)/libIPSec_MB.a $(I)/lib/ + +.PHONY: build-isal-crypto +build-isal-crypto: + mkdir -p $(I)/lib/ + cd $(ISA_L_CRYPTO_LIB_SOURCE) && ./autogen.sh && \ + ./configure --prefix=$(ISA_L_CRYPTO_INSTALL_DIR) \ + --libdir=$(ISA_L_CRYPTO_INSTALL_DIR)/lib CFLAGS='-fPIC -DPIC -O2' + make -C $(ISA_L_CRYPTO_LIB_SOURCE) -j install + cp $(ISA_L_CRYPTO_INSTALL_DIR)/lib/libisal_crypto.a $(I)/lib/ + +.PHONY: build-dpdk +build-dpdk: + @if [ ! -e $(B)/.config.ok ] ; then echo 'Please run "make config" first' && false ; fi + @make $(DPDK_MAKE_ARGS) install + +# Order matters +ifeq ($(AESNI),y) +BUILD_TARGETS += build-ipsec-mb +endif +ifeq ($(ISA_L_CRYPTO_LIB),y) +BUILD_TARGETS += build-isal-crypto +endif +BUILD_TARGETS += build-dpdk + +$(B)/.build.ok: $(BUILD_TARGETS) + @touch $@ + +.PHONY: build +build: $(B)/.build.ok + +.PHONY: install +install: $(B)/.build.ok + +.PHONY: clean +clean: + @rm -rf $(B) $(I) + +############################################################################## +# .deb packaging +############################################################################## + +DEB_ARCH=$(shell dpkg --print-architecture 2> /dev/null) +DEV_DEB=vpp-dpdk-dev_$(DPDK_VERSION)-$(PKG_SUFFIX)_$(DEB_ARCH).deb +INSTALLED_DEB_VER=$(shell dpkg-query --showformat='$${Version}' --show vpp-dpdk-dev 2> /dev/null) + +.PHONY: build-deb install-deb check-deb + +deb/debian/changelog: Makefile + @echo "vpp-dpdk ($(DPDK_VERSION)-$(PKG_SUFFIX)) unstable; urgency=low" > $@ + @echo "" >> $@ + @echo " * DPDK Release $(DPDK_VERSION)" >> $@ + @echo "" >> $@ + @echo " -- VPP Dev <vpp-dev@lists.fd.io> $(shell date -R)" >> $@ + +$(DEV_DEB): deb/debian/changelog + @cd deb && dpkg-buildpackage -b -uc -us + git clean -fdx deb + +build-deb: $(DEV_DEB) + +install-deb: +ifneq ($(INSTALLED_DEB_VER),$(DPDK_VERSION)-$(PKG_SUFFIX)) + @make $(DEV_DEB) + @sudo dpkg -i $(DEV_DEB) +else + @echo "==========================================================" + @echo " Up-to-date DPDK package already installed" + @echo "==========================================================" +endif + +check-deb: +ifneq ($(INSTALLED_DEB_VER),$(DPDK_VERSION)-$(PKG_SUFFIX)) + @echo "==========================================================" + @echo " Outdated DPDK package detected:" + @echo " Installed: vpp-dpdk-dev $(INSTALLED_DEB_VER)" + @echo " Current: vpp-dpdk-dev $(DPDK_VERSION)-$(PKG_SUFFIX)" + @echo "" + @echo " Please upgrade by invoking 'make dpdk-install-dev'" + @echo " from the top level directory." + @echo "==========================================================" +endif + +############################################################################## +# .rpm packaging +############################################################################## + +RPM_ARCH=$(shell rpm --eval "%{_arch}" 2> /dev/null) +DEV_RPM=vpp-dpdk-devel-$(DPDK_VERSION)-$(PKG_SUFFIX).$(RPM_ARCH).rpm +INSTALLED_RPM_VER=$(shell rpm -q --queryformat '%{VERSION}-%{RELEASE}' vpp-dpdk-devel 2> /dev/null | grep -v "not inst") + +.PHONY: build-rpm install-rpm check-rpm + +$(DEV_RPM): Makefile rpm/vpp-dpdk.spec + @rpmbuild -bb \ + --define "_topdir $(CURDIR)/rpm" \ + --define "_version $(DPDK_VERSION)" \ + --define "_release $(PKG_SUFFIX)" \ + $(CURDIR)/rpm/vpp-dpdk.spec + mv rpm/RPMS/$(RPM_ARCH)/*.rpm . + git clean -fdx rpm + +build-rpm: $(DEV_RPM) + +install-rpm: +ifneq ($(INSTALLED_RPM_VER),$(DPDK_VERSION)-$(PKG_SUFFIX)) + @$(MAKE) $(DEV_RPM) + sudo rpm -Uih $(DEV_RPM) +else + @echo "==========================================================" + @echo " Up-to-date DPDK package already installed" + @echo "==========================================================" +endif + +check-rpm: +ifneq ($(INSTALLED_RPM_VER),$(DPDK_VERSION)-$(PKG_SUFFIX)) + @echo "==========================================================" + @echo " Outdated DPDK package detected:" + @echo " Installed: vpp-dpdk-devel $(INSTALLED_RPM_VER)" + @echo " Current: vpp-dpdk-devel $(DPDK_VERSION)-$(PKG_SUFFIX)" + @echo "" + @echo " Please upgrade by invoking 'make dpdk-install-dev'" + @echo " from the top level directory." + @echo "==========================================================" +endif + +############################################################################## +# ebuild support +############################################################################## + +.PHONY: ebuild-build ebuild-install + +ebuild-build: +ifeq ($(INSTALLED_DEB_VER)$(INSTALLED_RPM_VER),) + @echo "==========================================================" + @echo "Building DPDK from source. Consider installing development" + @echo "package by invoking 'make dpdk-install-dev' from the" + @echo "top level directory" + @echo "==========================================================" + make config +else +ifneq ($(INSTALLED_DEB_VER),) + make check-deb +endif +ifneq ($(INSTALLED_RPM_VER),) + make check-rpm +endif +endif + +ebuild-install: +ifeq ($(INSTALLED_DEB_VER)$(INSTALLED_RPM_VER),) + make install +endif diff --git a/dpdk/README b/dpdk/README new file mode 100644 index 00000000..95154dc2 --- /dev/null +++ b/dpdk/README @@ -0,0 +1,34 @@ + +Changes needed to DPDK are stored here as git patch files. Maintaining these +files using “git format-patch” and “git am” will make it simpler to manage +these changes. Patches made to DPDK should only be temporary until they are +accepted upstream and made available in the next DPDK release. + +The following is the method used to generate these patches: + +1. Git clone the DPDK to a new directory: + # git clone http://dpdk.org/git/dpdk dpdk + +2. Create a branch based on the DPDK release you wish to patch. +Note, “git tag” will show you all the release tags. The following example is +for DPDK release tag “v2.2.0” and will create a branch named “two_dot_two”. + # cd dpdk + # git checkout -b two_dot_two v2.2.0 + +3. Apply all the existing VPP DPDK patches to this new branch. + # git am <VPP directory>/dpdk/dpdk-2.2.0_patches/* + +4. Make your changes and commit your change to your DPDK repository. + # <edit files> + # git commit -s + +5. Create the patch files with format-patch. This creates all the patch files +for your branch (two_dot_two), with your latest commits as the last ones. + # git format-patch master..two_dot_two + +6. Copy, add and commit the new patches into the VPP patches directory. + # cp <new patch files> <VPP directory>/dpdk/dpdk-2.2.0_patches + # cd <VPP directory> + # git add dpdk/dpdk-2.2.0_patches/<new patch files> + # git commit -s + diff --git a/dpdk/deb/debian/compat b/dpdk/deb/debian/compat new file mode 100755 index 00000000..ec635144 --- /dev/null +++ b/dpdk/deb/debian/compat @@ -0,0 +1 @@ +9 diff --git a/dpdk/deb/debian/control b/dpdk/deb/debian/control new file mode 100644 index 00000000..9ffa1a0a --- /dev/null +++ b/dpdk/deb/debian/control @@ -0,0 +1,18 @@ +Source: vpp-dpdk +Section: net +Priority: extra +Maintainer: vpp-dev@lists.fd.io +Build-Depends: debhelper (>= 9), dkms +Standards-Version: 3.9.4 + +Package: vpp-dpdk-dev +Architecture: any +Depends: ${shlibs:Depends} +Description: DPDK Development Package for VPP +Conflicts: dpdk, dpdk-dev, libdpdk-dev + +Package: vpp-dpdk-dkms +Architecture: any +Depends: ${misc:Depends} +Description: DPDK Development Package for VPP - Kernel Modules +Conflicts: dpdk-igb-uio-dkms diff --git a/dpdk/deb/debian/dkms/Makefile b/dpdk/deb/debian/dkms/Makefile new file mode 100644 index 00000000..452c7c26 --- /dev/null +++ b/dpdk/deb/debian/dkms/Makefile @@ -0,0 +1,10 @@ +obj-m:=igb_uio.o + +CONFIG_MODULE_SIG=n + +EXTRA_CFLAGS += -Winline -I$(PWD) + +default: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(shell pwd) modules +clean: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(shell pwd) clean diff --git a/dpdk/deb/debian/rules b/dpdk/deb/debian/rules new file mode 100755 index 00000000..98b1048b --- /dev/null +++ b/dpdk/deb/debian/rules @@ -0,0 +1,38 @@ +#!/usr/bin/make -f +DH_VERBOSE = 1 +PKG=vpp-dpdk + +VERSION = $(shell dpkg-parsechangelog | sed -nr '/^Version:/s/Version: //p') +BASE_VER = $(word 1, $(subst -, ,$(VERSION))) + +export DPDK_BUILD_DIR=$(CURDIR)/_build +export DPDK_INSTALL_DIR=$(CURDIR)/debian/tmp/usr + +SRC=_build/dpdk-$(BASE_VER) + +MAKE_ARGS=-C .. + +include /usr/share/dpkg/default.mk + +%: + dh $@ --with dkms + +override_dh_clean: + make $(MAKE_ARGS) clean + +override_dh_auto_configure: + make $(MAKE_ARGS) config + +override_dh_install: + make $(MAKE_ARGS) install + dh_install -p$(PKG)-dkms \ + $(SRC)/lib/librte_eal/common/include/rte_pci_dev_feature_defs.h \ + $(SRC)/lib/librte_eal/common/include/rte_pci_dev_features.h \ + $(SRC)/lib/librte_eal/linuxapp/igb_uio/igb_uio.c \ + $(SRC)/lib/librte_eal/linuxapp/igb_uio/compat.h \ + debian/dkms/Makefile \ + /usr/src/$(PKG)-dkms-$(VERSION) + dh_install -p$(PKG)-dev --autodest /usr + +override_dh_dkms: + dh_dkms -p$(PKG)-dkms -V $(VERSION) diff --git a/dpdk/deb/debian/vpp-dpdk-dkms.dkms b/dpdk/deb/debian/vpp-dpdk-dkms.dkms new file mode 100644 index 00000000..a166c313 --- /dev/null +++ b/dpdk/deb/debian/vpp-dpdk-dkms.dkms @@ -0,0 +1,8 @@ +PACKAGE_VERSION=#MODULE_VERSION# +PACKAGE_NAME="vpp-dpdk-dkms" +CLEAN="make clean" +BUILT_MODULE_NAME[0]="igb_uio" +BUILT_MODULE_LOCATION[0]="./" +DEST_MODULE_LOCATION[0]="/kernel/net" +MAKE[1]="make" +AUTOINSTALL="yes" diff --git a/dpdk/dpdk-17.02_patches/0001-dpdk-dev-net-mlx5-add-out-of-buffer-counter-to-extended-statistic.patch b/dpdk/dpdk-17.02_patches/0001-dpdk-dev-net-mlx5-add-out-of-buffer-counter-to-extended-statistic.patch new file mode 100644 index 00000000..3ebf5e8a --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0001-dpdk-dev-net-mlx5-add-out-of-buffer-counter-to-extended-statistic.patch @@ -0,0 +1,135 @@ +diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h +index 879da5e..2b4345a 100644 +--- a/drivers/net/mlx5/mlx5.h ++++ b/drivers/net/mlx5/mlx5.h +@@ -197,6 +197,8 @@ struct mlx5_secondary_data { + int mlx5_is_secondary(void); + int priv_get_ifname(const struct priv *, char (*)[IF_NAMESIZE]); + int priv_ifreq(const struct priv *, int req, struct ifreq *); ++int priv_is_ib_cntr(const char *); ++int priv_get_cntr_sysfs(struct priv *, const char *, uint64_t *); + int priv_get_num_vfs(struct priv *, uint16_t *); + int priv_get_mtu(struct priv *, uint16_t *); + int priv_set_flags(struct priv *, unsigned int, unsigned int); +diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c +index 2145965..6b64f44 100644 +--- a/drivers/net/mlx5/mlx5_ethdev.c ++++ b/drivers/net/mlx5/mlx5_ethdev.c +@@ -234,6 +234,23 @@ struct priv * + } + + /** ++ * Check if the counter is located on ib counters file. ++ * ++ * @param[in] cntr ++ * Counter name. ++ * ++ * @return ++ * 1 if counter is located on ib counters file , 0 otherwise. ++ */ ++int ++priv_is_ib_cntr(const char *cntr) ++{ ++ if (!strcmp(cntr, "out_of_buffer")) ++ return 1; ++ return 0; ++} ++ ++/** + * Read from sysfs entry. + * + * @param[in] priv +@@ -260,10 +277,15 @@ struct priv * + if (priv_get_ifname(priv, &ifname)) + return -1; + +- MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, +- ifname, entry); +- +- file = fopen(path, "rb"); ++ if (priv_is_ib_cntr(entry)) { ++ MKSTR(path, "%s/ports/1/hw_counters/%s", ++ priv->ctx->device->ibdev_path, entry); ++ file = fopen(path, "rb"); ++ } else { ++ MKSTR(path, "%s/device/net/%s/%s", ++ priv->ctx->device->ibdev_path, ifname, entry); ++ file = fopen(path, "rb"); ++ } + if (file == NULL) + return -1; + ret = fread(buf, 1, size, file); +@@ -469,6 +491,30 @@ struct priv * + } + + /** ++ * Read device counter from sysfs. ++ * ++ * @param priv ++ * Pointer to private structure. ++ * @param name ++ * Counter name. ++ * @param[out] cntr ++ * Counter output buffer. ++ * ++ * @return ++ * 0 on success, -1 on failure and errno is set. ++ */ ++int ++priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) ++{ ++ unsigned long ulong_ctr; ++ ++ if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) ++ return -1; ++ *cntr = ulong_ctr; ++ return 0; ++} ++ ++/** + * Set device MTU. + * + * @param priv +diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c +index 20c957e..a48ebea 100644 +--- a/drivers/net/mlx5/mlx5_stats.c ++++ b/drivers/net/mlx5/mlx5_stats.c +@@ -125,6 +125,10 @@ struct mlx5_counter_ctrl { + .dpdk_name = "tx_errors_phy", + .ctr_name = "tx_errors_phy", + }, ++ { ++ .dpdk_name = "rx_out_of_buffer", ++ .ctr_name = "out_of_buffer", ++ }, + }; + + static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); +@@ -159,9 +163,15 @@ struct mlx5_counter_ctrl { + WARN("unable to read statistic values from device"); + return -1; + } +- for (i = 0; i != xstats_n; ++i) +- stats[i] = (uint64_t) +- et_stats->data[xstats_ctrl->dev_table_idx[i]]; ++ for (i = 0; i != xstats_n; ++i) { ++ if (priv_is_ib_cntr(mlx5_counters_init[i].ctr_name)) ++ priv_get_cntr_sysfs(priv, ++ mlx5_counters_init[i].ctr_name, ++ &stats[i]); ++ else ++ stats[i] = (uint64_t) ++ et_stats->data[xstats_ctrl->dev_table_idx[i]]; ++ } + return 0; + } + +@@ -233,6 +243,8 @@ struct mlx5_counter_ctrl { + } + } + for (j = 0; j != xstats_n; ++j) { ++ if (priv_is_ib_cntr(mlx5_counters_init[i].ctr_name)) ++ continue; + if (xstats_ctrl->dev_table_idx[j] >= dev_stats_n) { + WARN("counter \"%s\" is not recognized", + mlx5_counters_init[j].dpdk_name); diff --git a/dpdk/dpdk-17.02_patches/0002-dpdk-dev-1-2-net-mlx5-remove-unused-interface-name-query.patch b/dpdk/dpdk-17.02_patches/0002-dpdk-dev-1-2-net-mlx5-remove-unused-interface-name-query.patch new file mode 100644 index 00000000..aa03639b --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0002-dpdk-dev-1-2-net-mlx5-remove-unused-interface-name-query.patch @@ -0,0 +1,22 @@ +diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c +index 20c957e..0c80e4f 100644 +--- a/drivers/net/mlx5/mlx5_stats.c ++++ b/drivers/net/mlx5/mlx5_stats.c +@@ -177,17 +177,12 @@ struct mlx5_counter_ctrl { + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + unsigned int i; + unsigned int j; +- char ifname[IF_NAMESIZE]; + struct ifreq ifr; + struct ethtool_drvinfo drvinfo; + struct ethtool_gstrings *strings = NULL; + unsigned int dev_stats_n; + unsigned int str_sz; + +- if (priv_get_ifname(priv, &ifname)) { +- WARN("unable to get interface name"); +- return; +- } + /* How many statistics are available. */ + drvinfo.cmd = ETHTOOL_GDRVINFO; + ifr.ifr_data = (caddr_t)&drvinfo; diff --git a/dpdk/dpdk-17.02_patches/0003-dpdk-dev-2-2-net-mlx5-fix-extended-statistics-wrong-number.patch b/dpdk/dpdk-17.02_patches/0003-dpdk-dev-2-2-net-mlx5-fix-extended-statistics-wrong-number.patch new file mode 100644 index 00000000..05c2e8df --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0003-dpdk-dev-2-2-net-mlx5-fix-extended-statistics-wrong-number.patch @@ -0,0 +1,87 @@ +diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c +index 0c80e4f..60ffbaa 100644 +--- a/drivers/net/mlx5/mlx5_stats.c ++++ b/drivers/net/mlx5/mlx5_stats.c +@@ -166,6 +166,29 @@ struct mlx5_counter_ctrl { + } + + /** ++ * Query the number of statistics provided by ETHTOOL. ++ * ++ * @param priv ++ * Pointer to private structure. ++ * ++ * @return ++ * Number of statistics on success, -1 on error. ++ */ ++static int ++priv_ethtool_get_stats_n(struct priv *priv) { ++ struct ethtool_drvinfo drvinfo; ++ struct ifreq ifr; ++ ++ drvinfo.cmd = ETHTOOL_GDRVINFO; ++ ifr.ifr_data = (caddr_t)&drvinfo; ++ if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { ++ WARN("unable to query number of statistics"); ++ return -1; ++ } ++ return drvinfo.n_stats; ++} ++ ++/** + * Init the structures to read device counters. + * + * @param priv +@@ -178,19 +201,11 @@ struct mlx5_counter_ctrl { + unsigned int i; + unsigned int j; + struct ifreq ifr; +- struct ethtool_drvinfo drvinfo; + struct ethtool_gstrings *strings = NULL; + unsigned int dev_stats_n; + unsigned int str_sz; + +- /* How many statistics are available. */ +- drvinfo.cmd = ETHTOOL_GDRVINFO; +- ifr.ifr_data = (caddr_t)&drvinfo; +- if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { +- WARN("unable to get driver info"); +- return; +- } +- dev_stats_n = drvinfo.n_stats; ++ dev_stats_n = priv_ethtool_get_stats_n(priv); + if (dev_stats_n < 1) { + WARN("no extended statistics available"); + return; +@@ -410,7 +425,15 @@ struct mlx5_counter_ctrl { + int ret = xstats_n; + + if (n >= xstats_n && stats) { ++ struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; ++ int stats_n; ++ + priv_lock(priv); ++ stats_n = priv_ethtool_get_stats_n(priv); ++ if (stats_n < 0) ++ return -1; ++ if (xstats_ctrl->stats_n != stats_n) ++ priv_xstats_init(priv); + ret = priv_xstats_get(priv, stats); + priv_unlock(priv); + } +@@ -427,8 +450,15 @@ struct mlx5_counter_ctrl { + mlx5_xstats_reset(struct rte_eth_dev *dev) + { + struct priv *priv = mlx5_get_priv(dev); ++ struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; ++ int stats_n; + + priv_lock(priv); ++ stats_n = priv_ethtool_get_stats_n(priv); ++ if (stats_n < 0) ++ return; ++ if (xstats_ctrl->stats_n != stats_n) ++ priv_xstats_init(priv); + priv_xstats_reset(priv); + priv_unlock(priv); + } diff --git a/dpdk/dpdk-17.02_patches/0004-dpdk-dev-net-mlx5-fix-extended-statistics-counters-identification.patch b/dpdk/dpdk-17.02_patches/0004-dpdk-dev-net-mlx5-fix-extended-statistics-counters-identification.patch new file mode 100644 index 00000000..8c066ad3 --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0004-dpdk-dev-net-mlx5-fix-extended-statistics-counters-identification.patch @@ -0,0 +1,13 @@ +diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c +index 1953293..703f48c 100644 +--- a/drivers/net/mlx5/mlx5_stats.c ++++ b/drivers/net/mlx5/mlx5_stats.c +@@ -253,7 +253,7 @@ struct mlx5_counter_ctrl { + } + } + for (j = 0; j != xstats_n; ++j) { +- if (priv_is_ib_cntr(mlx5_counters_init[i].ctr_name)) ++ if (priv_is_ib_cntr(mlx5_counters_init[j].ctr_name)) + continue; + if (xstats_ctrl->dev_table_idx[j] >= dev_stats_n) { + WARN("counter \"%s\" is not recognized", diff --git a/dpdk/dpdk-17.02_patches/0005-net-mlx5-fix-startup-when-flow-cannot-be-applied.patch b/dpdk/dpdk-17.02_patches/0005-net-mlx5-fix-startup-when-flow-cannot-be-applied.patch new file mode 100644 index 00000000..af928bb2 --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0005-net-mlx5-fix-startup-when-flow-cannot-be-applied.patch @@ -0,0 +1,57 @@ +From 0866d640e42d6c54b2b3f15ebde9930e756ba4d5 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?N=C3=A9lio=20Laranjeiro?= <nelio.laranjeiro@6wind.com> +Date: Wed, 22 Feb 2017 10:57:52 +0100 +Subject: [PATCH] net/mlx5: fix startup when flow cannot be applied + +When flows cannot be re-applied due to configuration modifications, the +start function should rollback the configuration done. + +Fixes: 2097d0d1e2cc ("net/mlx5: support basic flow items and actions") +Cc: stable@dpdk.org + +Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com> +--- + drivers/net/mlx5/mlx5_trigger.c | 21 ++++++++++++++++----- + 1 file changed, 16 insertions(+), 5 deletions(-) + +diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c +index 30addd2..0acbf28 100644 +--- a/drivers/net/mlx5/mlx5_trigger.c ++++ b/drivers/net/mlx5/mlx5_trigger.c +@@ -82,17 +82,28 @@ mlx5_dev_start(struct rte_eth_dev *dev) + ERROR("%p: an error occurred while configuring hash RX queues:" + " %s", + (void *)priv, strerror(err)); +- /* Rollback. */ +- priv_special_flow_disable_all(priv); +- priv_mac_addrs_disable(priv); +- priv_destroy_hash_rxqs(priv); ++ goto error; + } + if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) + priv_fdir_enable(priv); +- priv_dev_interrupt_handler_install(priv, dev); + err = priv_flow_start(priv); ++ if (err) { ++ priv->started = 0; ++ ERROR("%p: an error occurred while configuring flows:" ++ " %s", ++ (void *)priv, strerror(err)); ++ goto error; ++ } ++ priv_dev_interrupt_handler_install(priv, dev); + priv_xstats_init(priv); + priv_unlock(priv); ++ return 0; ++error: ++ /* Rollback. */ ++ priv_special_flow_disable_all(priv); ++ priv_mac_addrs_disable(priv); ++ priv_destroy_hash_rxqs(priv); ++ priv_flow_stop(priv); + return -err; + } + +-- +2.7.4 + diff --git a/dpdk/dpdk-17.02_patches/0006-net-mlx5-add-hardware-TSO-support.patch b/dpdk/dpdk-17.02_patches/0006-net-mlx5-add-hardware-TSO-support.patch new file mode 100644 index 00000000..929a6132 --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0006-net-mlx5-add-hardware-TSO-support.patch @@ -0,0 +1,385 @@ +From e25bad4a287924d26627ffe307f8a12824b87054 Mon Sep 17 00:00:00 2001 +From: Shahaf Shuler <shahafs@mellanox.com> +Date: Thu, 2 Mar 2017 11:01:31 +0200 +Subject: [PATCH] net/mlx5: add hardware TSO support + +Implement support for hardware TSO. + +Signed-off-by: Shahaf Shuler <shahafs@mellanox.com> +Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com> +--- + doc/guides/nics/features/mlx5.ini | 1 + + doc/guides/nics/mlx5.rst | 12 ++++ + drivers/net/mlx5/mlx5.c | 18 ++++++ + drivers/net/mlx5/mlx5.h | 2 + + drivers/net/mlx5/mlx5_defs.h | 3 + + drivers/net/mlx5/mlx5_ethdev.c | 2 + + drivers/net/mlx5/mlx5_rxtx.c | 123 +++++++++++++++++++++++++++++++++----- + drivers/net/mlx5/mlx5_rxtx.h | 2 + + drivers/net/mlx5/mlx5_txq.c | 13 ++++ + 9 files changed, 160 insertions(+), 16 deletions(-) + +diff --git a/doc/guides/nics/features/mlx5.ini b/doc/guides/nics/features/mlx5.ini +index f20d214..8df25ce 100644 +--- a/doc/guides/nics/features/mlx5.ini ++++ b/doc/guides/nics/features/mlx5.ini +@@ -11,6 +11,7 @@ Queue start/stop = Y + MTU update = Y + Jumbo frame = Y + Scattered Rx = Y ++TSO = Y + Promiscuous mode = Y + Allmulticast mode = Y + Unicast MAC filter = Y +diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst +index 5f6e594..9b0ba29 100644 +--- a/doc/guides/nics/mlx5.rst ++++ b/doc/guides/nics/mlx5.rst +@@ -90,6 +90,7 @@ Features + - Secondary process TX is supported. + - KVM and VMware ESX SR-IOV modes are supported. + - RSS hash result is supported. ++- Hardware TSO. + + Limitations + ----------- +@@ -186,9 +187,20 @@ Run-time configuration + save PCI bandwidth and improve performance at the cost of a slightly + higher CPU usage. + ++ This option cannot be used in conjunction with ``tso`` below. When ``tso`` ++ is set, ``txq_mpw_en`` is disabled. ++ + It is currently only supported on the ConnectX-4 Lx and ConnectX-5 + families of adapters. Enabled by default. + ++- ``tso`` parameter [int] ++ ++ A nonzero value enables hardware TSO. ++ When hardware TSO is enabled, packets marked with TCP segmentation ++ offload will be divided into segments by the hardware. ++ ++ Disabled by default. ++ + Prerequisites + ------------- + +diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c +index d4bd469..03ed3b3 100644 +--- a/drivers/net/mlx5/mlx5.c ++++ b/drivers/net/mlx5/mlx5.c +@@ -84,6 +84,9 @@ + /* Device parameter to enable multi-packet send WQEs. */ + #define MLX5_TXQ_MPW_EN "txq_mpw_en" + ++/* Device parameter to enable hardware TSO offload. */ ++#define MLX5_TSO "tso" ++ + /** + * Retrieve integer value from environment variable. + * +@@ -290,6 +293,8 @@ mlx5_args_check(const char *key, const char *val, void *opaque) + priv->txqs_inline = tmp; + } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { + priv->mps &= !!tmp; /* Enable MPW only if HW supports */ ++ } else if (strcmp(MLX5_TSO, key) == 0) { ++ priv->tso = !!tmp; + } else { + WARN("%s: unknown parameter", key); + return -EINVAL; +@@ -316,6 +321,7 @@ mlx5_args(struct priv *priv, struct rte_devargs *devargs) + MLX5_TXQ_INLINE, + MLX5_TXQS_MIN_INLINE, + MLX5_TXQ_MPW_EN, ++ MLX5_TSO, + NULL, + }; + struct rte_kvargs *kvlist; +@@ -479,6 +485,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + IBV_EXP_DEVICE_ATTR_RX_HASH | + IBV_EXP_DEVICE_ATTR_VLAN_OFFLOADS | + IBV_EXP_DEVICE_ATTR_RX_PAD_END_ALIGN | ++ IBV_EXP_DEVICE_ATTR_TSO_CAPS | + 0; + + DEBUG("using port %u (%08" PRIx32 ")", port, test); +@@ -580,11 +587,22 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + + priv_get_num_vfs(priv, &num_vfs); + priv->sriov = (num_vfs || sriov); ++ priv->tso = ((priv->tso) && ++ (exp_device_attr.tso_caps.max_tso > 0) && ++ (exp_device_attr.tso_caps.supported_qpts & ++ (1 << IBV_QPT_RAW_ETH))); ++ if (priv->tso) ++ priv->max_tso_payload_sz = ++ exp_device_attr.tso_caps.max_tso; + if (priv->mps && !mps) { + ERROR("multi-packet send not supported on this device" + " (" MLX5_TXQ_MPW_EN ")"); + err = ENOTSUP; + goto port_error; ++ } else if (priv->mps && priv->tso) { ++ WARN("multi-packet send not supported in conjunction " ++ "with TSO. MPS disabled"); ++ priv->mps = 0; + } + /* Allocate and register default RSS hash keys. */ + priv->rss_conf = rte_calloc(__func__, hash_rxq_init_n, +diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h +index 4c4b9d4..93f129b 100644 +--- a/drivers/net/mlx5/mlx5.h ++++ b/drivers/net/mlx5/mlx5.h +@@ -126,6 +126,8 @@ struct priv { + unsigned int mps:1; /* Whether multi-packet send is supported. */ + unsigned int cqe_comp:1; /* Whether CQE compression is enabled. */ + unsigned int pending_alarm:1; /* An alarm is pending. */ ++ unsigned int tso:1; /* Whether TSO is supported. */ ++ unsigned int max_tso_payload_sz; /* Maximum TCP payload for TSO. */ + unsigned int txq_inline; /* Maximum packet size for inlining. */ + unsigned int txqs_inline; /* Queue number threshold for inlining. */ + /* RX/TX queues. */ +diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h +index e91d245..eecb908 100644 +--- a/drivers/net/mlx5/mlx5_defs.h ++++ b/drivers/net/mlx5/mlx5_defs.h +@@ -79,4 +79,7 @@ + /* Maximum number of extended statistics counters. */ + #define MLX5_MAX_XSTATS 32 + ++/* Maximum Packet headers size (L2+L3+L4) for TSO. */ ++#define MLX5_MAX_TSO_HEADER 128 ++ + #endif /* RTE_PMD_MLX5_DEFS_H_ */ +diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c +index 5677f03..5542193 100644 +--- a/drivers/net/mlx5/mlx5_ethdev.c ++++ b/drivers/net/mlx5/mlx5_ethdev.c +@@ -693,6 +693,8 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM); ++ if (priv->tso) ++ info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; + if (priv_get_ifname(priv, &ifname) == 0) + info->if_index = if_nametoindex(ifname); + /* FIXME: RETA update/query API expects the callee to know the size of +diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c +index 4d5455b..98889f6 100644 +--- a/drivers/net/mlx5/mlx5_rxtx.c ++++ b/drivers/net/mlx5/mlx5_rxtx.c +@@ -365,6 +365,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) + const unsigned int elts_n = 1 << txq->elts_n; + unsigned int i = 0; + unsigned int j = 0; ++ unsigned int k = 0; + unsigned int max; + uint16_t max_wqe; + unsigned int comp; +@@ -392,8 +393,10 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) + uintptr_t addr; + uint64_t naddr; + uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2; ++ uint16_t tso_header_sz = 0; + uint16_t ehdr; + uint8_t cs_flags = 0; ++ uint64_t tso = 0; + #ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t total_length = 0; + #endif +@@ -465,14 +468,74 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) + length -= pkt_inline_sz; + addr += pkt_inline_sz; + } ++ if (txq->tso_en) { ++ tso = buf->ol_flags & PKT_TX_TCP_SEG; ++ if (tso) { ++ uintptr_t end = (uintptr_t) ++ (((uintptr_t)txq->wqes) + ++ (1 << txq->wqe_n) * ++ MLX5_WQE_SIZE); ++ unsigned int copy_b; ++ uint8_t vlan_sz = (buf->ol_flags & ++ PKT_TX_VLAN_PKT) ? 4 : 0; ++ ++ tso_header_sz = buf->l2_len + vlan_sz + ++ buf->l3_len + buf->l4_len; ++ ++ if (unlikely(tso_header_sz > ++ MLX5_MAX_TSO_HEADER)) ++ break; ++ copy_b = tso_header_sz - pkt_inline_sz; ++ /* First seg must contain all headers. */ ++ assert(copy_b <= length); ++ raw += MLX5_WQE_DWORD_SIZE; ++ if (copy_b && ++ ((end - (uintptr_t)raw) > copy_b)) { ++ uint16_t n = (MLX5_WQE_DS(copy_b) - ++ 1 + 3) / 4; ++ ++ if (unlikely(max_wqe < n)) ++ break; ++ max_wqe -= n; ++ rte_memcpy((void *)raw, ++ (void *)addr, copy_b); ++ addr += copy_b; ++ length -= copy_b; ++ pkt_inline_sz += copy_b; ++ /* ++ * Another DWORD will be added ++ * in the inline part. ++ */ ++ raw += MLX5_WQE_DS(copy_b) * ++ MLX5_WQE_DWORD_SIZE - ++ MLX5_WQE_DWORD_SIZE; ++ } else { ++ /* NOP WQE. */ ++ wqe->ctrl = (rte_v128u32_t){ ++ htonl(txq->wqe_ci << 8), ++ htonl(txq->qp_num_8s | 1), ++ 0, ++ 0, ++ }; ++ ds = 1; ++ total_length = 0; ++ pkts--; ++ pkts_n++; ++ elts_head = (elts_head - 1) & ++ (elts_n - 1); ++ k++; ++ goto next_wqe; ++ } ++ } ++ } + /* Inline if enough room. */ +- if (txq->max_inline) { ++ if (txq->inline_en || tso) { + uintptr_t end = (uintptr_t) + (((uintptr_t)txq->wqes) + + (1 << txq->wqe_n) * MLX5_WQE_SIZE); + unsigned int max_inline = txq->max_inline * + RTE_CACHE_LINE_SIZE - +- MLX5_WQE_DWORD_SIZE; ++ (pkt_inline_sz - 2); + uintptr_t addr_end = (addr + max_inline) & + ~(RTE_CACHE_LINE_SIZE - 1); + unsigned int copy_b = (addr_end > addr) ? +@@ -491,6 +554,18 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) + if (unlikely(max_wqe < n)) + break; + max_wqe -= n; ++ if (tso) { ++ uint32_t inl = ++ htonl(copy_b | MLX5_INLINE_SEG); ++ ++ pkt_inline_sz = ++ MLX5_WQE_DS(tso_header_sz) * ++ MLX5_WQE_DWORD_SIZE; ++ rte_memcpy((void *)raw, ++ (void *)&inl, sizeof(inl)); ++ raw += sizeof(inl); ++ pkt_inline_sz += sizeof(inl); ++ } + rte_memcpy((void *)raw, (void *)addr, copy_b); + addr += copy_b; + length -= copy_b; +@@ -591,18 +666,34 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) + next_pkt: + ++i; + /* Initialize known and common part of the WQE structure. */ +- wqe->ctrl = (rte_v128u32_t){ +- htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND), +- htonl(txq->qp_num_8s | ds), +- 0, +- 0, +- }; +- wqe->eseg = (rte_v128u32_t){ +- 0, +- cs_flags, +- 0, +- (ehdr << 16) | htons(pkt_inline_sz), +- }; ++ if (tso) { ++ wqe->ctrl = (rte_v128u32_t){ ++ htonl((txq->wqe_ci << 8) | MLX5_OPCODE_TSO), ++ htonl(txq->qp_num_8s | ds), ++ 0, ++ 0, ++ }; ++ wqe->eseg = (rte_v128u32_t){ ++ 0, ++ cs_flags | (htons(buf->tso_segsz) << 16), ++ 0, ++ (ehdr << 16) | htons(tso_header_sz), ++ }; ++ } else { ++ wqe->ctrl = (rte_v128u32_t){ ++ htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND), ++ htonl(txq->qp_num_8s | ds), ++ 0, ++ 0, ++ }; ++ wqe->eseg = (rte_v128u32_t){ ++ 0, ++ cs_flags, ++ 0, ++ (ehdr << 16) | htons(pkt_inline_sz), ++ }; ++ } ++next_wqe: + txq->wqe_ci += (ds + 3) / 4; + #ifdef MLX5_PMD_SOFT_COUNTERS + /* Increment sent bytes counter. */ +@@ -610,10 +701,10 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) + #endif + } while (pkts_n); + /* Take a shortcut if nothing must be sent. */ +- if (unlikely(i == 0)) ++ if (unlikely((i + k) == 0)) + return 0; + /* Check whether completion threshold has been reached. */ +- comp = txq->elts_comp + i + j; ++ comp = txq->elts_comp + i + j + k; + if (comp >= MLX5_TX_COMP_THRESH) { + volatile struct mlx5_wqe_ctrl *w = + (volatile struct mlx5_wqe_ctrl *)wqe; +diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h +index 41a34d7..6b328cf 100644 +--- a/drivers/net/mlx5/mlx5_rxtx.h ++++ b/drivers/net/mlx5/mlx5_rxtx.h +@@ -254,6 +254,8 @@ struct txq { + uint16_t cqe_n:4; /* Number of CQ elements (in log2). */ + uint16_t wqe_n:4; /* Number of of WQ elements (in log2). */ + uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ ++ uint16_t inline_en:1; /* When set inline is enabled. */ ++ uint16_t tso_en:1; /* When set hardware TSO is enabled. */ + uint32_t qp_num_8s; /* QP number shifted by 8. */ + volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ + volatile void *wqes; /* Work queue (use volatile to write into). */ +diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c +index 949035b..995b763 100644 +--- a/drivers/net/mlx5/mlx5_txq.c ++++ b/drivers/net/mlx5/mlx5_txq.c +@@ -342,6 +342,19 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, + RTE_CACHE_LINE_SIZE); + attr.init.cap.max_inline_data = + tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE; ++ tmpl.txq.inline_en = 1; ++ } ++ if (priv->tso) { ++ uint16_t max_tso_inline = ((MLX5_MAX_TSO_HEADER + ++ (RTE_CACHE_LINE_SIZE - 1)) / ++ RTE_CACHE_LINE_SIZE); ++ ++ attr.init.max_tso_header = ++ max_tso_inline * RTE_CACHE_LINE_SIZE; ++ attr.init.comp_mask |= IBV_EXP_QP_INIT_ATTR_MAX_TSO_HEADER; ++ tmpl.txq.max_inline = RTE_MAX(tmpl.txq.max_inline, ++ max_tso_inline); ++ tmpl.txq.tso_en = 1; + } + tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); + if (tmpl.qp == NULL) { +-- +2.7.4 + diff --git a/dpdk/dpdk-17.02_patches/0007-add-hardware-checksum-offload-for-tunnel-pa.patch b/dpdk/dpdk-17.02_patches/0007-add-hardware-checksum-offload-for-tunnel-pa.patch new file mode 100644 index 00000000..bbcce486 --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0007-add-hardware-checksum-offload-for-tunnel-pa.patch @@ -0,0 +1,194 @@ +From f0dda2ab16635894b1e3836d0b960b9270a3b491 Mon Sep 17 00:00:00 2001 +From: Shahaf Shuler <shahafs@mellanox.com> +Date: Thu, 2 Mar 2017 11:05:44 +0200 +Subject: [PATCH] net/mlx5: add hardware checksum offload for tunnel packets + +Prior to this commit Tx checksum offload was supported only for the +inner headers. +This commit adds support for the hardware to compute the checksum for the +outer headers as well. + +The support is for tunneling protocols GRE and VXLAN. + +Signed-off-by: Shahaf Shuler <shahafs@mellanox.com> +Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com> +--- + doc/guides/nics/features/mlx5.ini | 2 ++ + doc/guides/nics/mlx5.rst | 3 ++- + drivers/net/mlx5/mlx5.c | 7 +++++++ + drivers/net/mlx5/mlx5.h | 2 ++ + drivers/net/mlx5/mlx5_ethdev.c | 2 ++ + drivers/net/mlx5/mlx5_prm.h | 6 ++++++ + drivers/net/mlx5/mlx5_rxtx.c | 14 +++++++++++++- + drivers/net/mlx5/mlx5_rxtx.h | 2 ++ + drivers/net/mlx5/mlx5_txq.c | 2 ++ + 9 files changed, 38 insertions(+), 2 deletions(-) + +diff --git a/doc/guides/nics/features/mlx5.ini b/doc/guides/nics/features/mlx5.ini +index 8df25ce..1814f82 100644 +--- a/doc/guides/nics/features/mlx5.ini ++++ b/doc/guides/nics/features/mlx5.ini +@@ -27,6 +27,8 @@ CRC offload = Y + VLAN offload = Y + L3 checksum offload = Y + L4 checksum offload = Y ++Inner L3 checksum = Y ++Inner L4 checksum = Y + Packet type parsing = Y + Basic stats = Y + Stats per queue = Y +diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst +index 9b0ba29..41f3a47 100644 +--- a/doc/guides/nics/mlx5.rst ++++ b/doc/guides/nics/mlx5.rst +@@ -91,13 +91,14 @@ Features + - KVM and VMware ESX SR-IOV modes are supported. + - RSS hash result is supported. + - Hardware TSO. ++- Hardware checksum TX offload for VXLAN and GRE. + + Limitations + ----------- + + - Inner RSS for VXLAN frames is not supported yet. + - Port statistics through software counters only. +-- Hardware checksum offloads for VXLAN inner header are not supported yet. ++- Hardware checksum RX offloads for VXLAN inner header are not supported yet. + - Secondary process RX is not supported. + + Configuration +diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c +index 03ed3b3..6f42948 100644 +--- a/drivers/net/mlx5/mlx5.c ++++ b/drivers/net/mlx5/mlx5.c +@@ -375,6 +375,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + struct ibv_device_attr device_attr; + unsigned int sriov; + unsigned int mps; ++ unsigned int tunnel_en; + int idx; + int i; + +@@ -429,12 +430,17 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + * as all ConnectX-5 devices. + */ + switch (pci_dev->id.device_id) { ++ case PCI_DEVICE_ID_MELLANOX_CONNECTX4: ++ tunnel_en = 1; ++ mps = 0; ++ break; + case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: + mps = 1; ++ tunnel_en = 1; + break; + default: + mps = 0; +@@ -539,6 +545,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + priv->mtu = ETHER_MTU; + priv->mps = mps; /* Enable MPW by default if supported. */ + priv->cqe_comp = 1; /* Enable compression by default. */ ++ priv->tunnel_en = tunnel_en; + err = mlx5_args(priv, pci_dev->device.devargs); + if (err) { + ERROR("failed to process device arguments: %s", +diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h +index 93f129b..870e01f 100644 +--- a/drivers/net/mlx5/mlx5.h ++++ b/drivers/net/mlx5/mlx5.h +@@ -127,6 +127,8 @@ struct priv { + unsigned int cqe_comp:1; /* Whether CQE compression is enabled. */ + unsigned int pending_alarm:1; /* An alarm is pending. */ + unsigned int tso:1; /* Whether TSO is supported. */ ++ unsigned int tunnel_en:1; ++ /* Whether Tx offloads for tunneled packets are supported. */ + unsigned int max_tso_payload_sz; /* Maximum TCP payload for TSO. */ + unsigned int txq_inline; /* Maximum packet size for inlining. */ + unsigned int txqs_inline; /* Queue number threshold for inlining. */ +diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c +index 5542193..8be9e77 100644 +--- a/drivers/net/mlx5/mlx5_ethdev.c ++++ b/drivers/net/mlx5/mlx5_ethdev.c +@@ -695,6 +695,8 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) + DEV_TX_OFFLOAD_TCP_CKSUM); + if (priv->tso) + info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; ++ if (priv->tunnel_en) ++ info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + if (priv_get_ifname(priv, &ifname) == 0) + info->if_index = if_nametoindex(ifname); + /* FIXME: RETA update/query API expects the callee to know the size of +diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h +index 3318668..0a77f5b 100644 +--- a/drivers/net/mlx5/mlx5_prm.h ++++ b/drivers/net/mlx5/mlx5_prm.h +@@ -120,6 +120,12 @@ + /* Tunnel packet bit in the CQE. */ + #define MLX5_CQE_RX_TUNNEL_PACKET (1u << 0) + ++/* Inner L3 checksum offload (Tunneled packets only). */ ++#define MLX5_ETH_WQE_L3_INNER_CSUM (1u << 4) ++ ++/* Inner L4 checksum offload (Tunneled packets only). */ ++#define MLX5_ETH_WQE_L4_INNER_CSUM (1u << 5) ++ + /* INVALID is used by packets matching no flow rules. */ + #define MLX5_FLOW_MARK_INVALID 0 + +diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c +index 98889f6..c2eb891 100644 +--- a/drivers/net/mlx5/mlx5_rxtx.c ++++ b/drivers/net/mlx5/mlx5_rxtx.c +@@ -443,7 +443,19 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) + /* Should we enable HW CKSUM offload */ + if (buf->ol_flags & + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { +- cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; ++ const uint64_t is_tunneled = buf->ol_flags & ++ (PKT_TX_TUNNEL_GRE | ++ PKT_TX_TUNNEL_VXLAN); ++ ++ if (is_tunneled && txq->tunnel_en) { ++ cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | ++ MLX5_ETH_WQE_L4_INNER_CSUM; ++ if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM) ++ cs_flags |= MLX5_ETH_WQE_L3_CSUM; ++ } else { ++ cs_flags = MLX5_ETH_WQE_L3_CSUM | ++ MLX5_ETH_WQE_L4_CSUM; ++ } + } + raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE; + /* Replace the Ethernet type by the VLAN if necessary. */ +diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h +index 6b328cf..9669564 100644 +--- a/drivers/net/mlx5/mlx5_rxtx.h ++++ b/drivers/net/mlx5/mlx5_rxtx.h +@@ -256,6 +256,8 @@ struct txq { + uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ + uint16_t inline_en:1; /* When set inline is enabled. */ + uint16_t tso_en:1; /* When set hardware TSO is enabled. */ ++ uint16_t tunnel_en:1; ++ /* When set TX offload for tunneled packets are supported. */ + uint32_t qp_num_8s; /* QP number shifted by 8. */ + volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ + volatile void *wqes; /* Work queue (use volatile to write into). */ +diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c +index 995b763..9d0c00f 100644 +--- a/drivers/net/mlx5/mlx5_txq.c ++++ b/drivers/net/mlx5/mlx5_txq.c +@@ -356,6 +356,8 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, + max_tso_inline); + tmpl.txq.tso_en = 1; + } ++ if (priv->tunnel_en) ++ tmpl.txq.tunnel_en = 1; + tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); + if (tmpl.qp == NULL) { + ret = (errno ? errno : EINVAL); +-- +2.7.4 + diff --git a/dpdk/dpdk-17.02_patches/0008-net-mlx5-add-enhanced-multi-packet-send-for-ConnectX.patch b/dpdk/dpdk-17.02_patches/0008-net-mlx5-add-enhanced-multi-packet-send-for-ConnectX.patch new file mode 100644 index 00000000..6ff076c7 --- /dev/null +++ b/dpdk/dpdk-17.02_patches/0008-net-mlx5-add-enhanced-multi-packet-send-for-ConnectX.patch @@ -0,0 +1,809 @@ +From 7ca5c8de65acabe4cb60960adcfa9247efdd2a5c Mon Sep 17 00:00:00 2001 +From: Yongseok Koh <yskoh@mellanox.com> +Date: Wed, 15 Mar 2017 16:55:44 -0700 +Subject: [PATCH] net/mlx5: add enhanced multi-packet send for ConnectX-5 + +ConnectX-5 supports enhanced version of multi-packet send (MPS). An MPS Tx +descriptor can carry multiple packets either by including pointers of +packets or by inlining packets. Inlining packet data can be helpful to +better utilize PCIe bandwidth. In addition, Enhanced MPS supports hybrid +mode - mixing inlined packets and pointers in a descriptor. This feature is +enabled by default if supported by HW. + +Signed-off-by: Yongseok Koh <yskoh@mellanox.com> +--- + doc/guides/nics/mlx5.rst | 31 +++- + drivers/net/mlx5/mlx5.c | 37 +++- + drivers/net/mlx5/mlx5.h | 4 +- + drivers/net/mlx5/mlx5_defs.h | 7 + + drivers/net/mlx5/mlx5_ethdev.c | 6 +- + drivers/net/mlx5/mlx5_prm.h | 20 ++ + drivers/net/mlx5/mlx5_rxtx.c | 410 +++++++++++++++++++++++++++++++++++++++++ + drivers/net/mlx5/mlx5_rxtx.h | 7 +- + drivers/net/mlx5/mlx5_txq.c | 29 ++- + 9 files changed, 534 insertions(+), 17 deletions(-) + +diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst +index 41f3a47..0783aeb 100644 +--- a/doc/guides/nics/mlx5.rst ++++ b/doc/guides/nics/mlx5.rst +@@ -183,10 +183,17 @@ Run-time configuration + + - ``txq_mpw_en`` parameter [int] + +- A nonzero value enables multi-packet send. This feature allows the TX +- burst function to pack up to five packets in two descriptors in order to +- save PCI bandwidth and improve performance at the cost of a slightly +- higher CPU usage. ++ A nonzero value enables multi-packet send (MPS) for ConnectX-4 Lx and ++ enhanced multi-packet send (Enhanced MPS) for ConnectX-5. MPS allows the ++ TX burst function to pack up multiple packets in a single descriptor ++ session in order to save PCI bandwidth and improve performance at the ++ cost of a slightly higher CPU usage. When ``txq_inline`` is set along ++ with ``txq_mpw_en``, TX burst function tries to copy entire packet data ++ on to TX descriptor instead of including pointer of packet only if there ++ is enough room remained in the descriptor. ``txq_inline`` sets ++ per-descriptor space for either pointers or inlined packets. In addition, ++ Enhanced MPS supports hybrid mode - mixing inlined packets and pointers ++ in the same descriptor. + + This option cannot be used in conjunction with ``tso`` below. When ``tso`` + is set, ``txq_mpw_en`` is disabled. +@@ -194,6 +201,22 @@ Run-time configuration + It is currently only supported on the ConnectX-4 Lx and ConnectX-5 + families of adapters. Enabled by default. + ++- ``txq_mpw_hdr_dseg_en`` parameter [int] ++ ++ A nonzero value enables including two pointers in the first block of TX ++ descriptor. This can be used to lessen CPU load for memory copy. ++ ++ Effective only when Enhanced MPS is supported. Disabled by default. ++ ++- ``txq_max_inline_len`` parameter [int] ++ ++ Maximum size of packet to be inlined. This limits the size of packet to ++ be inlined. If the size of a packet is larger than configured value, the ++ packet isn't inlined even though there's enough space remained in the ++ descriptor. Instead, the packet is included with pointer. ++ ++ Effective only when Enhanced MPS is supported. The default value is 256. ++ + - ``tso`` parameter [int] + + A nonzero value enables hardware TSO. +diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c +index ebc7984..bc6a34f 100644 +--- a/drivers/net/mlx5/mlx5.c ++++ b/drivers/net/mlx5/mlx5.c +@@ -84,6 +84,12 @@ + /* Device parameter to enable multi-packet send WQEs. */ + #define MLX5_TXQ_MPW_EN "txq_mpw_en" + ++/* Device parameter to include 2 dsegs in the title WQEBB. */ ++#define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" ++ ++/* Device parameter to limit the size of inlining packet. */ ++#define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" ++ + /* Device parameter to enable hardware TSO offload. */ + #define MLX5_TSO "tso" + +@@ -294,7 +300,11 @@ mlx5_args_check(const char *key, const char *val, void *opaque) + } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { + priv->txqs_inline = tmp; + } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { +- priv->mps &= !!tmp; /* Enable MPW only if HW supports */ ++ priv->mps = !!tmp ? priv->mps : MLX5_MPW_DISABLED; ++ } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { ++ priv->mpw_hdr_dseg = !!tmp; ++ } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { ++ priv->inline_max_packet_sz = tmp; + } else if (strcmp(MLX5_TSO, key) == 0) { + priv->tso = !!tmp; + } else { +@@ -323,6 +333,8 @@ mlx5_args(struct priv *priv, struct rte_devargs *devargs) + MLX5_TXQ_INLINE, + MLX5_TXQS_MIN_INLINE, + MLX5_TXQ_MPW_EN, ++ MLX5_TXQ_MPW_HDR_DSEG_EN, ++ MLX5_TXQ_MAX_INLINE_LEN, + MLX5_TSO, + NULL, + }; +@@ -434,24 +446,27 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + switch (pci_dev->id.device_id) { + case PCI_DEVICE_ID_MELLANOX_CONNECTX4: + tunnel_en = 1; +- mps = 0; ++ mps = MLX5_MPW_DISABLED; + break; + case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: ++ mps = MLX5_MPW; ++ break; + case PCI_DEVICE_ID_MELLANOX_CONNECTX5: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: +- mps = 1; + tunnel_en = 1; ++ mps = MLX5_MPW_ENHANCED; + break; + default: +- mps = 0; ++ mps = MLX5_MPW_DISABLED; + } + INFO("PCI information matches, using device \"%s\"" +- " (SR-IOV: %s, MPS: %s)", ++ " (SR-IOV: %s, %sMPS: %s)", + list[i]->name, + sriov ? "true" : "false", +- mps ? "true" : "false"); ++ mps == MLX5_MPW_ENHANCED ? "Enhanced " : "", ++ mps != MLX5_MPW_DISABLED ? "true" : "false"); + attr_ctx = ibv_open_device(list[i]); + err = errno; + break; +@@ -546,6 +561,13 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + priv->pd = pd; + priv->mtu = ETHER_MTU; + priv->mps = mps; /* Enable MPW by default if supported. */ ++ /* Set default values for Enhanced MPW, a.k.a MPWv2. */ ++ if (mps == MLX5_MPW_ENHANCED) { ++ priv->mpw_hdr_dseg = 0; ++ priv->txqs_inline = MLX5_EMPW_MIN_TXQS; ++ priv->inline_max_packet_sz = MLX5_EMPW_MAX_INLINE_LEN; ++ priv->txq_inline = MLX5_WQE_SIZE_MAX - MLX5_WQE_SIZE; ++ } + priv->cqe_comp = 1; /* Enable compression by default. */ + priv->tunnel_en = tunnel_en; + err = mlx5_args(priv, pci_dev->device.devargs); +@@ -613,6 +635,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) + "with TSO. MPS disabled"); + priv->mps = 0; + } ++ INFO("%sMPS is %s", ++ priv->mps == MLX5_MPW_ENHANCED ? "Enhanced " : "", ++ priv->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); + /* Allocate and register default RSS hash keys. */ + priv->rss_conf = rte_calloc(__func__, hash_rxq_init_n, + sizeof((*priv->rss_conf)[0]), 0); +diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h +index 870e01f..d26d465 100644 +--- a/drivers/net/mlx5/mlx5.h ++++ b/drivers/net/mlx5/mlx5.h +@@ -123,7 +123,8 @@ struct priv { + unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */ + unsigned int hw_padding:1; /* End alignment padding is supported. */ + unsigned int sriov:1; /* This is a VF or PF with VF devices. */ +- unsigned int mps:1; /* Whether multi-packet send is supported. */ ++ unsigned int mps:2; /* Multi-packet send mode (0: disabled). */ ++ unsigned int mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ + unsigned int cqe_comp:1; /* Whether CQE compression is enabled. */ + unsigned int pending_alarm:1; /* An alarm is pending. */ + unsigned int tso:1; /* Whether TSO is supported. */ +@@ -132,6 +133,7 @@ struct priv { + unsigned int max_tso_payload_sz; /* Maximum TCP payload for TSO. */ + unsigned int txq_inline; /* Maximum packet size for inlining. */ + unsigned int txqs_inline; /* Queue number threshold for inlining. */ ++ unsigned int inline_max_packet_sz; /* Max packet size for inlining. */ + /* RX/TX queues. */ + unsigned int rxqs_n; /* RX queues array size. */ + unsigned int txqs_n; /* TX queues array size. */ +diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h +index eecb908..201bb33 100644 +--- a/drivers/net/mlx5/mlx5_defs.h ++++ b/drivers/net/mlx5/mlx5_defs.h +@@ -55,6 +55,13 @@ + #define MLX5_TX_COMP_THRESH 32 + + /* ++ * Request TX completion every time the total number of WQEBBs used for inlining ++ * packets exceeds the size of WQ divided by this divisor. Better to be power of ++ * two for performance. ++ */ ++#define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3) ++ ++/* + * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP + * from which buffers are to be transmitted will have to be mapped by this + * driver to their own Memory Region (MR). This is a slow operation. +diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c +index 5deb6e8..dd5fe5c 100644 +--- a/drivers/net/mlx5/mlx5_ethdev.c ++++ b/drivers/net/mlx5/mlx5_ethdev.c +@@ -1590,7 +1590,11 @@ priv_select_tx_function(struct priv *priv) + { + priv->dev->tx_pkt_burst = mlx5_tx_burst; + /* Select appropriate TX function. */ +- if (priv->mps && priv->txq_inline) { ++ if (priv->mps == MLX5_MPW_ENHANCED) { ++ priv->dev->tx_pkt_burst = ++ mlx5_tx_burst_empw; ++ DEBUG("selected Enhanced MPW TX function"); ++ } else if (priv->mps && priv->txq_inline) { + priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; + DEBUG("selected MPW inline TX function"); + } else if (priv->mps) { +diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h +index 0a77f5b..155bdba 100644 +--- a/drivers/net/mlx5/mlx5_prm.h ++++ b/drivers/net/mlx5/mlx5_prm.h +@@ -73,6 +73,9 @@ + /* WQE size */ + #define MLX5_WQE_SIZE (4 * MLX5_WQE_DWORD_SIZE) + ++/* Max size of a WQE session. */ ++#define MLX5_WQE_SIZE_MAX 960U ++ + /* Compute the number of DS. */ + #define MLX5_WQE_DS(n) \ + (((n) + MLX5_WQE_DWORD_SIZE - 1) / MLX5_WQE_DWORD_SIZE) +@@ -80,10 +83,19 @@ + /* Room for inline data in multi-packet WQE. */ + #define MLX5_MWQE64_INL_DATA 28 + ++/* Default minimum number of Tx queues for inlining packets. */ ++#define MLX5_EMPW_MIN_TXQS 8 ++ ++/* Default max packet length to be inlined. */ ++#define MLX5_EMPW_MAX_INLINE_LEN (4U * MLX5_WQE_SIZE) ++ + #ifndef HAVE_VERBS_MLX5_OPCODE_TSO + #define MLX5_OPCODE_TSO MLX5_OPCODE_LSO_MPW /* Compat with OFED 3.3. */ + #endif + ++#define MLX5_OPC_MOD_ENHANCED_MPSW 0 ++#define MLX5_OPCODE_ENHANCED_MPSW 0x29 ++ + /* CQE value to inform that VLAN is stripped. */ + #define MLX5_CQE_VLAN_STRIPPED (1u << 0) + +@@ -176,10 +188,18 @@ struct mlx5_wqe64 { + uint8_t raw[32]; + } __rte_aligned(MLX5_WQE_SIZE); + ++/* MPW mode. */ ++enum mlx5_mpw_mode { ++ MLX5_MPW_DISABLED, ++ MLX5_MPW, ++ MLX5_MPW_ENHANCED, /* Enhanced Multi-Packet Send WQE, a.k.a MPWv2. */ ++}; ++ + /* MPW session status. */ + enum mlx5_mpw_state { + MLX5_MPW_STATE_OPENED, + MLX5_MPW_INL_STATE_OPENED, ++ MLX5_MPW_ENHANCED_STATE_OPENED, + MLX5_MPW_STATE_CLOSED, + }; + +diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c +index 9fc433e..a1dd84a 100644 +--- a/drivers/net/mlx5/mlx5_rxtx.c ++++ b/drivers/net/mlx5/mlx5_rxtx.c +@@ -195,6 +195,62 @@ tx_mlx5_wqe(struct txq *txq, uint16_t ci) + } + + /** ++ * Return the size of tailroom of WQ. ++ * ++ * @param txq ++ * Pointer to TX queue structure. ++ * @param addr ++ * Pointer to tail of WQ. ++ * ++ * @return ++ * Size of tailroom. ++ */ ++static inline size_t ++tx_mlx5_wq_tailroom(struct txq *txq, void *addr) ++{ ++ size_t tailroom; ++ tailroom = (uintptr_t)(txq->wqes) + ++ (1 << txq->wqe_n) * MLX5_WQE_SIZE - ++ (uintptr_t)addr; ++ return tailroom; ++} ++ ++/** ++ * Copy data to tailroom of circular queue. ++ * ++ * @param dst ++ * Pointer to destination. ++ * @param src ++ * Pointer to source. ++ * @param n ++ * Number of bytes to copy. ++ * @param base ++ * Pointer to head of queue. ++ * @param tailroom ++ * Size of tailroom from dst. ++ * ++ * @return ++ * Pointer after copied data. ++ */ ++static inline void * ++mlx5_copy_to_wq(void *dst, const void *src, size_t n, ++ void *base, size_t tailroom) ++{ ++ void *ret; ++ ++ if (n > tailroom) { ++ rte_memcpy(dst, src, tailroom); ++ rte_memcpy(base, (void *)((uintptr_t)src + tailroom), ++ n - tailroom); ++ ret = (uint8_t *)base + n - tailroom; ++ } else { ++ rte_memcpy(dst, src, n); ++ ret = (n == tailroom) ? base : (uint8_t *)dst + n; ++ } ++ return ret; ++} ++ ++/** + * Manage TX completions. + * + * When sending a burst, mlx5_tx_burst() posts several WRs. +@@ -1269,6 +1325,360 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, + } + + /** ++ * Open an Enhanced MPW session. ++ * ++ * @param txq ++ * Pointer to TX queue structure. ++ * @param mpw ++ * Pointer to MPW session structure. ++ * @param length ++ * Packet length. ++ */ ++static inline void ++mlx5_empw_new(struct txq *txq, struct mlx5_mpw *mpw, int padding) ++{ ++ uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1); ++ ++ mpw->state = MLX5_MPW_ENHANCED_STATE_OPENED; ++ mpw->pkts_n = 0; ++ mpw->total_len = sizeof(struct mlx5_wqe); ++ mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx); ++ mpw->wqe->ctrl[0] = htonl((MLX5_OPC_MOD_ENHANCED_MPSW << 24) | ++ (txq->wqe_ci << 8) | ++ MLX5_OPCODE_ENHANCED_MPSW); ++ mpw->wqe->ctrl[2] = 0; ++ mpw->wqe->ctrl[3] = 0; ++ memset((void *)(uintptr_t)&mpw->wqe->eseg, 0, MLX5_WQE_DWORD_SIZE); ++ if (unlikely(padding)) { ++ uintptr_t addr = (uintptr_t)(mpw->wqe + 1); ++ ++ /* Pad the first 2 DWORDs with zero-length inline header. */ ++ *(volatile uint32_t *)addr = htonl(MLX5_INLINE_SEG); ++ *(volatile uint32_t *)(addr + MLX5_WQE_DWORD_SIZE) = ++ htonl(MLX5_INLINE_SEG); ++ mpw->total_len += 2 * MLX5_WQE_DWORD_SIZE; ++ /* Start from the next WQEBB. */ ++ mpw->data.raw = (volatile void *)(tx_mlx5_wqe(txq, idx + 1)); ++ } else { ++ mpw->data.raw = (volatile void *)(mpw->wqe + 1); ++ } ++} ++ ++/** ++ * Close an Enhanced MPW session. ++ * ++ * @param txq ++ * Pointer to TX queue structure. ++ * @param mpw ++ * Pointer to MPW session structure. ++ * ++ * @return ++ * Number of consumed WQEs. ++ */ ++static inline uint16_t ++mlx5_empw_close(struct txq *txq, struct mlx5_mpw *mpw) ++{ ++ uint16_t ret; ++ ++ /* Store size in multiple of 16 bytes. Control and Ethernet segments ++ * count as 2. ++ */ ++ mpw->wqe->ctrl[1] = htonl(txq->qp_num_8s | MLX5_WQE_DS(mpw->total_len)); ++ mpw->state = MLX5_MPW_STATE_CLOSED; ++ ret = (mpw->total_len + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE; ++ txq->wqe_ci += ret; ++ return ret; ++} ++ ++/** ++ * DPDK callback for TX with Enhanced MPW support. ++ * ++ * @param dpdk_txq ++ * Generic pointer to TX queue structure. ++ * @param[in] pkts ++ * Packets to transmit. ++ * @param pkts_n ++ * Number of packets in array. ++ * ++ * @return ++ * Number of packets successfully transmitted (<= pkts_n). ++ */ ++uint16_t ++mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) ++{ ++ struct txq *txq = (struct txq *)dpdk_txq; ++ uint16_t elts_head = txq->elts_head; ++ const unsigned int elts_n = 1 << txq->elts_n; ++ unsigned int i = 0; ++ unsigned int j = 0; ++ unsigned int max_elts; ++ uint16_t max_wqe; ++ unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE; ++ unsigned int mpw_room = 0; ++ unsigned int inl_pad = 0; ++ uint32_t inl_hdr; ++ struct mlx5_mpw mpw = { ++ .state = MLX5_MPW_STATE_CLOSED, ++ }; ++ ++ if (unlikely(!pkts_n)) ++ return 0; ++ /* Start processing. */ ++ txq_complete(txq); ++ max_elts = (elts_n - (elts_head - txq->elts_tail)); ++ if (max_elts > elts_n) ++ max_elts -= elts_n; ++ /* A CQE slot must always be available. */ ++ assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci)); ++ max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); ++ if (unlikely(!max_wqe)) ++ return 0; ++ do { ++ struct rte_mbuf *buf = *(pkts++); ++ unsigned int elts_head_next; ++ uintptr_t addr; ++ uint64_t naddr; ++ unsigned int n; ++ unsigned int do_inline = 0; /* Whether inline is possible. */ ++ uint32_t length; ++ unsigned int segs_n = buf->nb_segs; ++ uint32_t cs_flags = 0; ++ ++ /* ++ * Make sure there is enough room to store this packet and ++ * that one ring entry remains unused. ++ */ ++ assert(segs_n); ++ if (max_elts - j < segs_n + 1) ++ break; ++ /* Do not bother with large packets MPW cannot handle. */ ++ if (segs_n > MLX5_MPW_DSEG_MAX) ++ break; ++ /* Should we enable HW CKSUM offload. */ ++ if (buf->ol_flags & ++ (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) ++ cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; ++ /* Retrieve packet information. */ ++ length = PKT_LEN(buf); ++ /* Start new session if: ++ * - multi-segment packet ++ * - no space left even for a dseg ++ * - next packet can be inlined with a new WQE ++ * - cs_flag differs ++ * It can't be MLX5_MPW_STATE_OPENED as always have a single ++ * segmented packet. ++ */ ++ if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED) { ++ if ((segs_n != 1) || ++ (inl_pad + sizeof(struct mlx5_wqe_data_seg) > ++ mpw_room) || ++ (length <= txq->inline_max_packet_sz && ++ inl_pad + sizeof(inl_hdr) + length > ++ mpw_room) || ++ (mpw.wqe->eseg.cs_flags != cs_flags)) ++ max_wqe -= mlx5_empw_close(txq, &mpw); ++ } ++ if (unlikely(mpw.state == MLX5_MPW_STATE_CLOSED)) { ++ if (unlikely(segs_n != 1)) { ++ /* Fall back to legacy MPW. ++ * A MPW session consumes 2 WQEs at most to ++ * include MLX5_MPW_DSEG_MAX pointers. ++ */ ++ if (unlikely(max_wqe < 2)) ++ break; ++ mlx5_mpw_new(txq, &mpw, length); ++ } else { ++ /* In Enhanced MPW, inline as much as the budget ++ * is allowed. The remaining space is to be ++ * filled with dsegs. If the title WQEBB isn't ++ * padded, it will have 2 dsegs there. ++ */ ++ mpw_room = RTE_MIN(MLX5_WQE_SIZE_MAX, ++ (max_inline ? max_inline : ++ pkts_n * MLX5_WQE_DWORD_SIZE) + ++ MLX5_WQE_SIZE); ++ if (unlikely(max_wqe * MLX5_WQE_SIZE < ++ mpw_room)) ++ break; ++ /* Don't pad the title WQEBB to not waste WQ. */ ++ mlx5_empw_new(txq, &mpw, 0); ++ mpw_room -= mpw.total_len; ++ inl_pad = 0; ++ do_inline = ++ length <= txq->inline_max_packet_sz && ++ sizeof(inl_hdr) + length <= mpw_room && ++ !txq->mpw_hdr_dseg; ++ } ++ mpw.wqe->eseg.cs_flags = cs_flags; ++ } else { ++ /* Evaluate whether the next packet can be inlined. ++ * Inlininig is possible when: ++ * - length is less than configured value ++ * - length fits for remaining space ++ * - not required to fill the title WQEBB with dsegs ++ */ ++ do_inline = ++ length <= txq->inline_max_packet_sz && ++ inl_pad + sizeof(inl_hdr) + length <= ++ mpw_room && ++ (!txq->mpw_hdr_dseg || ++ mpw.total_len >= MLX5_WQE_SIZE); ++ } ++ /* Multi-segment packets must be alone in their MPW. */ ++ assert((segs_n == 1) || (mpw.pkts_n == 0)); ++ if (unlikely(mpw.state == MLX5_MPW_STATE_OPENED)) { ++#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) ++ length = 0; ++#endif ++ do { ++ volatile struct mlx5_wqe_data_seg *dseg; ++ ++ elts_head_next = ++ (elts_head + 1) & (elts_n - 1); ++ assert(buf); ++ (*txq->elts)[elts_head] = buf; ++ dseg = mpw.data.dseg[mpw.pkts_n]; ++ addr = rte_pktmbuf_mtod(buf, uintptr_t); ++ *dseg = (struct mlx5_wqe_data_seg){ ++ .byte_count = htonl(DATA_LEN(buf)), ++ .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), ++ .addr = htonll(addr), ++ }; ++ elts_head = elts_head_next; ++#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) ++ length += DATA_LEN(buf); ++#endif ++ buf = buf->next; ++ ++j; ++ ++mpw.pkts_n; ++ } while (--segs_n); ++ /* A multi-segmented packet takes one MPW session. ++ * TODO: Pack more multi-segmented packets if possible. ++ */ ++ mlx5_mpw_close(txq, &mpw); ++ if (mpw.pkts_n < 3) ++ max_wqe--; ++ else ++ max_wqe -= 2; ++ } else if (do_inline) { ++ /* Inline packet into WQE. */ ++ unsigned int max; ++ ++ assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED); ++ assert(length == DATA_LEN(buf)); ++ inl_hdr = htonl(length | MLX5_INLINE_SEG); ++ addr = rte_pktmbuf_mtod(buf, uintptr_t); ++ mpw.data.raw = (volatile void *) ++ ((uintptr_t)mpw.data.raw + inl_pad); ++ max = tx_mlx5_wq_tailroom(txq, ++ (void *)(uintptr_t)mpw.data.raw); ++ /* Copy inline header. */ ++ mpw.data.raw = (volatile void *) ++ mlx5_copy_to_wq( ++ (void *)(uintptr_t)mpw.data.raw, ++ &inl_hdr, ++ sizeof(inl_hdr), ++ (void *)(uintptr_t)txq->wqes, ++ max); ++ max = tx_mlx5_wq_tailroom(txq, ++ (void *)(uintptr_t)mpw.data.raw); ++ /* Copy packet data. */ ++ mpw.data.raw = (volatile void *) ++ mlx5_copy_to_wq( ++ (void *)(uintptr_t)mpw.data.raw, ++ (void *)addr, ++ length, ++ (void *)(uintptr_t)txq->wqes, ++ max); ++ ++mpw.pkts_n; ++ mpw.total_len += (inl_pad + sizeof(inl_hdr) + length); ++ /* No need to get completion as the entire packet is ++ * copied to WQ. Free the buf right away. ++ */ ++ elts_head_next = elts_head; ++ rte_pktmbuf_free_seg(buf); ++ mpw_room -= (inl_pad + sizeof(inl_hdr) + length); ++ /* Add pad in the next packet if any. */ ++ inl_pad = (((uintptr_t)mpw.data.raw + ++ (MLX5_WQE_DWORD_SIZE - 1)) & ++ ~(MLX5_WQE_DWORD_SIZE - 1)) - ++ (uintptr_t)mpw.data.raw; ++ } else { ++ /* No inline. Load a dseg of packet pointer. */ ++ volatile rte_v128u32_t *dseg; ++ ++ assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED); ++ assert((inl_pad + sizeof(*dseg)) <= mpw_room); ++ assert(length == DATA_LEN(buf)); ++ if (!tx_mlx5_wq_tailroom(txq, ++ (void *)((uintptr_t)mpw.data.raw ++ + inl_pad))) ++ dseg = (volatile void *)txq->wqes; ++ else ++ dseg = (volatile void *) ++ ((uintptr_t)mpw.data.raw + ++ inl_pad); ++ elts_head_next = (elts_head + 1) & (elts_n - 1); ++ (*txq->elts)[elts_head] = buf; ++ addr = rte_pktmbuf_mtod(buf, uintptr_t); ++ for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++) ++ rte_prefetch2((void *)(addr + ++ n * RTE_CACHE_LINE_SIZE)); ++ naddr = htonll(addr); ++ *dseg = (rte_v128u32_t) { ++ htonl(length), ++ txq_mp2mr(txq, txq_mb2mp(buf)), ++ naddr, ++ naddr >> 32, ++ }; ++ mpw.data.raw = (volatile void *)(dseg + 1); ++ mpw.total_len += (inl_pad + sizeof(*dseg)); ++ ++j; ++ ++mpw.pkts_n; ++ mpw_room -= (inl_pad + sizeof(*dseg)); ++ inl_pad = 0; ++ } ++ elts_head = elts_head_next; ++#ifdef MLX5_PMD_SOFT_COUNTERS ++ /* Increment sent bytes counter. */ ++ txq->stats.obytes += length; ++#endif ++ ++i; ++ } while (i < pkts_n); ++ /* Take a shortcut if nothing must be sent. */ ++ if (unlikely(i == 0)) ++ return 0; ++ /* Check whether completion threshold has been reached. */ ++ if (txq->elts_comp + j >= MLX5_TX_COMP_THRESH || ++ (uint16_t)(txq->wqe_ci - txq->mpw_comp) >= ++ (1 << txq->wqe_n) / MLX5_TX_COMP_THRESH_INLINE_DIV) { ++ volatile struct mlx5_wqe *wqe = mpw.wqe; ++ ++ /* Request completion on last WQE. */ ++ wqe->ctrl[2] = htonl(8); ++ /* Save elts_head in unused "immediate" field of WQE. */ ++ wqe->ctrl[3] = elts_head; ++ txq->elts_comp = 0; ++ txq->mpw_comp = txq->wqe_ci; ++ txq->cq_pi++; ++ } else { ++ txq->elts_comp += j; ++ } ++#ifdef MLX5_PMD_SOFT_COUNTERS ++ /* Increment sent packets counter. */ ++ txq->stats.opackets += i; ++#endif ++ if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED) ++ mlx5_empw_close(txq, &mpw); ++ else if (mpw.state == MLX5_MPW_STATE_OPENED) ++ mlx5_mpw_close(txq, &mpw); ++ /* Ring QP doorbell. */ ++ mlx5_tx_dbrec(txq, mpw.wqe); ++ txq->elts_head = elts_head; ++ return i; ++} ++ ++/** + * Translate RX completion flags to packet type. + * + * @param[in] cqe +diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h +index 0db810c..4a4bd84 100644 +--- a/drivers/net/mlx5/mlx5_rxtx.h ++++ b/drivers/net/mlx5/mlx5_rxtx.h +@@ -248,17 +248,21 @@ struct txq { + uint16_t elts_head; /* Current index in (*elts)[]. */ + uint16_t elts_tail; /* First element awaiting completion. */ + uint16_t elts_comp; /* Counter since last completion request. */ ++ uint16_t mpw_comp; /* WQ index since last completion request. */ + uint16_t cq_ci; /* Consumer index for completion queue. */ ++ uint16_t cq_pi; /* Producer index for completion queue. */ + uint16_t wqe_ci; /* Consumer index for work queue. */ + uint16_t wqe_pi; /* Producer index for work queue. */ + uint16_t elts_n:4; /* (*elts)[] length (in log2). */ + uint16_t cqe_n:4; /* Number of CQ elements (in log2). */ + uint16_t wqe_n:4; /* Number of of WQ elements (in log2). */ +- uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ + uint16_t inline_en:1; /* When set inline is enabled. */ + uint16_t tso_en:1; /* When set hardware TSO is enabled. */ + uint16_t tunnel_en:1; + /* When set TX offload for tunneled packets are supported. */ ++ uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ ++ uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ ++ uint16_t inline_max_packet_sz; /* Max packet size for inlining. */ + uint32_t qp_num_8s; /* QP number shifted by 8. */ + volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ + volatile void *wqes; /* Work queue (use volatile to write into). */ +@@ -329,6 +333,7 @@ uint16_t mlx5_tx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t); + uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t); + uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t); + uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t); ++uint16_t mlx5_tx_burst_empw(void *, struct rte_mbuf **, uint16_t); + uint16_t mlx5_rx_burst(void *, struct rte_mbuf **, uint16_t); + uint16_t removed_tx_burst(void *, struct rte_mbuf **, uint16_t); + uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); +diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c +index 9d0c00f..bbfce75 100644 +--- a/drivers/net/mlx5/mlx5_txq.c ++++ b/drivers/net/mlx5/mlx5_txq.c +@@ -266,6 +266,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, + struct ibv_exp_cq_attr cq_attr; + } attr; + enum ibv_exp_query_intf_status status; ++ unsigned int cqe_n; + int ret = 0; + + if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { +@@ -276,6 +277,8 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, + (void)conf; /* Thresholds configuration (ignored). */ + assert(desc > MLX5_TX_COMP_THRESH); + tmpl.txq.elts_n = log2above(desc); ++ if (priv->mps == MLX5_MPW_ENHANCED) ++ tmpl.txq.mpw_hdr_dseg = priv->mpw_hdr_dseg; + /* MRs will be registered in mp2mr[] later. */ + attr.rd = (struct ibv_exp_res_domain_init_attr){ + .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | +@@ -294,9 +297,12 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, + .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, + .res_domain = tmpl.rd, + }; ++ cqe_n = ((desc / MLX5_TX_COMP_THRESH) - 1) ? ++ ((desc / MLX5_TX_COMP_THRESH) - 1) : 1; ++ if (priv->mps == MLX5_MPW_ENHANCED) ++ cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV; + tmpl.cq = ibv_exp_create_cq(priv->ctx, +- (((desc / MLX5_TX_COMP_THRESH) - 1) ? +- ((desc / MLX5_TX_COMP_THRESH) - 1) : 1), ++ cqe_n, + NULL, NULL, 0, &attr.cq); + if (tmpl.cq == NULL) { + ret = ENOMEM; +@@ -340,9 +346,24 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, + tmpl.txq.max_inline = + ((priv->txq_inline + (RTE_CACHE_LINE_SIZE - 1)) / + RTE_CACHE_LINE_SIZE); +- attr.init.cap.max_inline_data = +- tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE; + tmpl.txq.inline_en = 1; ++ /* TSO and MPS can't be enabled concurrently. */ ++ assert(!priv->tso || !priv->mps); ++ if (priv->mps == MLX5_MPW_ENHANCED) { ++ tmpl.txq.inline_max_packet_sz = ++ priv->inline_max_packet_sz; ++ /* To minimize the size of data set, avoid requesting ++ * too large WQ. ++ */ ++ attr.init.cap.max_inline_data = ++ ((RTE_MIN(priv->txq_inline, ++ priv->inline_max_packet_sz) + ++ (RTE_CACHE_LINE_SIZE - 1)) / ++ RTE_CACHE_LINE_SIZE) * RTE_CACHE_LINE_SIZE; ++ } else { ++ attr.init.cap.max_inline_data = ++ tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE; ++ } + } + if (priv->tso) { + uint16_t max_tso_inline = ((MLX5_MAX_TSO_HEADER + +-- +2.7.4 + diff --git a/dpdk/dpdk-17.05_patches/0001-Revert-net-virtio-remove-redundant-MSI-X-detection.patch b/dpdk/dpdk-17.05_patches/0001-Revert-net-virtio-remove-redundant-MSI-X-detection.patch new file mode 100644 index 00000000..28dc68dd --- /dev/null +++ b/dpdk/dpdk-17.05_patches/0001-Revert-net-virtio-remove-redundant-MSI-X-detection.patch @@ -0,0 +1,58 @@ +From 3a1470e031ff33ac99da33b41dae0e9082d4da78 Mon Sep 17 00:00:00 2001 +From: Damjan Marion <damarion@cisco.com> +Date: Mon, 15 May 2017 12:27:37 +0200 +Subject: [PATCH] Revert "net/virtio: remove redundant MSI-X detection" + +This reverts commit ee1843bd89076c59e50cadbef5c935613f543765. +--- + drivers/net/virtio/virtio_pci.c | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/drivers/net/virtio/virtio_pci.c b/drivers/net/virtio/virtio_pci.c +index b7b3d6157..127f25791 100644 +--- a/drivers/net/virtio/virtio_pci.c ++++ b/drivers/net/virtio/virtio_pci.c +@@ -274,6 +274,32 @@ legacy_notify_queue(struct virtio_hw *hw, struct virtqueue *vq) + VIRTIO_PCI_QUEUE_NOTIFY); + } + ++#ifdef RTE_EXEC_ENV_LINUXAPP ++static int ++legacy_virtio_has_msix(const struct rte_pci_addr *loc) ++{ ++ DIR *d; ++ char dirname[PATH_MAX]; ++ ++ snprintf(dirname, sizeof(dirname), ++ "%s/" PCI_PRI_FMT "/msi_irqs", pci_get_sysfs_path(), ++ loc->domain, loc->bus, loc->devid, loc->function); ++ ++ d = opendir(dirname); ++ if (d) ++ closedir(d); ++ ++ return d != NULL; ++} ++#else ++static int ++legacy_virtio_has_msix(const struct rte_pci_addr *loc __rte_unused) ++{ ++ /* nic_uio does not enable interrupts, return 0 (false). */ ++ return 0; ++} ++#endif ++ + const struct virtio_pci_ops legacy_ops = { + .read_dev_cfg = legacy_read_dev_config, + .write_dev_cfg = legacy_write_dev_config, +@@ -694,6 +720,7 @@ vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw) + } + + virtio_hw_internal[hw->port_id].vtpci_ops = &legacy_ops; ++ hw->use_msix = legacy_virtio_has_msix(&dev->addr); + hw->modern = 0; + + return 0; +-- +2.11.0 + diff --git a/dpdk/dpdk-17.08_patches/0001-net-bonding-support-for-mlx.patch b/dpdk/dpdk-17.08_patches/0001-net-bonding-support-for-mlx.patch new file mode 100644 index 00000000..13a2ba67 --- /dev/null +++ b/dpdk/dpdk-17.08_patches/0001-net-bonding-support-for-mlx.patch @@ -0,0 +1,63 @@ +diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c +index bb634c6..7c65dda 100644 +--- a/drivers/net/bonding/rte_eth_bond_args.c ++++ b/drivers/net/bonding/rte_eth_bond_args.c +@@ -61,16 +61,6 @@ + unsigned i; + + for (i = 0; i < rte_eth_dev_count(); i++) { +- +- /* Currently populated by rte_eth_copy_pci_info(). +- * +- * TODO: Once the PCI bus has arrived we should have a better +- * way to test for being a PCI device or not. +- */ +- if (rte_eth_devices[i].data->kdrv == RTE_KDRV_UNKNOWN || +- rte_eth_devices[i].data->kdrv == RTE_KDRV_NONE) +- continue; +- + pci_dev = RTE_ETH_DEV_TO_PCI(&rte_eth_devices[i]); + eth_pci_addr = &pci_dev->addr; + +@@ -98,6 +88,16 @@ + return -1; + } + ++static inline int ++pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr) ++{ ++ struct rte_pci_device *pdev; ++ const struct rte_pci_addr *paddr = _pci_addr; ++ ++ pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev); ++ return rte_eal_compare_pci_addr(&pdev->addr, paddr); ++} ++ + /** + * Parses a port identifier string to a port id by pci address, then by name, + * and finally port id. +@@ -106,10 +106,23 @@ + parse_port_id(const char *port_str) + { + struct rte_pci_addr dev_addr; ++ struct rte_bus *pci_bus; ++ struct rte_device *dev; + int port_id; + ++ pci_bus = rte_bus_find_by_name("pci"); ++ if (pci_bus == NULL) { ++ RTE_LOG(ERR, PMD, "unable to find PCI bus\n"); ++ return -1; ++ } ++ + /* try parsing as pci address, physical devices */ +- if (eal_parse_pci_DomBDF(port_str, &dev_addr) == 0) { ++ if (pci_bus->parse(port_str, &dev_addr) == 0) { ++ dev = pci_bus->find_device(NULL, pci_addr_cmp, &dev_addr); ++ if (dev == NULL) { ++ RTE_LOG(ERR, PMD, "unable to find PCI device\n"); ++ return -1; ++ } + port_id = find_port_id_by_pci_addr(&dev_addr); + if (port_id < 0) + return -1; diff --git a/dpdk/rpm/vpp-dpdk.spec b/dpdk/rpm/vpp-dpdk.spec new file mode 100644 index 00000000..cc617681 --- /dev/null +++ b/dpdk/rpm/vpp-dpdk.spec @@ -0,0 +1,26 @@ +%define _make_args -C ../.. DPDK_BUILD_DIR=%{_topdir}/tmp DPDK_INSTALL_DIR=%{buildroot}/usr + +Name: vpp-dpdk +Version: %{_version} +Release: %{_release} +Summary: DPDK development packages for VPP +License: BSD + +%description + +%package devel +Summary: DPDK development package for VPP +Group: Development/Libraries + +%description devel + +%install +make %{_make_args} config +make %{_make_args} install + +%files devel +/usr/bin/* +/usr/include/dpdk +/usr/lib/* +/usr/sbin/* +/usr/share/dpdk |