diff options
Diffstat (limited to 'drivers')
404 files changed, 46080 insertions, 10669 deletions
diff --git a/drivers/Makefile b/drivers/Makefile index a04a01fb..7fef66d7 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -39,5 +39,6 @@ DEPDIRS-net := bus mempool DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += crypto DEPDIRS-crypto := mempool DIRS-$(CONFIG_RTE_LIBRTE_EVENTDEV) += event +DEPDIRS-event := bus include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile index 1e5b281c..02242148 100644 --- a/drivers/bus/Makefile +++ b/drivers/bus/Makefile @@ -1,7 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 NXP. All rights reserved. -# All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile index 973d279b..d1b790b5 100644 --- a/drivers/bus/fslmc/Makefile +++ b/drivers/bus/fslmc/Makefile @@ -1,7 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 NXP. -# All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -63,12 +62,16 @@ SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ qbman/qbman_portal.c SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ + mc/dpmng.c \ mc/dpbp.c \ mc/dpio.c \ - mc/mc_sys.c + mc/mc_sys.c \ + mc/dpcon.c \ + mc/dpci.c SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpio.c SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpbp.c +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpci.c SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += fslmc_vfio.c SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += fslmc_bus.c diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c index b24642dd..f71598d5 100644 --- a/drivers/bus/fslmc/fslmc_bus.c +++ b/drivers/bus/fslmc/fslmc_bus.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -32,6 +32,7 @@ #include <string.h> #include <dirent.h> +#include <stdbool.h> #include <rte_log.h> #include <rte_bus.h> @@ -105,6 +106,25 @@ rte_fslmc_probe(void) return ret; } +static struct rte_device * +rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_dpaa2_device *dev; + + TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { + if (start && &dev->device == start) { + start = NULL; /* starting point found */ + continue; + } + + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + + return NULL; +} + /*register a fslmc bus based dpaa2 driver */ void rte_fslmc_driver_register(struct rte_dpaa2_driver *driver) @@ -133,9 +153,10 @@ struct rte_fslmc_bus rte_fslmc_bus = { .bus = { .scan = rte_fslmc_scan, .probe = rte_fslmc_probe, + .find_device = rte_fslmc_find_device, }, .device_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.device_list), .driver_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.driver_list), }; -RTE_REGISTER_BUS(FSLMC_BUS_NAME, rte_fslmc_bus.bus); +RTE_REGISTER_BUS(fslmc, rte_fslmc_bus.bus); diff --git a/drivers/bus/fslmc/fslmc_logs.h b/drivers/bus/fslmc/fslmc_logs.h index a890e6c3..1f7c24b3 100644 --- a/drivers/bus/fslmc/fslmc_logs.h +++ b/drivers/bus/fslmc/fslmc_logs.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c index 5d4ac67c..45e59277 100644 --- a/drivers/bus/fslmc/fslmc_vfio.c +++ b/drivers/bus/fslmc/fslmc_vfio.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -40,7 +40,6 @@ #include <errno.h> #include <sys/ioctl.h> #include <sys/stat.h> -#include <sys/types.h> #include <sys/mman.h> #include <sys/vfs.h> #include <libgen.h> @@ -55,7 +54,6 @@ #include <rte_cycles.h> #include <rte_kvargs.h> #include <rte_dev.h> -#include <rte_ethdev.h> #include <rte_bus.h> #include "rte_fslmc.h" @@ -80,6 +78,17 @@ static uint32_t *msi_intr_vaddr; void *(*rte_mcp_ptr_list); static uint32_t mcp_id; static int is_dma_done; +static struct rte_fslmc_object_list fslmc_obj_list = + TAILQ_HEAD_INITIALIZER(fslmc_obj_list); + +/*register a fslmc bus based dpaa2 driver */ +void +rte_fslmc_object_register(struct rte_dpaa2_object *object) +{ + RTE_VERIFY(object); + + TAILQ_INSERT_TAIL(&fslmc_obj_list, object, next); +} static int vfio_connect_container(struct fslmc_vfio_group *vfio_group) { @@ -91,9 +100,9 @@ static int vfio_connect_container(struct fslmc_vfio_group *vfio_group) container = &vfio_containers[i]; if (!ioctl(vfio_group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) { - FSLMC_VFIO_LOG(INFO, "Container pre-exists with" - " FD[0x%x] for this group", - container->fd); + FSLMC_VFIO_LOG(INFO, + "Container pre-exists with FD[0x%x] for this group", + container->fd); vfio_group->container = container; return 0; } @@ -132,7 +141,6 @@ static int vfio_connect_container(struct fslmc_vfio_group *vfio_group) for (i = 0; i < VFIO_MAX_CONTAINERS; i++) { if (vfio_containers[i].used) continue; - FSLMC_VFIO_LOG(DEBUG, "Unused container at index %d", i); container = &vfio_containers[i]; } if (!container) { @@ -178,29 +186,6 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group) return -errno; } -int vfio_dmamap_mem_region(uint64_t vaddr, - uint64_t iova, - uint64_t size) -{ - struct fslmc_vfio_group *group; - struct vfio_iommu_type1_dma_map dma_map = { - .argsz = sizeof(dma_map), - .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, - }; - - dma_map.vaddr = vaddr; - dma_map.size = size; - dma_map.iova = iova; - - /* SET DMA MAP for IOMMU */ - group = &vfio_groups[0]; - if (ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &dma_map)) { - FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA (errno = %d)", errno); - return -1; - } - return 0; -} - int rte_fslmc_vfio_dmamap(void) { int ret; @@ -215,17 +200,18 @@ int rte_fslmc_vfio_dmamap(void) if (is_dma_done) return 0; - is_dma_done = 1; - for (i = 0; i < RTE_MAX_MEMSEG; i++) { - memseg = rte_eal_get_physmem_layout(); - if (memseg == NULL) { - FSLMC_VFIO_LOG(ERR, "Cannot get physical layout."); - return -ENODEV; - } + memseg = rte_eal_get_physmem_layout(); + if (memseg == NULL) { + FSLMC_VFIO_LOG(ERR, "Cannot get physical layout."); + return -ENODEV; + } - if (memseg[i].addr == NULL && memseg[i].len == 0) + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if (memseg[i].addr == NULL && memseg[i].len == 0) { + FSLMC_VFIO_LOG(DEBUG, "Total %d segments found.", i); break; + } dma_map.size = memseg[i].len; dma_map.vaddr = memseg[i].addr_64; @@ -245,16 +231,20 @@ int rte_fslmc_vfio_dmamap(void) FSLMC_VFIO_LOG(DEBUG, "-->Initial SHM Virtual ADDR %llX", dma_map.vaddr); - FSLMC_VFIO_LOG(DEBUG, "-----> DMA size 0x%llX\n", dma_map.size); + FSLMC_VFIO_LOG(DEBUG, "-----> DMA size 0x%llX", dma_map.size); ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &dma_map); if (ret) { - FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA API" - "(errno = %d)", errno); + FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA API(errno = %d)", + errno); return ret; } - FSLMC_VFIO_LOG(DEBUG, "-----> dma_map.vaddr = 0x%llX", - dma_map.vaddr); + } + + /* Verifying that at least single segment is available */ + if (i <= 0) { + FSLMC_VFIO_LOG(ERR, "No Segments found for VFIO Mapping"); + return -1; } /* TODO - This is a W.A. as VFIO currently does not add the mapping of @@ -263,6 +253,8 @@ int rte_fslmc_vfio_dmamap(void) */ vfio_map_irq_region(group); + is_dma_done = 1; + return 0; } @@ -277,8 +269,8 @@ static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj) /* getting the mcp object's fd*/ mc_fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, mcp_obj); if (mc_fd < 0) { - FSLMC_VFIO_LOG(ERR, "error in VFIO get device %s fd from group" - " %d", mcp_obj, group->fd); + FSLMC_VFIO_LOG(ERR, "error in VFIO get dev %s fd from group %d", + mcp_obj, group->fd); return v_addr; } @@ -297,7 +289,7 @@ static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj) } FSLMC_VFIO_LOG(DEBUG, "region offset = %llx , region size = %llx", - reg_info.offset, reg_info.size); + reg_info.offset, reg_info.size); v_addr = (uint64_t)mmap(NULL, reg_info.size, PROT_WRITE | PROT_READ, MAP_SHARED, @@ -351,6 +343,40 @@ fslmc_bus_add_device(struct rte_dpaa2_device *dev) } } +#define IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + sizeof(int)) + +int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, + uint32_t index) +{ + struct vfio_irq_set *irq_set; + char irq_set_buf[IRQ_SET_BUF_LEN]; + int *fd_ptr, fd, ret; + + /* Prepare vfio_irq_set structure and SET the IRQ in VFIO */ + /* Give the eventfd to VFIO */ + fd = eventfd(0, 0); + irq_set = (struct vfio_irq_set *)irq_set_buf; + irq_set->argsz = sizeof(irq_set_buf); + irq_set->count = 1; + irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | + VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set->index = index; + irq_set->start = 0; + fd_ptr = (int *)&irq_set->data; + *fd_ptr = fd; + + ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set); + if (ret < 0) { + FSLMC_VFIO_LOG(ERR, "Unable to set IRQ in VFIO, ret: %d\n", + ret); + return -1; + } + + /* Set the FD and update the flags */ + intr_handle->fd = fd; + return 0; +} + /* Following function shall fetch total available list of MC devices * from VFIO container & populate private list of devices and other * data structures @@ -366,14 +392,13 @@ int fslmc_vfio_process_group(void) char path[PATH_MAX]; int64_t v_addr; int ndev_count; - int dpio_count = 0, dpbp_count = 0; struct fslmc_vfio_group *group = &vfio_groups[0]; static int process_once; /* if already done once */ if (process_once) { - FSLMC_VFIO_LOG(DEBUG, "Already scanned once - re-scan " - "not supported"); + FSLMC_VFIO_LOG(DEBUG, + "Already scanned once - re-scan not supported"); return 0; } process_once = 0; @@ -397,8 +422,8 @@ int fslmc_vfio_process_group(void) free(mcp_obj); mcp_obj = malloc(sizeof(dir->d_name)); if (!mcp_obj) { - FSLMC_VFIO_LOG(ERR, "mcp obj:Unable to" - " allocate memory"); + FSLMC_VFIO_LOG(ERR, + "mcp obj:alloc failed"); closedir(d); return -ENOMEM; } @@ -441,8 +466,6 @@ int fslmc_vfio_process_group(void) goto FAILURE; } - FSLMC_VFIO_LOG(DEBUG, "DPAA2 MC has VIR_ADD = %ld", v_addr); - rte_mcp_ptr_list[0] = (void *)v_addr; d = opendir(path); @@ -452,7 +475,6 @@ int fslmc_vfio_process_group(void) } i = 0; - FSLMC_VFIO_LOG(DEBUG, "DPAA2 - Parsing devices:"); /* Parsing each object and initiating them*/ while ((dir = readdir(d)) != NULL) { if (dir->d_type != DT_LNK) @@ -469,14 +491,13 @@ int fslmc_vfio_process_group(void) object_type = strtok(dir->d_name, "."); temp_obj = strtok(NULL, "."); sscanf(temp_obj, "%d", &object_id); - FSLMC_VFIO_LOG(DEBUG, " - %s ", dev_name); /* getting the device fd*/ dev_fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, dev_name); if (dev_fd < 0) { - FSLMC_VFIO_LOG(ERR, "VFIO_GROUP_GET_DEVICE_FD error" - " Device fd: %s, Group: %d", - dev_name, group->fd); + FSLMC_VFIO_LOG(ERR, + "GET_DEVICE_FD error fd: %s, Group: %d", + dev_name, group->fd); free(dev_name); goto FAILURE; } @@ -505,22 +526,22 @@ int fslmc_vfio_process_group(void) dev->dev_type = (strcmp(object_type, "dpseci")) ? DPAA2_MC_DPNI_DEVID : DPAA2_MC_DPSECI_DEVID; - FSLMC_VFIO_LOG(DEBUG, "DPAA2: Added [%s-%d]\n", - object_type, object_id); + sprintf(dev->name, "%s.%d", object_type, object_id); + dev->device.name = dev->name; fslmc_bus_add_device(dev); - } - if (!strcmp(object_type, "dpio")) { - ret = dpaa2_create_dpio_device(vdev, - &device_info, + FSLMC_VFIO_LOG(DEBUG, "DPAA2: Added %s", dev->name); + } else { + /* Parse all other objects */ + struct rte_dpaa2_object *object; + + TAILQ_FOREACH(object, &fslmc_obj_list, next) { + if (!strcmp(object_type, object->name)) + object->create(vdev, &device_info, object_id); - if (!ret) - dpio_count++; - } - if (!strcmp(object_type, "dpbp")) { - ret = dpaa2_create_dpbp_device(object_id); - if (!ret) - dpbp_count++; + else + continue; + } } } closedir(d); @@ -529,8 +550,6 @@ int fslmc_vfio_process_group(void) if (ret) FSLMC_VFIO_LOG(DEBUG, "Error in affining qbman swp %d", ret); - FSLMC_VFIO_LOG(DEBUG, "DPAA2: Added dpbp_count = %d dpio_count=%d\n", - dpbp_count, dpio_count); return 0; FAILURE: diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h index 53dd0b74..0aff9b1f 100644 --- a/drivers/bus/fslmc/fslmc_vfio.h +++ b/drivers/bus/fslmc/fslmc_vfio.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -39,6 +39,10 @@ #define DPAA2_VENDOR_ID 0x1957 #define DPAA2_MC_DPNI_DEVID 7 #define DPAA2_MC_DPSECI_DEVID 3 +#define DPAA2_MC_DPCON_DEVID 5 +#define DPAA2_MC_DPIO_DEVID 9 +#define DPAA2_MC_DPBP_DEVID 10 +#define DPAA2_MC_DPCI_DEVID 11 #define VFIO_MAX_GRP 1 @@ -63,20 +67,48 @@ typedef struct fslmc_vfio_container { struct fslmc_vfio_group *group_list[VFIO_MAX_GRP]; } fslmc_vfio_container; -int vfio_dmamap_mem_region( - uint64_t vaddr, - uint64_t iova, - uint64_t size); +struct rte_dpaa2_object; + +TAILQ_HEAD(rte_fslmc_object_list, rte_dpaa2_object); + +typedef int (*rte_fslmc_obj_create_t)(struct fslmc_vfio_device *vdev, + struct vfio_device_info *obj_info, + int object_id); + +/** + * A structure describing a DPAA2 driver. + */ +struct rte_dpaa2_object { + TAILQ_ENTRY(rte_dpaa2_object) next; /**< Next in list. */ + const char *name; /**< Name of Object. */ + uint16_t object_id; /**< DPAA2 Object ID */ + rte_fslmc_obj_create_t create; +}; + +int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, + uint32_t index); int fslmc_vfio_setup_group(void); int fslmc_vfio_process_group(void); int rte_fslmc_vfio_dmamap(void); -/* create dpio device */ -int dpaa2_create_dpio_device(struct fslmc_vfio_device *vdev, - struct vfio_device_info *obj_info, - int object_id); +/** + * Register a DPAA2 MC Object driver. + * + * @param mc_object + * A pointer to a rte_dpaa_object structure describing the mc object + * to be registered. + */ +void rte_fslmc_object_register(struct rte_dpaa2_object *object); -int dpaa2_create_dpbp_device(int dpbp_id); +/** Helper for DPAA2 object registration */ +#define RTE_PMD_REGISTER_DPAA2_OBJECT(nm, dpaa2_obj) \ +RTE_INIT(dpaa2objinitfn_ ##nm); \ +static void dpaa2objinitfn_ ##nm(void) \ +{\ + (dpaa2_obj).name = RTE_STR(nm);\ + rte_fslmc_object_register(&dpaa2_obj); \ +} \ +RTE_PMD_EXPORT_NAME(nm, __COUNTER__) #endif /* _FSLMC_VFIO_H_ */ diff --git a/drivers/bus/fslmc/mc/dpbp.c b/drivers/bus/fslmc/mc/dpbp.c index 12f9180a..fd9a52d9 100644 --- a/drivers/bus/fslmc/mc/dpbp.c +++ b/drivers/bus/fslmc/mc/dpbp.c @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/bus/fslmc/mc/dpci.c b/drivers/bus/fslmc/mc/dpci.c new file mode 100644 index 00000000..0ea78379 --- /dev/null +++ b/drivers/bus/fslmc/mc/dpci.c @@ -0,0 +1,307 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include <fsl_mc_sys.h> +#include <fsl_mc_cmd.h> +#include <fsl_dpci.h> +#include <fsl_dpci_cmd.h> + +int dpci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpci_id, + uint16_t *token) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_OPEN, + cmd_flags, + 0); + DPCI_CMD_OPEN(cmd, dpci_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = MC_CMD_HDR_READ_TOKEN(cmd.header); + + return 0; +} + +int dpci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_CLOSE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpci_cfg *cfg, + uint32_t *obj_id) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_CREATE, + cmd_flags, + dprc_token); + DPCI_CMD_CREATE(cmd, cfg); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + CMD_CREATE_RSP_GET_OBJ_ID_PARAM0(cmd, *obj_id); + + return 0; +} + +int dpci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_DESTROY, + cmd_flags, + dprc_token); + /* set object id to destroy */ + CMD_DESTROY_SET_OBJ_ID_PARAM0(cmd, object_id); + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpci_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpci_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct mc_command cmd = { 0 }; + int err; + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_IS_ENABLED, cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPCI_RSP_IS_ENABLED(cmd, *en); + + return 0; +} + +int dpci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_RESET, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_attr *attr) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPCI_RSP_GET_ATTRIBUTES(cmd, attr); + + return 0; +} + +int dpci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + const struct dpci_rx_queue_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_SET_RX_QUEUE, + cmd_flags, + token); + DPCI_CMD_SET_RX_QUEUE(cmd, priority, cfg); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_rx_queue_attr *attr) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_RX_QUEUE, + cmd_flags, + token); + DPCI_CMD_GET_RX_QUEUE(cmd, priority); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPCI_RSP_GET_RX_QUEUE(cmd, attr); + + return 0; +} + +int dpci_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_tx_queue_attr *attr) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_TX_QUEUE, + cmd_flags, + token); + DPCI_CMD_GET_TX_QUEUE(cmd, priority); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPCI_RSP_GET_TX_QUEUE(cmd, attr); + + return 0; +} + +int dpci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct mc_command cmd = { 0 }; + int err; + + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_API_VERSION, + cmd_flags, + 0); + + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + DPCI_RSP_GET_API_VERSION(cmd, *major_ver, *minor_ver); + + return 0; +} diff --git a/drivers/bus/fslmc/mc/dpcon.c b/drivers/bus/fslmc/mc/dpcon.c new file mode 100644 index 00000000..b078dff8 --- /dev/null +++ b/drivers/bus/fslmc/mc/dpcon.c @@ -0,0 +1,230 @@ +/* Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include <fsl_mc_sys.h> +#include <fsl_mc_cmd.h> +#include <fsl_dpcon.h> +#include <fsl_dpcon_cmd.h> + +int dpcon_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpcon_id, + uint16_t *token) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_OPEN, + cmd_flags, + 0); + DPCON_CMD_OPEN(cmd, dpcon_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = MC_CMD_HDR_READ_TOKEN(cmd.header); + + return 0; +} + +int dpcon_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_CLOSE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpcon_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpcon_cfg *cfg, + uint32_t *obj_id) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_CREATE, + cmd_flags, + dprc_token); + DPCON_CMD_CREATE(cmd, cfg); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + CMD_CREATE_RSP_GET_OBJ_ID_PARAM0(cmd, *obj_id); + + return 0; +} + +int dpcon_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_DESTROY, + cmd_flags, + dprc_token); + /* set object id to destroy */ + CMD_DESTROY_SET_OBJ_ID_PARAM0(cmd, object_id); + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpcon_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpcon_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpcon_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct mc_command cmd = { 0 }; + int err; + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_IS_ENABLED, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPCON_RSP_IS_ENABLED(cmd, *en); + + return 0; +} + +int dpcon_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_RESET, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpcon_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_attr *attr) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPCON_RSP_GET_ATTR(cmd, attr); + + return 0; +} + +int dpcon_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct mc_command cmd = { 0 }; + int err; + + cmd.header = mc_encode_cmd_header(DPCON_CMDID_GET_API_VERSION, + cmd_flags, + 0); + + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + DPCON_RSP_GET_API_VERSION(cmd, *major_ver, *minor_ver); + + return 0; +} diff --git a/drivers/bus/fslmc/mc/dpio.c b/drivers/bus/fslmc/mc/dpio.c index d84232a3..608b57a7 100644 --- a/drivers/bus/fslmc/mc/dpio.c +++ b/drivers/bus/fslmc/mc/dpio.c @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -257,6 +257,50 @@ int dpio_get_stashing_destination(struct fsl_mc_io *mc_io, return 0; } +int dpio_add_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id, + uint8_t *channel_index) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_ADD_STATIC_DEQUEUE_CHANNEL, + cmd_flags, + token); + DPIO_CMD_ADD_STATIC_DEQUEUE_CHANNEL(cmd, dpcon_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPIO_RSP_ADD_STATIC_DEQUEUE_CHANNEL(cmd, *channel_index); + + return 0; +} + +int dpio_remove_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPIO_CMDID_REMOVE_STATIC_DEQUEUE_CHANNEL, + cmd_flags, + token); + DPIO_CMD_REMOVE_STATIC_DEQUEUE_CHANNEL(cmd, dpcon_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + int dpio_get_api_version(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t *major_ver, diff --git a/drivers/bus/fslmc/mc/dpmng.c b/drivers/bus/fslmc/mc/dpmng.c new file mode 100644 index 00000000..dd1c3ac0 --- /dev/null +++ b/drivers/bus/fslmc/mc/dpmng.c @@ -0,0 +1,88 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include <fsl_mc_sys.h> +#include <fsl_mc_cmd.h> +#include <fsl_dpmng.h> +#include <fsl_dpmng_cmd.h> + +int mc_get_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_version *mc_ver_info) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPMNG_CMDID_GET_VERSION, + cmd_flags, + 0); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPMNG_RSP_GET_VERSION(cmd, mc_ver_info); + + return 0; +} + +int mc_get_soc_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_soc_version *mc_platform_info) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPMNG_CMDID_GET_SOC_VERSION, + cmd_flags, + 0); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPMNG_RSP_GET_SOC_VERSION(cmd, mc_platform_info); + + return 0; +} diff --git a/drivers/bus/fslmc/mc/fsl_dpbp.h b/drivers/bus/fslmc/mc/fsl_dpbp.h index d14e25d1..32bb9aa3 100644 --- a/drivers/bus/fslmc/mc/fsl_dpbp.h +++ b/drivers/bus/fslmc/mc/fsl_dpbp.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h index 1428dc6e..f0ee65a6 100644 --- a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/bus/fslmc/mc/fsl_dpci.h b/drivers/bus/fslmc/mc/fsl_dpci.h new file mode 100644 index 00000000..1e155dd7 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpci.h @@ -0,0 +1,404 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPCI_H +#define __FSL_DPCI_H + +/* Data Path Communication Interface API + * Contains initialization APIs and runtime control APIs for DPCI + */ + +struct fsl_mc_io; + +/** General DPCI macros */ + +/** + * Maximum number of Tx/Rx priorities per DPCI object + */ +#define DPCI_PRIO_NUM 2 + +/** + * Indicates an invalid frame queue + */ +#define DPCI_FQID_NOT_VALID (uint32_t)(-1) + +/** + * All queues considered; see dpci_set_rx_queue() + */ +#define DPCI_ALL_QUEUES (uint8_t)(-1) + +/** + * dpci_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpci_id: DPCI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpci_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpci_id, + uint16_t *token); + +/** + * dpci_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * Enable the Order Restoration support + */ +#define DPCI_OPT_HAS_OPR 0x000040 + +/** + * Order Point Records are shared for the entire DPCI + */ +#define DPCI_OPT_OPR_SHARED 0x000080 + +/** + * struct dpci_cfg - Structure representing DPCI configuration + * @options: Any combination of the following options: + * DPCI_OPT_HAS_OPR + * DPCI_OPT_OPR_SHARED + * @num_of_priorities: Number of receive priorities (queues) for the DPCI; + * note, that the number of transmit priorities (queues) + * is determined by the number of receive priorities of + * the peer DPCI object + */ +struct dpci_cfg { + uint32_t options; + uint8_t num_of_priorities; +}; + +/** + * dpci_create() - Create the DPCI object. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: returned object id + * + * Create the DPCI object, allocate required resources and perform required + * initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpci_cfg *cfg, + uint32_t *obj_id); + +/** + * dpci_destroy() - Destroy the DPCI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; error code otherwise. + */ +int dpci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); + +/** + * dpci_enable() - Enable the DPCI, allow sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * dpci_disable() - Disable the DPCI, stop sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * dpci_is_enabled() - Check if the DPCI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +/** + * dpci_reset() - Reset the DPCI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpci_attr - Structure representing DPCI attributes + * @id: DPCI object ID + * @num_of_priorities: Number of receive priorities + */ +struct dpci_attr { + int id; + uint8_t num_of_priorities; +}; + +/** + * dpci_get_attributes() - Retrieve DPCI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_attr *attr); + +/** + * enum dpci_dest - DPCI destination types + * @DPCI_DEST_NONE: Unassigned destination; The queue is set in parked mode + * and does not generate FQDAN notifications; user is + * expected to dequeue from the queue based on polling or + * other user-defined method + * @DPCI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected + * to dequeue from the queue only after notification is + * received + * @DPCI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified + * DPCON object; + * user is expected to dequeue from the DPCON channel + */ +enum dpci_dest { + DPCI_DEST_NONE = 0, + DPCI_DEST_DPIO = 1, + DPCI_DEST_DPCON = 2 +}; + +/** + * struct dpci_dest_cfg - Structure representing DPCI destination configuration + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid + * values are 0-1 or 0-7, depending on the number of priorities + * in that channel; not relevant for 'DPCI_DEST_NONE' option + */ +struct dpci_dest_cfg { + enum dpci_dest dest_type; + int dest_id; + uint8_t priority; +}; + +/** DPCI queue modification options */ + +/** + * Select to modify the user's context associated with the queue + */ +#define DPCI_QUEUE_OPT_USER_CTX 0x00000001 + +/** + * Select to modify the queue's destination + */ +#define DPCI_QUEUE_OPT_DEST 0x00000002 + +/** + * struct dpci_rx_queue_cfg - Structure representing RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPCI_QUEUE_OPT_<X>' flags + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; + * valid only if 'DPCI_QUEUE_OPT_USER_CTX' is contained in + * 'options' + * @dest_cfg: Queue destination parameters; + * valid only if 'DPCI_QUEUE_OPT_DEST' is contained in 'options' + */ +struct dpci_rx_queue_cfg { + uint32_t options; + uint64_t user_ctx; + struct dpci_dest_cfg dest_cfg; +}; + +/** + * dpci_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @priority: Select the queue relative to number of + * priorities configured at DPCI creation; use + * DPCI_ALL_QUEUES to configure all Rx queues + * identically. + * @cfg: Rx queue configuration + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + const struct dpci_rx_queue_cfg *cfg); + +/** + * struct dpci_rx_queue_attr - Structure representing Rx queue attributes + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations + */ +struct dpci_rx_queue_attr { + uint64_t user_ctx; + struct dpci_dest_cfg dest_cfg; + uint32_t fqid; +}; + +/** + * dpci_get_rx_queue() - Retrieve Rx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @priority: Select the queue relative to number of + * priorities configured at DPCI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_rx_queue_attr *attr); + +/** + * struct dpci_tx_queue_attr - Structure representing attributes of Tx queues + * @fqid: Virtual FQID to be used for sending frames to peer DPCI; + * returns 'DPCI_FQID_NOT_VALID' if a no peer is connected or if + * the selected priority exceeds the number of priorities of the + * peer DPCI object + */ +struct dpci_tx_queue_attr { + uint32_t fqid; +}; + +/** + * dpci_get_tx_queue() - Retrieve Tx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @priority: Select the queue relative to number of + * priorities of the peer DPCI object + * @attr: Returned Tx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_tx_queue_attr *attr); + +/** + * dpci_get_api_version() - Get communication interface API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path communication interface API + * @minor_ver: Minor version of data path communication interface API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); + +#endif /* __FSL_DPCI_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpci_cmd.h b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h new file mode 100644 index 00000000..6d4e2730 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h @@ -0,0 +1,147 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPCI_CMD_H +#define _FSL_DPCI_CMD_H + +/* DPCI Version */ +#define DPCI_VER_MAJOR 3 +#define DPCI_VER_MINOR 3 + +/* Command IDs */ +#define DPCI_CMDID_CLOSE 0x8001 +#define DPCI_CMDID_OPEN 0x8071 +#define DPCI_CMDID_CREATE 0x9072 +#define DPCI_CMDID_DESTROY 0x9871 +#define DPCI_CMDID_GET_API_VERSION 0xa071 + +#define DPCI_CMDID_ENABLE 0x0021 +#define DPCI_CMDID_DISABLE 0x0031 +#define DPCI_CMDID_GET_ATTR 0x0041 +#define DPCI_CMDID_RESET 0x0051 +#define DPCI_CMDID_IS_ENABLED 0x0061 + +#define DPCI_CMDID_SET_IRQ_ENABLE 0x0121 +#define DPCI_CMDID_GET_IRQ_ENABLE 0x0131 +#define DPCI_CMDID_SET_IRQ_MASK 0x0141 +#define DPCI_CMDID_GET_IRQ_MASK 0x0151 +#define DPCI_CMDID_GET_IRQ_STATUS 0x0161 +#define DPCI_CMDID_CLEAR_IRQ_STATUS 0x0171 + +#define DPCI_CMDID_SET_RX_QUEUE 0x0e01 +#define DPCI_CMDID_GET_LINK_STATE 0x0e11 +#define DPCI_CMDID_GET_PEER_ATTR 0x0e21 +#define DPCI_CMDID_GET_RX_QUEUE 0x0e31 +#define DPCI_CMDID_GET_TX_QUEUE 0x0e41 +#define DPCI_CMDID_SET_OPR 0x0e51 +#define DPCI_CMDID_GET_OPR 0x0e61 + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_CMD_OPEN(cmd, dpci_id) \ + MC_CMD_OP(cmd, 0, 0, 32, int, dpci_id) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_CMD_CREATE(cmd, cfg) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, cfg->num_of_priorities);\ + MC_CMD_OP(cmd, 2, 0, 32, uint32_t, cfg->options);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_RSP_IS_ENABLED(cmd, en) \ + MC_RSP_OP(cmd, 0, 0, 1, int, en) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_RSP_GET_ATTRIBUTES(cmd, attr) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 32, int, (attr)->id);\ + MC_RSP_OP(cmd, 0, 48, 8, uint8_t, (attr)->num_of_priorities);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_RSP_GET_PEER_ATTR(cmd, attr) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 32, int, attr->peer_id);\ + MC_RSP_OP(cmd, 1, 0, 8, uint8_t, attr->num_of_priorities);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_RSP_GET_LINK_STATE(cmd, up) \ + MC_RSP_OP(cmd, 0, 0, 1, int, up) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_CMD_SET_RX_QUEUE(cmd, priority, cfg) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 32, int, cfg->dest_cfg.dest_id);\ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->dest_cfg.priority);\ + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority);\ + MC_CMD_OP(cmd, 0, 48, 4, enum dpci_dest, cfg->dest_cfg.dest_type);\ + MC_CMD_OP(cmd, 1, 0, 64, uint64_t, cfg->user_ctx);\ + MC_CMD_OP(cmd, 2, 0, 32, uint32_t, cfg->options);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_CMD_GET_RX_QUEUE(cmd, priority) \ + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_RSP_GET_RX_QUEUE(cmd, attr) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 32, int, attr->dest_cfg.dest_id);\ + MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->dest_cfg.priority);\ + MC_RSP_OP(cmd, 0, 48, 4, enum dpci_dest, attr->dest_cfg.dest_type);\ + MC_RSP_OP(cmd, 1, 0, 8, uint64_t, attr->user_ctx);\ + MC_RSP_OP(cmd, 2, 0, 32, uint32_t, attr->fqid);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_CMD_GET_TX_QUEUE(cmd, priority) \ + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_RSP_GET_TX_QUEUE(cmd, attr) \ + MC_RSP_OP(cmd, 0, 32, 32, uint32_t, attr->fqid) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCI_RSP_GET_API_VERSION(cmd, major, minor) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 16, uint16_t, major);\ + MC_RSP_OP(cmd, 0, 16, 16, uint16_t, minor);\ +} while (0) + +#endif /* _FSL_DPCI_CMD_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h new file mode 100644 index 00000000..0ed9db56 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpcon.h @@ -0,0 +1,238 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPCON_H +#define __FSL_DPCON_H + +/* Data Path Concentrator API + * Contains initialization APIs and runtime control APIs for DPCON + */ + +struct fsl_mc_io; + +/** General DPCON macros */ + +/** + * Use it to disable notifications; see dpcon_set_notification() + */ +#define DPCON_INVALID_DPIO_ID (int)(-1) + +/** + * dpcon_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpcon_id: DPCON unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpcon_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpcon_id, + uint16_t *token); + +/** + * dpcon_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpcon_cfg - Structure representing DPCON configuration + * @num_priorities: Number of priorities for the DPCON channel (1-8) + */ +struct dpcon_cfg { + uint8_t num_priorities; +}; + +/** + * dpcon_create() - Create the DPCON object. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: returned object id + * + * Create the DPCON object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpcon_cfg *cfg, + uint32_t *obj_id); + +/** + * dpcon_destroy() - Destroy the DPCON object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; error code otherwise. + */ +int dpcon_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); + +/** + * dpcon_enable() - Enable the DPCON + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * Return: '0' on Success; Error code otherwise + */ +int dpcon_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * dpcon_disable() - Disable the DPCON + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * Return: '0' on Success; Error code otherwise + */ +int dpcon_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * dpcon_is_enabled() - Check if the DPCON is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +/** + * dpcon_reset() - Reset the DPCON, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpcon_attr - Structure representing DPCON attributes + * @id: DPCON object ID + * @qbman_ch_id: Channel ID to be used by dequeue operation + * @num_priorities: Number of priorities for the DPCON channel (1-8) + */ +struct dpcon_attr { + int id; + uint16_t qbman_ch_id; + uint8_t num_priorities; +}; + +/** + * dpcon_get_attributes() - Retrieve DPCON attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * @attr: Object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_attr *attr); + +/** + * dpcon_get_api_version() - Get Data Path Concentrator API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path concentrator API + * @minor_ver: Minor version of data path concentrator API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); + +#endif /* __FSL_DPCON_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h b/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h new file mode 100644 index 00000000..f7f76902 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h @@ -0,0 +1,175 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPCON_CMD_H +#define _FSL_DPCON_CMD_H + +/* DPCON Version */ +#define DPCON_VER_MAJOR 3 +#define DPCON_VER_MINOR 2 + +/* Command IDs */ +#define DPCON_CMDID_CLOSE ((0x800 << 4) | (0x1)) +#define DPCON_CMDID_OPEN ((0x808 << 4) | (0x1)) +#define DPCON_CMDID_CREATE ((0x908 << 4) | (0x1)) +#define DPCON_CMDID_DESTROY ((0x988 << 4) | (0x1)) +#define DPCON_CMDID_GET_API_VERSION ((0xa08 << 4) | (0x1)) + +#define DPCON_CMDID_ENABLE ((0x002 << 4) | (0x1)) +#define DPCON_CMDID_DISABLE ((0x003 << 4) | (0x1)) +#define DPCON_CMDID_GET_ATTR ((0x004 << 4) | (0x1)) +#define DPCON_CMDID_RESET ((0x005 << 4) | (0x1)) +#define DPCON_CMDID_IS_ENABLED ((0x006 << 4) | (0x1)) + +#define DPCON_CMDID_SET_IRQ ((0x010 << 4) | (0x1)) +#define DPCON_CMDID_GET_IRQ ((0x011 << 4) | (0x1)) +#define DPCON_CMDID_SET_IRQ_ENABLE ((0x012 << 4) | (0x1)) +#define DPCON_CMDID_GET_IRQ_ENABLE ((0x013 << 4) | (0x1)) +#define DPCON_CMDID_SET_IRQ_MASK ((0x014 << 4) | (0x1)) +#define DPCON_CMDID_GET_IRQ_MASK ((0x015 << 4) | (0x1)) +#define DPCON_CMDID_GET_IRQ_STATUS ((0x016 << 4) | (0x1)) +#define DPCON_CMDID_CLEAR_IRQ_STATUS ((0x017 << 4) | (0x1)) + +#define DPCON_CMDID_SET_NOTIFICATION ((0x100 << 4) | (0x1)) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_OPEN(cmd, dpcon_id) \ + MC_CMD_OP(cmd, 0, 0, 32, int, dpcon_id) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_CREATE(cmd, cfg) \ + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, cfg->num_priorities) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_RSP_IS_ENABLED(cmd, en) \ + MC_RSP_OP(cmd, 0, 0, 1, int, en) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_SET_IRQ(cmd, irq_index, irq_cfg) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, irq_index);\ + MC_CMD_OP(cmd, 0, 32, 32, uint32_t, irq_cfg->val);\ + MC_CMD_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ + MC_CMD_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_GET_IRQ(cmd, irq_index) \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_RSP_GET_IRQ(cmd, type, irq_cfg) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, irq_cfg->val);\ + MC_RSP_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ + MC_RSP_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ + MC_RSP_OP(cmd, 2, 32, 32, int, type);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_SET_IRQ_ENABLE(cmd, irq_index, en) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, en); \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_GET_IRQ_ENABLE(cmd, irq_index) \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_RSP_GET_IRQ_ENABLE(cmd, en) \ + MC_RSP_OP(cmd, 0, 0, 8, uint8_t, en) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_SET_IRQ_MASK(cmd, irq_index, mask) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, mask); \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_GET_IRQ_MASK(cmd, irq_index) \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_RSP_GET_IRQ_MASK(cmd, mask) \ + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mask) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_GET_IRQ_STATUS(cmd, irq_index, status) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status);\ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_RSP_GET_IRQ_STATUS(cmd, status) \ + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, status) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status); \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_RSP_GET_ATTR(cmd, attr) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 32, int, attr->id);\ + MC_RSP_OP(cmd, 0, 32, 16, uint16_t, attr->qbman_ch_id);\ + MC_RSP_OP(cmd, 0, 48, 8, uint8_t, attr->num_priorities);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_CMD_SET_NOTIFICATION(cmd, cfg) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 32, int, cfg->dpio_id);\ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->priority);\ + MC_CMD_OP(cmd, 1, 0, 64, uint64_t, cfg->user_ctx);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPCON_RSP_GET_API_VERSION(cmd, major, minor) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 16, uint16_t, major);\ + MC_RSP_OP(cmd, 0, 16, 16, uint16_t, minor);\ +} while (0) + +#endif /* _FSL_DPCON_CMD_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpio.h b/drivers/bus/fslmc/mc/fsl_dpio.h index 6d86f07d..4448cca5 100644 --- a/drivers/bus/fslmc/mc/fsl_dpio.h +++ b/drivers/bus/fslmc/mc/fsl_dpio.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -230,6 +230,36 @@ int dpio_get_stashing_destination(struct fsl_mc_io *mc_io, uint8_t *sdest); /** + * dpio_add_static_dequeue_channel() - Add a static dequeue channel. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @dpcon_id: DPCON object ID + * @channel_index: Returned channel index to be used in qbman API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_add_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id, + uint8_t *channel_index); + +/** + * dpio_remove_static_dequeue_channel() - Remove a static dequeue channel. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @dpcon_id: DPCON object ID + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_remove_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id); + +/** * struct dpio_attr - Structure representing DPIO attributes * @id: DPIO object ID * @qbman_portal_ce_offset: offset of the software portal cache-enabled area diff --git a/drivers/bus/fslmc/mc/fsl_dpio_cmd.h b/drivers/bus/fslmc/mc/fsl_dpio_cmd.h index b1147de2..d7575077 100644 --- a/drivers/bus/fslmc/mc/fsl_dpio_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpio_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/bus/fslmc/mc/fsl_dpmng.h b/drivers/bus/fslmc/mc/fsl_dpmng.h new file mode 100644 index 00000000..c2ddde09 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpmng.h @@ -0,0 +1,106 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPMNG_H +#define __FSL_DPMNG_H + +/* Management Complex General API + * Contains general API for the Management Complex firmware + */ + +struct fsl_mc_io; + +/** + * Management Complex firmware version information + */ +#define MC_VER_MAJOR 10 +#define MC_VER_MINOR 1 + +/** + * struct mc_versoin + * @major: Major version number: incremented on API compatibility changes + * @minor: Minor version number: incremented on API additions (that are + * backward compatible); reset when major version is incremented + * @revision: Internal revision number: incremented on implementation changes + * and/or bug fixes that have no impact on API + */ +struct mc_version { + uint32_t major; + uint32_t minor; + uint32_t revision; +}; + +/** + * struct mc_platform + * @svr: system version (content of platform SVR register) + * @pvr: processor version (content of platform PVR register) + */ +struct mc_soc_version { + uint32_t svr; + uint32_t pvr; +}; + +/** + * mc_get_version() - Retrieves the Management Complex firmware + * version information + * @mc_io: Pointer to opaque I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @mc_ver_info: Returned version information structure + * + * Return: '0' on Success; Error code otherwise. + */ +int mc_get_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_version *mc_ver_info); + +/** + * mc_get_soc_version() - Retrieves the Management Complex firmware + * version information + * @mc_io: Pointer to opaque I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @mc_platform_info: Returned version information structure. The structure + * contains the values of SVR and PVR registers. Please consult platform + * specific reference manual for detailed information. + * + * Return: '0' on Success; Error code otherwise. + */ +int mc_get_soc_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_soc_version *mc_platform_info); + +#endif /* __FSL_DPMNG_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h b/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h new file mode 100644 index 00000000..3a36b6dc --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h @@ -0,0 +1,61 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPMNG_CMD_H +#define __FSL_DPMNG_CMD_H + +/* Command IDs */ +#define DPMNG_CMDID_GET_VERSION 0x8311 +#define DPMNG_CMDID_GET_SOC_VERSION 0x8321 + +/* cmd, param, offset, width, type, arg_name */ +#define DPMNG_RSP_GET_VERSION(cmd, mc_ver_info) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mc_ver_info->revision); \ + MC_RSP_OP(cmd, 0, 32, 32, uint32_t, mc_ver_info->major); \ + MC_RSP_OP(cmd, 1, 0, 32, uint32_t, mc_ver_info->minor); \ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPMNG_RSP_GET_SOC_VERSION(cmd, mc_soc_version) \ +do { \ + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mc_soc_version->svr); \ + MC_RSP_OP(cmd, 0, 32, 32, uint32_t, mc_soc_version->pvr); \ +} while (0) + +#endif /* __FSL_DPMNG_CMD_H */ diff --git a/drivers/bus/fslmc/mc/fsl_mc_cmd.h b/drivers/bus/fslmc/mc/fsl_mc_cmd.h index d2252228..0ca4345c 100644 --- a/drivers/bus/fslmc/mc/fsl_mc_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_mc_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c index 2fb285c1..33f9eedf 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -53,29 +53,20 @@ #include "portal/dpaa2_hw_pvt.h" #include "portal/dpaa2_hw_dpio.h" -TAILQ_HEAD(dpbp_device_list, dpaa2_dpbp_dev); -static struct dpbp_device_list *dpbp_dev_list; /*!< DPBP device list */ +TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev); +static struct dpbp_dev_list dpbp_dev_list + = TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */ -int -dpaa2_create_dpbp_device( - int dpbp_id) +static int +dpaa2_create_dpbp_device(struct fslmc_vfio_device *vdev __rte_unused, + struct vfio_device_info *obj_info __rte_unused, + int dpbp_id) { struct dpaa2_dpbp_dev *dpbp_node; int ret; - if (!dpbp_dev_list) { - dpbp_dev_list = malloc(sizeof(struct dpbp_device_list)); - if (!dpbp_dev_list) { - PMD_INIT_LOG(ERR, "Memory alloc failed in DPBP list\n"); - return -1; - } - /* Initialize the DPBP List */ - TAILQ_INIT(dpbp_dev_list); - } - /* Allocate DPAA2 dpbp handle */ - dpbp_node = (struct dpaa2_dpbp_dev *) - malloc(sizeof(struct dpaa2_dpbp_dev)); + dpbp_node = rte_malloc(NULL, sizeof(struct dpaa2_dpbp_dev), 0); if (!dpbp_node) { PMD_INIT_LOG(ERR, "Memory allocation failed for DPBP Device"); return -1; @@ -88,7 +79,7 @@ dpaa2_create_dpbp_device( if (ret) { PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d", ret); - free(dpbp_node); + rte_free(dpbp_node); return -1; } @@ -98,16 +89,16 @@ dpaa2_create_dpbp_device( PMD_INIT_LOG(ERR, "Failure cleaning dpbp device with" " error code %d\n", ret); dpbp_close(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token); - free(dpbp_node); + rte_free(dpbp_node); return -1; } dpbp_node->dpbp_id = dpbp_id; rte_atomic16_init(&dpbp_node->in_use); - TAILQ_INSERT_HEAD(dpbp_dev_list, dpbp_node, next); + TAILQ_INSERT_TAIL(&dpbp_dev_list, dpbp_node, next); - PMD_INIT_LOG(DEBUG, "Buffer pool resource initialized %d", dpbp_id); + PMD_INIT_LOG(DEBUG, "DPAA2: Added [dpbp.%d]", dpbp_id); return 0; } @@ -117,7 +108,7 @@ struct dpaa2_dpbp_dev *dpaa2_alloc_dpbp_dev(void) struct dpaa2_dpbp_dev *dpbp_dev = NULL; /* Get DPBP dev handle from list using index */ - TAILQ_FOREACH(dpbp_dev, dpbp_dev_list, next) { + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { if (dpbp_dev && rte_atomic16_test_and_set(&dpbp_dev->in_use)) break; } @@ -130,10 +121,17 @@ void dpaa2_free_dpbp_dev(struct dpaa2_dpbp_dev *dpbp) struct dpaa2_dpbp_dev *dpbp_dev = NULL; /* Match DPBP handle and mark it free */ - TAILQ_FOREACH(dpbp_dev, dpbp_dev_list, next) { + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { if (dpbp_dev == dpbp) { rte_atomic16_dec(&dpbp_dev->in_use); return; } } } + +static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = { + .object_id = DPAA2_MC_DPBP_DEVID, + .create = dpaa2_create_dpbp_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj); diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c new file mode 100644 index 00000000..478e4f7e --- /dev/null +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c @@ -0,0 +1,179 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <unistd.h> +#include <stdio.h> +#include <sys/types.h> +#include <string.h> +#include <stdlib.h> +#include <fcntl.h> +#include <errno.h> + +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_string_fns.h> +#include <rte_cycles.h> +#include <rte_kvargs.h> +#include <rte_dev.h> +#include <rte_ethdev.h> + +#include <fslmc_logs.h> +#include <fslmc_vfio.h> +#include <mc/fsl_dpci.h> +#include "portal/dpaa2_hw_pvt.h" +#include "portal/dpaa2_hw_dpio.h" + +TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev); +static struct dpci_dev_list dpci_dev_list + = TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */ + +static int +rte_dpaa2_create_dpci_device(struct fslmc_vfio_device *vdev __rte_unused, + struct vfio_device_info *obj_info __rte_unused, + int dpci_id) +{ + struct dpaa2_dpci_dev *dpci_node; + struct dpci_attr attr; + struct dpci_rx_queue_cfg rx_queue_cfg; + struct dpci_rx_queue_attr rx_attr; + int ret, i; + + /* Allocate DPAA2 dpci handle */ + dpci_node = rte_malloc(NULL, sizeof(struct dpaa2_dpci_dev), 0); + if (!dpci_node) { + PMD_INIT_LOG(ERR, "Memory allocation failed for DPCI Device"); + return -1; + } + + /* Open the dpci object */ + dpci_node->dpci.regs = rte_mcp_ptr_list[MC_PORTAL_INDEX]; + ret = dpci_open(&dpci_node->dpci, + CMD_PRI_LOW, dpci_id, &dpci_node->token); + if (ret) { + PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + /* Get the device attributes */ + ret = dpci_get_attributes(&dpci_node->dpci, + CMD_PRI_LOW, dpci_node->token, &attr); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Reading device failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + /* Set up the Rx Queue */ + memset(&rx_queue_cfg, 0, sizeof(struct dpci_rx_queue_cfg)); + ret = dpci_set_rx_queue(&dpci_node->dpci, + CMD_PRI_LOW, + dpci_node->token, + 0, &rx_queue_cfg); + if (ret) { + PMD_INIT_LOG(ERR, "Setting Rx queue failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + /* Enable the device */ + ret = dpci_enable(&dpci_node->dpci, + CMD_PRI_LOW, dpci_node->token); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Enabling device failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) { + /* Get the Rx FQID's */ + ret = dpci_get_rx_queue(&dpci_node->dpci, + CMD_PRI_LOW, + dpci_node->token, i, + &rx_attr); + if (ret != 0) { + PMD_INIT_LOG(ERR, + "Reading device failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + dpci_node->queue[i].fqid = rx_attr.fqid; + } + + dpci_node->dpci_id = dpci_id; + rte_atomic16_init(&dpci_node->in_use); + + TAILQ_INSERT_TAIL(&dpci_dev_list, dpci_node, next); + + PMD_INIT_LOG(DEBUG, "DPAA2: Added [dpci.%d]", dpci_id); + + return 0; +} + +struct dpaa2_dpci_dev *rte_dpaa2_alloc_dpci_dev(void) +{ + struct dpaa2_dpci_dev *dpci_dev = NULL; + + /* Get DPCI dev handle from list using index */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { + if (dpci_dev && rte_atomic16_test_and_set(&dpci_dev->in_use)) + break; + } + + return dpci_dev; +} + +void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci) +{ + struct dpaa2_dpci_dev *dpci_dev = NULL; + + /* Match DPCI handle and mark it free */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { + if (dpci_dev == dpci) { + rte_atomic16_dec(&dpci_dev->in_use); + return; + } + } +} + +static struct rte_dpaa2_object rte_dpaa2_dpci_obj = { + .object_id = DPAA2_MC_DPCI_DEVID, + .create = rte_dpaa2_create_dpci_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj); diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c index a1a58b9c..283441b4 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -46,6 +46,8 @@ #include <sys/stat.h> #include <sys/mman.h> #include <sys/syscall.h> +#include <sys/epoll.h> +#include<sys/eventfd.h> #include <rte_mbuf.h> #include <rte_ethdev.h> @@ -55,20 +57,23 @@ #include <rte_cycles.h> #include <rte_kvargs.h> #include <rte_dev.h> -#include <rte_ethdev.h> #include <fslmc_logs.h> #include <fslmc_vfio.h> #include "dpaa2_hw_pvt.h" #include "dpaa2_hw_dpio.h" +#include <mc/fsl_dpmng.h> #define NUM_HOST_CPUS RTE_MAX_LCORE struct dpaa2_io_portal_t dpaa2_io_portal[RTE_MAX_LCORE]; RTE_DEFINE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io); -TAILQ_HEAD(dpio_device_list, dpaa2_dpio_dev); -static struct dpio_device_list *dpio_dev_list; /*!< DPIO device list */ +struct swp_active_dqs rte_global_active_dqs_list[NUM_MAX_SWP]; + +TAILQ_HEAD(dpio_dev_list, dpaa2_dpio_dev); +static struct dpio_dev_list dpio_dev_list + = TAILQ_HEAD_INITIALIZER(dpio_dev_list); /*!< DPIO device list */ static uint32_t io_space_count; /*Stashing Macros default for LS208x*/ @@ -102,6 +107,95 @@ dpaa2_core_cluster_sdest(int cpu_id) return dpaa2_core_cluster_base + x; } +static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id) +{ +#define STRING_LEN 28 +#define COMMAND_LEN 50 + uint32_t cpu_mask = 1; + int ret; + size_t len = 0; + char *temp = NULL, *token = NULL; + char string[STRING_LEN], command[COMMAND_LEN]; + FILE *file; + + snprintf(string, STRING_LEN, "dpio.%d", dpio_id); + file = fopen("/proc/interrupts", "r"); + if (!file) { + PMD_DRV_LOG(WARNING, "Failed to open /proc/interrupts file\n"); + return; + } + while (getline(&temp, &len, file) != -1) { + if ((strstr(temp, string)) != NULL) { + token = strtok(temp, ":"); + break; + } + } + + if (!token) { + PMD_DRV_LOG(WARNING, "Failed to get interrupt id for dpio.%d\n", + dpio_id); + if (temp) + free(temp); + fclose(file); + return; + } + + cpu_mask = cpu_mask << rte_lcore_id(); + snprintf(command, COMMAND_LEN, "echo %X > /proc/irq/%s/smp_affinity", + cpu_mask, token); + ret = system(command); + if (ret < 0) + PMD_DRV_LOG(WARNING, + "Failed to affine interrupts on respective core\n"); + else + PMD_DRV_LOG(WARNING, " %s command is executed\n", command); + + free(temp); + fclose(file); +} + +static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev) +{ + struct epoll_event epoll_ev; + int eventfd, dpio_epoll_fd, ret; + int threshold = 0x3, timeout = 0xFF; + + dpio_epoll_fd = epoll_create(1); + ret = rte_dpaa2_intr_enable(&dpio_dev->intr_handle, 0); + if (ret) { + PMD_DRV_LOG(ERR, "Interrupt registeration failed\n"); + return -1; + } + + if (getenv("DPAA2_PORTAL_INTR_THRESHOLD")) + threshold = atoi(getenv("DPAA2_PORTAL_INTR_THRESHOLD")); + + if (getenv("DPAA2_PORTAL_INTR_TIMEOUT")) + sscanf(getenv("DPAA2_PORTAL_INTR_TIMEOUT"), "%x", &timeout); + + qbman_swp_interrupt_set_trigger(dpio_dev->sw_portal, + QBMAN_SWP_INTERRUPT_DQRI); + qbman_swp_interrupt_clear_status(dpio_dev->sw_portal, 0xffffffff); + qbman_swp_interrupt_set_inhibit(dpio_dev->sw_portal, 0); + qbman_swp_dqrr_thrshld_write(dpio_dev->sw_portal, threshold); + qbman_swp_intr_timeout_write(dpio_dev->sw_portal, timeout); + + eventfd = dpio_dev->intr_handle.fd; + epoll_ev.events = EPOLLIN | EPOLLPRI | EPOLLET; + epoll_ev.data.fd = eventfd; + + ret = epoll_ctl(dpio_epoll_fd, EPOLL_CTL_ADD, eventfd, &epoll_ev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "epoll_ctl failed\n"); + return -1; + } + dpio_dev->epoll_fd = dpio_epoll_fd; + + dpaa2_affine_dpio_intr_to_respective_core(dpio_dev->hw_id); + + return 0; +} + static int configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) { @@ -147,8 +241,6 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) } PMD_INIT_LOG(DEBUG, "Qbman Portal ID %d", attr.qbman_portal_id); - PMD_INIT_LOG(DEBUG, "Portal CE adr 0x%lX", attr.qbman_portal_ce_offset); - PMD_INIT_LOG(DEBUG, "Portal CI adr 0x%lX", attr.qbman_portal_ci_offset); /* Configure & setup SW portal */ p_des.block = NULL; @@ -166,19 +258,31 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) return -1; } - PMD_INIT_LOG(DEBUG, "QBMan SW Portal 0x%p\n", dpio_dev->sw_portal); - return 0; } static int -dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev) +dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id) { - int sdest; - int cpu_id, ret; + int sdest, ret; + static int first_time; + + /* find the SoC type for the first time */ + if (!first_time) { + struct mc_soc_version mc_plat_info = {0}; + + if (mc_get_soc_version(dpio_dev->dpio, + CMD_PRI_LOW, &mc_plat_info)) { + PMD_INIT_LOG(ERR, "\tmc_get_soc_version failed\n"); + } else if ((mc_plat_info.svr & 0xffff0000) == SVR_LS1080A) { + dpaa2_core_cluster_base = 0x02; + dpaa2_cluster_sz = 4; + PMD_INIT_LOG(DEBUG, "\tLS108x (A53) Platform Detected"); + } + first_time = 1; + } /* Set the Stashing Destination */ - cpu_id = rte_lcore_id(); if (cpu_id < 0) { cpu_id = rte_get_master_lcore(); if (cpu_id < 0) { @@ -188,8 +292,6 @@ dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev) } /* Set the STASH Destination depending on Current CPU ID. * Valid values of SDEST are 4,5,6,7. Where, - * CPU 0-1 will have SDEST 4 - * CPU 2-3 will have SDEST 5.....and so on. */ sdest = dpaa2_core_cluster_sdest(cpu_id); @@ -203,16 +305,21 @@ dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev) return -1; } + if (dpaa2_dpio_intr_init(dpio_dev)) { + PMD_DRV_LOG(ERR, "Interrupt registration failed for dpio\n"); + return -1; + } + return 0; } -static inline struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(void) +struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id) { struct dpaa2_dpio_dev *dpio_dev = NULL; int ret; /* Get DPIO dev handle from list using index */ - TAILQ_FOREACH(dpio_dev, dpio_dev_list, next) { + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) { if (dpio_dev && rte_atomic16_test_and_set(&dpio_dev->ref_count)) break; } @@ -222,7 +329,7 @@ static inline struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(void) PMD_DRV_LOG(DEBUG, "New Portal=0x%x (%d) affined thread - %lu", dpio_dev, dpio_dev->index, syscall(SYS_gettid)); - ret = dpaa2_configure_stashing(dpio_dev); + ret = dpaa2_configure_stashing(dpio_dev, cpu_id); if (ret) PMD_DRV_LOG(ERR, "dpaa2_configure_stashing failed"); @@ -262,7 +369,7 @@ dpaa2_affine_qbman_swp(void) } /* Populate the dpaa2_io_portal structure */ - dpaa2_io_portal[lcore_id].dpio_dev = dpaa2_get_qbman_swp(); + dpaa2_io_portal[lcore_id].dpio_dev = dpaa2_get_qbman_swp(lcore_id); if (dpaa2_io_portal[lcore_id].dpio_dev) { RTE_PER_LCORE(_dpaa2_io).dpio_dev @@ -308,7 +415,7 @@ dpaa2_affine_qbman_swp_sec(void) } /* Populate the dpaa2_io_portal structure */ - dpaa2_io_portal[lcore_id].sec_dpio_dev = dpaa2_get_qbman_swp(); + dpaa2_io_portal[lcore_id].sec_dpio_dev = dpaa2_get_qbman_swp(lcore_id); if (dpaa2_io_portal[lcore_id].sec_dpio_dev) { RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev @@ -320,13 +427,14 @@ dpaa2_affine_qbman_swp_sec(void) } } -int +static int dpaa2_create_dpio_device(struct fslmc_vfio_device *vdev, struct vfio_device_info *obj_info, - int object_id) + int object_id) { struct dpaa2_dpio_dev *dpio_dev; struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)}; + int vfio_dev_fd; if (obj_info->num_regions < NUM_DPIO_REGIONS) { PMD_INIT_LOG(ERR, "ERROR, Not sufficient number " @@ -334,80 +442,57 @@ dpaa2_create_dpio_device(struct fslmc_vfio_device *vdev, return -1; } - if (!dpio_dev_list) { - dpio_dev_list = malloc(sizeof(struct dpio_device_list)); - if (!dpio_dev_list) { - PMD_INIT_LOG(ERR, "Memory alloc failed in DPIO list\n"); - return -1; - } - - /* Initialize the DPIO List */ - TAILQ_INIT(dpio_dev_list); - } - - dpio_dev = malloc(sizeof(struct dpaa2_dpio_dev)); + dpio_dev = rte_malloc(NULL, sizeof(struct dpaa2_dpio_dev), + RTE_CACHE_LINE_SIZE); if (!dpio_dev) { PMD_INIT_LOG(ERR, "Memory allocation failed for DPIO Device\n"); return -1; } - PMD_DRV_LOG(INFO, "\t Aloocated DPIO [%p]", dpio_dev); dpio_dev->dpio = NULL; dpio_dev->hw_id = object_id; - dpio_dev->vfio_fd = vdev->fd; + dpio_dev->intr_handle.vfio_dev_fd = vdev->fd; rte_atomic16_init(&dpio_dev->ref_count); /* Using single portal for all devices */ dpio_dev->mc_portal = rte_mcp_ptr_list[MC_PORTAL_INDEX]; reg_info.index = 0; - if (ioctl(dpio_dev->vfio_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + vfio_dev_fd = dpio_dev->intr_handle.vfio_dev_fd; + if (ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { PMD_INIT_LOG(ERR, "vfio: error getting region info\n"); - free(dpio_dev); + rte_free(dpio_dev); return -1; } - PMD_DRV_LOG(DEBUG, "\t Region Offset = %llx", reg_info.offset); - PMD_DRV_LOG(DEBUG, "\t Region Size = %llx", reg_info.size); dpio_dev->ce_size = reg_info.size; dpio_dev->qbman_portal_ce_paddr = (uint64_t)mmap(NULL, reg_info.size, PROT_WRITE | PROT_READ, MAP_SHARED, - dpio_dev->vfio_fd, reg_info.offset); - - /* Create Mapping for QBMan Cache Enabled area. This is a fix for - * SMMU fault for DQRR statshing transaction. - */ - if (vfio_dmamap_mem_region(dpio_dev->qbman_portal_ce_paddr, - reg_info.offset, reg_info.size)) { - PMD_INIT_LOG(ERR, "DMAMAP for Portal CE area failed.\n"); - free(dpio_dev); - return -1; - } + vfio_dev_fd, reg_info.offset); reg_info.index = 1; - if (ioctl(dpio_dev->vfio_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + if (ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { PMD_INIT_LOG(ERR, "vfio: error getting region info\n"); - free(dpio_dev); + rte_free(dpio_dev); return -1; } - PMD_DRV_LOG(DEBUG, "\t Region Offset = %llx", reg_info.offset); - PMD_DRV_LOG(DEBUG, "\t Region Size = %llx", reg_info.size); dpio_dev->ci_size = reg_info.size; dpio_dev->qbman_portal_ci_paddr = (uint64_t)mmap(NULL, reg_info.size, PROT_WRITE | PROT_READ, MAP_SHARED, - dpio_dev->vfio_fd, reg_info.offset); + vfio_dev_fd, reg_info.offset); if (configure_dpio_qbman_swp(dpio_dev)) { PMD_INIT_LOG(ERR, "Fail to configure the dpio qbman portal for %d\n", dpio_dev->hw_id); - free(dpio_dev); + rte_free(dpio_dev); return -1; } io_space_count++; dpio_dev->index = io_space_count; - TAILQ_INSERT_HEAD(dpio_dev_list, dpio_dev, next); + TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next); + PMD_INIT_LOG(DEBUG, "DPAA2: Added [dpio.%d]", object_id); return 0; } @@ -437,9 +522,15 @@ dpaa2_alloc_dq_storage(struct queue_storage_info_t *q_storage) } return 0; fail: - i -= 1; - while (i >= 0) + while (--i >= 0) rte_free(q_storage->dq_storage[i]); return -1; } + +static struct rte_dpaa2_object rte_dpaa2_dpio_obj = { + .object_id = DPAA2_MC_DPIO_DEVID, + .create = dpaa2_create_dpio_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj); diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h index f2e11680..e845340c 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -42,6 +42,7 @@ struct dpaa2_io_portal_t { struct dpaa2_dpio_dev *sec_dpio_dev; uint64_t net_tid; uint64_t sec_tid; + void *eventdev; }; /*! Global per thread DPIO portal */ @@ -53,6 +54,10 @@ RTE_DECLARE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io); #define DPAA2_PER_LCORE_SEC_DPIO RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev #define DPAA2_PER_LCORE_SEC_PORTAL DPAA2_PER_LCORE_SEC_DPIO->sw_portal +extern struct dpaa2_io_portal_t dpaa2_io_portal[RTE_MAX_LCORE]; + +struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id); + /* Affine a DPIO portal to current processing thread */ int dpaa2_affine_qbman_swp(void); diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h index c0223734..5d7a8282 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,6 +34,8 @@ #ifndef _DPAA2_HW_PVT_H_ #define _DPAA2_HW_PVT_H_ +#include <rte_eventdev.h> + #include <mc/fsl_mc_sys.h> #include <fsl_qbman_portal.h> @@ -46,6 +48,10 @@ #define lower_32_bits(x) ((uint32_t)(x)) #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) +#define SVR_LS1080A 0x87030000 +#define SVR_LS2080A 0x87010000 +#define SVR_LS2088A 0x87090000 + #ifndef ETH_VLAN_HLEN #define ETH_VLAN_HLEN 4 /** < Vlan Header Length */ #endif @@ -65,7 +71,7 @@ #define MAX_BPID 256 #define DPAA2_MBUF_HW_ANNOTATION 64 -#define DPAA2_FD_PTA_SIZE 64 +#define DPAA2_FD_PTA_SIZE 0 #if (DPAA2_MBUF_HW_ANNOTATION + DPAA2_FD_PTA_SIZE) > RTE_PKTMBUF_HEADROOM #error "Annotation requirement is more than RTE_PKTMBUF_HEADROOM" @@ -75,6 +81,8 @@ #define DPAA2_HW_BUF_RESERVE 0 #define DPAA2_PACKET_LAYOUT_ALIGN 64 /*changing from 256 */ +#define DPAA2_DPCI_MAX_QUEUES 2 + struct dpaa2_dpio_dev { TAILQ_ENTRY(dpaa2_dpio_dev) next; /**< Pointer to Next device instance */ @@ -93,8 +101,11 @@ struct dpaa2_dpio_dev { uintptr_t qbman_portal_ci_paddr; /**< Physical address of Cache Inhibit Area */ uintptr_t ci_size; /**< Size of the CI region */ - int32_t vfio_fd; /**< File descriptor received via VFIO */ + struct rte_intr_handle intr_handle; /* Interrupt related info */ + int32_t epoll_fd; /**< File descriptor created for interrupt polling */ int32_t hw_id; /**< An unique ID of this DPIO device instance */ + uint64_t dqrr_held; + uint8_t dqrr_size; }; struct dpaa2_dpbp_dev { @@ -108,8 +119,16 @@ struct dpaa2_dpbp_dev { struct queue_storage_info_t { struct qbman_result *dq_storage[NUM_DQS_PER_QUEUE]; + struct qbman_result *active_dqs; + int active_dpio_id; + int toggle; }; +typedef void (dpaa2_queue_cb_dqrr_t)(struct qbman_swp *swp, + const struct qbman_fd *fd, + const struct qbman_result *dq, + struct rte_event *ev); + struct dpaa2_queue { struct rte_mempool *mb_pool; /**< mbuf pool to populate RX ring. */ void *dev; @@ -120,7 +139,30 @@ struct dpaa2_queue { uint64_t rx_pkts; uint64_t tx_pkts; uint64_t err_pkts; - struct queue_storage_info_t *q_storage; + union { + struct queue_storage_info_t *q_storage; + struct qbman_result *cscn; + }; + dpaa2_queue_cb_dqrr_t *cb; +}; + +struct swp_active_dqs { + struct qbman_result *global_active_dqs; + uint64_t reserved[7]; +}; + +#define NUM_MAX_SWP 64 + +extern struct swp_active_dqs rte_global_active_dqs_list[NUM_MAX_SWP]; + +struct dpaa2_dpci_dev { + TAILQ_ENTRY(dpaa2_dpci_dev) next; + /**< Pointer to Next device instance */ + struct fsl_mc_io dpci; /** handle to DPCI portal object */ + uint16_t token; + rte_atomic16_t in_use; + uint32_t dpci_id; /*HW ID for DPCI object */ + struct dpaa2_queue queue[DPAA2_DPCI_MAX_QUEUES]; }; /*! Global MCP list */ @@ -137,6 +179,19 @@ struct qbman_fle { uint32_t reserved[3]; /* Not used currently */ }; +struct qbman_sge { + uint32_t addr_lo; + uint32_t addr_hi; + uint32_t length; + uint32_t fin_bpid_offset; +}; + +/* There are three types of frames: Single, Scatter Gather and Frame Lists */ +enum qbman_fd_format { + qbman_fd_single = 0, + qbman_fd_list, + qbman_fd_sg +}; /*Macros to define operations on FD*/ #define DPAA2_SET_FD_ADDR(fd, addr) do { \ fd->simple.addr_lo = lower_32_bits((uint64_t)(addr)); \ @@ -163,10 +218,17 @@ struct qbman_fle { fle->addr_lo = lower_32_bits((uint64_t)addr); \ fle->addr_hi = upper_32_bits((uint64_t)addr); \ } while (0) +#define DPAA2_GET_FLE_CTXT(fle) \ + (uint64_t)((((uint64_t)((fle)->reserved[1])) << 32) + \ + (fle)->reserved[0]) +#define DPAA2_FLE_SAVE_CTXT(fle, addr) do { \ + fle->reserved[0] = lower_32_bits((uint64_t)addr); \ + fle->reserved[1] = upper_32_bits((uint64_t)addr); \ +} while (0) #define DPAA2_SET_FLE_OFFSET(fle, offset) \ ((fle)->fin_bpid_offset |= (uint32_t)(offset) << 16) #define DPAA2_SET_FLE_BPID(fle, bpid) ((fle)->fin_bpid_offset |= (uint64_t)bpid) -#define DPAA2_GET_FLE_BPID(fle, bpid) (fle->fin_bpid_offset & 0x000000ff) +#define DPAA2_GET_FLE_BPID(fle) ((fle)->fin_bpid_offset & 0x000000ff) #define DPAA2_SET_FLE_FIN(fle) (fle->fin_bpid_offset |= (uint64_t)1 << 31) #define DPAA2_SET_FLE_IVP(fle) (((fle)->fin_bpid_offset |= 0x00004000)) #define DPAA2_SET_FD_COMPOUND_FMT(fd) \ @@ -178,6 +240,7 @@ struct qbman_fle { #define DPAA2_GET_FD_BPID(fd) (((fd)->simple.bpid_offset & 0x00003FFF)) #define DPAA2_GET_FD_IVP(fd) ((fd->simple.bpid_offset & 0x00004000) >> 14) #define DPAA2_GET_FD_OFFSET(fd) (((fd)->simple.bpid_offset & 0x0FFF0000) >> 16) +#define DPAA2_GET_FLE_OFFSET(fle) (((fle)->fin_bpid_offset & 0x0FFF0000) >> 16) #define DPAA2_SET_FLE_SG_EXT(fle) (fle->fin_bpid_offset |= (uint64_t)1 << 29) #define DPAA2_IS_SET_FLE_SG_EXT(fle) \ ((fle->fin_bpid_offset & ((uint64_t)1 << 29)) ? 1 : 0) @@ -187,6 +250,17 @@ struct qbman_fle { #define DPAA2_ASAL_VAL (DPAA2_MBUF_HW_ANNOTATION / 64) +#define DPAA2_FD_SET_FORMAT(fd, format) do { \ + (fd)->simple.bpid_offset &= 0xCFFFFFFF; \ + (fd)->simple.bpid_offset |= (uint32_t)format << 28; \ +} while (0) +#define DPAA2_FD_GET_FORMAT(fd) (((fd)->simple.bpid_offset >> 28) & 0x3) + +#define DPAA2_SG_SET_FINAL(sg, fin) do { \ + (sg)->fin_bpid_offset &= 0x7FFFFFFF; \ + (sg)->fin_bpid_offset |= (uint32_t)fin << 31; \ +} while (0) +#define DPAA2_SG_IS_FINAL(sg) (!!((sg)->fin_bpid_offset >> 31)) /* Only Enqueue Error responses will be * pushed on FQID_ERR of Enqueue FQ */ @@ -231,7 +305,7 @@ static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) /** * When we are using Physical addresses as IO Virtual Addresses, * Need to call conversion routines dpaa2_mem_vtop & dpaa2_mem_ptov - * whereever required. + * wherever required. * These routines are called with help of below MACRO's */ @@ -264,7 +338,35 @@ static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) #endif /* RTE_LIBRTE_DPAA2_USE_PHYS_IOVA */ +static inline +int check_swp_active_dqs(uint16_t dpio_index) +{ + if (rte_global_active_dqs_list[dpio_index].global_active_dqs != NULL) + return 1; + return 0; +} + +static inline +void clear_swp_active_dqs(uint16_t dpio_index) +{ + rte_global_active_dqs_list[dpio_index].global_active_dqs = NULL; +} + +static inline +struct qbman_result *get_swp_active_dqs(uint16_t dpio_index) +{ + return rte_global_active_dqs_list[dpio_index].global_active_dqs; +} + +static inline +void set_swp_active_dqs(uint16_t dpio_index, struct qbman_result *dqs) +{ + rte_global_active_dqs_list[dpio_index].global_active_dqs = dqs; +} struct dpaa2_dpbp_dev *dpaa2_alloc_dpbp_dev(void); void dpaa2_free_dpbp_dev(struct dpaa2_dpbp_dev *dpbp); +struct dpaa2_dpci_dev *rte_dpaa2_alloc_dpci_dev(void); +void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci); + #endif diff --git a/drivers/bus/fslmc/qbman/include/compat.h b/drivers/bus/fslmc/qbman/include/compat.h index 41effe37..529f1ea3 100644 --- a/drivers/bus/fslmc/qbman/include/compat.h +++ b/drivers/bus/fslmc/qbman/include/compat.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2008-2016 Freescale Semiconductor, Inc. - * All rights reserved. + * Copyright 2017 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h index 77317723..9e9047e2 100644 --- a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h +++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h @@ -124,6 +124,36 @@ uint32_t qbman_swp_interrupt_read_status(struct qbman_swp *p); void qbman_swp_interrupt_clear_status(struct qbman_swp *p, uint32_t mask); /** + * qbman_swp_dqrr_thrshld_read_status() - Get the data in software portal + * DQRR interrupt threshold register. + * @p: the given software portal object. + */ +uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p); + +/** + * qbman_swp_dqrr_thrshld_write() - Set the data in software portal + * DQRR interrupt threshold register. + * @p: the given software portal object. + * @mask: The value to set in SWP_DQRR_ITR register. + */ +void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask); + +/** + * qbman_swp_intr_timeout_read_status() - Get the data in software portal + * Interrupt Time-Out period register. + * @p: the given software portal object. + */ +uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p); + +/** + * qbman_swp_intr_timeout_write() - Set the data in software portal + * Interrupt Time-Out period register. + * @p: the given software portal object. + * @mask: The value to set in SWP_ITPR register. + */ +void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask); + +/** * qbman_swp_interrupt_get_trigger() - Get the data in software portal * interrupt enable register. * @p: the given software portal object. @@ -349,7 +379,7 @@ void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct qbman_result *dq); * * Return dqrr index. */ -uint8_t qbman_get_dqrr_idx(struct qbman_result *dqrr); +uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr); /** * qbman_get_dqrr_from_idx() - Use index to get the dqrr entry from the @@ -883,6 +913,20 @@ void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, */ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, const struct qbman_fd *fd); +/** + * qbman_swp_enqueue_multiple_eqdesc() - Enqueue multiple frames with separte + * enqueue descriptors. + * @s: the software portal used for enqueue. + * @d: the enqueue descriptors + * @fd: the frame descriptor to be enqueued. + * @num_frames: the number of the frames to be enqueued. + * + * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. + */ +int qbman_swp_enqueue_multiple_eqdesc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); /* TODO: * qbman_swp_enqueue_thresh() - Set threshold for EQRI interrupt. diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c index 5d407cc0..dd62e9af 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.c +++ b/drivers/bus/fslmc/qbman/qbman_portal.c @@ -44,6 +44,8 @@ #define QBMAN_CINH_SWP_IER 0xe40 #define QBMAN_CINH_SWP_ISDR 0xe80 #define QBMAN_CINH_SWP_IIR 0xec0 +#define QBMAN_CINH_SWP_DQRR_ITR 0xa80 +#define QBMAN_CINH_SWP_ITPR 0xf40 /* CENA register offsets */ #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6)) @@ -218,6 +220,26 @@ void qbman_swp_interrupt_clear_status(struct qbman_swp *p, uint32_t mask) qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISR, mask); } +uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQRR_ITR); +} + +void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_DQRR_ITR, mask); +} + +uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ITPR); +} + +void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ITPR, mask); +} + uint32_t qbman_swp_interrupt_get_trigger(struct qbman_swp *p) { return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IER); @@ -288,7 +310,7 @@ void *qbman_swp_mc_result(struct qbman_swp *p) qbman_cena_invalidate_prefetch(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); - /* Remove the valid-bit - command completed iff the rest is non-zero */ + /* Remove the valid-bit - command completed if the rest is non-zero */ verb = ret[0] & ~QB_VALID_BIT; if (!verb) return NULL; @@ -574,6 +596,76 @@ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, return qbman_swp_enqueue_ring_mode(s, d, fd); } +int qbman_swp_enqueue_multiple_eqdesc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqcr_ci, eqcr_pi; + uint8_t diff; + int i, num_enqueued = 0; + uint64_t addr_cena; + + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cena_read_reg(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & 0xF; + diff = qm_cyc_diff(QBMAN_EQCR_SIZE, + eqcr_ci, s->eqcr.ci); + s->eqcr.available += diff; + if (!diff) + return 0; + } + + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + eqcr_pi &= 0xF; + /*Pointing to the next enqueue descriptor*/ + cl += (sizeof(struct qbman_eq_desc) / sizeof(uint32_t)); + } + + lwsync(); + + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + cl = qb_cl(d); + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + p[0] = cl[0] | s->eqcr.pi_vb; + eqcr_pi++; + eqcr_pi &= 0xF; + if (!(eqcr_pi & 7)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + /*Pointing to the next enqueue descriptor*/ + cl += (sizeof(struct qbman_eq_desc) / sizeof(uint32_t)); + } + + /* Flush all the cacheline without load/store in between */ + eqcr_pi = s->eqcr.pi; + addr_cena = (uint64_t)s->sys.addr_cena; + for (i = 0; i < num_enqueued; i++) { + dcbf((uint64_t *)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + eqcr_pi++; + eqcr_pi &= 0xF; + } + s->eqcr.pi = eqcr_pi; + + return num_enqueued; +} + /*************************/ /* Static (push) dequeue */ /*************************/ @@ -769,7 +861,7 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) */ uint32_t dqpi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI); uint32_t pi = qb_attr_code_decode(&code_dqpi_pi, &dqpi); - /* there are new entries iff pi != next_idx */ + /* there are new entries if pi != next_idx */ if (pi == s->dqrr.next_idx) return NULL; /* if next_idx is/was the last ring index, and 'pi' is @@ -1393,7 +1485,7 @@ int qbman_swp_CDAN_set_context_enable(struct qbman_swp *s, uint16_t channelid, 1, ctx); } -uint8_t qbman_get_dqrr_idx(struct qbman_result *dqrr) +uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr) { return QBMAN_IDX_FROM_DQRR(dqrr); } diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map index 2db0fcef..3cdf14ef 100644 --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map @@ -49,3 +49,31 @@ DPDK_17.05 { local: *; }; + +DPDK_17.08 { + global: + + dpaa2_io_portal; + dpaa2_get_qbman_swp; + dpci_set_rx_queue; + dpcon_open; + dpcon_get_attributes; + dpio_add_static_dequeue_channel; + dpio_remove_static_dequeue_channel; + mc_get_soc_version; + mc_get_version; + qbman_eq_desc_set_dca; + qbman_get_dqrr_from_idx; + qbman_get_dqrr_idx; + qbman_result_DQ_fqd_ctx; + qbman_result_SCN_state_in_mem; + qbman_swp_dqrr_consume; + qbman_swp_dqrr_next; + qbman_swp_enqueue_multiple_eqdesc; + qbman_swp_interrupt_clear_status; + qbman_swp_push_set; + rte_dpaa2_alloc_dpci_dev; + rte_fslmc_object_register; + rte_global_active_dqs_list; + +} DPDK_17.05; diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h index 040ab958..e60d6eba 100644 --- a/drivers/bus/fslmc/rte_fslmc.h +++ b/drivers/bus/fslmc/rte_fslmc.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,9 +56,6 @@ extern "C" { #include <rte_dev.h> #include <rte_bus.h> -/** Name of FSLMC Bus */ -#define FSLMC_BUS_NAME "FSLMC" - struct rte_dpaa2_driver; /* DPAA2 Device and Driver lists for FSLMC bus */ @@ -81,6 +78,7 @@ struct rte_dpaa2_device { uint16_t object_id; /**< DPAA2 Object ID */ struct rte_intr_handle intr_handle; /**< Interrupt handle */ struct rte_dpaa2_driver *driver; /**< Associated driver */ + char name[32]; /**< DPAA2 Object name*/ }; typedef int (*rte_dpaa2_probe_t)(struct rte_dpaa2_driver *dpaa2_drv, diff --git a/drivers/crypto/aesni_gcm/Makefile b/drivers/crypto/aesni_gcm/Makefile index 59a7c6a9..6fca5e1c 100644 --- a/drivers/crypto/aesni_gcm/Makefile +++ b/drivers/crypto/aesni_gcm/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 Intel Corporation. All rights reserved. +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,6 +31,9 @@ include $(RTE_SDK)/mk/rte.vars.mk ifneq ($(MAKECMDGOALS),clean) +ifeq ($(AESNI_MULTI_BUFFER_LIB_PATH),) +$(error "Please define AESNI_MULTI_BUFFER_LIB_PATH environment variable") +endif endif # library name @@ -47,7 +50,9 @@ LIBABIVER := 1 EXPORT_MAP := rte_pmd_aesni_gcm_version.map # external library dependencies -LDLIBS += -lisal_crypto +CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH) +CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH)/include +LDLIBS += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm_pmd.c diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h index e9de6546..d6a18efc 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,26 +37,109 @@ #define LINUX #endif -#include <isa-l_crypto/aes_gcm.h> +#include <gcm_defines.h> +#include <aux_funcs.h> -typedef void (*aesni_gcm_init_t)(struct gcm_data *my_ctx_data, - uint8_t *iv, +/** Supported vector modes */ +enum aesni_gcm_vector_mode { + RTE_AESNI_GCM_NOT_SUPPORTED = 0, + RTE_AESNI_GCM_SSE, + RTE_AESNI_GCM_AVX, + RTE_AESNI_GCM_AVX2, + RTE_AESNI_GCM_VECTOR_NUM +}; + +enum aesni_gcm_key { + AESNI_GCM_KEY_128, + AESNI_GCM_KEY_192, + AESNI_GCM_KEY_256, + AESNI_GCM_KEY_NUM +}; + + +typedef void (*aesni_gcm_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, uint8_t *out, + const uint8_t *in, uint64_t plaintext_len, const uint8_t *iv, + const uint8_t *aad, uint64_t aad_len, + uint8_t *auth_tag, uint64_t auth_tag_len); + +typedef void (*aesni_gcm_precomp_t)(const void *key, struct gcm_key_data *gcm_data); + +typedef void (*aesni_gcm_init_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, + const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); -typedef void (*aesni_gcm_update_t)(struct gcm_data *my_ctx_data, +typedef void (*aesni_gcm_update_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, uint8_t *out, const uint8_t *in, uint64_t plaintext_len); -typedef void (*aesni_gcm_finalize_t)(struct gcm_data *my_ctx_data, +typedef void (*aesni_gcm_finalize_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** GCM library function pointer table */ struct aesni_gcm_ops { + aesni_gcm_t enc; /**< GCM encode function pointer */ + aesni_gcm_t dec; /**< GCM decode function pointer */ + aesni_gcm_precomp_t precomp; /**< GCM pre-compute */ aesni_gcm_init_t init; - aesni_gcm_update_t update; + aesni_gcm_update_t update_enc; + aesni_gcm_update_t update_dec; aesni_gcm_finalize_t finalize; }; +#define AES_GCM_FN(keylen, arch) \ +aes_gcm_enc_##keylen##_##arch,\ +aes_gcm_dec_##keylen##_##arch,\ +aes_gcm_pre_##keylen##_##arch,\ +aes_gcm_init_##keylen##_##arch,\ +aes_gcm_enc_##keylen##_update_##arch,\ +aes_gcm_dec_##keylen##_update_##arch,\ +aes_gcm_enc_##keylen##_finalize_##arch, + +static const struct aesni_gcm_ops gcm_ops[RTE_AESNI_GCM_VECTOR_NUM][AESNI_GCM_KEY_NUM] = { + [RTE_AESNI_GCM_NOT_SUPPORTED] = { + [AESNI_GCM_KEY_128] = {NULL}, + [AESNI_GCM_KEY_192] = {NULL}, + [AESNI_GCM_KEY_256] = {NULL} + }, + [RTE_AESNI_GCM_SSE] = { + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, sse) + }, + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, sse) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, sse) + } + }, + [RTE_AESNI_GCM_AVX] = { + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, avx_gen2) + }, + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, avx_gen2) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, avx_gen2) + } + }, + [RTE_AESNI_GCM_AVX2] = { + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, avx_gen4) + }, + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, avx_gen4) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, avx_gen4) + } + } +}; #endif /* _AESNI_GCM_OPS_H_ */ diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index 101ef98b..d9c91d06 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -42,133 +43,155 @@ #include "aesni_gcm_pmd_private.h" -/** GCM encode functions pointer table */ -static const struct aesni_gcm_ops aesni_gcm_enc[] = { - [AESNI_GCM_KEY_128] = { - aesni_gcm128_init, - aesni_gcm128_enc_update, - aesni_gcm128_enc_finalize - }, - [AESNI_GCM_KEY_256] = { - aesni_gcm256_init, - aesni_gcm256_enc_update, - aesni_gcm256_enc_finalize - } -}; - -/** GCM decode functions pointer table */ -static const struct aesni_gcm_ops aesni_gcm_dec[] = { - [AESNI_GCM_KEY_128] = { - aesni_gcm128_init, - aesni_gcm128_dec_update, - aesni_gcm128_dec_finalize - }, - [AESNI_GCM_KEY_256] = { - aesni_gcm256_init, - aesni_gcm256_dec_update, - aesni_gcm256_dec_finalize - } -}; +static uint8_t cryptodev_driver_id; /** Parse crypto xform chain and set private session parameters */ int -aesni_gcm_set_session_parameters(struct aesni_gcm_session *sess, +aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, + struct aesni_gcm_session *sess, const struct rte_crypto_sym_xform *xform) { const struct rte_crypto_sym_xform *auth_xform; - const struct rte_crypto_sym_xform *cipher_xform; - - if (xform->next == NULL || xform->next->next != NULL) { - GCM_LOG_ERR("Two and only two chained xform required"); - return -EINVAL; - } + const struct rte_crypto_sym_xform *aead_xform; + uint16_t digest_length; + uint8_t key_length; + uint8_t *key; - if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && - xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { - auth_xform = xform->next; - cipher_xform = xform; - } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && - xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + /* AES-GMAC */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { auth_xform = xform; - cipher_xform = xform->next; + if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) { + GCM_LOG_ERR("Only AES GMAC is supported as an " + "authentication only algorithm"); + return -ENOTSUP; + } + /* Set IV parameters */ + sess->iv.offset = auth_xform->auth.iv.offset; + sess->iv.length = auth_xform->auth.iv.length; + + /* Select Crypto operation */ + if (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) + sess->op = AESNI_GMAC_OP_GENERATE; + else + sess->op = AESNI_GMAC_OP_VERIFY; + + key_length = auth_xform->auth.key.length; + key = auth_xform->auth.key.data; + digest_length = auth_xform->auth.digest_length; + + /* AES-GCM */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + aead_xform = xform; + + if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) { + GCM_LOG_ERR("The only combined operation " + "supported is AES GCM"); + return -ENOTSUP; + } + + /* Set IV parameters */ + sess->iv.offset = aead_xform->aead.iv.offset; + sess->iv.length = aead_xform->aead.iv.length; + + /* Select Crypto operation */ + if (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; + else + sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; + + key_length = aead_xform->aead.key.length; + key = aead_xform->aead.key.data; + + sess->aad_length = aead_xform->aead.aad_length; + digest_length = aead_xform->aead.digest_length; } else { - GCM_LOG_ERR("Cipher and auth xform required"); - return -EINVAL; + GCM_LOG_ERR("Wrong xform type, has to be AEAD or authentication"); + return -ENOTSUP; } - if (!(cipher_xform->cipher.algo == RTE_CRYPTO_CIPHER_AES_GCM && - (auth_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GCM || - auth_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC))) { - GCM_LOG_ERR("We only support AES GCM and AES GMAC"); - return -EINVAL; - } - /* Select Crypto operation */ - if (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT && - auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) - sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; - else if (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT && - auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) - sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; - else { - GCM_LOG_ERR("Cipher/Auth operations: Encrypt/Generate or" - " Decrypt/Verify are valid only"); + /* IV check */ + if (sess->iv.length != 16 && sess->iv.length != 12 && + sess->iv.length != 0) { + GCM_LOG_ERR("Wrong IV length"); return -EINVAL; } /* Check key length and calculate GCM pre-compute. */ - switch (cipher_xform->cipher.key.length) { + switch (key_length) { case 16: - aesni_gcm128_pre(cipher_xform->cipher.key.data, &sess->gdata); sess->key = AESNI_GCM_KEY_128; - + break; + case 24: + sess->key = AESNI_GCM_KEY_192; break; case 32: - aesni_gcm256_pre(cipher_xform->cipher.key.data, &sess->gdata); sess->key = AESNI_GCM_KEY_256; - break; default: - GCM_LOG_ERR("Unsupported cipher key length"); + GCM_LOG_ERR("Invalid key length"); + return -EINVAL; + } + + gcm_ops[sess->key].precomp(key, &sess->gdata_key); + + /* Digest check */ + if (digest_length != 16 && + digest_length != 12 && + digest_length != 8) { + GCM_LOG_ERR("digest"); return -EINVAL; } + sess->digest_length = digest_length; return 0; } /** Get gcm session */ static struct aesni_gcm_session * -aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op) +aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op) { struct aesni_gcm_session *sess = NULL; - - if (op->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->session->dev_type - != RTE_CRYPTODEV_AESNI_GCM_PMD)) - return sess; - - sess = (struct aesni_gcm_session *)op->session->_private; + struct rte_crypto_sym_op *sym_op = op->sym; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(sym_op->session != NULL)) + sess = (struct aesni_gcm_session *) + get_session_private_data( + sym_op->session, + cryptodev_driver_id); } else { void *_sess; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, &_sess)) - return sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; - sess = (struct aesni_gcm_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + sess = (struct aesni_gcm_session *)_sess_private_data; - if (unlikely(aesni_gcm_set_session_parameters(sess, - op->xform) != 0)) { + if (unlikely(aesni_gcm_set_session_parameters(qp->ops, + sess, sym_op->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } + + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } /** - * Process a crypto operation and complete a JOB_AES_HMAC job structure for - * submission to the multi buffer library for processing. + * Process a crypto operation, calling + * the GCM API from the multi buffer library. * * @param qp queue pair * @param op symmetric crypto operation @@ -178,14 +201,27 @@ aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op) * */ static int -process_gcm_crypto_op(struct rte_crypto_sym_op *op, +process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, struct aesni_gcm_session *session) { uint8_t *src, *dst; - struct rte_mbuf *m_src = op->m_src; - uint32_t offset = op->cipher.data.offset; + uint8_t *iv_ptr; + struct rte_crypto_sym_op *sym_op = op->sym; + struct rte_mbuf *m_src = sym_op->m_src; + uint32_t offset, data_offset, data_length; uint32_t part_len, total_len, data_len; + if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION || + session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + offset = sym_op->aead.data.offset; + data_offset = offset; + data_length = sym_op->aead.data.length; + } else { + offset = sym_op->auth.data.offset; + data_offset = offset; + data_length = sym_op->auth.data.length; + } + RTE_ASSERT(m_src != NULL); while (offset >= m_src->data_len) { @@ -196,60 +232,50 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, } data_len = m_src->data_len - offset; - part_len = (data_len < op->cipher.data.length) ? data_len : - op->cipher.data.length; + part_len = (data_len < data_length) ? data_len : + data_length; /* Destination buffer is required when segmented source buffer */ - RTE_ASSERT((part_len == op->cipher.data.length) || - ((part_len != op->cipher.data.length) && - (op->m_dst != NULL))); + RTE_ASSERT((part_len == data_length) || + ((part_len != data_length) && + (sym_op->m_dst != NULL))); /* Segmented destination buffer is not supported */ - RTE_ASSERT((op->m_dst == NULL) || - ((op->m_dst != NULL) && - rte_pktmbuf_is_contiguous(op->m_dst))); + RTE_ASSERT((sym_op->m_dst == NULL) || + ((sym_op->m_dst != NULL) && + rte_pktmbuf_is_contiguous(sym_op->m_dst))); - dst = op->m_dst ? - rte_pktmbuf_mtod_offset(op->m_dst, uint8_t *, - op->cipher.data.offset) : - rte_pktmbuf_mtod_offset(op->m_src, uint8_t *, - op->cipher.data.offset); + dst = sym_op->m_dst ? + rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *, + data_offset) : + rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *, + data_offset); src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset); - /* sanity checks */ - if (op->cipher.iv.length != 16 && op->cipher.iv.length != 12 && - op->cipher.iv.length != 0) { - GCM_LOG_ERR("iv"); - return -1; - } - + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); /* * GCM working in 12B IV mode => 16B pre-counter block we need * to set BE LSB to 1, driver expects that 16B is allocated */ - if (op->cipher.iv.length == 12) { - uint32_t *iv_padd = (uint32_t *)&op->cipher.iv.data[12]; + if (session->iv.length == 12) { + uint32_t *iv_padd = (uint32_t *)&(iv_ptr[12]); *iv_padd = rte_bswap32(1); } - if (op->auth.digest.length != 16 && - op->auth.digest.length != 12 && - op->auth.digest.length != 8) { - GCM_LOG_ERR("digest"); - return -1; - } - if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION) { - aesni_gcm_enc[session->key].init(&session->gdata, - op->cipher.iv.data, - op->auth.aad.data, - (uint64_t)op->auth.aad.length); + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + sym_op->aead.aad.data, + (uint64_t)session->aad_length); - aesni_gcm_enc[session->key].update(&session->gdata, dst, src, + qp->ops[session->key].update_enc(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); - total_len = op->cipher.data.length - part_len; + total_len = data_length - part_len; while (total_len) { dst += part_len; @@ -261,33 +287,36 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, part_len = (m_src->data_len < total_len) ? m_src->data_len : total_len; - aesni_gcm_enc[session->key].update(&session->gdata, - dst, src, + qp->ops[session->key].update_enc(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); total_len -= part_len; } - aesni_gcm_enc[session->key].finalize(&session->gdata, - op->auth.digest.data, - (uint64_t)op->auth.digest.length); - } else { /* session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION */ - uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(op->m_dst ? - op->m_dst : op->m_src, - op->auth.digest.length); + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + sym_op->aead.digest.data, + (uint64_t)session->digest_length); + } else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(sym_op->m_dst ? + sym_op->m_dst : sym_op->m_src, + session->digest_length); if (!auth_tag) { GCM_LOG_ERR("auth_tag"); return -1; } - aesni_gcm_dec[session->key].init(&session->gdata, - op->cipher.iv.data, - op->auth.aad.data, - (uint64_t)op->auth.aad.length); + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + sym_op->aead.aad.data, + (uint64_t)session->aad_length); - aesni_gcm_dec[session->key].update(&session->gdata, dst, src, + qp->ops[session->key].update_dec(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); - total_len = op->cipher.data.length - part_len; + total_len = data_length - part_len; while (total_len) { dst += part_len; @@ -299,15 +328,47 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, part_len = (m_src->data_len < total_len) ? m_src->data_len : total_len; - aesni_gcm_dec[session->key].update(&session->gdata, + qp->ops[session->key].update_dec(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); total_len -= part_len; } - aesni_gcm_dec[session->key].finalize(&session->gdata, + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, auth_tag, - (uint64_t)op->auth.digest.length); + (uint64_t)session->digest_length); + } else if (session->op == AESNI_GMAC_OP_GENERATE) { + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + src, + (uint64_t)data_length); + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + sym_op->auth.digest.data, + (uint64_t)session->digest_length); + } else { /* AESNI_GMAC_OP_VERIFY */ + uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(sym_op->m_dst ? + sym_op->m_dst : sym_op->m_src, + session->digest_length); + + if (!auth_tag) { + GCM_LOG_ERR("auth_tag"); + return -1; + } + + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + src, + (uint64_t)data_length); + + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + auth_tag, + (uint64_t)session->digest_length); } return 0; @@ -324,34 +385,38 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, * - Returns NULL on invalid job */ static void -post_process_gcm_crypto_op(struct rte_crypto_op *op) +post_process_gcm_crypto_op(struct rte_crypto_op *op, + struct aesni_gcm_session *session) { struct rte_mbuf *m = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; - struct aesni_gcm_session *session = - (struct aesni_gcm_session *)op->sym->session->_private; - op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Verify digest if required */ - if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION || + session->op == AESNI_GMAC_OP_VERIFY) { + uint8_t *digest; uint8_t *tag = rte_pktmbuf_mtod_offset(m, uint8_t *, - m->data_len - op->sym->auth.digest.length); + m->data_len - session->digest_length); + + if (session->op == AESNI_GMAC_OP_VERIFY) + digest = op->sym->auth.digest.data; + else + digest = op->sym->aead.digest.data; #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG rte_hexdump(stdout, "auth tag (orig):", - op->sym->auth.digest.data, op->sym->auth.digest.length); + digest, session->digest_length); rte_hexdump(stdout, "auth tag (calc):", - tag, op->sym->auth.digest.length); + tag, session->digest_length); #endif - if (memcmp(tag, op->sym->auth.digest.data, - op->sym->auth.digest.length) != 0) + if (memcmp(tag, digest, session->digest_length) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* trim area used for digest from mbuf */ - rte_pktmbuf_trim(m, op->sym->auth.digest.length); + rte_pktmbuf_trim(m, session->digest_length); } } @@ -359,6 +424,7 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) * Process a completed GCM request * * @param qp Queue Pair to process + * @param op Crypto operation * @param job JOB_AES_HMAC job * * @return @@ -366,12 +432,17 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) */ static void handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp, - struct rte_crypto_op *op) + struct rte_crypto_op *op, + struct aesni_gcm_session *sess) { - post_process_gcm_crypto_op(op); + post_process_gcm_crypto_op(op, sess); /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_gcm_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -392,21 +463,21 @@ aesni_gcm_pmd_dequeue_burst(void *queue_pair, for (i = 0; i < nb_dequeued; i++) { - sess = aesni_gcm_get_session(qp, ops[i]->sym); + sess = aesni_gcm_get_session(qp, ops[i]); if (unlikely(sess == NULL)) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; qp->qp_stats.dequeue_err_count++; break; } - retval = process_gcm_crypto_op(ops[i]->sym, sess); + retval = process_gcm_crypto_op(qp, ops[i], sess); if (retval < 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; qp->qp_stats.dequeue_err_count++; break; } - handle_completed_gcm_crypto_op(qp, ops[i]); + handle_completed_gcm_crypto_op(qp, ops[i], sess); } qp->qp_stats.dequeued_count += i; @@ -438,6 +509,7 @@ aesni_gcm_create(const char *name, { struct rte_cryptodev *dev; struct aesni_gcm_private *internals; + enum aesni_gcm_vector_mode vector_mode; if (init_params->name[0] == '\0') snprintf(init_params->name, sizeof(init_params->name), @@ -449,14 +521,23 @@ aesni_gcm_create(const char *name, return -EFAULT; } - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct aesni_gcm_private), init_params->socket_id); + /* Check CPU for supported vector instruction set */ + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) + vector_mode = RTE_AESNI_GCM_AVX2; + else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) + vector_mode = RTE_AESNI_GCM_AVX; + else + vector_mode = RTE_AESNI_GCM_SSE; + + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct aesni_gcm_private), init_params->socket_id, + vdev); if (dev == NULL) { GCM_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_AESNI_GCM_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_gcm_pmd_ops; /* register rx/tx burst functions for data path */ @@ -468,8 +549,24 @@ aesni_gcm_create(const char *name, RTE_CRYPTODEV_FF_CPU_AESNI | RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; + switch (vector_mode) { + case RTE_AESNI_GCM_SSE: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + break; + case RTE_AESNI_GCM_AVX: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + break; + case RTE_AESNI_GCM_AVX2: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + break; + default: + break; + } + internals = dev->data->dev_private; + internals->vector_mode = vector_mode; + internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; internals->max_nb_sessions = init_params->max_nb_sessions; @@ -498,7 +595,7 @@ aesni_gcm_probe(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -539,3 +636,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index 1fc047be..48400ac2 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -49,32 +49,32 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = { .key_size = { .min = 16, .max = 32, - .increment = 16 + .increment = 8 }, .digest_size = { .min = 8, .max = 16, .increment = 4 }, - .aad_size = { - .min = 0, - .max = 65535, - .increment = 1 + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 } }, } }, } }, - { /* AES GCM (AUTH) */ + { /* AES GCM */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_AES_GCM, + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, .block_size = 16, .key_size = { .min = 16, .max = 32, - .increment = 16 + .increment = 8 }, .digest_size = { .min = 8, @@ -85,21 +85,6 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = { .min = 0, .max = 65535, .increment = 1 - } - }, } - }, } - }, - { /* AES GCM (CIPHER) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_GCM, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 16 }, .iv_size = { .min = 12, @@ -181,9 +166,9 @@ aesni_gcm_pmd_info_get(struct rte_cryptodev *dev, struct aesni_gcm_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; - dev_info->feature_flags = dev->feature_flags; - dev_info->capabilities = aesni_gcm_pmd_capabilities; + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = aesni_gcm_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; @@ -244,9 +229,10 @@ aesni_gcm_pmd_qp_create_processed_pkts_ring(struct aesni_gcm_qp *qp, static int aesni_gcm_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct aesni_gcm_qp *qp = NULL; + struct aesni_gcm_private *internals = dev->data->dev_private; /* Free memory prior to re-allocation if needed. */ if (dev->data->queue_pairs[qp_id] != NULL) @@ -264,12 +250,14 @@ aesni_gcm_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (aesni_gcm_pmd_qp_set_unique_name(dev, qp)) goto qp_setup_cleanup; + qp->ops = (const struct aesni_gcm_ops *)gcm_ops[internals->vector_mode]; + qp->processed_pkts = aesni_gcm_pmd_qp_create_processed_pkts_ring(qp, qp_conf->nb_descriptors, socket_id); if (qp->processed_pkts == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -313,29 +301,57 @@ aesni_gcm_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni gcm session from a crypto xform chain */ -static void * +static int aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + struct aesni_gcm_private *internals = dev->data->dev_private; + if (unlikely(sess == NULL)) { GCM_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (aesni_gcm_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + ret = aesni_gcm_set_session_parameters(gcm_ops[internals->vector_mode], + sess_private_data, xform); + if (ret != 0) { GCM_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct aesni_gcm_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_gcm_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_gcm_pmd_ops = { diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h index 0496b447..7e155729 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include "aesni_gcm_ops.h" +#define CRYPTODEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm +/**< AES-NI GCM PMD device name */ + #define GCM_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), \ @@ -58,6 +61,8 @@ /** private data structure for each virtual AESNI GCM device */ struct aesni_gcm_private { + enum aesni_gcm_vector_mode vector_mode; + /**< Vector mode */ unsigned max_nb_queue_pairs; /**< Max number of queue pairs supported by device */ unsigned max_nb_sessions; @@ -65,36 +70,46 @@ struct aesni_gcm_private { }; struct aesni_gcm_qp { - uint16_t id; - /**< Queue Pair Identifier */ - char name[RTE_CRYPTODEV_NAME_LEN]; - /**< Unique Queue Pair Name */ + const struct aesni_gcm_ops *ops; + /**< Architecture dependent function pointer table of the gcm APIs */ struct rte_ring *processed_pkts; /**< Ring for placing process packets */ + struct gcm_context_data gdata_ctx; /* (16 * 5) + 8 = 88 B */ + /**< GCM parameters */ + struct rte_cryptodev_stats qp_stats; /* 8 * 4 = 32 B */ + /**< Queue pair statistics */ struct rte_mempool *sess_mp; /**< Session Mempool */ - struct rte_cryptodev_stats qp_stats; - /**< Queue pair statistics */ + uint16_t id; + /**< Queue Pair Identifier */ + char name[RTE_CRYPTODEV_NAME_LEN]; + /**< Unique Queue Pair Name */ } __rte_cache_aligned; enum aesni_gcm_operation { AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION, - AESNI_GCM_OP_AUTHENTICATED_DECRYPTION -}; - -enum aesni_gcm_key { - AESNI_GCM_KEY_128, - AESNI_GCM_KEY_256 + AESNI_GCM_OP_AUTHENTICATED_DECRYPTION, + AESNI_GMAC_OP_GENERATE, + AESNI_GMAC_OP_VERIFY }; /** AESNI GCM private session structure */ struct aesni_gcm_session { + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ + uint16_t aad_length; + /**< AAD length */ + uint16_t digest_length; + /**< Digest length */ enum aesni_gcm_operation op; /**< GCM operation type */ enum aesni_gcm_key key; /**< GCM key type */ - struct gcm_data gdata __rte_cache_aligned; + struct gcm_key_data gdata_key; /**< GCM parameters */ }; @@ -109,7 +124,8 @@ struct aesni_gcm_session { * - On failure returns error code < 0 */ extern int -aesni_gcm_set_session_parameters(struct aesni_gcm_session *sess, +aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *ops, + struct aesni_gcm_session *sess, const struct rte_crypto_sym_xform *xform); diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 45b25c9d..16e14512 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,12 +34,15 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> #include "rte_aesni_mb_pmd_private.h" +static uint8_t cryptodev_driver_id; + typedef void (*hash_one_block_t)(const void *data, void *digest); typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys); @@ -166,7 +169,7 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, break; default: MB_LOG_ERR("Unsupported authentication algorithm selection"); - return -1; + return -ENOTSUP; } /* Calculate Authentication precomputes */ @@ -194,7 +197,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) { MB_LOG_ERR("Crypto xform struct not of type cipher"); - return -1; + return -EINVAL; } /* Select cipher direction */ @@ -206,8 +209,8 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, sess->cipher.direction = DECRYPT; break; default: - MB_LOG_ERR("Unsupported cipher operation parameter"); - return -1; + MB_LOG_ERR("Invalid cipher operation parameter"); + return -EINVAL; } /* Select cipher mode */ @@ -223,7 +226,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, break; default: MB_LOG_ERR("Unsupported cipher mode parameter"); - return -1; + return -ENOTSUP; } /* Check key length and choose key expansion function */ @@ -241,10 +244,14 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, aes_keyexp_fn = mb_ops->aux.keyexp.aes256; break; default: - MB_LOG_ERR("Unsupported cipher key length"); - return -1; + MB_LOG_ERR("Invalid cipher key length"); + return -EINVAL; } + /* Set IV parameters */ + sess->iv.offset = xform->cipher.iv.offset; + sess->iv.length = xform->cipher.iv.length; + /* Expanded cipher keys */ (*aes_keyexp_fn)(xform->cipher.key.data, sess->cipher.expanded_aes_keys.encode, @@ -261,6 +268,7 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, { const struct rte_crypto_sym_xform *auth_xform = NULL; const struct rte_crypto_sym_xform *cipher_xform = NULL; + int ret; /* Select Crypto operation - hash then cipher / cipher then hash */ switch (aesni_mb_get_chain_order(xform)) { @@ -296,19 +304,25 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, case AESNI_MB_OP_NOT_SUPPORTED: default: MB_LOG_ERR("Unsupported operation chain order parameter"); - return -1; + return -ENOTSUP; } - if (aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform)) { + /* Default IV length = 0 */ + sess->iv.length = 0; + + ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform); + if (ret != 0) { MB_LOG_ERR("Invalid/unsupported authentication parameters"); - return -1; + return ret; } - if (aesni_mb_set_session_cipher_parameters(mb_ops, sess, - cipher_xform)) { + ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess, + cipher_xform); + if (ret != 0) { MB_LOG_ERR("Invalid/unsupported cipher parameters"); - return -1; + return ret; } + return 0; } @@ -344,30 +358,38 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) { struct aesni_mb_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_AESNI_MB_PMD)) { - return NULL; - } - - sess = (struct aesni_mb_session *)op->sym->session->_private; - } else { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct aesni_mb_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { void *_sess = NULL; + void *_sess_private_data = NULL; if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct aesni_mb_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct aesni_mb_session *)_sess_private_data; if (unlikely(aesni_mb_set_session_parameters(qp->op_fns, sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -396,7 +418,6 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; return -1; } - op->status = RTE_CRYPTO_OP_STATUS_ENQUEUED; /* Set crypto operation */ job->chain_order = session->chain_order; @@ -470,8 +491,9 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, get_truncated_digest_byte_length(job->hash_alg); /* Set IV parameters */ - job->iv = op->sym->cipher.iv.data; - job->iv_len_in_bytes = op->sym->cipher.iv.length; + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + job->iv_len_in_bytes = session->iv.length; /* Data Parameter */ job->src = rte_pktmbuf_mtod(m_src, uint8_t *); @@ -494,8 +516,6 @@ static inline void verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { struct rte_mbuf *m_dst = (struct rte_mbuf *)job->user_data2; - RTE_ASSERT(m_dst == NULL); - /* Verify digest if required */ if (memcmp(job->auth_tag_output, op->sym->auth.digest.data, job->auth_tag_output_len_in_bytes) != 0) @@ -508,31 +528,28 @@ verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { /** * Process a completed job and return rte_mbuf which job processed * + * @param qp Queue Pair to process * @param job JOB_AES_HMAC job to process * * @return - * - Returns processed mbuf which is trimmed of output digest used in - * verification of supplied digest in the case of a HASH_CIPHER operation + * - Returns processed crypto operation which mbuf is trimmed of output digest + * used in verification of supplied digest. * - Returns NULL on invalid job */ static inline struct rte_crypto_op * post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) { struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data; + struct aesni_mb_session *sess = get_session_private_data( + op->sym->session, + cryptodev_driver_id); - struct aesni_mb_session *sess; - - RTE_ASSERT(op == NULL); - - if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ENQUEUED)) { + if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) { switch (job->status) { case STS_COMPLETED: op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; if (job->hash_alg != NULL_HASH) { - sess = (struct aesni_mb_session *) - op->sym->session->_private; - if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) verify_digest(job, op); @@ -544,7 +561,11 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) } /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_mb_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -569,7 +590,7 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job, struct rte_crypto_op *op = NULL; unsigned processed_jobs = 0; - while (job != NULL && processed_jobs < nb_ops) { + while (job != NULL) { op = post_process_mb_job(qp, job); if (op) { @@ -579,6 +600,8 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job, qp->stats.dequeue_err_count++; break; } + if (processed_jobs == nb_ops) + break; job = (*qp->op_fns->job.get_completed_job)(&qp->mb_mgr); } @@ -624,6 +647,9 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, int retval, processed_jobs = 0; + if (unlikely(nb_ops == 0)) + return 0; + do { /* Get next operation to process from ingress queue */ retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op); @@ -691,21 +717,18 @@ cryptodev_aesni_mb_create(const char *name, vector_mode = RTE_AESNI_MB_AVX2; else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) vector_mode = RTE_AESNI_MB_AVX; - else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) + else vector_mode = RTE_AESNI_MB_SSE; - else { - MB_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct aesni_mb_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct aesni_mb_private), init_params->socket_id, + vdev); if (dev == NULL) { MB_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_AESNI_MB_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_mb_pmd_ops; /* register rx/tx burst functions for data path */ @@ -765,7 +788,7 @@ cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -806,3 +829,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_aesni_mb_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index d1bc28e0..692b354f 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -48,16 +48,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_MD5_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 12, .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -69,16 +69,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 12, .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -90,16 +90,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 14, .max = 14, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -111,16 +111,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 16, .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -132,16 +132,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 24, .max = 24, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -153,16 +153,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -183,7 +183,7 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -220,9 +220,9 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .increment = 8 }, .iv_size = { - .min = 16, + .min = 12, .max = 16, - .increment = 0 + .increment = 4 } }, } }, } @@ -321,7 +321,7 @@ aesni_mb_pmd_info_get(struct rte_cryptodev *dev, struct aesni_mb_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = aesni_mb_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; @@ -397,7 +397,7 @@ aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, static int aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct aesni_mb_qp *qp = NULL; struct aesni_mb_private *internals = dev->data->dev_private; @@ -426,7 +426,7 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->ingress_queue == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->stats, 0, sizeof(qp->stats)); @@ -472,36 +472,58 @@ aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni multi-buffer session from a crypto xform chain */ -static void * +static int aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; struct aesni_mb_private *internals = dev->data->dev_private; + int ret; if (unlikely(sess == NULL)) { MB_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], - sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], + sess_private_data, xform); + if (ret != 0) { MB_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_mb_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_mb_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct aesni_mb_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_mb_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_mb_pmd_ops = { diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index 0d82699c..6676948e 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -35,6 +35,9 @@ #include "aesni_mb_ops.h" +#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb +/**< AES-NI Multi buffer PMD device name */ + #define MB_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD), \ @@ -167,6 +170,11 @@ struct aesni_mb_qp { /** AES-NI multi-buffer private session structure */ struct aesni_mb_session { JOB_CHAIN_ORDER chain_order; + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ /** Cipher Parameters */ struct { diff --git a/drivers/crypto/armv8/Makefile b/drivers/crypto/armv8/Makefile index 1474951c..86611fa2 100644 --- a/drivers/crypto/armv8/Makefile +++ b/drivers/crypto/armv8/Makefile @@ -1,7 +1,7 @@ # # BSD LICENSE # -# Copyright (C) Cavium networks Ltd. 2017. +# Copyright (C) Cavium, Inc. 2017. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # diff --git a/drivers/crypto/armv8/rte_armv8_pmd.c b/drivers/crypto/armv8/rte_armv8_pmd.c index 3d603a5a..a5c39c9b 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd.c +++ b/drivers/crypto/armv8/rte_armv8_pmd.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -36,6 +36,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -44,6 +45,8 @@ #include "rte_armv8_pmd_private.h" +static uint8_t cryptodev_driver_id; + static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev); /** @@ -288,27 +291,14 @@ auth_set_prerequisites(struct armv8_crypto_session *sess, * Generate authentication key, i_key_pad and o_key_pad. */ /* Zero memory under key */ - memset(sess->auth.hmac.key, 0, SHA1_AUTH_KEY_LENGTH); - - if (xform->auth.key.length > SHA1_AUTH_KEY_LENGTH) { - /* - * In case the key is longer than 160 bits - * the algorithm will use SHA1(key) instead. - */ - error = sha1_block(NULL, xform->auth.key.data, - sess->auth.hmac.key, xform->auth.key.length); - if (error != 0) - return -1; - } else { - /* - * Now copy the given authentication key to the session - * key assuming that the session key is zeroed there is - * no need for additional zero padding if the key is - * shorter than SHA1_AUTH_KEY_LENGTH. - */ - rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, - xform->auth.key.length); - } + memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE); + + /* + * Now copy the given authentication key to the session + * key. + */ + rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, + xform->auth.key.length); /* Prepare HMAC padding: key|pattern */ auth_hmac_pad_prepare(sess, xform); @@ -334,27 +324,14 @@ auth_set_prerequisites(struct armv8_crypto_session *sess, * Generate authentication key, i_key_pad and o_key_pad. */ /* Zero memory under key */ - memset(sess->auth.hmac.key, 0, SHA256_AUTH_KEY_LENGTH); - - if (xform->auth.key.length > SHA256_AUTH_KEY_LENGTH) { - /* - * In case the key is longer than 256 bits - * the algorithm will use SHA256(key) instead. - */ - error = sha256_block(NULL, xform->auth.key.data, - sess->auth.hmac.key, xform->auth.key.length); - if (error != 0) - return -1; - } else { - /* - * Now copy the given authentication key to the session - * key assuming that the session key is zeroed there is - * no need for additional zero padding if the key is - * shorter than SHA256_AUTH_KEY_LENGTH. - */ - rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, - xform->auth.key.length); - } + memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE); + + /* + * Now copy the given authentication key to the session + * key. + */ + rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, + xform->auth.key.length); /* Prepare HMAC padding: key|pattern */ auth_hmac_pad_prepare(sess, xform); @@ -414,7 +391,7 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, order = sess->chain_order; break; default: - return -EINVAL; + return -ENOTSUP; } /* Select cipher direction */ sess->cipher.direction = cipher_xform->cipher.op; @@ -431,10 +408,10 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, case RTE_CRYPTO_CIPHER_AES_CBC: sess->cipher.algo = calg; /* IV len is always 16 bytes (block size) for AES CBC */ - sess->cipher.iv_len = 16; + sess->cipher.iv.length = 16; break; default: - return -EINVAL; + return -ENOTSUP; } /* Select auth generate/verify */ sess->auth.operation = auth_xform->auth.op; @@ -448,9 +425,12 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC; break; default: - return -EINVAL; + return -ENOTSUP; } + /* Set the digest length */ + sess->auth.digest_length = auth_xform->auth.digest_length; + /* Verify supported key lengths and extract proper algorithm */ switch (cipher_xform->cipher.key.length << 3) { case 128: @@ -465,7 +445,7 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, default: /* Fall through */ sess->crypto_func = NULL; sess->cipher.key_sched = NULL; - return -EINVAL; + return -ENOTSUP; } if (unlikely(sess->crypto_func == NULL)) { @@ -519,20 +499,23 @@ armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess, break; default: is_chained_op = false; - return -EINVAL; + return -ENOTSUP; } + /* Set IV offset */ + sess->cipher.iv.offset = cipher_xform->cipher.iv.offset; + if (is_chained_op) { ret = armv8_crypto_set_session_chained_parameters(sess, cipher_xform, auth_xform); if (unlikely(ret != 0)) { ARMV8_CRYPTO_LOG_ERR( "Invalid/unsupported chained (cipher/auth) parameters"); - return -EINVAL; + return ret; } } else { ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation"); - return -EINVAL; + return -ENOTSUP; } return 0; @@ -544,30 +527,36 @@ get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) { struct armv8_crypto_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { /* get existing session */ - if (likely(op->sym->session != NULL && - op->sym->session->dev_type == - RTE_CRYPTODEV_ARMV8_PMD)) { + if (likely(op->sym->session != NULL)) { sess = (struct armv8_crypto_session *) - op->sym->session->_private; + get_session_private_data( + op->sym->session, + cryptodev_driver_id); } } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct armv8_crypto_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(armv8_crypto_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct armv8_crypto_session *)_sess_private_data; + + if (unlikely(armv8_crypto_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (unlikely(sess == NULL)) @@ -645,15 +634,11 @@ process_armv8_chained_op } } else { adst = (uint8_t *)rte_pktmbuf_append(m_asrc, - op->sym->auth.digest.length); - } - - if (unlikely(op->sym->cipher.iv.length != sess->cipher.iv_len)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - return; + sess->auth.digest_length); } - arg.cipher.iv = op->sym->cipher.iv.data; + arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->cipher.iv.offset); arg.cipher.key = sess->cipher.key.data; /* Acquire combined mode function */ crypto_func = sess->crypto_func; @@ -667,12 +652,12 @@ process_armv8_chained_op op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { if (memcmp(adst, op->sym->auth.digest.data, - op->sym->auth.digest.length) != 0) { + sess->auth.digest_length) != 0) { op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(m_asrc, - op->sym->auth.digest.length); + sess->auth.digest_length); } } @@ -699,8 +684,11 @@ process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op, } /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { memset(sess, 0, sizeof(struct armv8_crypto_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -806,15 +794,16 @@ cryptodev_armv8_crypto_create(const char *name, snprintf(init_params->name, sizeof(init_params->name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, + dev = rte_cryptodev_vdev_pmd_init(init_params->name, sizeof(struct armv8_crypto_private), - init_params->socket_id); + init_params->socket_id, + vdev); if (dev == NULL) { ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_ARMV8_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_armv8_crypto_pmd_ops; /* register rx/tx burst functions for data path */ @@ -860,7 +849,7 @@ cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -904,3 +893,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, cryptodev_driver_id); diff --git a/drivers/crypto/armv8/rte_armv8_pmd_ops.c b/drivers/crypto/armv8/rte_armv8_pmd_ops.c index 4d9ccbfb..00297beb 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_ops.c +++ b/drivers/crypto/armv8/rte_armv8_pmd_ops.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -50,16 +50,16 @@ static const struct rte_cryptodev_capabilities .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 16, - .max = 128, - .increment = 0 + .min = 1, + .max = 64, + .increment = 1 }, .digest_size = { .min = 20, .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -71,16 +71,16 @@ static const struct rte_cryptodev_capabilities .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 16, - .max = 128, - .increment = 0 + .min = 1, + .max = 64, + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -178,7 +178,7 @@ armv8_crypto_pmd_info_get(struct rte_cryptodev *dev, struct armv8_crypto_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = armv8_crypto_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; @@ -247,7 +247,7 @@ armv8_crypto_pmd_qp_create_processed_ops_ring(struct armv8_crypto_qp *qp, static int armv8_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct armv8_crypto_qp *qp = NULL; @@ -272,7 +272,7 @@ armv8_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->stats, 0, sizeof(qp->stats)); @@ -316,33 +316,56 @@ armv8_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * -armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) +static int +armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { ARMV8_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (armv8_crypto_set_session_parameters( - sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = armv8_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { ARMV8_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); /* Zero out the whole structure */ - if (sess) - memset(sess, 0, sizeof(struct armv8_crypto_session)); + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct armv8_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops armv8_crypto_pmd_ops = { diff --git a/drivers/crypto/armv8/rte_armv8_pmd_private.h b/drivers/crypto/armv8/rte_armv8_pmd_private.h index b75107f2..d02992a6 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_private.h +++ b/drivers/crypto/armv8/rte_armv8_pmd_private.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -33,6 +33,9 @@ #ifndef _RTE_ARMV8_PMD_PRIVATE_H_ #define _RTE_ARMV8_PMD_PRIVATE_H_ +#define CRYPTODEV_NAME_ARMV8_PMD crypto_armv8 +/**< ARMv8 Crypto PMD device name */ + #define ARMV8_CRYPTO_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_ARMV8_CRYPTO_PMD), \ @@ -159,8 +162,11 @@ struct armv8_crypto_session { /**< cipher operation direction */ enum rte_crypto_cipher_algorithm algo; /**< cipher algorithm */ - int iv_len; - /**< IV length */ + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ struct { uint8_t data[256]; @@ -192,10 +198,12 @@ struct armv8_crypto_session { uint8_t o_key_pad[SHA_BLOCK_MAX] __rte_cache_aligned; /**< outer pad (max supported block length) */ - uint8_t key[SHA_AUTH_KEY_MAX]; - /**< HMAC key (max supported length)*/ + uint8_t key[SHA_BLOCK_MAX]; + /**< HMAC key (max supported block length)*/ } hmac; }; + uint16_t digest_length; + /* Digest length */ } auth; } __rte_cache_aligned; diff --git a/drivers/crypto/dpaa2_sec/Makefile b/drivers/crypto/dpaa2_sec/Makefile index 11c7c785..ae15c99f 100644 --- a/drivers/crypto/dpaa2_sec/Makefile +++ b/drivers/crypto/dpaa2_sec/Makefile @@ -1,7 +1,7 @@ # BSD LICENSE # # Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -# Copyright (c) 2016 NXP. All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -46,6 +46,12 @@ CFLAGS += $(WERROR_FLAGS) endif CFLAGS += -D _GNU_SOURCE +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +ifeq ($(shell test $(GCC_VERSION) -gt 70 && echo 1), 1) +CFLAGS += -Wno-implicit-fallthrough +endif +endif + CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/ CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/mc CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/ diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 4e01fe81..e0f6cfca 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -69,10 +69,158 @@ #define FSL_MC_DPSECI_DEVID 3 #define NO_PREFETCH 0 -#define TDES_CBC_IV_LEN 8 -#define AES_CBC_IV_LEN 16 +/* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */ +#define FLE_POOL_NUM_BUFS 32000 +#define FLE_POOL_BUF_SIZE 256 +#define FLE_POOL_CACHE_SIZE 512 + enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8; +static uint8_t cryptodev_driver_id; + +static inline int +build_authenc_gcm_fd(dpaa2_sec_session *sess, + struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct ctxt_priv *priv = sess->ctxt; + struct qbman_fle *fle, *sge; + struct sec_flow_context *flc; + uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; + int icv_len = sess->digest_length, retval; + uint8_t *old_icv; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + + PMD_INIT_FUNC_TRACE(); + + /* TODO we are using the first FLE entry to store Mbuf and session ctxt. + * Currently we donot know which FLE has the mbuf stored. + * So while retreiving we can go back 1 FLE from the FD -ADDR + * to get the MBUF Addr from the previous FLE. + * We can have a better approach to use the inline Mbuf + */ + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); + return -1; + } + memset(fle, 0, FLE_POOL_BUF_SIZE); + DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); + fle = fle + 1; + sge = fle + 2; + if (likely(bpid < MAX_BPID)) { + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FLE_BPID(fle, bpid); + DPAA2_SET_FLE_BPID(fle + 1, bpid); + DPAA2_SET_FLE_BPID(sge, bpid); + DPAA2_SET_FLE_BPID(sge + 1, bpid); + DPAA2_SET_FLE_BPID(sge + 2, bpid); + DPAA2_SET_FLE_BPID(sge + 3, bpid); + } else { + DPAA2_SET_FD_IVP(fd); + DPAA2_SET_FLE_IVP(fle); + DPAA2_SET_FLE_IVP((fle + 1)); + DPAA2_SET_FLE_IVP(sge); + DPAA2_SET_FLE_IVP((sge + 1)); + DPAA2_SET_FLE_IVP((sge + 2)); + DPAA2_SET_FLE_IVP((sge + 3)); + } + + /* Save the shared descriptor */ + flc = &priv->flc_desc[0].flc; + /* Configure FD as a FRAME LIST */ + DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); + DPAA2_SET_FD_COMPOUND_FMT(fd); + DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); + + PMD_TX_LOG(DEBUG, "auth_off: 0x%x/length %d, digest-len=%d\n" + "iv-len=%d data_off: 0x%x\n", + sym_op->aead.data.offset, + sym_op->aead.data.length, + sym_op->aead.digest.length, + sess->iv.length, + sym_op->m_src->data_off); + + /* Configure Output FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + if (auth_only_len) + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->aead.data.length + icv_len + auth_only_len) : + sym_op->aead.data.length + auth_only_len; + + DPAA2_SET_FLE_SG_EXT(fle); + + /* Configure Output SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + + sym_op->m_src->data_off - auth_only_len); + sge->length = sym_op->aead.data.length + auth_only_len; + + if (sess->dir == DIR_ENC) { + sge++; + DPAA2_SET_FLE_ADDR(sge, + DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + + sess->iv.length + auth_only_len)); + } + DPAA2_SET_FLE_FIN(sge); + + sge++; + fle++; + + /* Configure Input FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + DPAA2_SET_FLE_SG_EXT(fle); + DPAA2_SET_FLE_FIN(fle); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->aead.data.length + sess->iv.length + auth_only_len) : + (sym_op->aead.data.length + sess->iv.length + auth_only_len + + sess->digest_length); + + /* Configure Input SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr)); + sge->length = sess->iv.length; + sge++; + if (auth_only_len) { + DPAA2_SET_FLE_ADDR(sge, + DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data)); + sge->length = auth_only_len; + DPAA2_SET_FLE_BPID(sge, bpid); + sge++; + } + + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + + sym_op->m_src->data_off); + sge->length = sym_op->aead.data.length; + if (sess->dir == DIR_DEC) { + sge++; + old_icv = (uint8_t *)(sge + 1); + memcpy(old_icv, sym_op->aead.digest.data, + sess->digest_length); + memset(sym_op->aead.digest.data, 0, sess->digest_length); + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + + sess->digest_length + + sess->iv.length + + auth_only_len)); + } + DPAA2_SET_FLE_FIN(sge); + + if (auth_only_len) { + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); + } + + return 0; +} + static inline int build_authenc_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, @@ -84,9 +232,10 @@ build_authenc_fd(dpaa2_sec_session *sess, struct sec_flow_context *flc; uint32_t auth_only_len = sym_op->auth.data.length - sym_op->cipher.data.length; - int icv_len = sym_op->auth.digest.length; + int icv_len = sess->digest_length, retval; uint8_t *old_icv; - uint32_t mem_len = (7 * sizeof(struct qbman_fle)) + icv_len; + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); PMD_INIT_FUNC_TRACE(); @@ -96,12 +245,14 @@ build_authenc_fd(dpaa2_sec_session *sess, * to get the MBUF Addr from the previous FLE. * We can have a better approach to use the inline Mbuf */ - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; sge = fle + 2; if (likely(bpid < MAX_BPID)) { @@ -133,10 +284,10 @@ build_authenc_fd(dpaa2_sec_session *sess, "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n", sym_op->auth.data.offset, sym_op->auth.data.length, - sym_op->auth.digest.length, + sess->digest_length, sym_op->cipher.data.offset, sym_op->cipher.data.length, - sym_op->cipher.iv.length, + sess->iv.length, sym_op->m_src->data_off); /* Configure Output FLE with Scatter/Gather Entry */ @@ -159,9 +310,9 @@ build_authenc_fd(dpaa2_sec_session *sess, sge++; DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); - sge->length = sym_op->auth.digest.length; + sge->length = sess->digest_length; DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + - sym_op->cipher.iv.length)); + sess->iv.length)); } DPAA2_SET_FLE_FIN(sge); @@ -173,13 +324,13 @@ build_authenc_fd(dpaa2_sec_session *sess, DPAA2_SET_FLE_SG_EXT(fle); DPAA2_SET_FLE_FIN(fle); fle->length = (sess->dir == DIR_ENC) ? - (sym_op->auth.data.length + sym_op->cipher.iv.length) : - (sym_op->auth.data.length + sym_op->cipher.iv.length + - sym_op->auth.digest.length); + (sym_op->auth.data.length + sess->iv.length) : + (sym_op->auth.data.length + sess->iv.length + + sess->digest_length); /* Configure Input SGE for Encap/Decap */ - DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(sym_op->cipher.iv.data)); - sge->length = sym_op->cipher.iv.length; + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); + sge->length = sess->iv.length; sge++; DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); @@ -190,13 +341,13 @@ build_authenc_fd(dpaa2_sec_session *sess, sge++; old_icv = (uint8_t *)(sge + 1); memcpy(old_icv, sym_op->auth.digest.data, - sym_op->auth.digest.length); - memset(sym_op->auth.digest.data, 0, sym_op->auth.digest.length); + sess->digest_length); + memset(sym_op->auth.digest.data, 0, sess->digest_length); DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); - sge->length = sym_op->auth.digest.length; + sge->length = sess->digest_length; DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + - sym_op->auth.digest.length + - sym_op->cipher.iv.length)); + sess->digest_length + + sess->iv.length)); } DPAA2_SET_FLE_FIN(sge); if (auth_only_len) { @@ -212,21 +363,19 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, { struct rte_crypto_sym_op *sym_op = op->sym; struct qbman_fle *fle, *sge; - uint32_t mem_len = (sess->dir == DIR_ENC) ? - (3 * sizeof(struct qbman_fle)) : - (5 * sizeof(struct qbman_fle) + - sym_op->auth.digest.length); struct sec_flow_context *flc; struct ctxt_priv *priv = sess->ctxt; uint8_t *old_digest; + int retval; PMD_INIT_FUNC_TRACE(); - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { - RTE_LOG(ERR, PMD, "Memory alloc failed for FLE\n"); + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); /* TODO we are using the first FLE entry to store Mbuf. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -234,6 +383,7 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, * We can have a better approach to use the inline Mbuf */ DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; if (likely(bpid < MAX_BPID)) { @@ -249,7 +399,7 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); - fle->length = sym_op->auth.digest.length; + fle->length = sess->digest_length; DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); DPAA2_SET_FD_COMPOUND_FMT(fd); @@ -280,17 +430,17 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, sym_op->m_src->data_off); DPAA2_SET_FD_LEN(fd, sym_op->auth.data.length + - sym_op->auth.digest.length); + sess->digest_length); sge->length = sym_op->auth.data.length; sge++; old_digest = (uint8_t *)(sge + 1); rte_memcpy(old_digest, sym_op->auth.digest.data, - sym_op->auth.digest.length); - memset(sym_op->auth.digest.data, 0, sym_op->auth.digest.length); + sess->digest_length); + memset(sym_op->auth.digest.data, 0, sess->digest_length); DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); - sge->length = sym_op->auth.digest.length; + sge->length = sess->digest_length; fle->length = sym_op->auth.data.length + - sym_op->auth.digest.length; + sess->digest_length; DPAA2_SET_FLE_FIN(sge); } DPAA2_SET_FLE_FIN(fle); @@ -304,18 +454,20 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, { struct rte_crypto_sym_op *sym_op = op->sym; struct qbman_fle *fle, *sge; - uint32_t mem_len = (5 * sizeof(struct qbman_fle)); + int retval; struct sec_flow_context *flc; struct ctxt_priv *priv = sess->ctxt; + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); PMD_INIT_FUNC_TRACE(); - /* todo - we can use some mempool to avoid malloc here */ - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); /* TODO we are using the first FLE entry to store Mbuf. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -323,6 +475,7 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, * We can have a better approach to use the inline Mbuf */ DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; sge = fle + 2; @@ -343,21 +496,21 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, flc = &priv->flc_desc[0].flc; DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); DPAA2_SET_FD_LEN(fd, sym_op->cipher.data.length + - sym_op->cipher.iv.length); + sess->iv.length); DPAA2_SET_FD_COMPOUND_FMT(fd); DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); PMD_TX_LOG(DEBUG, "cipher_off: 0x%x/length %d,ivlen=%d data_off: 0x%x", sym_op->cipher.data.offset, sym_op->cipher.data.length, - sym_op->cipher.iv.length, + sess->iv.length, sym_op->m_src->data_off); DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); DPAA2_SET_FLE_OFFSET(fle, sym_op->cipher.data.offset + sym_op->m_src->data_off); - fle->length = sym_op->cipher.data.length + sym_op->cipher.iv.length; + fle->length = sym_op->cipher.data.length + sess->iv.length; PMD_TX_LOG(DEBUG, "1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d", flc, fle, fle->addr_hi, fle->addr_lo, fle->length); @@ -365,12 +518,12 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, fle++; DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); - fle->length = sym_op->cipher.data.length + sym_op->cipher.iv.length; + fle->length = sym_op->cipher.data.length + sess->iv.length; DPAA2_SET_FLE_SG_EXT(fle); - DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(sym_op->cipher.iv.data)); - sge->length = sym_op->cipher.iv.length; + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); + sge->length = sess->iv.length; sge++; DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); @@ -406,6 +559,9 @@ build_sec_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, case DPAA2_SEC_AUTH: ret = build_auth_fd(sess, op, fd, bpid); break; + case DPAA2_SEC_AEAD: + ret = build_authenc_gcm_fd(sess, op, fd, bpid); + break; case DPAA2_SEC_CIPHER_HASH: ret = build_authenc_fd(sess, op, fd, bpid); break; @@ -437,7 +593,7 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, if (unlikely(nb_ops == 0)) return 0; - if (ops[0]->sym->sess_type != RTE_CRYPTO_SYM_OP_WITH_SESSION) { + if (ops[0]->sess_type != RTE_CRYPTO_OP_WITH_SESSION) { RTE_LOG(ERR, PMD, "sessionless crypto op not supported\n"); return 0; } @@ -463,7 +619,9 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, /*Clear the unused FD fields before sending*/ memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); sess = (dpaa2_sec_session *) - (*ops)->sym->session->_private; + get_session_private_data( + (*ops)->sym->session, + cryptodev_driver_id); mb_pool = (*ops)->sym->m_src->pool; bpid = mempool_to_bpid(mb_pool); ret = build_sec_fd(sess, *ops, &fd_arr[loop], bpid); @@ -495,6 +653,7 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) { struct qbman_fle *fle; struct rte_crypto_op *op; + struct ctxt_priv *priv; fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); @@ -510,7 +669,7 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) if (unlikely(DPAA2_GET_FD_IVP(fd))) { /* TODO complete it. */ - RTE_LOG(ERR, PMD, "error: Non inline buffer - WHAT to DO?"); + RTE_LOG(ERR, PMD, "error: Non inline buffer - WHAT to DO?\n"); return NULL; } op = (struct rte_crypto_op *)DPAA2_IOVA_TO_VADDR( @@ -530,7 +689,8 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) DPAA2_GET_FD_LEN(fd)); /* free the fle memory */ - rte_free(fle - 1); + priv = (struct ctxt_priv *)DPAA2_GET_FLE_CTXT(fle - 1); + rte_mempool_put(priv->fle_pool, (void *)(fle - 1)); return op; } @@ -571,8 +731,8 @@ dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, /*Issue a volatile dequeue command. */ while (1) { if (qbman_swp_pull(swp, &pulldesc)) { - RTE_LOG(WARNING, PMD, "SEC VDQ command is not issued." - "QBMAN is busy\n"); + RTE_LOG(WARNING, PMD, + "SEC VDQ command is not issued : QBMAN busy\n"); /* Portal was busy, try again */ continue; } @@ -656,7 +816,8 @@ dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) static int dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, - __rte_unused int socket_id) + __rte_unused int socket_id, + __rte_unused struct rte_mempool *session_pool) { struct dpaa2_sec_dev_private *priv = dev->data->dev_private; struct dpaa2_sec_qp *qp; @@ -747,19 +908,12 @@ dpaa2_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) return sizeof(dpaa2_sec_session); } -static void -dpaa2_sec_session_initialize(struct rte_mempool *mp __rte_unused, - void *sess __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); -} - static int dpaa2_sec_cipher_init(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, dpaa2_sec_session *session) { - struct dpaa2_sec_cipher_ctxt *ctxt = &session->ext_params.cipher_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo cipherdata; int bufsize, i; struct ctxt_priv *priv; @@ -772,16 +926,18 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - RTE_LOG(ERR, PMD, "No Memory for priv CTXT"); + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); return -1; } + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[0].flc; session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, RTE_CACHE_LINE_SIZE); if (session->cipher_key.data == NULL) { - RTE_LOG(ERR, PMD, "No Memory for cipher key"); + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); rte_free(priv); return -1; } @@ -794,23 +950,27 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, cipherdata.key_enc_flags = 0; cipherdata.key_type = RTA_DATA_IMM; + /* Set IV parameters */ + session->iv.offset = xform->cipher.iv.offset; + session->iv.length = xform->cipher.iv.length; + switch (xform->cipher.algo) { case RTE_CRYPTO_CIPHER_AES_CBC: cipherdata.algtype = OP_ALG_ALGSEL_AES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; - ctxt->iv.length = AES_CBC_IV_LEN; break; case RTE_CRYPTO_CIPHER_3DES_CBC: cipherdata.algtype = OP_ALG_ALGSEL_3DES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; - ctxt->iv.length = TDES_CBC_IV_LEN; break; case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; case RTE_CRYPTO_CIPHER_3DES_CTR: - case RTE_CRYPTO_CIPHER_AES_GCM: - case RTE_CRYPTO_CIPHER_AES_CCM: case RTE_CRYPTO_CIPHER_AES_ECB: case RTE_CRYPTO_CIPHER_3DES_ECB: case RTE_CRYPTO_CIPHER_AES_XTS: @@ -820,7 +980,7 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: case RTE_CRYPTO_CIPHER_ZUC_EEA3: case RTE_CRYPTO_CIPHER_NULL: - RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", xform->cipher.algo); goto error_out; default: @@ -832,8 +992,8 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, DIR_ENC : DIR_DEC; bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, - &cipherdata, NULL, ctxt->iv.length, - session->dir); + &cipherdata, NULL, session->iv.length, + session->dir); if (bufsize < 0) { RTE_LOG(ERR, PMD, "Crypto: Descriptor build failed\n"); goto error_out; @@ -868,9 +1028,9 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, dpaa2_sec_session *session) { - struct dpaa2_sec_auth_ctxt *ctxt = &session->ext_params.auth_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo authdata; - unsigned int bufsize; + unsigned int bufsize, i; struct ctxt_priv *priv; struct sec_flow_context *flc; @@ -882,16 +1042,17 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, sizeof(struct sec_flc_desc), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - RTE_LOG(ERR, PMD, "No Memory for priv CTXT"); + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); return -1; } + priv->fle_pool = dev_priv->fle_pool; flc = &priv->flc_desc[DESC_INITFINAL].flc; session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, RTE_CACHE_LINE_SIZE); if (session->auth_key.data == NULL) { - RTE_LOG(ERR, PMD, "No Memory for auth key"); + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); rte_free(priv); return -1; } @@ -904,6 +1065,8 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, authdata.key_enc_flags = 0; authdata.key_type = RTA_DATA_IMM; + session->digest_length = xform->auth.digest_length; + switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_SHA1_HMAC: authdata.algtype = OP_ALG_ALGSEL_SHA1; @@ -936,7 +1099,6 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; break; case RTE_CRYPTO_AUTH_AES_XCBC_MAC: - case RTE_CRYPTO_AUTH_AES_GCM: case RTE_CRYPTO_AUTH_SNOW3G_UIA2: case RTE_CRYPTO_AUTH_NULL: case RTE_CRYPTO_AUTH_SHA1: @@ -945,13 +1107,12 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_SHA224: case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_MD5: - case RTE_CRYPTO_AUTH_AES_CCM: case RTE_CRYPTO_AUTH_AES_GMAC: case RTE_CRYPTO_AUTH_KASUMI_F9: case RTE_CRYPTO_AUTH_AES_CMAC: case RTE_CRYPTO_AUTH_AES_CBC_MAC: case RTE_CRYPTO_AUTH_ZUC_EIA3: - RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", xform->auth.algo); goto error_out; default: @@ -964,7 +1125,7 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 1, 0, &authdata, !session->dir, - ctxt->trunc_len); + session->digest_length); flc->word1_sdl = (uint8_t)bufsize; flc->word2_rflc_31_0 = lower_32_bits( @@ -974,6 +1135,10 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, (uint64_t)&(((struct dpaa2_sec_qp *) dev->data->queue_pairs[0])->rx_vq)); session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[DESC_INITFINAL].desc[i]); + return 0; @@ -989,8 +1154,129 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, dpaa2_sec_session *session) { struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; + struct alginfo aeaddata; + unsigned int bufsize, i; + struct ctxt_priv *priv; + struct sec_flow_context *flc; + struct rte_crypto_aead_xform *aead_xform = &xform->aead; + int err; + + PMD_INIT_FUNC_TRACE(); + + /* Set IV parameters */ + session->iv.offset = aead_xform->iv.offset; + session->iv.length = aead_xform->iv.length; + session->ctxt_type = DPAA2_SEC_AEAD; + + /* For SEC AEAD only one descriptor is required */ + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + if (priv == NULL) { + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); + return -1; + } + + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[0].flc; + + session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->aead_key.data == NULL && aead_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for aead key\n"); + rte_free(priv); + return -1; + } + memcpy(session->aead_key.data, aead_xform->key.data, + aead_xform->key.length); + + session->digest_length = aead_xform->digest_length; + session->aead_key.length = aead_xform->key.length; + ctxt->auth_only_len = aead_xform->aad_length; + + aeaddata.key = (uint64_t)session->aead_key.data; + aeaddata.keylen = session->aead_key.length; + aeaddata.key_enc_flags = 0; + aeaddata.key_type = RTA_DATA_IMM; + + switch (aead_xform->algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + aeaddata.algtype = OP_ALG_ALGSEL_AES; + aeaddata.algmode = OP_ALG_AAI_GCM; + session->cipher_alg = RTE_CRYPTO_AEAD_AES_GCM; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + RTE_LOG(ERR, PMD, "Crypto: Unsupported AEAD alg %u\n", + aead_xform->algo); + goto error_out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined AEAD specified %u\n", + aead_xform->algo); + goto error_out; + } + session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + priv->flc_desc[0].desc[0] = aeaddata.keylen; + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)priv->flc_desc[0].desc, + &priv->flc_desc[0].desc[1], 1); + + if (err < 0) { + PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths\n"); + goto error_out; + } + if (priv->flc_desc[0].desc[1] & 1) { + aeaddata.key_type = RTA_DATA_IMM; + } else { + aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key); + aeaddata.key_type = RTA_DATA_PTR; + } + priv->flc_desc[0].desc[0] = 0; + priv->flc_desc[0].desc[1] = 0; + + if (session->dir == DIR_ENC) + bufsize = cnstr_shdsc_gcm_encap( + priv->flc_desc[0].desc, 1, 0, + &aeaddata, session->iv.length, + session->digest_length); + else + bufsize = cnstr_shdsc_gcm_decap( + priv->flc_desc[0].desc, 1, 0, + &aeaddata, session->iv.length, + session->digest_length); + flc->word1_sdl = (uint8_t)bufsize; + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); + + return 0; + +error_out: + rte_free(session->aead_key.data); + rte_free(priv); + return -1; +} + + +static int +dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + dpaa2_sec_session *session) +{ + struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo authdata, cipherdata; - unsigned int bufsize; + unsigned int bufsize, i; struct ctxt_priv *priv; struct sec_flow_context *flc; struct rte_crypto_cipher_xform *cipher_xform; @@ -1012,21 +1298,27 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH; } + + /* Set IV parameters */ + session->iv.offset = cipher_xform->iv.offset; + session->iv.length = cipher_xform->iv.length; + /* For SEC AEAD only one descriptor is required */ priv = (struct ctxt_priv *)rte_zmalloc(NULL, sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - RTE_LOG(ERR, PMD, "No Memory for priv CTXT"); + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); return -1; } + priv->fle_pool = dev_priv->fle_pool; flc = &priv->flc_desc[0].flc; session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, RTE_CACHE_LINE_SIZE); if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) { - RTE_LOG(ERR, PMD, "No Memory for cipher key"); + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); rte_free(priv); return -1; } @@ -1034,7 +1326,7 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, RTE_CACHE_LINE_SIZE); if (session->auth_key.data == NULL && auth_xform->key.length > 0) { - RTE_LOG(ERR, PMD, "No Memory for auth key"); + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); rte_free(session->cipher_key.data); rte_free(priv); return -1; @@ -1045,12 +1337,13 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, memcpy(session->auth_key.data, auth_xform->key.data, auth_xform->key.length); - ctxt->trunc_len = auth_xform->digest_length; authdata.key = (uint64_t)session->auth_key.data; authdata.keylen = session->auth_key.length; authdata.key_enc_flags = 0; authdata.key_type = RTA_DATA_IMM; + session->digest_length = auth_xform->digest_length; + switch (auth_xform->algo) { case RTE_CRYPTO_AUTH_SHA1_HMAC: authdata.algtype = OP_ALG_ALGSEL_SHA1; @@ -1083,7 +1376,6 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; break; case RTE_CRYPTO_AUTH_AES_XCBC_MAC: - case RTE_CRYPTO_AUTH_AES_GCM: case RTE_CRYPTO_AUTH_SNOW3G_UIA2: case RTE_CRYPTO_AUTH_NULL: case RTE_CRYPTO_AUTH_SHA1: @@ -1092,13 +1384,12 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_SHA224: case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_MD5: - case RTE_CRYPTO_AUTH_AES_CCM: case RTE_CRYPTO_AUTH_AES_GMAC: case RTE_CRYPTO_AUTH_KASUMI_F9: case RTE_CRYPTO_AUTH_AES_CMAC: case RTE_CRYPTO_AUTH_AES_CBC_MAC: case RTE_CRYPTO_AUTH_ZUC_EIA3: - RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", auth_xform->algo); goto error_out; default: @@ -1116,23 +1407,23 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, cipherdata.algtype = OP_ALG_ALGSEL_AES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; - ctxt->iv.length = AES_CBC_IV_LEN; break; case RTE_CRYPTO_CIPHER_3DES_CBC: cipherdata.algtype = OP_ALG_ALGSEL_3DES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; - ctxt->iv.length = TDES_CBC_IV_LEN; break; - case RTE_CRYPTO_CIPHER_AES_GCM: + case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: case RTE_CRYPTO_CIPHER_NULL: case RTE_CRYPTO_CIPHER_3DES_ECB: case RTE_CRYPTO_CIPHER_AES_ECB: - case RTE_CRYPTO_CIPHER_AES_CTR: - case RTE_CRYPTO_CIPHER_AES_CCM: case RTE_CRYPTO_CIPHER_KASUMI_F8: - RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", cipher_xform->algo); goto error_out; default: @@ -1151,7 +1442,7 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, &priv->flc_desc[0].desc[2], 2); if (err < 0) { - PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths"); + PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths\n"); goto error_out; } if (priv->flc_desc[0].desc[2] & 1) { @@ -1173,12 +1464,12 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) { bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1, 0, &cipherdata, &authdata, - ctxt->iv.length, + session->iv.length, ctxt->auth_only_len, - ctxt->trunc_len, + session->digest_length, session->dir); } else { - RTE_LOG(ERR, PMD, "Hash before cipher not supported"); + RTE_LOG(ERR, PMD, "Hash before cipher not supported\n"); goto error_out; } @@ -1190,6 +1481,9 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, (uint64_t)&(((struct dpaa2_sec_qp *) dev->data->queue_pairs[0])->rx_vq)); session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); return 0; @@ -1200,8 +1494,8 @@ error_out: return -1; } -static void * -dpaa2_sec_session_configure(struct rte_cryptodev *dev, +static int +dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *sess) { dpaa2_sec_session *session = sess; @@ -1209,9 +1503,13 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, PMD_INIT_FUNC_TRACE(); if (unlikely(sess == NULL)) { - RTE_LOG(ERR, PMD, "invalid session struct"); - return NULL; + RTE_LOG(ERR, PMD, "invalid session struct\n"); + return -1; } + + /* Default IV length = 0 */ + session->iv.length = 0; + /* Cipher Only */ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { session->ctxt_type = DPAA2_SEC_CIPHER; @@ -1227,33 +1525,76 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { session->ext_params.aead_ctxt.auth_cipher_text = true; - dpaa2_sec_aead_init(dev, xform, session); + dpaa2_sec_aead_chain_init(dev, xform, session); /* Authenticate then Cipher */ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { session->ext_params.aead_ctxt.auth_cipher_text = false; + dpaa2_sec_aead_chain_init(dev, xform, session); + + /* AEAD operation for AES-GCM kind of Algorithms */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && + xform->next == NULL) { dpaa2_sec_aead_init(dev, xform, session); + } else { - RTE_LOG(ERR, PMD, "Invalid crypto type"); - return NULL; + RTE_LOG(ERR, PMD, "Invalid crypto type\n"); + return -EINVAL; + } + + return 0; +} + +static int +dpaa2_sec_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - return session; + ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "DPAA2 PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -dpaa2_sec_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +dpaa2_sec_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { PMD_INIT_FUNC_TRACE(); - dpaa2_sec_session *s = (dpaa2_sec_session *)sess; + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; - if (s) { + if (sess_priv) { rte_free(s->ctxt); rte_free(s->cipher_key.data); rte_free(s->auth_key.data); memset(sess, 0, sizeof(dpaa2_sec_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } @@ -1263,7 +1604,7 @@ dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, { PMD_INIT_FUNC_TRACE(); - return -ENOTSUP; + return 0; } static int @@ -1366,7 +1707,7 @@ dpaa2_sec_dev_close(struct rte_cryptodev *dev) /*Free the allocated memory for ethernet private data and dpseci*/ priv->hw = NULL; - free(dpseci); + rte_free(dpseci); return 0; } @@ -1383,7 +1724,7 @@ dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev, info->feature_flags = dev->feature_flags; info->capabilities = dpaa2_sec_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->dev_type = RTE_CRYPTODEV_DPAA2_SEC_PMD; + info->driver_id = cryptodev_driver_id; } } @@ -1475,15 +1816,17 @@ static struct rte_cryptodev_ops crypto_ops = { .queue_pair_stop = dpaa2_sec_queue_pair_stop, .queue_pair_count = dpaa2_sec_queue_pair_count, .session_get_size = dpaa2_sec_session_get_size, - .session_initialize = dpaa2_sec_session_initialize, .session_configure = dpaa2_sec_session_configure, .session_clear = dpaa2_sec_session_clear, }; static int -dpaa2_sec_uninit(const struct rte_cryptodev_driver *crypto_drv __rte_unused, - struct rte_cryptodev *dev) +dpaa2_sec_uninit(const struct rte_cryptodev *dev) { + struct dpaa2_sec_dev_private *internals = dev->data->dev_private; + + rte_mempool_free(internals->fle_pool); + PMD_INIT_LOG(INFO, "Closing DPAA2_SEC device %s on numa socket %u\n", dev->data->name, rte_socket_id()); @@ -1500,6 +1843,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) uint16_t token; struct dpseci_attr attr; int retcode, hw_id; + char str[20]; PMD_INIT_FUNC_TRACE(); dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device); @@ -1509,7 +1853,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) } hw_id = dpaa2_dev->object_id; - cryptodev->dev_type = RTE_CRYPTODEV_DPAA2_SEC_PMD; + cryptodev->driver_id = cryptodev_driver_id; cryptodev->dev_ops = &crypto_ops; cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst; @@ -1560,6 +1904,18 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) internals->hw = dpseci; internals->token = token; + sprintf(str, "fle_pool_%d", cryptodev->data->dev_id); + internals->fle_pool = rte_mempool_create((const char *)str, + FLE_POOL_NUM_BUFS, + FLE_POOL_BUF_SIZE, + FLE_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!internals->fle_pool) { + RTE_LOG(ERR, PMD, "%s create failed\n", str); + goto init_error; + } + PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); return 0; @@ -1599,7 +1955,7 @@ cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv, dpaa2_dev->cryptodev = cryptodev; cryptodev->device = &dpaa2_dev->device; - cryptodev->driver = (struct rte_cryptodev_driver *)dpaa2_drv; + cryptodev->device->driver = &dpaa2_drv->driver; /* init user callbacks */ TAILQ_INIT(&(cryptodev->link_intr_cbs)); @@ -1627,7 +1983,7 @@ cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) if (cryptodev == NULL) return -ENODEV; - ret = dpaa2_sec_uninit(NULL, cryptodev); + ret = dpaa2_sec_uninit(cryptodev); if (ret) return ret; @@ -1638,7 +1994,6 @@ cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) rte_free(cryptodev->data->dev_private); cryptodev->device = NULL; - cryptodev->driver = NULL; cryptodev->data = NULL; return 0; @@ -1653,4 +2008,5 @@ static struct rte_dpaa2_driver rte_dpaa2_sec_driver = { .remove = cryptodev_dpaa2_sec_remove, }; -RTE_PMD_REGISTER_DPAA2(dpaa2_sec_pmd, rte_dpaa2_sec_driver); +RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver); +RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_dpaa2_sec_driver, cryptodev_driver_id); diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h index 03d4c703..b5e99f80 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h index f5c61694..3849a052 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,12 +34,16 @@ #ifndef _RTE_DPAA2_SEC_PMD_PRIVATE_H_ #define _RTE_DPAA2_SEC_PMD_PRIVATE_H_ +#define CRYPTODEV_NAME_DPAA2_SEC_PMD crypto_dpaa2_sec +/**< NXP DPAA2 - SEC PMD device name */ + #define MAX_QUEUES 64 #define MAX_DESC_SIZE 64 /** private data structure for each DPAA2_SEC device */ struct dpaa2_sec_dev_private { void *mc_portal; /**< MC Portal for configuring this device */ void *hw; /**< Hardware handle for this device.Used by NADK framework */ + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ int32_t hw_id; /**< An unique ID of this device instance */ int32_t vfio_fd; /**< File descriptor received via VFIO */ uint16_t token; /**< Token required by DPxxx objects */ @@ -128,6 +132,7 @@ struct sec_flc_desc { }; struct ctxt_priv { + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ struct sec_flc_desc flc_desc[0]; }; @@ -135,6 +140,7 @@ enum dpaa2_sec_op_type { DPAA2_SEC_NONE, /*!< No Cipher operations*/ DPAA2_SEC_CIPHER,/*!< CIPHER operations */ DPAA2_SEC_AUTH, /*!< Authentication Operations */ + DPAA2_SEC_AEAD, /*!< AEAD (AES-GCM/CCM) type operations */ DPAA2_SEC_CIPHER_HASH, /*!< Authenticated Encryption with * associated data */ @@ -147,30 +153,9 @@ enum dpaa2_sec_op_type { DPAA2_SEC_MAX }; -struct dpaa2_sec_cipher_ctxt { - struct { - uint8_t *data; - uint16_t length; - } iv; /**< Initialisation vector parameters */ - uint8_t *init_counter; /*!< Set initial counter for CTR mode */ -}; - -struct dpaa2_sec_auth_ctxt { - uint8_t trunc_len; /*!< Length for output ICV, should - * be 0 if no truncation required - */ -}; - struct dpaa2_sec_aead_ctxt { - struct { - uint8_t *data; - uint16_t length; - } iv; /**< Initialisation vector parameters */ uint16_t auth_only_len; /*!< Length of data for Auth only */ uint8_t auth_cipher_text; /**< Authenticate/cipher ordering */ - uint8_t trunc_len; /*!< Length for output ICV, should - * be 0 if no truncation required - */ }; typedef struct dpaa2_sec_session_entry { @@ -179,18 +164,29 @@ typedef struct dpaa2_sec_session_entry { uint8_t dir; /*!< Operation Direction */ enum rte_crypto_cipher_algorithm cipher_alg; /*!< Cipher Algorithm*/ enum rte_crypto_auth_algorithm auth_alg; /*!< Authentication Algorithm*/ + union { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } aead_key; + struct { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } cipher_key; + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } auth_key; + }; + }; struct { - uint8_t *data; /**< pointer to key data */ - size_t length; /**< key length in bytes */ - } cipher_key; - struct { - uint8_t *data; /**< pointer to key data */ - size_t length; /**< key length in bytes */ - } auth_key; + uint16_t length; /**< IV length in bytes */ + uint16_t offset; /**< IV offset in bytes */ + } iv; + uint16_t digest_length; uint8_t status; union { - struct dpaa2_sec_cipher_ctxt cipher_ctxt; - struct dpaa2_sec_auth_ctxt auth_ctxt; struct dpaa2_sec_aead_ctxt aead_ctxt; } ext_params; } dpaa2_sec_session; @@ -204,16 +200,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_MD5_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 16, .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -225,16 +221,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 20, .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -246,16 +242,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 28, .max = 28, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -267,16 +263,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 - }, - .aad_size = { 0 } + }, + .iv_size = { 0 } }, } }, } }, @@ -288,16 +284,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 48, .max = 48, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -309,16 +305,46 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 64, .max = 64, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 240, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + }, }, } }, } }, @@ -342,6 +368,26 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { }, } }, } }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, { /* 3DES CBC */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { diff --git a/drivers/crypto/dpaa2_sec/hw/compat.h b/drivers/crypto/dpaa2_sec/hw/compat.h index 11fdaa8e..b17b27ef 100644 --- a/drivers/crypto/dpaa2_sec/hw/compat.h +++ b/drivers/crypto/dpaa2_sec/hw/compat.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,7 +49,9 @@ #include <stdlib.h> #include <stdio.h> #include <stdbool.h> + #include <rte_byteorder.h> +#include <rte_common.h> #ifndef __BYTE_ORDER__ #error "Undefined endianness" @@ -60,7 +62,7 @@ #endif #ifndef __always_inline -#define __always_inline (inline __attribute__((always_inline))) +#define __always_inline __rte_always_inline #endif #ifndef __always_unused diff --git a/drivers/crypto/dpaa2_sec/hw/desc.h b/drivers/crypto/dpaa2_sec/hw/desc.h index 5185be22..0be4439c 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc.h +++ b/drivers/crypto/dpaa2_sec/hw/desc.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/desc/algo.h b/drivers/crypto/dpaa2_sec/hw/desc/algo.h index c71ada07..5ae3a1ac 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc/algo.h +++ b/drivers/crypto/dpaa2_sec/hw/desc/algo.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -159,6 +159,10 @@ cnstr_shdsc_snow_f9(uint32_t *descbuf, bool ps, bool swap, * @ps: if 36/40bit addressing is desired, this parameter must be true * @swap: must be true when core endianness doesn't match SEC endianness * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values one of OP_ALG_ALGSEL_* {DES, 3DES, AES} + * Valid modes for: + * AES: OP_ALG_AAI_* {CBC, CTR} + * DES, 3DES: OP_ALG_AAI_CBC * @iv: IV data; if NULL, "ivlen" bytes from the input frame will be read as IV * @ivlen: IV length * @dir: DIR_ENC/DIR_DEC @@ -172,8 +176,10 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, { struct program prg; struct program *p = &prg; - const bool is_aes_dec = (dir == DIR_DEC) && - (cipherdata->algtype == OP_ALG_ALGSEL_AES); + uint32_t iv_off = 0; + const bool need_dk = (dir == DIR_DEC) && + (cipherdata->algtype == OP_ALG_ALGSEL_AES) && + (cipherdata->algmode == OP_ALG_AAI_CBC); LABEL(keyjmp); LABEL(skipdk); REFERENCE(pkeyjmp); @@ -191,7 +197,7 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, cipherdata->keylen, INLINE_KEY(cipherdata)); - if (is_aes_dec) { + if (need_dk) { ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); @@ -199,7 +205,7 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, } SET_LABEL(p, keyjmp); - if (is_aes_dec) { + if (need_dk) { ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode | OP_ALG_AAI_DK, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); @@ -209,12 +215,15 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); } + if (cipherdata->algmode == OP_ALG_AAI_CTR) + iv_off = 16; + if (iv) /* IV load, convert size */ - LOAD(p, (uintptr_t)iv, CONTEXT1, 0, ivlen, IMMED | COPY); + LOAD(p, (uintptr_t)iv, CONTEXT1, iv_off, ivlen, IMMED | COPY); else /* IV is present first before the actual message */ - SEQLOAD(p, CONTEXT1, 0, ivlen, 0); + SEQLOAD(p, CONTEXT1, iv_off, ivlen, 0); MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); MATHB(p, SEQINSZ, SUB, MATH2, VSEQOUTSZ, 4, 0); @@ -224,7 +233,7 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, SEQFIFOSTORE(p, MSG, 0, 0, VLF); PATCH_JUMP(p, pkeyjmp, keyjmp); - if (is_aes_dec) + if (need_dk) PATCH_JUMP(p, pskipdk, skipdk); return PROGRAM_FINALIZE(p); @@ -434,6 +443,211 @@ cnstr_shdsc_kasumi_f9(uint32_t *descbuf, bool ps, bool swap, } /** + * cnstr_shdsc_gcm_encap - AES-GCM encap as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with + * OP_ALG_AAI_GCM. + * @ivlen: Initialization vector length + * @icvsize: integrity check value (ICV) size (truncated or full) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_gcm_encap(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, + uint32_t ivlen, uint32_t icvsize) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + LABEL(zeroassocjump2); + LABEL(zeroassocjump1); + LABEL(zeropayloadjump); + REFERENCE(pkeyjmp); + REFERENCE(pzeroassocjump2); + REFERENCE(pzeroassocjump1); + REFERENCE(pzeropayloadjump); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SELF | SHRD); + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + SET_LABEL(p, keyjmp); + + /* class 1 operation */ + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, DIR_ENC); + + MATHB(p, DPOVRD, AND, 0x7fffffff, MATH3, 4, IMMED2); + + /* if assoclen + cryptlen is ZERO, skip to ICV write */ + MATHB(p, SEQINSZ, SUB, ivlen, VSEQOUTSZ, 4, IMMED2); + pzeroassocjump2 = JUMP(p, zeroassocjump2, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + MATHB(p, ZERO, ADD, MATH3, VSEQINSZ, 4, 0); + pzeroassocjump1 = JUMP(p, zeroassocjump1, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + MATHB(p, ZERO, ADD, MATH3, VSEQOUTSZ, 4, 0); + + /* skip assoc data */ + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + + /* cryptlen = seqinlen - assoclen */ + MATHB(p, SEQINSZ, SUB, MATH3, VSEQOUTSZ, 4, 0); + + /* if cryptlen is ZERO jump to zero-payload commands */ + pzeropayloadjump = JUMP(p, zeropayloadjump, LOCAL_JUMP, ALL_TRUE, + MATH_Z); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | FLUSH1); + SET_LABEL(p, zeroassocjump1); + + MATHB(p, SEQINSZ, SUB, MATH0, VSEQINSZ, 4, 0); + + /* write encrypted data */ + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + /* read payload data */ + SEQFIFOLOAD(p, MSG1, 0, CLASS1 | VLF | LAST1); + + /* jump the zero-payload commands */ + JUMP(p, 4, LOCAL_JUMP, ALL_TRUE, 0); + + /* zero-payload commands */ + SET_LABEL(p, zeropayloadjump); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | LAST1); + + JUMP(p, 2, LOCAL_JUMP, ALL_TRUE, 0); + + /* There is no input data */ + SET_LABEL(p, zeroassocjump2); + + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1 | LAST1); + + /* write ICV */ + SEQSTORE(p, CONTEXT1, 0, icvsize, 0); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pzeroassocjump2, zeroassocjump2); + PATCH_JUMP(p, pzeroassocjump1, zeroassocjump1); + PATCH_JUMP(p, pzeropayloadjump, zeropayloadjump); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_gcm_decap - AES-GCM decap as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with + * OP_ALG_AAI_GCM. + * @icvsize: integrity check value (ICV) size (truncated or full) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_gcm_decap(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, + uint32_t ivlen, uint32_t icvsize) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + LABEL(zeroassocjump1); + LABEL(zeropayloadjump); + REFERENCE(pkeyjmp); + REFERENCE(pzeroassocjump1); + REFERENCE(pzeropayloadjump); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SELF | SHRD); + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + SET_LABEL(p, keyjmp); + + /* class 1 operation */ + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_ENABLE, DIR_DEC); + + MATHB(p, DPOVRD, AND, 0x7fffffff, MATH3, 4, IMMED2); + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + MATHB(p, ZERO, ADD, MATH3, VSEQINSZ, 4, 0); + pzeroassocjump1 = JUMP(p, zeroassocjump1, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + MATHB(p, ZERO, ADD, MATH3, VSEQOUTSZ, 4, 0); + + /* skip assoc data */ + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | FLUSH1); + + SET_LABEL(p, zeroassocjump1); + + /* cryptlen = seqoutlen - assoclen */ + MATHB(p, SEQOUTSZ, SUB, MATH0, VSEQINSZ, 4, 0); + + /* jump to zero-payload command if cryptlen is zero */ + pzeropayloadjump = JUMP(p, zeropayloadjump, LOCAL_JUMP, ALL_TRUE, + MATH_Z); + + MATHB(p, SEQOUTSZ, SUB, MATH0, VSEQOUTSZ, 4, 0); + + /* store encrypted data */ + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + /* read payload data */ + SEQFIFOLOAD(p, MSG1, 0, CLASS1 | VLF | FLUSH1); + + /* zero-payload command */ + SET_LABEL(p, zeropayloadjump); + + /* read ICV */ + SEQFIFOLOAD(p, ICV1, icvsize, CLASS1 | LAST1); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pzeroassocjump1, zeroassocjump1); + PATCH_JUMP(p, pzeropayloadjump, zeropayloadjump); + + return PROGRAM_FINALIZE(p); +} + +/** * cnstr_shdsc_crc - CRC32 Accelerator (IEEE 802 CRC32 protocol mode) * @descbuf: pointer to descriptor-under-construction buffer * @swap: must be true when core endianness doesn't match SEC endianness diff --git a/drivers/crypto/dpaa2_sec/hw/desc/common.h b/drivers/crypto/dpaa2_sec/hw/desc/common.h index 6b254908..c2ac99be 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc/common.h +++ b/drivers/crypto/dpaa2_sec/hw/desc/common.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h index c63d0dac..cc637361 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h +++ b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -1311,8 +1311,11 @@ cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps, * @descbuf: pointer to buffer used for descriptor construction * @ps: if 36/40bit addressing is desired, this parameter must be true * @swap: if true, perform descriptor byte swapping on a 4-byte boundary - * @cipherdata: ointer to block cipher transform definitions. + * @cipherdata: pointer to block cipher transform definitions. * Valid algorithm values one of OP_ALG_ALGSEL_* {DES, 3DES, AES} + * Valid modes for: + * AES: OP_ALG_AAI_* {CBC, CTR} + * DES, 3DES: OP_ALG_AAI_CBC * @authdata: pointer to authentication transform definitions. * Valid algorithm values - one of OP_ALG_ALGSEL_* {MD5, SHA1, * SHA224, SHA256, SHA384, SHA512} @@ -1379,8 +1382,9 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, { struct program prg; struct program *p = &prg; - const bool is_aes_dec = (dir == DIR_DEC) && - (cipherdata->algtype == OP_ALG_ALGSEL_AES); + const bool need_dk = (dir == DIR_DEC) && + (cipherdata->algtype == OP_ALG_ALGSEL_AES) && + (cipherdata->algmode == OP_ALG_AAI_CBC); LABEL(skip_patch_len); LABEL(keyjmp); @@ -1466,7 +1470,7 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, dir == DIR_ENC ? ICV_CHECK_DISABLE : ICV_CHECK_ENABLE, dir); - if (is_aes_dec) + if (need_dk) ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); pskipkeys = JUMP(p, skipkeys, LOCAL_JUMP, ALL_TRUE, 0); @@ -1478,7 +1482,7 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, dir == DIR_ENC ? ICV_CHECK_DISABLE : ICV_CHECK_ENABLE, dir); - if (is_aes_dec) { + if (need_dk) { ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode | OP_ALG_AAI_DK, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); @@ -1503,7 +1507,10 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, SET_LABEL(p, aonly_len_offset); /* Read IV */ - SEQLOAD(p, CONTEXT1, 0, ivlen, 0); + if (cipherdata->algmode == OP_ALG_AAI_CTR) + SEQLOAD(p, CONTEXT1, 16, ivlen, 0); + else + SEQLOAD(p, CONTEXT1, 0, ivlen, 0); /* * Read data needed only for authentication. This is overwritten above diff --git a/drivers/crypto/dpaa2_sec/hw/rta.h b/drivers/crypto/dpaa2_sec/hw/rta.h index 7cf4c8a9..283ad600 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta.h +++ b/drivers/crypto/dpaa2_sec/hw/rta.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h index 79a48da2..dc0a3342 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h index c2a27a2d..96c9f1e0 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h index 2c85beec..cba65073 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h index 9bef1155..8e097856 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h index 1db55b33..2e786a20 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h index a9b9091d..64a84b47 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h index 10d2e193..3b9ba792 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h index 30be0825..f1ead2da 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h index 5e88fb46..698ff530 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h index 2e7b2f2d..f134235a 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h b/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h index c12edb0c..11995aa2 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h index 8d421a5d..4aea5770 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h index ac4f3ff8..00597f30 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h index 8fd01801..1b75692e 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/mc/dpseci.c b/drivers/crypto/dpaa2_sec/mc/dpseci.c index 6b3411f0..4a109620 100644 --- a/drivers/crypto/dpaa2_sec/mc/dpseci.c +++ b/drivers/crypto/dpaa2_sec/mc/dpseci.c @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h index 0fb47f73..6cc14a66 100644 --- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h +++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h index 1fb18cc3..3f9f4743 100644 --- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h +++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd.c b/drivers/crypto/kasumi/rte_kasumi_pmd.c index 9da9e897..38cd8a9b 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -47,6 +48,8 @@ #define KASUMI_MAX_BURST 4 #define BYTE_LEN 8 +static uint8_t cryptodev_driver_id; + /** Get xform chain order. */ static enum kasumi_operation kasumi_get_mode(const struct rte_crypto_sym_xform *xform) @@ -108,13 +111,20 @@ kasumi_set_session_parameters(struct kasumi_session *sess, case KASUMI_OP_NOT_SUPPORTED: default: KASUMI_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only KASUMI F8 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_KASUMI_F8) + return -ENOTSUP; + + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + if (cipher_xform->cipher.iv.length != KASUMI_IV_LENGTH) { + KASUMI_LOG_ERR("Wrong IV length"); return -EINVAL; + } + /* Initialize key */ sso_kasumi_init_f8_key_sched(cipher_xform->cipher.key.data, &sess->pKeySched_cipher); @@ -123,8 +133,15 @@ kasumi_set_session_parameters(struct kasumi_session *sess, if (auth_xform) { /* Only KASUMI F9 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_KASUMI_F9) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != KASUMI_DIGEST_LENGTH) { + KASUMI_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + /* Initialize key */ sso_kasumi_init_f9_key_sched(auth_xform->auth.key.data, &sess->pKeySched_hash); @@ -140,27 +157,40 @@ kasumi_set_session_parameters(struct kasumi_session *sess, static struct kasumi_session * kasumi_get_session(struct kasumi_qp *qp, struct rte_crypto_op *op) { - struct kasumi_session *sess; - - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_KASUMI_PMD)) + struct kasumi_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct kasumi_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct kasumi_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct kasumi_session *)c_sess->_private; + sess = (struct kasumi_session *)_sess_private_data; if (unlikely(kasumi_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -173,17 +203,11 @@ process_kasumi_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[num_ops], *dst[num_ops]; - uint64_t IV[num_ops]; + uint8_t *iv_ptr; + uint64_t iv[num_ops]; uint32_t num_bytes[num_ops]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (ops[i]->sym->cipher.iv.length != KASUMI_IV_LENGTH) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("iv"); - break; - } - src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); dst[i] = ops[i]->sym->m_dst ? @@ -191,14 +215,16 @@ process_kasumi_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = *((uint64_t *)(ops[i]->sym->cipher.iv.data)); + iv_ptr = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); + iv[i] = *((uint64_t *)(iv_ptr)); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; processed_ops++; } if (processed_ops != 0) - sso_kasumi_f8_n_buffer(&session->pKeySched_cipher, IV, + sso_kasumi_f8_n_buffer(&session->pKeySched_cipher, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -210,16 +236,10 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, struct kasumi_session *session) { uint8_t *src, *dst; - uint64_t IV; + uint8_t *iv_ptr; + uint64_t iv; uint32_t length_in_bits, offset_in_bits; - /* Sanity checks. */ - if (unlikely(op->sym->cipher.iv.length != KASUMI_IV_LENGTH)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("iv"); - return 0; - } - offset_in_bits = op->sym->cipher.data.offset; src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); if (op->sym->m_dst == NULL) { @@ -228,10 +248,12 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, return 0; } dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); - IV = *((uint64_t *)(op->sym->cipher.iv.data)); + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + session->cipher_iv_offset); + iv = *((uint64_t *)(iv_ptr)); length_in_bits = op->sym->cipher.data.length; - sso_kasumi_f8_1_buffer_bit(&session->pKeySched_cipher, IV, + sso_kasumi_f8_1_buffer_bit(&session->pKeySched_cipher, iv, src, dst, length_in_bits, offset_in_bits); return 1; @@ -248,23 +270,8 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, uint8_t *src, *dst; uint32_t length_in_bits; uint32_t num_bytes; - uint32_t shift_bits; - uint64_t IV; - uint8_t direction; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != KASUMI_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != KASUMI_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -276,34 +283,28 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); - /* IV from AAD */ - IV = *((uint64_t *)(ops[i]->sym->auth.aad.data)); /* Direction from next bit after end of message */ - num_bytes = (length_in_bits >> 3) + 1; - shift_bits = (BYTE_LEN - 1 - length_in_bits) % BYTE_LEN; - direction = (src[num_bytes - 1] >> shift_bits) & 0x01; + num_bytes = length_in_bits >> 3; if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + KASUMI_DIGEST_LENGTH); + sso_kasumi_f9_1_buffer(&session->pKeySched_hash, src, + num_bytes, dst); - sso_kasumi_f9_1_buffer_user(&session->pKeySched_hash, - IV, src, - length_in_bits, dst, direction); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + KASUMI_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + KASUMI_DIGEST_LENGTH); } else { dst = ops[i]->sym->auth.digest.data; - sso_kasumi_f9_1_buffer_user(&session->pKeySched_hash, - IV, src, - length_in_bits, dst, direction); + sso_kasumi_f9_1_buffer(&session->pKeySched_hash, src, + num_bytes, dst); } processed_ops++; } @@ -352,7 +353,11 @@ process_ops(struct rte_crypto_op **ops, struct kasumi_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct kasumi_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -404,8 +409,9 @@ process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session, op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, sizeof(struct kasumi_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } @@ -566,21 +572,18 @@ cryptodev_kasumi_create(const char *name, /* Check CPU for supported vector instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) cpu_flags |= RTE_CRYPTODEV_FF_CPU_AVX; - else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) + else cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - KASUMI_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct kasumi_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct kasumi_private), init_params->socket_id, + vdev); if (dev == NULL) { KASUMI_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_KASUMI_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_kasumi_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -622,7 +625,7 @@ cryptodev_kasumi_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -664,3 +667,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_KASUMI_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_kasumi_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c index 62ebdbd2..8033114b 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,11 +56,7 @@ static const struct rte_cryptodev_capabilities kasumi_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { - .min = 8, - .max = 8, - .increment = 0 - } + .iv_size = { 0 } }, } }, } }, @@ -156,7 +152,7 @@ kasumi_pmd_info_get(struct rte_cryptodev *dev, struct kasumi_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -223,7 +219,7 @@ kasumi_pmd_qp_create_processed_ops_ring(struct kasumi_qp *qp, static int kasumi_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct kasumi_qp *qp = NULL; @@ -248,7 +244,7 @@ kasumi_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -291,33 +287,56 @@ kasumi_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a KASUMI session from a crypto xform chain */ -static void * +static int kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { KASUMI_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (kasumi_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = kasumi_set_session_parameters(sess_private_data, xform); + if (ret != 0) { KASUMI_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -kasumi_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +kasumi_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct kasumi_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct kasumi_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops kasumi_pmd_ops = { diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h index fb586caa..0ce2a2e3 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include <sso_kasumi.h> +#define CRYPTODEV_NAME_KASUMI_PMD crypto_kasumi +/**< KASUMI PMD device name */ + #define KASUMI_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), \ @@ -92,6 +95,7 @@ struct kasumi_session { sso_kasumi_key_sched_t pKeySched_hash; enum kasumi_operation op; enum rte_crypto_auth_operation auth_op; + uint16_t cipher_iv_offset; } __rte_cache_aligned; diff --git a/drivers/crypto/null/null_crypto_pmd.c b/drivers/crypto/null/null_crypto_pmd.c index 023450a3..2c827257 100644 --- a/drivers/crypto/null/null_crypto_pmd.c +++ b/drivers/crypto/null/null_crypto_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -33,11 +33,14 @@ #include <rte_common.h> #include <rte_config.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include "null_crypto_pmd_private.h" +static uint8_t cryptodev_driver_id; + /** verify and set session parameters */ int null_crypto_set_session_parameters( @@ -45,7 +48,7 @@ null_crypto_set_session_parameters( const struct rte_crypto_sym_xform *xform) { if (xform == NULL) { - return -1; + return -EINVAL; } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL) { /* Authentication Only */ @@ -70,7 +73,7 @@ null_crypto_set_session_parameters( return 0; } - return -1; + return -ENOTSUP; } /** Process crypto operation for mbuf */ @@ -81,6 +84,14 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, /* set status as successful by default */ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + /* Free session if a session-less crypto op. */ + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, + sizeof(struct null_crypto_session)); + rte_cryptodev_sym_session_free(op->sym->session); + op->sym->session = NULL; + } + /* * if crypto session and operation are valid just enqueue the packet * in the processed ring @@ -89,26 +100,37 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, } static struct null_crypto_session * -get_session(struct null_crypto_qp *qp, struct rte_crypto_sym_op *op) +get_session(struct null_crypto_qp *qp, struct rte_crypto_op *op) { - struct null_crypto_session *sess; - - if (op->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->session == NULL || - op->session->dev_type != RTE_CRYPTODEV_NULL_PMD)) + struct null_crypto_session *sess = NULL; + struct rte_crypto_sym_op *sym_op = op->sym; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(sym_op->session != NULL)) + sess = (struct null_crypto_session *) + get_session_private_data( + sym_op->session, cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct null_crypto_session *)op->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct null_crypto_session *)c_sess->_private; - - if (null_crypto_set_session_parameters(sess, op->xform) != 0) - return NULL; + sess = (struct null_crypto_session *)_sess_private_data; + + if (unlikely(null_crypto_set_session_parameters(sess, + sym_op->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } return sess; @@ -125,7 +147,7 @@ null_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, int i, retval; for (i = 0; i < nb_ops; i++) { - sess = get_session(qp, ops[i]->sym); + sess = get_session(qp, ops[i]); if (unlikely(sess == NULL)) goto enqueue_err; @@ -166,6 +188,7 @@ static int cryptodev_null_remove(const char *name); /** Create crypto device */ static int cryptodev_null_create(const char *name, + struct rte_vdev_device *vdev, struct rte_crypto_vdev_init_params *init_params) { struct rte_cryptodev *dev; @@ -175,15 +198,16 @@ cryptodev_null_create(const char *name, snprintf(init_params->name, sizeof(init_params->name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, + dev = rte_cryptodev_vdev_pmd_init(init_params->name, sizeof(struct null_crypto_private), - init_params->socket_id); + init_params->socket_id, + vdev); if (dev == NULL) { NULL_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_NULL_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = null_crypto_pmd_ops; /* register rx/tx burst functions for data path */ @@ -235,7 +259,7 @@ cryptodev_null_probe(struct rte_vdev_device *dev) RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", init_params.max_nb_sessions); - return cryptodev_null_create(name, &init_params); + return cryptodev_null_create(name, dev, &init_params); } /** Uninitialise null crypto device */ @@ -268,3 +292,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_NULL_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_null_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/null/null_crypto_pmd_ops.c b/drivers/crypto/null/null_crypto_pmd_ops.c index 12c946c9..76153203 100644 --- a/drivers/crypto/null/null_crypto_pmd_ops.c +++ b/drivers/crypto/null/null_crypto_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities null_crypto_pmd_capabilities[] = .max = 0, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, }, }, }, }, @@ -72,11 +72,7 @@ static const struct rte_cryptodev_capabilities null_crypto_pmd_capabilities[] = .max = 0, .increment = 0 }, - .iv_size = { - .min = 0, - .max = 0, - .increment = 0 - } + .iv_size = { 0 } }, }, }, } }, @@ -151,7 +147,7 @@ null_crypto_pmd_info_get(struct rte_cryptodev *dev, struct null_crypto_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -215,7 +211,7 @@ null_crypto_pmd_qp_create_processed_pkts_ring(struct null_crypto_qp *qp, static int null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct null_crypto_private *internals = dev->data->dev_private; struct null_crypto_qp *qp; @@ -258,7 +254,7 @@ null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, goto qp_setup_cleanup; } - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -302,33 +298,56 @@ null_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a null crypto session from a crypto xform chain */ -static void * +static int null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mp) { - int retval; + void *sess_private_data; + int ret; if (unlikely(sess == NULL)) { NULL_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; + } + + if (rte_mempool_get(mp, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - retval = null_crypto_set_session_parameters( - (struct null_crypto_session *)sess, xform); - if (retval != 0) { + + ret = null_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { NULL_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mp, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -null_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +null_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct null_crypto_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct null_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops pmd_ops = { diff --git a/drivers/crypto/null/null_crypto_pmd_private.h b/drivers/crypto/null/null_crypto_pmd_private.h index acebc973..4d1c3c9e 100644 --- a/drivers/crypto/null/null_crypto_pmd_private.h +++ b/drivers/crypto/null/null_crypto_pmd_private.h @@ -35,6 +35,9 @@ #include "rte_config.h" +#define CRYPTODEV_NAME_NULL_PMD crypto_null +/**< Null crypto PMD device name */ + #define NULL_CRYPTO_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_NULL_PMD), \ diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index f0c5ca3c..0bd5f98e 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,6 +34,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -44,6 +45,8 @@ #define DES_BLOCK_SIZE 8 +static uint8_t cryptodev_driver_id; + static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); /*----------------------------------------------------------------------------*/ @@ -88,6 +91,8 @@ openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) res = OPENSSL_CHAIN_CIPHER_AUTH; } + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) + res = OPENSSL_CHAIN_COMBINED; } return res; @@ -183,21 +188,6 @@ get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, res = -EINVAL; } break; - case RTE_CRYPTO_CIPHER_AES_GCM: - switch (keylen) { - case 16: - *algo = EVP_aes_128_gcm(); - break; - case 24: - *algo = EVP_aes_192_gcm(); - break; - case 32: - *algo = EVP_aes_256_gcm(); - break; - default: - res = -EINVAL; - } - break; default: res = -EINVAL; break; @@ -253,6 +243,41 @@ get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, return res; } +/** Get adequate openssl function for input cipher algorithm */ +static uint8_t +get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, + const EVP_CIPHER **algo) +{ + int res = 0; + + if (algo != NULL) { + switch (sess_algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + switch (keylen) { + case 16: + *algo = EVP_aes_128_gcm(); + break; + case 24: + *algo = EVP_aes_192_gcm(); + break; + case 32: + *algo = EVP_aes_256_gcm(); + break; + default: + res = -EINVAL; + } + break; + default: + res = -EINVAL; + break; + } + } else { + res = -EINVAL; + } + + return res; +} + /** Set session cipher parameters */ static int openssl_set_session_cipher_parameters(struct openssl_session *sess, @@ -263,12 +288,15 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, /* Select cipher key */ sess->cipher.key.length = xform->cipher.key.length; + /* Set IV parameters */ + sess->iv.offset = xform->cipher.iv.offset; + sess->iv.length = xform->cipher.iv.length; + /* Select cipher algo */ switch (xform->cipher.algo) { case RTE_CRYPTO_CIPHER_3DES_CBC: case RTE_CRYPTO_CIPHER_AES_CBC: case RTE_CRYPTO_CIPHER_AES_CTR: - case RTE_CRYPTO_CIPHER_AES_GCM: sess->cipher.mode = OPENSSL_CIPHER_LIB; sess->cipher.algo = xform->cipher.algo; sess->cipher.ctx = EVP_CIPHER_CTX_new(); @@ -308,7 +336,7 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, break; default: sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; - return -EINVAL; + return -ENOTSUP; } return 0; @@ -326,11 +354,33 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, /* Select auth algo */ switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_AES_GMAC: - case RTE_CRYPTO_AUTH_AES_GCM: - /* Check additional condition for AES_GMAC/GCM */ - if (sess->cipher.algo != RTE_CRYPTO_CIPHER_AES_GCM) - return -EINVAL; sess->chain_order = OPENSSL_CHAIN_COMBINED; + + /* Set IV parameters */ + sess->iv.offset = xform->auth.iv.offset; + sess->iv.length = xform->auth.iv.length; + + /* + * OpenSSL requires GMAC to be a GCM operation + * with no cipher data length + */ + sess->cipher.mode = OPENSSL_CIPHER_LIB; + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + else + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; + + sess->cipher.key.length = xform->auth.key.length; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM, + sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(xform->auth.key.data, xform->auth.key.length, + sess->cipher.key.data); + break; case RTE_CRYPTO_AUTH_MD5: @@ -362,9 +412,55 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, break; default: - return -EINVAL; + return -ENOTSUP; } + sess->auth.digest_length = xform->auth.digest_length; + + return 0; +} + +/* Set session AEAD parameters */ +static int +openssl_set_session_aead_parameters(struct openssl_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + /* Select cipher direction */ + sess->cipher.direction = xform->cipher.op; + /* Select cipher key */ + sess->cipher.key.length = xform->aead.key.length; + + /* Set IV parameters */ + sess->iv.offset = xform->aead.iv.offset; + sess->iv.length = xform->aead.iv.length; + + /* Select auth generate/verify */ + sess->auth.operation = xform->auth.op; + sess->auth.algo = xform->auth.algo; + + /* Select auth algo */ + switch (xform->aead.algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + sess->cipher.mode = OPENSSL_CIPHER_LIB; + sess->aead_algo = xform->aead.algo; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(sess->aead_algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, + sess->cipher.key.data); + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + break; + default: + return -ENOTSUP; + } + + sess->auth.aad_length = xform->aead.aad_length; + sess->auth.digest_length = xform->aead.digest_length; + return 0; } @@ -375,6 +471,8 @@ openssl_set_session_parameters(struct openssl_session *sess, { const struct rte_crypto_sym_xform *cipher_xform = NULL; const struct rte_crypto_sym_xform *auth_xform = NULL; + const struct rte_crypto_sym_xform *aead_xform = NULL; + int ret; sess->chain_order = openssl_get_chain_order(xform); switch (sess->chain_order) { @@ -392,25 +490,42 @@ openssl_set_session_parameters(struct openssl_session *sess, auth_xform = xform; cipher_xform = xform->next; break; + case OPENSSL_CHAIN_COMBINED: + aead_xform = xform; + break; default: return -EINVAL; } + /* Default IV length = 0 */ + sess->iv.length = 0; + /* cipher_xform must be check before auth_xform */ if (cipher_xform) { - if (openssl_set_session_cipher_parameters( - sess, cipher_xform)) { + ret = openssl_set_session_cipher_parameters( + sess, cipher_xform); + if (ret != 0) { OPENSSL_LOG_ERR( "Invalid/unsupported cipher parameters"); - return -EINVAL; + return ret; } } if (auth_xform) { - if (openssl_set_session_auth_parameters(sess, auth_xform)) { + ret = openssl_set_session_auth_parameters(sess, auth_xform); + if (ret != 0) { OPENSSL_LOG_ERR( "Invalid/unsupported auth parameters"); - return -EINVAL; + return ret; + } + } + + if (aead_xform) { + ret = openssl_set_session_aead_parameters(sess, aead_xform); + if (ret != 0) { + OPENSSL_LOG_ERR( + "Invalid/unsupported AEAD parameters"); + return ret; } } @@ -445,29 +560,35 @@ get_session(struct openssl_qp *qp, struct rte_crypto_op *op) { struct openssl_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { /* get existing session */ - if (likely(op->sym->session != NULL && - op->sym->session->dev_type == - RTE_CRYPTODEV_OPENSSL_PMD)) + if (likely(op->sym->session != NULL)) sess = (struct openssl_session *) - op->sym->session->_private; - } else { + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct openssl_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(openssl_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct openssl_session *)_sess_private_data; + + if (unlikely(openssl_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (sess == NULL) @@ -912,6 +1033,7 @@ process_openssl_combined_op /* cipher */ uint8_t *dst = NULL, *iv, *tag, *aad; int srclen, ivlen, aadlen, status = -1; + uint32_t offset; /* * Segmented destination buffer is not supported for @@ -922,34 +1044,41 @@ process_openssl_combined_op return; } - iv = op->sym->cipher.iv.data; - ivlen = op->sym->cipher.iv.length; - aad = op->sym->auth.aad.data; - aadlen = op->sym->auth.aad.length; - - tag = op->sym->auth.digest.data; - if (tag == NULL) - tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, - op->sym->cipher.data.offset + - op->sym->cipher.data.length); - - if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + ivlen = sess->iv.length; + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { srclen = 0; - else { - srclen = op->sym->cipher.data.length; + offset = op->sym->auth.data.offset; + aadlen = op->sym->auth.data.length; + aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, + op->sym->auth.data.offset); + tag = op->sym->auth.digest.data; + if (tag == NULL) + tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + offset + aadlen); + } else { + srclen = op->sym->aead.data.length; dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, - op->sym->cipher.data.offset); + op->sym->aead.data.offset); + offset = op->sym->aead.data.offset; + aad = op->sym->aead.aad.data; + aadlen = sess->auth.aad_length; + tag = op->sym->aead.digest.data; + if (tag == NULL) + tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + offset + srclen); } if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) status = process_openssl_auth_encryption_gcm( - mbuf_src, op->sym->cipher.data.offset, srclen, + mbuf_src, offset, srclen, aad, aadlen, iv, ivlen, sess->cipher.key.data, dst, tag, sess->cipher.ctx, sess->cipher.evp_algo); else status = process_openssl_auth_decryption_gcm( - mbuf_src, op->sym->cipher.data.offset, srclen, + mbuf_src, offset, srclen, aad, aadlen, iv, ivlen, sess->cipher.key.data, dst, tag, sess->cipher.ctx, sess->cipher.evp_algo); @@ -986,7 +1115,8 @@ process_openssl_cipher_op dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); - iv = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); if (sess->cipher.mode == OPENSSL_CIPHER_LIB) if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) @@ -1027,7 +1157,8 @@ process_openssl_docsis_bpi_op(struct rte_crypto_op *op, dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); - iv = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); block_size = DES_BLOCK_SIZE; @@ -1085,7 +1216,8 @@ process_openssl_docsis_bpi_op(struct rte_crypto_op *op, dst, iv, last_block_len, sess->cipher.bpi_ctx); /* Prepare parameters for CBC mode op */ - iv = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); dst += last_block_len - srclen; srclen -= last_block_len; } @@ -1116,7 +1248,7 @@ process_openssl_auth_op if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, - op->sym->auth.digest.length); + sess->auth.digest_length); else { dst = op->sym->auth.digest.data; if (dst == NULL) @@ -1144,11 +1276,11 @@ process_openssl_auth_op if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { if (memcmp(dst, op->sym->auth.digest.data, - op->sym->auth.digest.length) != 0) { + sess->auth.digest_length) != 0) { op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(mbuf_src, op->sym->auth.digest.length); + rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length); } if (status != 0) @@ -1195,9 +1327,12 @@ process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, } /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { openssl_reset_session(sess); memset(sess, 0, sizeof(struct openssl_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -1275,15 +1410,16 @@ cryptodev_openssl_create(const char *name, snprintf(init_params->name, sizeof(init_params->name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, + dev = rte_cryptodev_vdev_pmd_init(init_params->name, sizeof(struct openssl_private), - init_params->socket_id); + init_params->socket_id, + vdev); if (dev == NULL) { OPENSSL_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_OPENSSL_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_openssl_pmd_ops; /* register rx/tx burst functions for data path */ @@ -1329,7 +1465,7 @@ cryptodev_openssl_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -1372,3 +1508,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_openssl_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c index 22a68730..8cdd0b2e 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c +++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -48,16 +48,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_MD5_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 16, .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -78,7 +78,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -90,16 +90,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 20, .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -120,7 +120,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -132,16 +132,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 28, .max = 28, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -162,7 +162,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 28, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -174,40 +174,40 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, { /* SHA256 */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_SHA256, - .block_size = 64, - .key_size = { - .min = 0, - .max = 0, - .increment = 0 - }, - .digest_size = { - .min = 32, - .max = 32, - .increment = 0 - }, - .aad_size = { 0 } - }, } + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + .iv_size = { 0 } }, } - }, + }, } + }, { /* SHA384 HMAC */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { @@ -216,16 +216,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 48, .max = 48, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -246,7 +246,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 48, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -258,16 +258,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 64, .max = 64, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -288,7 +288,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 64, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -332,12 +332,12 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { }, } }, } }, - { /* AES GCM (AUTH) */ + { /* AES GCM */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_AES_GCM, + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, .block_size = 16, .key_size = { .min = 16, @@ -353,27 +353,12 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .min = 0, .max = 65535, .increment = 1 - } - }, } - }, } - }, - { /* AES GCM (CIPHER) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_GCM, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 8 }, .iv_size = { .min = 12, .max = 16, .increment = 4 - } + }, }, } }, } }, @@ -394,9 +379,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 16, .increment = 0 }, - .aad_size = { - .min = 8, - .max = 65532, + .iv_size = { + .min = 12, + .max = 16, .increment = 4 } }, } @@ -536,7 +521,7 @@ openssl_pmd_info_get(struct rte_cryptodev *dev, struct openssl_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = openssl_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; @@ -602,7 +587,7 @@ openssl_pmd_qp_create_processed_ops_ring(struct openssl_qp *qp, static int openssl_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct openssl_qp *qp = NULL; @@ -627,7 +612,7 @@ openssl_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->stats, 0, sizeof(qp->stats)); @@ -671,36 +656,57 @@ openssl_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * +static int openssl_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { OPENSSL_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - if (openssl_set_session_parameters( - sess, xform) != 0) { + ret = openssl_set_session_parameters(sess_private_data, xform); + if (ret != 0) { OPENSSL_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -openssl_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +openssl_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) { - openssl_reset_session(sess); - memset(sess, 0, sizeof(struct openssl_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + openssl_reset_session(sess_priv); + memset(sess_priv, 0, sizeof(struct openssl_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } diff --git a/drivers/crypto/openssl/rte_openssl_pmd_private.h b/drivers/crypto/openssl/rte_openssl_pmd_private.h index 4d820c51..b7f74752 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_private.h +++ b/drivers/crypto/openssl/rte_openssl_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -36,6 +36,8 @@ #include <openssl/evp.h> #include <openssl/des.h> +#define CRYPTODEV_NAME_OPENSSL_PMD crypto_openssl +/**< Open SSL Crypto PMD device name */ #define OPENSSL_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ @@ -108,6 +110,15 @@ struct openssl_session { enum openssl_chain_order chain_order; /**< chain order mode */ + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ + + enum rte_crypto_aead_algorithm aead_algo; + /**< AEAD algorithm */ + /** Cipher Parameters */ struct { enum rte_crypto_cipher_operation direction; @@ -157,6 +168,11 @@ struct openssl_session { /**< pointer to EVP context structure */ } hmac; }; + + uint16_t aad_length; + /**< AAD length */ + uint16_t digest_length; + /**< digest length */ } auth; } __rte_cache_aligned; diff --git a/drivers/crypto/qat/qat_adf/qat_algs.h b/drivers/crypto/qat/qat_adf/qat_algs.h index 5c63406b..2c8e03c0 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs.h +++ b/drivers/crypto/qat/qat_adf/qat_algs.h @@ -17,7 +17,7 @@ * qat-linux@intel.com * * BSD LICENSE - * Copyright(c) 2015-2016 Intel Corporation. + * Copyright(c) 2015-2017 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -51,6 +51,7 @@ #include "icp_qat_hw.h" #include "icp_qat_fw.h" #include "icp_qat_fw_la.h" +#include "../qat_crypto.h" /* * Key Modifier (KM) value used in KASUMI algorithm in F9 mode to XOR @@ -127,15 +128,17 @@ struct qat_session { struct icp_qat_fw_la_bulk_req fw_req; uint8_t aad_len; struct qat_crypto_instance *inst; + struct { + uint16_t offset; + uint16_t length; + } cipher_iv; + struct { + uint16_t offset; + uint16_t length; + } auth_iv; + uint16_t digest_length; rte_spinlock_t lock; /* protects this struct */ -}; - -struct qat_alg_ablkcipher_cd { - struct icp_qat_hw_cipher_algo_blk *cd; - phys_addr_t cd_paddr; - struct icp_qat_fw_la_bulk_req fw_req; - struct qat_crypto_instance *inst; - rte_spinlock_t lock; /* protects this struct */ + enum qat_device_gen min_qat_dev_gen; }; int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg); @@ -147,21 +150,13 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cd, int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, uint8_t *authkey, uint32_t authkeylen, - uint32_t add_auth_data_length, + uint32_t aad_length, uint32_t digestsize, unsigned int operation); void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, enum qat_crypto_proto_flag proto_flags); -void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_cd *cd, - int alg, const uint8_t *key, - unsigned int keylen); - -void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_cd *cd, - int alg, const uint8_t *key, - unsigned int keylen); - int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg); int qat_alg_validate_aes_docsisbpi_key(int key_len, enum icp_qat_hw_cipher_algo *alg); diff --git a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c index 154e1ddd..2d16c9e2 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c +++ b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c @@ -17,7 +17,7 @@ * qat-linux@intel.com * * BSD LICENSE - * Copyright(c) 2015-2016 Intel Corporation. + * Copyright(c) 2015-2017 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -121,6 +121,9 @@ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg) case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: return QAT_HW_ROUND_UP(ICP_QAT_HW_KASUMI_F9_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_NULL: + return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum state1 size in this case */ return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, @@ -603,6 +606,7 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc, cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; + cdesc->min_qat_dev_gen = QAT_GEN2; } else { total_key_size = cipherkeylen; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; @@ -661,7 +665,7 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc, int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, uint8_t *authkey, uint32_t authkeylen, - uint32_t add_auth_data_length, + uint32_t aad_length, uint32_t digestsize, unsigned int operation) { @@ -810,13 +814,14 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, * in big-endian format. This field is 8 bytes */ auth_param->u2.aad_sz = - RTE_ALIGN_CEIL(add_auth_data_length, 16); + RTE_ALIGN_CEIL(aad_length, 16); auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3; aad_len = (uint32_t *)(cdesc->cd_cur_ptr + ICP_QAT_HW_GALOIS_128_STATE1_SZ + ICP_QAT_HW_GALOIS_H_SZ); - *aad_len = rte_bswap32(add_auth_data_length); + *aad_len = rte_bswap32(aad_length); + cdesc->aad_len = aad_length; break; case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; @@ -837,8 +842,7 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, 0, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + authkeylen + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; - auth_param->hash_state_sz = - RTE_ALIGN_CEIL(add_auth_data_length, 16) >> 3; + auth_param->hash_state_sz = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; break; case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: hash->auth_config.config = @@ -854,8 +858,8 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); cdesc->cd_cur_ptr += state1_size + state2_size + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; - auth_param->hash_state_sz = - RTE_ALIGN_CEIL(add_auth_data_length, 16) >> 3; + auth_param->hash_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; + cdesc->min_qat_dev_gen = QAT_GEN2; break; case ICP_QAT_HW_AUTH_ALGO_MD5: @@ -868,6 +872,9 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, state2_size = ICP_QAT_HW_MD5_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_NULL: + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_NULL); + state2_size = ICP_QAT_HW_NULL_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: state1_size = qat_hash_get_state1_size( diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index 386aa453..1f52cabf 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -60,6 +60,7 @@ #include <rte_spinlock.h> #include <rte_hexdump.h> #include <rte_crypto_sym.h> +#include <rte_cryptodev_pci.h> #include <openssl/evp.h> #include "qat_logs.h" @@ -170,16 +171,19 @@ cipher_decrypt_err: /** Creates a context in either AES or DES in ECB mode * Depends on openssl libcrypto */ -static void * +static int bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, enum rte_crypto_cipher_operation direction __rte_unused, - uint8_t *key) + uint8_t *key, void **ctx) { const EVP_CIPHER *algo = NULL; - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + int ret; + *ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) + if (*ctx == NULL) { + ret = -ENOMEM; goto ctx_init_err; + } if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI) algo = EVP_des_ecb(); @@ -187,15 +191,17 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, algo = EVP_aes_128_ecb(); /* IV will be ECB encrypted whether direction is encrypt or decrypt*/ - if (EVP_EncryptInit_ex(ctx, algo, NULL, key, 0) != 1) + if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) { + ret = -EINVAL; goto ctx_init_err; + } - return ctx; + return 0; ctx_init_err: - if (ctx != NULL) - EVP_CIPHER_CTX_free(ctx); - return NULL; + if (*ctx != NULL) + EVP_CIPHER_CTX_free(*ctx); + return ret; } /** Frees a context previously created @@ -213,25 +219,25 @@ adf_modulo(uint32_t data, uint32_t shift); static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie); + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp); -void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, - void *session) +void +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - struct qat_session *sess = session; - phys_addr_t cd_paddr; - PMD_INIT_FUNC_TRACE(); - if (sess) { - if (sess->bpi_ctx) { - bpi_cipher_ctx_free(sess->bpi_ctx); - sess->bpi_ctx = NULL; - } - cd_paddr = sess->cd_paddr; - memset(sess, 0, qat_crypto_sym_get_session_private_size(dev)); - sess->cd_paddr = cd_paddr; - } else - PMD_DRV_LOG(ERR, "NULL session"); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + struct qat_session *s = (struct qat_session *)sess_priv; + + if (sess_priv) { + if (s->bpi_ctx) + bpi_cipher_ctx_free(s->bpi_ctx); + memset(s, 0, qat_crypto_sym_get_session_private_size(dev)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } static int @@ -245,6 +251,14 @@ qat_get_cmd_id(const struct rte_crypto_sym_xform *xform) if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL) return ICP_QAT_FW_LA_CMD_AUTH; + /* AEAD */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + else + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + } + if (xform->next == NULL) return -1; @@ -286,38 +300,37 @@ qat_get_cipher_xform(struct rte_crypto_sym_xform *xform) return NULL; } -void * + +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private) + struct rte_crypto_sym_xform *xform, + struct qat_session *session) { - struct qat_session *session = session_private; struct qat_pmd_private *internals = dev->data->dev_private; struct rte_crypto_cipher_xform *cipher_xform = NULL; + int ret; /* Get cipher xform from crypto xform chain */ cipher_xform = qat_get_cipher_xform(xform); + session->cipher_iv.offset = cipher_xform->iv.offset; + session->cipher_iv.length = cipher_xform->iv.length; + switch (cipher_xform->algo) { case RTE_CRYPTO_CIPHER_AES_CBC: if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; break; - case RTE_CRYPTO_CIPHER_AES_GCM: - if (qat_alg_validate_aes_key(cipher_xform->key.length, - &session->qat_cipher_alg) != 0) { - PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); - goto error_out; - } - session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; - break; case RTE_CRYPTO_CIPHER_AES_CTR: if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; @@ -326,6 +339,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_snow3g_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; @@ -337,6 +351,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_kasumi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE; @@ -345,6 +360,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -353,6 +369,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -361,38 +378,43 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; break; case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create DES BPI ctx"); goto error_out; } if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; break; case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create AES BPI ctx"); goto error_out; } if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -403,27 +425,30 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_cipher_algorithm_strings [cipher_xform->algo]); + ret = -ENOTSUP; goto error_out; } if (qat_alg_validate_zuc_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; break; case RTE_CRYPTO_CIPHER_3DES_ECB: case RTE_CRYPTO_CIPHER_AES_ECB: - case RTE_CRYPTO_CIPHER_AES_CCM: case RTE_CRYPTO_CIPHER_AES_F8: case RTE_CRYPTO_CIPHER_AES_XTS: case RTE_CRYPTO_CIPHER_ARC4: PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u", cipher_xform->algo); + ret = -ENOTSUP; goto error_out; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n", cipher_xform->algo); + ret = -EINVAL; goto error_out; } @@ -434,50 +459,119 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_aead_session_create_content_desc_cipher(session, cipher_xform->key.data, - cipher_xform->key.length)) + cipher_xform->key.length)) { + ret = -EINVAL; goto error_out; + } - return session; + return 0; error_out: if (session->bpi_ctx) { bpi_cipher_ctx_free(session->bpi_ctx); session->bpi_ctx = NULL; } - return NULL; + return ret; } - -void * +int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_session *session = session_private; + int ret; int qat_cmd_id; PMD_INIT_FUNC_TRACE(); + /* Set context descriptor physical address */ + session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + offsetof(struct qat_session, cd); + + session->min_qat_dev_gen = QAT_GEN1; + /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) { PMD_DRV_LOG(ERR, "Unsupported xform chain requested"); - goto error_out; + return -ENOTSUP; } session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id; switch (session->qat_cmd) { case ICP_QAT_FW_LA_CMD_CIPHER: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_AUTH: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_CIPHER_HASH: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, + session); + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_cipher(dev, + xform, session); + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_auth(dev, + xform, session); + if (ret < 0) + return ret; + } break; case ICP_QAT_FW_LA_CMD_HASH_CIPHER: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, + session); + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_auth(dev, + xform, session); + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_cipher(dev, + xform, session); + if (ret < 0) + return ret; + } break; case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM: case ICP_QAT_FW_LA_CMD_TRNG_TEST: @@ -490,30 +584,26 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, case ICP_QAT_FW_LA_CMD_DELIMITER: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; } - return session; - -error_out: - return NULL; + return 0; } -struct qat_session * +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private) + struct qat_session *session) { - - struct qat_session *session = session_private; struct rte_crypto_auth_xform *auth_xform = NULL; - struct rte_crypto_cipher_xform *cipher_xform = NULL; struct qat_pmd_private *internals = dev->data->dev_private; auth_xform = qat_get_auth_xform(xform); + uint8_t *key_data = auth_xform->key.data; + uint8_t key_length = auth_xform->key.length; switch (auth_xform->algo) { case RTE_CRYPTO_AUTH_SHA1_HMAC: @@ -534,11 +624,15 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_AES_XCBC_MAC: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC; break; - case RTE_CRYPTO_AUTH_AES_GCM: - session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; - break; case RTE_CRYPTO_AUTH_AES_GMAC: + if (qat_alg_validate_aes_key(auth_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; + break; case RTE_CRYPTO_AUTH_SNOW3G_UIA2: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2; @@ -557,7 +651,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_auth_algorithm_strings [auth_xform->algo]); - goto error_out; + return -ENOTSUP; } session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3; break; @@ -567,42 +661,149 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_SHA224: case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_MD5: - case RTE_CRYPTO_AUTH_AES_CCM: case RTE_CRYPTO_AUTH_AES_CMAC: case RTE_CRYPTO_AUTH_AES_CBC_MAC: PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u", auth_xform->algo); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified", auth_xform->algo); - goto error_out; + return -EINVAL; } - cipher_xform = qat_get_cipher_xform(xform); - if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) || - (session->qat_hash_alg == - ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) { + session->auth_iv.offset = auth_xform->iv.offset; + session->auth_iv.length = auth_xform->iv.length; + + if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) { + if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) { + session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + /* + * It needs to create cipher desc content first, + * then authentication + */ + if (qat_alg_aead_session_create_content_desc_cipher(session, + auth_xform->key.data, + auth_xform->key.length)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_auth(session, + key_data, + key_length, + 0, + auth_xform->digest_length, + auth_xform->op)) + return -EINVAL; + } else { + session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + /* + * It needs to create authentication desc content first, + * then cipher + */ + if (qat_alg_aead_session_create_content_desc_auth(session, + key_data, + key_length, + 0, + auth_xform->digest_length, + auth_xform->op)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_cipher(session, + auth_xform->key.data, + auth_xform->key.length)) + return -EINVAL; + } + /* Restore to authentication only only */ + session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH; + } else { if (qat_alg_aead_session_create_content_desc_auth(session, - cipher_xform->key.data, - cipher_xform->key.length, - auth_xform->add_auth_data_length, + key_data, + key_length, + 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; + } + + session->digest_length = auth_xform->digest_length; + return 0; +} + +int +qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, + struct qat_session *session) +{ + struct rte_crypto_aead_xform *aead_xform = &xform->aead; + + /* + * Store AEAD IV parameters as cipher IV, + * to avoid unnecessary memory usage + */ + session->cipher_iv.offset = xform->aead.iv.offset; + session->cipher_iv.length = xform->aead.iv.length; + + switch (aead_xform->algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + if (qat_alg_validate_aes_key(aead_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported AEAD alg %u", + aead_xform->algo); + return -ENOTSUP; + default: + PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", + aead_xform->algo); + return -EINVAL; + } + + if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + /* + * It needs to create cipher desc content first, + * then authentication + */ + if (qat_alg_aead_session_create_content_desc_cipher(session, + aead_xform->key.data, + aead_xform->key.length)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_auth(session, + aead_xform->key.data, + aead_xform->key.length, + aead_xform->aad_length, + aead_xform->digest_length, + RTE_CRYPTO_AUTH_OP_GENERATE)) + return -EINVAL; } else { + session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + /* + * It needs to create authentication desc content first, + * then cipher + */ if (qat_alg_aead_session_create_content_desc_auth(session, - auth_xform->key.data, - auth_xform->key.length, - auth_xform->add_auth_data_length, - auth_xform->digest_length, - auth_xform->op)) - goto error_out; + aead_xform->key.data, + aead_xform->key.length, + aead_xform->aad_length, + aead_xform->digest_length, + RTE_CRYPTO_AUTH_OP_VERIFY)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_cipher(session, + aead_xform->key.data, + aead_xform->key.length)) + return -EINVAL; } - return session; -error_out: - return NULL; + session->digest_length = aead_xform->digest_length; + return 0; } unsigned qat_crypto_sym_get_session_private_size( @@ -617,7 +818,8 @@ qat_bpicipher_preprocess(struct qat_session *ctx, { uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = sym_op->cipher.data.length % block_len; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) { @@ -641,7 +843,8 @@ qat_bpicipher_preprocess(struct qat_session *ctx, iv = last_block - block_len; else /* runt block, i.e. less than one full block */ - iv = sym_op->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX rte_hexdump(stdout, "BPI: src before pre-process:", last_block, @@ -670,7 +873,8 @@ qat_bpicipher_postprocess(struct qat_session *ctx, { uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = sym_op->cipher.data.length % block_len; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; if (last_block_len > 0 && ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { @@ -696,7 +900,8 @@ qat_bpicipher_postprocess(struct qat_session *ctx, iv = dst - block_len; else /* runt block, i.e. less than one full block */ - iv = sym_op->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX rte_hexdump(stdout, "BPI: src before post-process:", last_block, @@ -752,12 +957,12 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, while (nb_ops_sent != nb_ops_possible) { ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail, - tmp_qp->op_cookies[tail / queue->msg_size]); + tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp); if (ret != 0) { tmp_qp->stats.enqueue_err_count++; /* * This message cannot be enqueued, - * decrease number of ops that wasnt sent + * decrease number of ops that wasn't sent */ rte_atomic16_sub(&tmp_qp->inflights16, nb_ops_possible - nb_ops_sent); @@ -808,7 +1013,10 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { struct qat_session *sess = (struct qat_session *) - (rx_op->sym->session->_private); + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + if (sess->bpi_ctx) qat_bpicipher_postprocess(sess, rx_op); rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -882,23 +1090,44 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, return 0; } +static inline void +set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, + struct icp_qat_fw_la_cipher_req_params *cipher_param, + struct rte_crypto_op *op, + struct icp_qat_fw_la_bulk_req *qat_req) +{ + /* copy IV into request if it fits */ + if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) { + rte_memcpy(cipher_param->u.cipher_IV_array, + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset), + iv_length); + } else { + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_CIPH_IV_64BIT_PTR); + cipher_param->u.s.cipher_IV_ptr = + rte_crypto_op_ctophys_offset(op, + iv_offset); + } +} + static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie) + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp) { int ret = 0; struct qat_session *ctx; struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; register struct icp_qat_fw_la_bulk_req *qat_req; - uint8_t do_auth = 0, do_cipher = 0; + uint8_t do_auth = 0, do_cipher = 0, do_aead = 0; uint32_t cipher_len = 0, cipher_ofs = 0; uint32_t auth_len = 0, auth_ofs = 0; uint32_t min_ofs = 0; uint64_t src_buf_start = 0, dst_buf_start = 0; uint8_t do_sgl = 0; - #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) { PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto " @@ -907,18 +1136,26 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } #endif - if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) { + if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented" " requests, op (%p) is sessionless.", op); return -EINVAL; } - if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) { + ctx = (struct qat_session *)get_session_private_data( + op->sym->session, cryptodev_qat_driver_id); + + if (unlikely(ctx == NULL)) { PMD_DRV_LOG(ERR, "Session was not created for this device"); return -EINVAL; } - ctx = (struct qat_session *)op->sym->session->_private; + if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) { + PMD_DRV_LOG(ERR, "Session alg not supported on this device gen"); + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -EINVAL; + } + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -926,9 +1163,15 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || - ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { - do_auth = 1; - do_cipher = 1; + ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { + /* AES-GCM */ + if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + do_aead = 1; + } else { + do_auth = 1; + do_cipher = 1; + } } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { do_auth = 1; do_cipher = 0; @@ -970,26 +1213,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, cipher_ofs = op->sym->cipher.data.offset; } - /* copy IV into request if it fits */ - /* - * If IV length is zero do not copy anything but still - * use request descriptor embedded IV - * - */ - if (op->sym->cipher.iv.length) { - if (op->sym->cipher.iv.length <= - sizeof(cipher_param->u.cipher_IV_array)) { - rte_memcpy(cipher_param->u.cipher_IV_array, - op->sym->cipher.iv.data, - op->sym->cipher.iv.length); - } else { - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_CIPH_IV_64BIT_PTR); - cipher_param->u.s.cipher_IV_ptr = - op->sym->cipher.iv.phys_addr; - } - } + set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, + cipher_param, op, qat_req); min_ofs = cipher_ofs; } @@ -1009,34 +1234,70 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_ofs = op->sym->auth.data.offset >> 3; auth_len = op->sym->auth.data.length >> 3; - if (ctx->qat_hash_alg == - ICP_QAT_HW_AUTH_ALGO_KASUMI_F9) { - if (do_cipher) { - auth_len = auth_len + auth_ofs + 1 - - ICP_QAT_HW_KASUMI_BLK_SZ; - auth_ofs = ICP_QAT_HW_KASUMI_BLK_SZ; - } else { - auth_len = auth_len + auth_ofs + 1; - auth_ofs = 0; - } - } + auth_param->u1.aad_adr = + rte_crypto_op_ctophys_offset(op, + ctx->auth_iv.offset); } else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { - auth_ofs = op->sym->cipher.data.offset; - auth_len = op->sym->cipher.data.length; + /* AES-GMAC */ + set_cipher_iv(ctx->auth_iv.length, + ctx->auth_iv.offset, + cipher_param, op, qat_req); + auth_ofs = op->sym->auth.data.offset; + auth_len = op->sym->auth.data.length; + + auth_param->u1.aad_adr = 0; + auth_param->u2.aad_sz = 0; + + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->auth_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + + } } else { auth_ofs = op->sym->auth.data.offset; auth_len = op->sym->auth.data.length; + } min_ofs = auth_ofs; auth_param->auth_res_addr = op->sym->auth.digest.phys_addr; - auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr; + } + if (do_aead) { + if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->cipher_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + } + + } + + cipher_len = op->sym->aead.data.length; + cipher_ofs = op->sym->aead.data.offset; + auth_len = op->sym->aead.data.length; + auth_ofs = op->sym->aead.data.offset; + + auth_param->u1.aad_adr = op->sym->aead.aad.phys_addr; + auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; + set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, + cipher_param, op, qat_req); + min_ofs = op->sym->aead.data.offset; } if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next)) @@ -1080,7 +1341,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, dst_buf_start = src_buf_start; } - if (do_cipher) { + if (do_cipher || do_aead) { cipher_param->cipher_offset = (uint32_t)rte_pktmbuf_mtophys_offset( op->sym->m_src, cipher_ofs) - src_buf_start; @@ -1089,7 +1350,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, cipher_param->cipher_offset = 0; cipher_param->cipher_length = 0; } - if (do_auth) { + + if (do_auth || do_aead) { auth_param->auth_off = (uint32_t)rte_pktmbuf_mtophys_offset( op->sym->m_src, auth_ofs) - src_buf_start; auth_param->auth_len = auth_len; @@ -1097,6 +1359,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_param->auth_off = 0; auth_param->auth_len = 0; } + qat_req->comn_mid.dst_length = qat_req->comn_mid.src_length = (cipher_param->cipher_offset + cipher_param->cipher_length) @@ -1142,48 +1405,38 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, qat_req->comn_mid.dest_data_addr = dst_buf_start; } - if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || - ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { - if (op->sym->cipher.iv.length == 12) { - /* - * For GCM a 12 bit IV is allowed, - * but we need to inform the f/w - */ - ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); - } - if (op->sym->cipher.data.length == 0) { - /* - * GMAC - */ - qat_req->comn_mid.dest_data_addr = - qat_req->comn_mid.src_data_addr = - op->sym->auth.aad.phys_addr; - qat_req->comn_mid.dst_length = - qat_req->comn_mid.src_length = - rte_pktmbuf_data_len(op->sym->m_src); - cipher_param->cipher_length = 0; - cipher_param->cipher_offset = 0; - auth_param->u1.aad_adr = 0; - auth_param->auth_len = op->sym->auth.aad.length; - auth_param->auth_off = op->sym->auth.data.offset; - auth_param->u2.aad_sz = 0; - } - } - #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX rte_hexdump(stdout, "qat_req:", qat_req, sizeof(struct icp_qat_fw_la_bulk_req)); rte_hexdump(stdout, "src_data:", rte_pktmbuf_mtod(op->sym->m_src, uint8_t*), rte_pktmbuf_data_len(op->sym->m_src)); - rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data, - op->sym->cipher.iv.length); - rte_hexdump(stdout, "digest:", op->sym->auth.digest.data, - op->sym->auth.digest.length); - rte_hexdump(stdout, "aad:", op->sym->auth.aad.data, - op->sym->auth.aad.length); + if (do_cipher) { + uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op, + uint8_t *, + ctx->cipher_iv.offset); + rte_hexdump(stdout, "cipher iv:", cipher_iv_ptr, + ctx->cipher_iv.length); + } + + if (do_auth) { + if (ctx->auth_iv.length) { + uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op, + uint8_t *, + ctx->auth_iv.offset); + rte_hexdump(stdout, "auth iv:", auth_iv_ptr, + ctx->auth_iv.length); + } + rte_hexdump(stdout, "digest:", op->sym->auth.digest.data, + ctx->digest_length); + } + + if (do_aead) { + rte_hexdump(stdout, "digest:", op->sym->aead.digest.data, + ctx->digest_length); + rte_hexdump(stdout, "aad:", op->sym->aead.aad.data, + ctx->aad_len); + } #endif return 0; } @@ -1196,17 +1449,6 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) return data - mult; } -void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess) -{ - struct rte_cryptodev_sym_session *sess = sym_sess; - struct qat_session *s = (void *)sess->_private; - - PMD_INIT_FUNC_TRACE(); - s->cd_paddr = rte_mempool_virt2phy(mp, sess) + - offsetof(struct qat_session, cd) + - offsetof(struct rte_cryptodev_sym_session, _private); -} - int qat_dev_config(__rte_unused struct rte_cryptodev *dev, __rte_unused struct rte_cryptodev_config *config) { @@ -1240,8 +1482,8 @@ int qat_dev_close(struct rte_cryptodev *dev) return 0; } -void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev, - struct rte_cryptodev_info *info) +void qat_dev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) { struct qat_pmd_private *internals = dev->data->dev_private; @@ -1253,7 +1495,8 @@ void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev, info->feature_flags = dev->feature_flags; info->capabilities = internals->qat_dev_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD; + info->driver_id = cryptodev_qat_driver_id; + info->pci_dev = RTE_DEV_TO_PCI(dev->device); } } diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index b740d6b0..3f35a00e 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -39,6 +39,9 @@ #include "qat_crypto_capabilities.h" +#define CRYPTODEV_NAME_QAT_SYM_PMD crypto_qat +/**< Intel QAT Symmetric Crypto PMD device name */ + /* * This macro rounds up a number to a be a multiple of * the alignment when the alignment is a power of 2 @@ -47,6 +50,13 @@ (((num) + (align) - 1) & ~((align) - 1)) #define QAT_64_BTYE_ALIGN_MASK (~0x3f) +struct qat_session; + +enum qat_device_gen { + QAT_GEN1 = 1, + QAT_GEN2, +}; + /** * Structure associated with each queue. */ @@ -74,6 +84,7 @@ struct qat_qp { struct rte_mempool *op_cookie_pool; void **op_cookies; uint32_t nb_descriptors; + enum qat_device_gen qat_dev_gen; } __rte_cache_aligned; /** private data structure for each QAT device */ @@ -82,9 +93,13 @@ struct qat_pmd_private { /**< Max number of queue pairs supported by device */ unsigned max_nb_sessions; /**< Max number of sessions supported by device */ + enum qat_device_gen qat_dev_gen; + /**< QAT device generation */ const struct rte_cryptodev_capabilities *qat_dev_capabilities; }; +extern uint8_t cryptodev_qat_driver_id; + int qat_dev_config(struct rte_cryptodev *dev, struct rte_cryptodev_config *config); int qat_dev_start(struct rte_cryptodev *dev); @@ -98,7 +113,8 @@ void qat_crypto_sym_stats_get(struct rte_cryptodev *dev, void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev); int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, - const struct rte_cryptodev_qp_conf *rx_conf, int socket_id); + const struct rte_cryptodev_qp_conf *rx_conf, int socket_id, + struct rte_mempool *session_pool); int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id); @@ -109,26 +125,35 @@ qat_pmd_session_mempool_create(struct rte_cryptodev *dev, extern unsigned qat_crypto_sym_get_session_private_size(struct rte_cryptodev *dev); -extern void -qat_crypto_sym_session_init(struct rte_mempool *mempool, void *priv_sess); - -extern void * +extern int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool); + + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private); -struct qat_session * +int +qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, + struct qat_session *session); + +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private); + struct qat_session *session); -void * +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private); + struct rte_crypto_sym_xform *xform, + struct qat_session *session); extern void -qat_crypto_sym_clear_session(struct rte_cryptodev *dev, void *session); - +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *session); extern uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, diff --git a/drivers/crypto/qat/qat_crypto_capabilities.h b/drivers/crypto/qat/qat_crypto_capabilities.h index 1294f247..70120072 100644 --- a/drivers/crypto/qat/qat_crypto_capabilities.h +++ b/drivers/crypto/qat/qat_crypto_capabilities.h @@ -34,7 +34,7 @@ #ifndef _QAT_CRYPTO_CAPABILITIES_H_ #define _QAT_CRYPTO_CAPABILITIES_H_ -#define QAT_BASE_CPM16_SYM_CAPABILITIES \ +#define QAT_BASE_GEN1_SYM_CAPABILITIES \ { /* SHA1 HMAC */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ @@ -43,16 +43,16 @@ .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 20, \ .max = 20, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -64,16 +64,16 @@ .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 28, \ .max = 28, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -85,16 +85,16 @@ .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 32, \ .max = 32, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -104,18 +104,18 @@ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ {.auth = { \ .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, \ - .block_size = 64, \ + .block_size = 128, \ .key_size = { \ - .min = 128, \ + .min = 1, \ .max = 128, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 48, \ .max = 48, \ .increment = 0 \ - }, \ - .aad_size = { 0 } \ + }, \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -127,16 +127,16 @@ .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, \ .block_size = 128, \ .key_size = { \ - .min = 128, \ + .min = 1, \ .max = 128, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 64, \ .max = 64, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -148,16 +148,16 @@ .algo = RTE_CRYPTO_AUTH_MD5_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 8, \ + .min = 1, \ .max = 64, \ - .increment = 8 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 16, \ .max = 16, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -178,16 +178,17 @@ .max = 16, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .aad_size = { 0 }, \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ - { /* AES GCM (AUTH) */ \ + { /* AES GCM */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ - {.auth = { \ - .algo = RTE_CRYPTO_AUTH_AES_GCM, \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \ + {.aead = { \ + .algo = RTE_CRYPTO_AEAD_AES_GCM, \ .block_size = 16, \ .key_size = { \ .min = 16, \ @@ -203,7 +204,12 @@ .min = 0, \ .max = 240, \ .increment = 1 \ - } \ + }, \ + .iv_size = { \ + .min = 12, \ + .max = 12, \ + .increment = 0 \ + }, \ }, } \ }, } \ }, \ @@ -224,10 +230,10 @@ .max = 16, \ .increment = 4 \ }, \ - .aad_size = { \ - .min = 1, \ - .max = 65535, \ - .increment = 1 \ + .iv_size = { \ + .min = 12, \ + .max = 12, \ + .increment = 0 \ } \ }, } \ }, } \ @@ -249,7 +255,7 @@ .max = 4, \ .increment = 0 \ }, \ - .aad_size = { \ + .iv_size = { \ .min = 16, \ .max = 16, \ .increment = 0 \ @@ -257,26 +263,6 @@ }, } \ }, } \ }, \ - { /* AES GCM (CIPHER) */ \ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ - {.sym = { \ - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ - {.cipher = { \ - .algo = RTE_CRYPTO_CIPHER_AES_GCM, \ - .block_size = 16, \ - .key_size = { \ - .min = 16, \ - .max = 32, \ - .increment = 8 \ - }, \ - .iv_size = { \ - .min = 12, \ - .max = 12, \ - .increment = 0 \ - } \ - }, } \ - }, } \ - }, \ { /* AES CBC */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ @@ -374,7 +360,7 @@ .max = 0, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, }, \ }, }, \ }, \ @@ -435,11 +421,7 @@ .max = 4, \ .increment = 0 \ }, \ - .aad_size = { \ - .min = 8, \ - .max = 8, \ - .increment = 0 \ - } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -524,7 +506,7 @@ }, } \ } -#define QAT_EXTRA_CPM17_SYM_CAPABILITIES \ +#define QAT_EXTRA_GEN2_SYM_CAPABILITIES \ { /* ZUC (EEA3) */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ @@ -562,7 +544,7 @@ .max = 4, \ .increment = 0 \ }, \ - .aad_size = { \ + .iv_size = { \ .min = 16, \ .max = 16, \ .increment = 0 \ diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c index a358ccd7..5048d214 100644 --- a/drivers/crypto/qat/qat_qp.c +++ b/drivers/crypto/qat/qat_qp.c @@ -36,6 +36,7 @@ #include <rte_malloc.h> #include <rte_memzone.h> #include <rte_cryptodev_pmd.h> +#include <rte_pci.h> #include <rte_atomic.h> #include <rte_prefetch.h> @@ -133,7 +134,7 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool __rte_unused) { struct qat_qp *qp; struct rte_pci_device *pci_dev; @@ -205,7 +206,7 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, adf_configure_queues(qp); adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr); snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "%s_qp_op_%d_%hu", - dev->driver->pci_drv.driver.name, dev->data->dev_id, + pci_dev->driver->driver.name, dev->data->dev_id, queue_pair_id); qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name); @@ -242,6 +243,11 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, offsetof(struct qat_crypto_op_cookie, qat_sgl_list_dst); } + + struct qat_pmd_private *internals + = dev->data->dev_private; + qp->qat_dev_gen = internals->qat_dev_gen; + dev->data->queue_pairs[queue_pair_id] = qp; return 0; @@ -355,11 +361,13 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, return -EINVAL; } + pci_dev = RTE_DEV_TO_PCI(dev->device); + /* * Allocate a memzone for the queue - create a unique name. */ snprintf(queue->memz_name, sizeof(queue->memz_name), "%s_%s_%d_%d_%d", - dev->driver->pci_drv.driver.name, "qp_mem", dev->data->dev_id, + pci_dev->driver->driver.name, "qp_mem", dev->data->dev_id, queue->hw_bundle_number, queue->hw_queue_number); qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes, socket_id); @@ -408,7 +416,6 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, queue->queue_size); - pci_dev = RTE_DEV_TO_PCI(dev->device); io_addr = pci_dev->mem_resource[0].addr; diff --git a/drivers/crypto/qat/rte_qat_cryptodev.c b/drivers/crypto/qat/rte_qat_cryptodev.c index 1bdd30da..7d56fca4 100644 --- a/drivers/crypto/qat/rte_qat_cryptodev.c +++ b/drivers/crypto/qat/rte_qat_cryptodev.c @@ -35,18 +35,21 @@ #include <rte_dev.h> #include <rte_malloc.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_pci.h> #include "qat_crypto.h" #include "qat_logs.h" -static const struct rte_cryptodev_capabilities qat_cpm16_capabilities[] = { - QAT_BASE_CPM16_SYM_CAPABILITIES, +uint8_t cryptodev_qat_driver_id; + +static const struct rte_cryptodev_capabilities qat_gen1_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; -static const struct rte_cryptodev_capabilities qat_cpm17_capabilities[] = { - QAT_BASE_CPM16_SYM_CAPABILITIES, - QAT_EXTRA_CPM17_SYM_CAPABILITIES, +static const struct rte_cryptodev_capabilities qat_gen2_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, + QAT_EXTRA_GEN2_SYM_CAPABILITIES, RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; @@ -70,7 +73,6 @@ static struct rte_cryptodev_ops crypto_qat_ops = { /* Crypto related operations */ .session_get_size = qat_crypto_sym_get_session_private_size, .session_configure = qat_crypto_sym_configure_session, - .session_initialize = qat_crypto_sym_session_init, .session_clear = qat_crypto_sym_clear_session }; @@ -95,8 +97,7 @@ static const struct rte_pci_id pci_id_qat_map[] = { }; static int -crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_drv, - struct rte_cryptodev *cryptodev) +crypto_qat_dev_init(struct rte_cryptodev *cryptodev) { struct qat_pmd_private *internals; @@ -106,7 +107,7 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ RTE_DEV_TO_PCI(cryptodev->device)->addr.devid, RTE_DEV_TO_PCI(cryptodev->device)->addr.function); - cryptodev->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD; + cryptodev->driver_id = cryptodev_qat_driver_id; cryptodev->dev_ops = &crypto_qat_ops; cryptodev->enqueue_burst = qat_pmd_enqueue_op_burst; @@ -121,12 +122,14 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ internals->max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS; switch (RTE_DEV_TO_PCI(cryptodev->device)->id.device_id) { case 0x0443: - internals->qat_dev_capabilities = qat_cpm16_capabilities; + internals->qat_dev_gen = QAT_GEN1; + internals->qat_dev_capabilities = qat_gen1_capabilities; break; case 0x37c9: case 0x19e3: case 0x6f55: - internals->qat_dev_capabilities = qat_cpm17_capabilities; + internals->qat_dev_gen = QAT_GEN2; + internals->qat_dev_capabilities = qat_gen2_capabilities; break; default: PMD_DRV_LOG(ERR, @@ -147,17 +150,25 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ return 0; } -static struct rte_cryptodev_driver rte_qat_pmd = { - .pci_drv = { - .id_table = pci_id_qat_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - .probe = rte_cryptodev_pci_probe, - .remove = rte_cryptodev_pci_remove, - }, - .cryptodev_init = crypto_qat_dev_init, - .dev_private_size = sizeof(struct qat_pmd_private), +static int crypto_qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_cryptodev_pci_generic_probe(pci_dev, + sizeof(struct qat_pmd_private), crypto_qat_dev_init); +} + +static int crypto_qat_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_cryptodev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_qat_pmd = { + .id_table = pci_id_qat_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = crypto_qat_pci_probe, + .remove = crypto_qat_pci_remove }; -RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd.pci_drv); +RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd); RTE_PMD_REGISTER_PCI_TABLE(CRYPTODEV_NAME_QAT_SYM_PMD, pci_id_qat_map); - +RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_qat_pmd, cryptodev_qat_driver_id); diff --git a/drivers/crypto/scheduler/Makefile b/drivers/crypto/scheduler/Makefile index c273e784..b045410c 100644 --- a/drivers/crypto/scheduler/Makefile +++ b/drivers/crypto/scheduler/Makefile @@ -56,5 +56,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += rte_cryptodev_scheduler.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_roundrobin.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_pkt_size_distr.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_failover.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_multicore.c include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c index 319dcf0a..df8634ad 100644 --- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c +++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c @@ -198,7 +198,7 @@ rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -226,12 +226,12 @@ rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) rte_cryptodev_info_get(slave_id, &dev_info); slave->dev_id = slave_id; - slave->dev_type = dev_info.dev_type; + slave->driver_id = dev_info.driver_id; sched_ctx->nb_slaves++; if (update_scheduler_capability(sched_ctx) < 0) { slave->dev_id = 0; - slave->dev_type = 0; + slave->driver_id = 0; sched_ctx->nb_slaves--; CS_LOG_ERR("capabilities update failed"); @@ -257,7 +257,7 @@ rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -314,7 +314,7 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -351,6 +351,13 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, return -1; } break; + case CDEV_SCHED_MODE_MULTICORE: + if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, + multicore_scheduler) < 0) { + CS_LOG_ERR("Failed to load scheduler"); + return -1; + } + break; default: CS_LOG_ERR("Not yet supported"); return -ENOTSUP; @@ -359,13 +366,6 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, return 0; } -int -rte_crpytodev_scheduler_mode_set(uint8_t scheduler_id, - enum rte_cryptodev_scheduler_mode mode) -{ - return rte_cryptodev_scheduler_mode_set(scheduler_id, mode); -} - enum rte_cryptodev_scheduler_mode rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) { @@ -377,7 +377,7 @@ rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -387,12 +387,6 @@ rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) return sched_ctx->mode; } -enum rte_cryptodev_scheduler_mode -rte_crpytodev_scheduler_mode_get(uint8_t scheduler_id) -{ - return rte_cryptodev_scheduler_mode_get(scheduler_id); -} - int rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, uint32_t enable_reorder) @@ -405,7 +399,7 @@ rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -433,7 +427,7 @@ rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -455,7 +449,7 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -467,8 +461,22 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, sched_ctx = dev->data->dev_private; + if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) { + CS_LOG_ERR("Invalid name %s, should be less than " + "%u bytes.\n", scheduler->name, + RTE_CRYPTODEV_NAME_MAX_LEN); + return -EINVAL; + } strncpy(sched_ctx->name, scheduler->name, RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); + + if (strlen(scheduler->description) > + RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) { + CS_LOG_ERR("Invalid description %s, should be less than " + "%u bytes.\n", scheduler->description, + RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1); + return -EINVAL; + } strncpy(sched_ctx->description, scheduler->description, RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN); @@ -512,7 +520,7 @@ rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -580,7 +588,7 @@ rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, return -EINVAL; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h index 2ba6e470..df22f2a9 100644 --- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h +++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h @@ -58,12 +58,17 @@ extern "C" { #define RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES (8) #endif +/** Maximum number of multi-core worker cores */ +#define RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES (64) + /** Round-robin scheduling mode string */ #define SCHEDULER_MODE_NAME_ROUND_ROBIN round-robin /** Packet-size based distribution scheduling mode string */ #define SCHEDULER_MODE_NAME_PKT_SIZE_DISTR packet-size-distr /** Fail-over scheduling mode string */ #define SCHEDULER_MODE_NAME_FAIL_OVER fail-over +/** multi-core scheduling mode string */ +#define SCHEDULER_MODE_NAME_MULTI_CORE multi-core /** * Crypto scheduler PMD operation modes @@ -78,6 +83,8 @@ enum rte_cryptodev_scheduler_mode { CDEV_SCHED_MODE_PKT_SIZE_DISTR, /** Fail-over mode */ CDEV_SCHED_MODE_FAILOVER, + /** multi-core mode */ + CDEV_SCHED_MODE_MULTICORE, CDEV_SCHED_MODE_COUNT /**< number of modes */ }; @@ -116,6 +123,7 @@ struct rte_cryptodev_scheduler; * - 0 if the scheduler is successfully loaded * - -ENOTSUP if the operation is not supported. * - -EBUSY if device is started. + * - -EINVAL if input values are invalid. */ int rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, @@ -186,38 +194,6 @@ enum rte_cryptodev_scheduler_mode rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id); /** - * @deprecated - * Set the scheduling mode - * - * @param scheduler_id - * The target scheduler device ID - * @param mode - * The scheduling mode - * - * @return - * 0 if attaching successful, negative integer if otherwise. - */ -__rte_deprecated -int -rte_crpytodev_scheduler_mode_set(uint8_t scheduler_id, - enum rte_cryptodev_scheduler_mode mode); - -/** - * @deprecated - * Get the current scheduling mode - * - * @param scheduler_id - * The target scheduler device ID - * - * @return - * If successful, returns the scheduling mode, negative integer - * otherwise - */ -__rte_deprecated -enum rte_cryptodev_scheduler_mode -rte_crpytodev_scheduler_mode_get(uint8_t scheduler_id); - -/** * Set the crypto ops reordering feature on/off * * @param scheduler_id @@ -327,6 +303,8 @@ extern struct rte_cryptodev_scheduler *roundrobin_scheduler; extern struct rte_cryptodev_scheduler *pkt_size_based_distr_scheduler; /** Fail-over mode scheduler */ extern struct rte_cryptodev_scheduler *failover_scheduler; +/** multi-core mode scheduler */ +extern struct rte_cryptodev_scheduler *multicore_scheduler; #ifdef __cplusplus } diff --git a/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map b/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map index 0a8b4716..5c43127c 100644 --- a/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map +++ b/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map @@ -4,8 +4,6 @@ DPDK_17.02 { rte_cryptodev_scheduler_load_user_scheduler; rte_cryptodev_scheduler_slave_attach; rte_cryptodev_scheduler_slave_detach; - rte_crpytodev_scheduler_mode_set; - rte_crpytodev_scheduler_mode_get; rte_cryptodev_scheduler_ordering_set; rte_cryptodev_scheduler_ordering_get; diff --git a/drivers/crypto/scheduler/scheduler_failover.c b/drivers/crypto/scheduler/scheduler_failover.c index 2471a5f1..2aa13f8e 100644 --- a/drivers/crypto/scheduler/scheduler_failover.c +++ b/drivers/crypto/scheduler/scheduler_failover.c @@ -48,58 +48,19 @@ struct fo_scheduler_qp_ctx { uint8_t deq_idx; }; -static inline uint16_t __attribute__((always_inline)) -failover_slave_enqueue(struct scheduler_slave *slave, uint8_t slave_idx, +static __rte_always_inline uint16_t +failover_slave_enqueue(struct scheduler_slave *slave, struct rte_crypto_op **ops, uint16_t nb_ops) { uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); slave->nb_inflight_cops += processed_ops; - if (unlikely(processed_ops < nb_ops)) - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - return processed_ops; } @@ -114,11 +75,11 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) return 0; enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave, - PRIMARY_SLAVE_IDX, ops, nb_ops); + ops, nb_ops); if (enqueued_ops < nb_ops) enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave, - SECONDARY_SLAVE_IDX, &ops[enqueued_ops], + &ops[enqueued_ops], nb_ops - enqueued_ops); return enqueued_ops; diff --git a/drivers/crypto/scheduler/scheduler_multicore.c b/drivers/crypto/scheduler/scheduler_multicore.c new file mode 100644 index 00000000..0cd5bce5 --- /dev/null +++ b/drivers/crypto/scheduler/scheduler_multicore.c @@ -0,0 +1,415 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include <unistd.h> + +#include <rte_cryptodev.h> +#include <rte_malloc.h> + +#include "rte_cryptodev_scheduler_operations.h" +#include "scheduler_pmd_private.h" + +#define MC_SCHED_ENQ_RING_NAME_PREFIX "MCS_ENQR_" +#define MC_SCHED_DEQ_RING_NAME_PREFIX "MCS_DEQR_" + +#define MC_SCHED_BUFFER_SIZE 32 + +#define CRYPTO_OP_STATUS_BIT_COMPLETE 0x80 + +/** multi-core scheduler context */ +struct mc_scheduler_ctx { + uint32_t num_workers; /**< Number of workers polling */ + uint32_t stop_signal; + + struct rte_ring *sched_enq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + struct rte_ring *sched_deq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; +}; + +struct mc_scheduler_qp_ctx { + struct scheduler_slave slaves[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; + uint32_t nb_slaves; + + uint32_t last_enq_worker_idx; + uint32_t last_deq_worker_idx; + + struct mc_scheduler_ctx *mc_private_ctx; +}; + +static uint16_t +schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct mc_scheduler_qp_ctx *mc_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct mc_scheduler_ctx *mc_ctx = mc_qp_ctx->mc_private_ctx; + uint32_t worker_idx = mc_qp_ctx->last_enq_worker_idx; + uint16_t i, processed_ops = 0; + + if (unlikely(nb_ops == 0)) + return 0; + + for (i = 0; i < mc_ctx->num_workers && nb_ops != 0; i++) { + struct rte_ring *enq_ring = mc_ctx->sched_enq_ring[worker_idx]; + uint16_t nb_queue_ops = rte_ring_enqueue_burst(enq_ring, + (void *)(&ops[processed_ops]), nb_ops, NULL); + + nb_ops -= nb_queue_ops; + processed_ops += nb_queue_ops; + + if (++worker_idx == mc_ctx->num_workers) + worker_idx = 0; + } + mc_qp_ctx->last_enq_worker_idx = worker_idx; + + return processed_ops; +} + +static uint16_t +schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, + nb_ops); + uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, + nb_ops_to_enq); + + scheduler_order_insert(order_ring, ops, nb_ops_enqd); + + return nb_ops_enqd; +} + + +static uint16_t +schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct mc_scheduler_qp_ctx *mc_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct mc_scheduler_ctx *mc_ctx = mc_qp_ctx->mc_private_ctx; + uint32_t worker_idx = mc_qp_ctx->last_deq_worker_idx; + uint16_t i, processed_ops = 0; + + for (i = 0; i < mc_ctx->num_workers && nb_ops != 0; i++) { + struct rte_ring *deq_ring = mc_ctx->sched_deq_ring[worker_idx]; + uint16_t nb_deq_ops = rte_ring_dequeue_burst(deq_ring, + (void *)(&ops[processed_ops]), nb_ops, NULL); + + nb_ops -= nb_deq_ops; + processed_ops += nb_deq_ops; + if (++worker_idx == mc_ctx->num_workers) + worker_idx = 0; + } + + mc_qp_ctx->last_deq_worker_idx = worker_idx; + + return processed_ops; + +} + +static uint16_t +schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = ((struct scheduler_qp_ctx *)qp)->order_ring; + struct rte_crypto_op *op; + uint32_t nb_objs = rte_ring_count(order_ring); + uint32_t nb_ops_to_deq = 0; + uint32_t nb_ops_deqd = 0; + + if (nb_objs > nb_ops) + nb_objs = nb_ops; + + while (nb_ops_to_deq < nb_objs) { + SCHEDULER_GET_RING_OBJ(order_ring, nb_ops_to_deq, op); + + if (!(op->status & CRYPTO_OP_STATUS_BIT_COMPLETE)) + break; + + op->status &= ~CRYPTO_OP_STATUS_BIT_COMPLETE; + nb_ops_to_deq++; + } + + if (nb_ops_to_deq) { + nb_ops_deqd = rte_ring_sc_dequeue_bulk(order_ring, + (void **)ops, nb_ops_to_deq, NULL); + } + + return nb_ops_deqd; +} + +static int +slave_attach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +slave_detach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +mc_scheduler_worker(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + struct rte_ring *enq_ring; + struct rte_ring *deq_ring; + uint32_t core_id = rte_lcore_id(); + int i, worker_idx = -1; + struct scheduler_slave *slave; + struct rte_crypto_op *enq_ops[MC_SCHED_BUFFER_SIZE]; + struct rte_crypto_op *deq_ops[MC_SCHED_BUFFER_SIZE]; + uint16_t processed_ops; + uint16_t pending_enq_ops = 0; + uint16_t pending_enq_ops_idx = 0; + uint16_t pending_deq_ops = 0; + uint16_t pending_deq_ops_idx = 0; + uint16_t inflight_ops = 0; + const uint8_t reordering_enabled = sched_ctx->reordering_enabled; + + for (i = 0; i < (int)sched_ctx->nb_wc; i++) { + if (sched_ctx->wc_pool[i] == core_id) { + worker_idx = i; + break; + } + } + if (worker_idx == -1) { + CS_LOG_ERR("worker on core %u:cannot find worker index!\n", core_id); + return -1; + } + + slave = &sched_ctx->slaves[worker_idx]; + enq_ring = mc_ctx->sched_enq_ring[worker_idx]; + deq_ring = mc_ctx->sched_deq_ring[worker_idx]; + + while (!mc_ctx->stop_signal) { + if (pending_enq_ops) { + processed_ops = + rte_cryptodev_enqueue_burst(slave->dev_id, + slave->qp_id, &enq_ops[pending_enq_ops_idx], + pending_enq_ops); + pending_enq_ops -= processed_ops; + pending_enq_ops_idx += processed_ops; + inflight_ops += processed_ops; + } else { + processed_ops = rte_ring_dequeue_burst(enq_ring, (void *)enq_ops, + MC_SCHED_BUFFER_SIZE, NULL); + if (processed_ops) { + pending_enq_ops_idx = rte_cryptodev_enqueue_burst( + slave->dev_id, slave->qp_id, + enq_ops, processed_ops); + pending_enq_ops = processed_ops - pending_enq_ops_idx; + inflight_ops += pending_enq_ops_idx; + } + } + + if (pending_deq_ops) { + processed_ops = rte_ring_enqueue_burst( + deq_ring, (void *)&deq_ops[pending_deq_ops_idx], + pending_deq_ops, NULL); + pending_deq_ops -= processed_ops; + pending_deq_ops_idx += processed_ops; + } else if (inflight_ops) { + processed_ops = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, deq_ops, MC_SCHED_BUFFER_SIZE); + if (processed_ops) { + inflight_ops -= processed_ops; + if (reordering_enabled) { + uint16_t j; + + for (j = 0; j < processed_ops; j++) { + deq_ops[j]->status |= + CRYPTO_OP_STATUS_BIT_COMPLETE; + } + } else { + pending_deq_ops_idx = rte_ring_enqueue_burst( + deq_ring, (void *)deq_ops, processed_ops, + NULL); + pending_deq_ops = processed_ops - + pending_deq_ops_idx; + } + } + } + + rte_pause(); + } + + return 0; +} + +static int +scheduler_start(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + uint16_t i; + + mc_ctx->stop_signal = 0; + + for (i = 0; i < sched_ctx->nb_wc; i++) + rte_eal_remote_launch( + (lcore_function_t *)mc_scheduler_worker, dev, + sched_ctx->wc_pool[i]); + + if (sched_ctx->reordering_enabled) { + dev->enqueue_burst = &schedule_enqueue_ordering; + dev->dequeue_burst = &schedule_dequeue_ordering; + } else { + dev->enqueue_burst = &schedule_enqueue; + dev->dequeue_burst = &schedule_dequeue; + } + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; + struct mc_scheduler_qp_ctx *mc_qp_ctx = + qp_ctx->private_qp_ctx; + uint32_t j; + + memset(mc_qp_ctx->slaves, 0, + RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES * + sizeof(struct scheduler_slave)); + for (j = 0; j < sched_ctx->nb_slaves; j++) { + mc_qp_ctx->slaves[j].dev_id = + sched_ctx->slaves[j].dev_id; + mc_qp_ctx->slaves[j].qp_id = i; + } + + mc_qp_ctx->nb_slaves = sched_ctx->nb_slaves; + + mc_qp_ctx->last_enq_worker_idx = 0; + mc_qp_ctx->last_deq_worker_idx = 0; + } + + return 0; +} + +static int +scheduler_stop(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + uint16_t i; + + mc_ctx->stop_signal = 1; + + for (i = 0; i < sched_ctx->nb_wc; i++) + rte_eal_wait_lcore(sched_ctx->wc_pool[i]); + + return 0; +} + +static int +scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + struct mc_scheduler_qp_ctx *mc_qp_ctx; + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + + mc_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*mc_qp_ctx), 0, + rte_socket_id()); + if (!mc_qp_ctx) { + CS_LOG_ERR("failed allocate memory for private queue pair"); + return -ENOMEM; + } + + mc_qp_ctx->mc_private_ctx = mc_ctx; + qp_ctx->private_qp_ctx = (void *)mc_qp_ctx; + + + return 0; +} + +static int +scheduler_create_private_ctx(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx; + uint16_t i; + + if (sched_ctx->private_ctx) + rte_free(sched_ctx->private_ctx); + + mc_ctx = rte_zmalloc_socket(NULL, sizeof(struct mc_scheduler_ctx), 0, + rte_socket_id()); + if (!mc_ctx) { + CS_LOG_ERR("failed allocate memory"); + return -ENOMEM; + } + + mc_ctx->num_workers = sched_ctx->nb_wc; + for (i = 0; i < sched_ctx->nb_wc; i++) { + char r_name[16]; + + snprintf(r_name, sizeof(r_name), MC_SCHED_ENQ_RING_NAME_PREFIX "%u", i); + mc_ctx->sched_enq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_enq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", i); + return -1; + } + snprintf(r_name, sizeof(r_name), MC_SCHED_DEQ_RING_NAME_PREFIX "%u", i); + mc_ctx->sched_deq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_deq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", i); + return -1; + } + } + + sched_ctx->private_ctx = (void *)mc_ctx; + + return 0; +} + +struct rte_cryptodev_scheduler_ops scheduler_mc_ops = { + slave_attach, + slave_detach, + scheduler_start, + scheduler_stop, + scheduler_config_qp, + scheduler_create_private_ctx, + NULL, /* option_set */ + NULL /* option_get */ +}; + +struct rte_cryptodev_scheduler mc_scheduler = { + .name = "multicore-scheduler", + .description = "scheduler which will run burst across multiple cpu cores", + .mode = CDEV_SCHED_MODE_MULTICORE, + .ops = &scheduler_mc_ops +}; + +struct rte_cryptodev_scheduler *multicore_scheduler = &mc_scheduler; diff --git a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c index 6b628dfa..1dd1bc32 100644 --- a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c +++ b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c @@ -67,7 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) struct scheduler_qp_ctx *qp_ctx = qp; struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx; struct rte_crypto_op *sched_ops[NB_PKT_SIZE_SLAVES][nb_ops]; - struct scheduler_session *sess; uint32_t in_flight_ops[NB_PKT_SIZE_SLAVES] = { psd_qp_ctx->primary_slave.nb_inflight_cops, psd_qp_ctx->secondary_slave.nb_inflight_cops @@ -97,8 +96,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rte_prefetch0(ops[i + 7]->sym); rte_prefetch0(ops[i + 7]->sym->session); - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; /* job_len is initialized as cipher data length, once * it is 0, equals to auth data length */ @@ -118,11 +115,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+1]->sym->session->_private; job_len = ops[i+1]->sym->cipher.data.length; job_len += (ops[i+1]->sym->cipher.data.length == 0) * ops[i+1]->sym->auth.data.length; @@ -135,11 +129,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+1]; - ops[i+1]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+2]->sym->session->_private; job_len = ops[i+2]->sym->cipher.data.length; job_len += (ops[i+2]->sym->cipher.data.length == 0) * ops[i+2]->sym->auth.data.length; @@ -152,12 +143,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+2]; - ops[i+2]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - job_len = ops[i+3]->sym->cipher.data.length; job_len += (ops[i+3]->sym->cipher.data.length == 0) * ops[i+3]->sym->auth.data.length; @@ -170,14 +157,10 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+3]; - ops[i+3]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } for (; i < nb_ops; i++) { - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; - job_len = ops[i]->sym->cipher.data.length; job_len += (ops[i]->sym->cipher.data.length == 0) * ops[i]->sym->auth.data.length; @@ -190,7 +173,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } diff --git a/drivers/crypto/scheduler/scheduler_pmd.c b/drivers/crypto/scheduler/scheduler_pmd.c index 0b63c20b..400fc4f1 100644 --- a/drivers/crypto/scheduler/scheduler_pmd.c +++ b/drivers/crypto/scheduler/scheduler_pmd.c @@ -33,6 +33,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -41,11 +42,14 @@ #include "rte_cryptodev_scheduler.h" #include "scheduler_pmd_private.h" +uint8_t cryptodev_driver_id; + struct scheduler_init_params { struct rte_crypto_vdev_init_params def_p; uint32_t nb_slaves; enum rte_cryptodev_scheduler_mode mode; uint32_t enable_ordering; + uint64_t wcmask; char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; }; @@ -57,6 +61,8 @@ struct scheduler_init_params { #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG ("max_nb_sessions") #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") +#define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") +#define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") const char *scheduler_valid_params[] = { RTE_CRYPTODEV_VDEV_NAME, @@ -65,7 +71,9 @@ const char *scheduler_valid_params[] = { RTE_CRYPTODEV_VDEV_ORDERING, RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, - RTE_CRYPTODEV_VDEV_SOCKET_ID + RTE_CRYPTODEV_VDEV_SOCKET_ID, + RTE_CRYPTODEV_VDEV_COREMASK, + RTE_CRYPTODEV_VDEV_CORELIST }; struct scheduler_parse_map { @@ -79,7 +87,9 @@ const struct scheduler_parse_map scheduler_mode_map[] = { {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), CDEV_SCHED_MODE_PKT_SIZE_DISTR}, {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), - CDEV_SCHED_MODE_FAILOVER} + CDEV_SCHED_MODE_FAILOVER}, + {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), + CDEV_SCHED_MODE_MULTICORE} }; const struct scheduler_parse_map scheduler_ordering_map[] = { @@ -89,7 +99,8 @@ const struct scheduler_parse_map scheduler_ordering_map[] = { static int cryptodev_scheduler_create(const char *name, - struct scheduler_init_params *init_params) + struct rte_vdev_device *vdev, + struct scheduler_init_params *init_params) { struct rte_cryptodev *dev; struct scheduler_ctx *sched_ctx; @@ -101,22 +112,38 @@ cryptodev_scheduler_create(const char *name, sizeof(init_params->def_p.name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->def_p.name, + dev = rte_cryptodev_vdev_pmd_init(init_params->def_p.name, sizeof(struct scheduler_ctx), - init_params->def_p.socket_id); + init_params->def_p.socket_id, + vdev); if (dev == NULL) { CS_LOG_ERR("driver %s: failed to create cryptodev vdev", name); return -EFAULT; } - dev->dev_type = RTE_CRYPTODEV_SCHEDULER_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_crypto_scheduler_pmd_ops; sched_ctx = dev->data->dev_private; sched_ctx->max_nb_queue_pairs = init_params->def_p.max_nb_queue_pairs; + if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { + uint16_t i; + + sched_ctx->nb_wc = 0; + + for (i = 0; i < RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES; i++) { + if (init_params->wcmask & (1ULL << i)) { + sched_ctx->wc_pool[sched_ctx->nb_wc++] = i; + RTE_LOG(INFO, PMD, + " Worker core[%u]=%u added\n", + sched_ctx->nb_wc-1, i); + } + } + } + if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && init_params->mode < CDEV_SCHED_MODE_COUNT) { ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, @@ -219,22 +246,6 @@ cryptodev_scheduler_remove(struct rte_vdev_device *vdev) return 0; } -static uint8_t -number_of_sockets(void) -{ - int sockets = 0; - int i; - const struct rte_memseg *ms = rte_eal_get_physmem_layout(); - - for (i = 0; ((i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL)); i++) { - if (sockets < ms[i].socket_id) - sockets = ms[i].socket_id; - } - - /* Number of sockets = maximum socket_id + 1 */ - return ++sockets; -} - /** Parse integer from integer argument */ static int parse_integer_arg(const char *key __rte_unused, @@ -251,6 +262,43 @@ parse_integer_arg(const char *key __rte_unused, return 0; } +/** Parse integer from hexadecimal integer argument */ +static int +parse_coremask_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *params = extra_args; + + params->wcmask = strtoull(value, NULL, 16); + + return 0; +} + +/** Parse integer from list of integers argument */ +static int +parse_corelist_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *params = extra_args; + + params->wcmask = 0ULL; + + const char *token = value; + + while (isdigit(token[0])) { + char *rval; + unsigned int core = strtoul(token, &rval, 10); + + params->wcmask |= 1ULL << core; + token = (const char *)rval; + if (token[0] == '\0') + break; + token++; + } + + return 0; +} + /** Parse name */ static int parse_name_arg(const char *key __rte_unused, @@ -277,7 +325,7 @@ parse_slave_arg(const char *key __rte_unused, { struct scheduler_init_params *param = extra_args; - if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES - 1) { + if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { CS_LOG_ERR("Too many slaves.\n"); return -ENOMEM; } @@ -370,6 +418,18 @@ scheduler_parse_init_params(struct scheduler_init_params *params, if (ret < 0) goto free_kvlist; + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, + &parse_coremask_arg, + params); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, + &parse_corelist_arg, + params); + if (ret < 0) + goto free_kvlist; + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, &parse_name_arg, ¶ms->def_p); @@ -390,12 +450,6 @@ scheduler_parse_init_params(struct scheduler_init_params *params, &parse_ordering_arg, params); if (ret < 0) goto free_kvlist; - - if (params->def_p.socket_id >= number_of_sockets()) { - CDEV_LOG_ERR("Invalid socket id specified to create " - "the virtual crypto device on"); - goto free_kvlist; - } } free_kvlist: @@ -437,8 +491,12 @@ cryptodev_scheduler_probe(struct rte_vdev_device *vdev) if (init_params.def_p.name[0] != '\0') RTE_LOG(INFO, PMD, " User defined name = %s\n", init_params.def_p.name); + if (init_params.wcmask != 0) + RTE_LOG(INFO, PMD, " workers core mask = %"PRIx64"\n", + init_params.wcmask); return cryptodev_scheduler_create(name, + vdev, &init_params); } @@ -454,3 +512,5 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, "max_nb_sessions=<int> " "socket_id=<int> " "slave=<name>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_scheduler_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c index 2b5858df..d3795346 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_ops.c +++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c @@ -37,6 +37,7 @@ #include <rte_dev.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_reorder.h> #include "scheduler_pmd_private.h" @@ -368,7 +369,7 @@ scheduler_pmd_info_get(struct rte_cryptodev *dev, max_nb_sessions; } - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = sched_ctx->capabilities; dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs; @@ -398,7 +399,8 @@ scheduler_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) /** Setup a queue pair */ static int scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, - const struct rte_cryptodev_qp_conf *qp_conf, int socket_id) + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id, + struct rte_mempool *session_pool) { struct scheduler_ctx *sched_ctx = dev->data->dev_private; struct scheduler_qp_ctx *qp_ctx; @@ -420,8 +422,13 @@ scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, for (i = 0; i < sched_ctx->nb_slaves; i++) { uint8_t slave_id = sched_ctx->slaves[i].dev_id; + /* + * All slaves will share the same session mempool + * for session-less operations, so the objects + * must be big enough for all the drivers used. + */ ret = rte_cryptodev_queue_pair_setup(slave_id, qp_id, - qp_conf, socket_id); + qp_conf, socket_id, session_pool); if (ret < 0) return ret; } @@ -483,37 +490,41 @@ scheduler_pmd_qp_count(struct rte_cryptodev *dev) static uint32_t scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) { - return sizeof(struct scheduler_session); + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint8_t i = 0; + uint32_t max_priv_sess_size = 0; + + /* Check what is the maximum private session size for all slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id]; + uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + + if (max_priv_sess_size < priv_sess_size) + max_priv_sess_size = priv_sess_size; + } + + return max_priv_sess_size; } static int -config_slave_sess(struct scheduler_ctx *sched_ctx, - struct rte_crypto_sym_xform *xform, - struct scheduler_session *sess, - uint32_t create) +scheduler_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + struct scheduler_ctx *sched_ctx = dev->data->dev_private; uint32_t i; + int ret; for (i = 0; i < sched_ctx->nb_slaves; i++) { struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (sess->sessions[i]) { - if (create) - continue; - /* !create */ - sess->sessions[i] = rte_cryptodev_sym_session_free( - slave->dev_id, sess->sessions[i]); - } else { - if (!create) - continue; - /* create */ - sess->sessions[i] = - rte_cryptodev_sym_session_create( - slave->dev_id, xform); - if (!sess->sessions[i]) { - config_slave_sess(sched_ctx, NULL, sess, 0); - return -1; - } + ret = rte_cryptodev_sym_session_init(slave->dev_id, sess, + xform, mempool); + if (ret < 0) { + CS_LOG_ERR("unabled to config sym session"); + return ret; } } @@ -523,27 +534,17 @@ config_slave_sess(struct scheduler_ctx *sched_ctx, /** Clear the memory of session so it doesn't leave key material behind */ static void scheduler_pmd_session_clear(struct rte_cryptodev *dev, - void *sess) + struct rte_cryptodev_sym_session *sess) { struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; - config_slave_sess(sched_ctx, NULL, sess, 0); - - memset(sess, 0, sizeof(struct scheduler_session)); -} - -static void * -scheduler_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) -{ - struct scheduler_ctx *sched_ctx = dev->data->dev_private; + /* Clear private data of slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (config_slave_sess(sched_ctx, xform, sess, 1) < 0) { - CS_LOG_ERR("unabled to config sym session"); - return NULL; + rte_cryptodev_sym_session_clear(slave->dev_id, sess); } - - return sess; } struct rte_cryptodev_ops scheduler_pmd_ops = { diff --git a/drivers/crypto/scheduler/scheduler_pmd_private.h b/drivers/crypto/scheduler/scheduler_pmd_private.h index 421dae37..e606716a 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_private.h +++ b/drivers/crypto/scheduler/scheduler_pmd_private.h @@ -36,6 +36,9 @@ #include "rte_cryptodev_scheduler.h" +#define CRYPTODEV_NAME_SCHEDULER_PMD crypto_scheduler +/**< Scheduler Crypto PMD device name */ + #define PER_SLAVE_BUFF_SIZE (256) #define CS_LOG_ERR(fmt, args...) \ @@ -63,7 +66,7 @@ struct scheduler_slave { uint16_t qp_id; uint32_t nb_inflight_cops; - enum rte_cryptodev_type dev_type; + uint8_t driver_id; }; struct scheduler_ctx { @@ -86,6 +89,8 @@ struct scheduler_ctx { char name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; char description[RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN]; + uint16_t wc_pool[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + uint16_t nb_wc; char *init_slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; int nb_init_slaves; @@ -100,12 +105,10 @@ struct scheduler_qp_ctx { uint32_t seqn; } __rte_cache_aligned; -struct scheduler_session { - struct rte_cryptodev_sym_session *sessions[ - RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; -}; -static inline uint16_t __attribute__((always_inline)) +extern uint8_t cryptodev_driver_id; + +static __rte_always_inline uint16_t get_max_enqueue_order_count(struct rte_ring *order_ring, uint16_t nb_ops) { uint32_t count = rte_ring_free_count(order_ring); @@ -113,7 +116,7 @@ get_max_enqueue_order_count(struct rte_ring *order_ring, uint16_t nb_ops) return count > nb_ops ? nb_ops : count; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void scheduler_order_insert(struct rte_ring *order_ring, struct rte_crypto_op **ops, uint16_t nb_ops) { @@ -125,7 +128,7 @@ scheduler_order_insert(struct rte_ring *order_ring, op = ring[(order_ring->cons.head + pos) & order_ring->mask]; \ } while (0) -static inline uint16_t __attribute__((always_inline)) +static __rte_always_inline uint16_t scheduler_order_drain(struct rte_ring *order_ring, struct rte_crypto_op **ops, uint16_t nb_ops) { diff --git a/drivers/crypto/scheduler/scheduler_roundrobin.c b/drivers/crypto/scheduler/scheduler_roundrobin.c index 01162764..4a847281 100644 --- a/drivers/crypto/scheduler/scheduler_roundrobin.c +++ b/drivers/crypto/scheduler/scheduler_roundrobin.c @@ -52,8 +52,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) uint32_t slave_idx = rr_qp_ctx->last_enq_slave_idx; struct scheduler_slave *slave = &rr_qp_ctx->slaves[slave_idx]; uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; if (unlikely(nb_ops == 0)) return 0; @@ -61,39 +59,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); @@ -102,12 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rr_qp_ctx->last_enq_slave_idx += 1; rr_qp_ctx->last_enq_slave_idx %= rr_qp_ctx->nb_slaves; - /* recover session if enqueue is failed */ - if (unlikely(processed_ops < nb_ops)) { - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - } - return processed_ops; } diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd.c b/drivers/crypto/snow3g/rte_snow3g_pmd.c index 960956ca..dad45068 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -46,6 +47,8 @@ #define SNOW3G_MAX_BURST 8 #define BYTE_LEN 8 +static uint8_t cryptodev_driver_id; + /** Get xform chain order. */ static enum snow3g_operation snow3g_get_mode(const struct rte_crypto_sym_xform *xform) @@ -108,13 +111,20 @@ snow3g_set_session_parameters(struct snow3g_session *sess, case SNOW3G_OP_NOT_SUPPORTED: default: SNOW3G_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only SNOW 3G UEA2 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_SNOW3G_UEA2) + return -ENOTSUP; + + if (cipher_xform->cipher.iv.length != SNOW3G_IV_LENGTH) { + SNOW3G_LOG_ERR("Wrong IV length"); return -EINVAL; + } + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + /* Initialize key */ sso_snow3g_init_key_sched(cipher_xform->cipher.key.data, &sess->pKeySched_cipher); @@ -123,8 +133,21 @@ snow3g_set_session_parameters(struct snow3g_session *sess, if (auth_xform) { /* Only SNOW 3G UIA2 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_SNOW3G_UIA2) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != SNOW3G_DIGEST_LENGTH) { + SNOW3G_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + + if (auth_xform->auth.iv.length != SNOW3G_IV_LENGTH) { + SNOW3G_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + sess->auth_iv_offset = auth_xform->auth.iv.offset; + /* Initialize key */ sso_snow3g_init_key_sched(auth_xform->auth.key.data, &sess->pKeySched_hash); @@ -140,27 +163,41 @@ snow3g_set_session_parameters(struct snow3g_session *sess, static struct snow3g_session * snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op) { - struct snow3g_session *sess; - - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_SNOW3G_PMD)) + struct snow3g_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct snow3g_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct snow3g_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct snow3g_session *)c_sess->_private; + sess = (struct snow3g_session *)_sess_private_data; if (unlikely(snow3g_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -173,17 +210,10 @@ process_snow3g_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[SNOW3G_MAX_BURST], *dst[SNOW3G_MAX_BURST]; - uint8_t *IV[SNOW3G_MAX_BURST]; + uint8_t *iv[SNOW3G_MAX_BURST]; uint32_t num_bytes[SNOW3G_MAX_BURST]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (unlikely(ops[i]->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("iv"); - break; - } - src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); dst[i] = ops[i]->sym->m_dst ? @@ -191,13 +221,14 @@ process_snow3g_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = ops[i]->sym->cipher.iv.data; + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; processed_ops++; } - sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, IV, src, dst, + sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -209,16 +240,9 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, struct snow3g_session *session) { uint8_t *src, *dst; - uint8_t *IV; + uint8_t *iv; uint32_t length_in_bits, offset_in_bits; - /* Sanity checks. */ - if (unlikely(op->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("iv"); - return 0; - } - offset_in_bits = op->sym->cipher.data.offset; src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); if (op->sym->m_dst == NULL) { @@ -227,10 +251,11 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, return 0; } dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); - IV = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->cipher_iv_offset); length_in_bits = op->sym->cipher.data.length; - sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, IV, + sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, iv, src, dst, length_in_bits, offset_in_bits); return 1; @@ -246,20 +271,9 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, uint8_t processed_ops = 0; uint8_t *src, *dst; uint32_t length_in_bits; + uint8_t *iv; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != SNOW3G_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != SNOW3G_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -271,27 +285,29 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); + iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->auth_iv_offset); if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + SNOW3G_DIGEST_LENGTH); sso_snow3g_f9_1_buffer(&session->pKeySched_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + SNOW3G_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + SNOW3G_DIGEST_LENGTH); } else { dst = ops[i]->sym->auth.digest.data; sso_snow3g_f9_1_buffer(&session->pKeySched_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); } processed_ops++; @@ -356,7 +372,11 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct snow3g_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -408,8 +428,9 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, sizeof(struct snow3g_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } @@ -548,28 +569,21 @@ cryptodev_snow3g_create(const char *name, { struct rte_cryptodev *dev; struct snow3g_private *internals; - uint64_t cpu_flags = 0; + uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; if (init_params->name[0] == '\0') snprintf(init_params->name, sizeof(init_params->name), "%s", name); - /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - SNOW3G_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct snow3g_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct snow3g_private), init_params->socket_id, + vdev); if (dev == NULL) { SNOW3G_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_SNOW3G_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_snow3g_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -611,7 +625,7 @@ cryptodev_snow3g_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -653,3 +667,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_snow3g_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c index 7ce96be9..ae9569c8 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities snow3g_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { + .iv_size = { .min = 16, .max = 16, .increment = 0 @@ -156,7 +156,7 @@ snow3g_pmd_info_get(struct rte_cryptodev *dev, struct snow3g_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -220,7 +220,7 @@ snow3g_pmd_qp_create_processed_ops_ring(struct snow3g_qp *qp, static int snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct snow3g_qp *qp = NULL; @@ -245,7 +245,7 @@ snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -289,33 +289,56 @@ snow3g_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a SNOW 3G session from a crypto xform chain */ -static void * +static int snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { SNOW3G_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - if (snow3g_set_session_parameters(sess, xform) != 0) { + ret = snow3g_set_session_parameters(sess_private_data, xform); + if (ret != 0) { SNOW3G_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -snow3g_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +snow3g_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct snow3g_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct snow3g_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops snow3g_pmd_ops = { diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h index 03973b97..fba3cb86 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include <sso_snow3g.h> +#define CRYPTODEV_NAME_SNOW3G_PMD crypto_snow3g +/**< SNOW 3G PMD device name */ + #define SNOW3G_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), \ @@ -91,6 +94,8 @@ struct snow3g_session { enum rte_crypto_auth_operation auth_op; sso_snow3g_key_schedule_t pKeySched_cipher; sso_snow3g_key_schedule_t pKeySched_hash; + uint16_t cipher_iv_offset; + uint16_t auth_iv_offset; } __rte_cache_aligned; diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c index 1020544b..b301711e 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/drivers/crypto/zuc/rte_zuc_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -45,6 +46,8 @@ #define ZUC_MAX_BURST 8 #define BYTE_LEN 8 +static uint8_t cryptodev_driver_id; + /** Get xform chain order. */ static enum zuc_operation zuc_get_mode(const struct rte_crypto_sym_xform *xform) @@ -107,13 +110,20 @@ zuc_set_session_parameters(struct zuc_session *sess, case ZUC_OP_NOT_SUPPORTED: default: ZUC_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only ZUC EEA3 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_ZUC_EEA3) + return -ENOTSUP; + + if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) { + ZUC_LOG_ERR("Wrong IV length"); return -EINVAL; + } + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + /* Copy the key */ memcpy(sess->pKey_cipher, cipher_xform->cipher.key.data, ZUC_IV_KEY_LENGTH); @@ -122,8 +132,21 @@ zuc_set_session_parameters(struct zuc_session *sess, if (auth_xform) { /* Only ZUC EIA3 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_ZUC_EIA3) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) { + ZUC_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + + if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) { + ZUC_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + sess->auth_iv_offset = auth_xform->auth.iv.offset; + /* Copy the key */ memcpy(sess->pKey_hash, auth_xform->auth.key.data, ZUC_IV_KEY_LENGTH); @@ -139,27 +162,40 @@ zuc_set_session_parameters(struct zuc_session *sess, static struct zuc_session * zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) { - struct zuc_session *sess; - - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_ZUC_PMD)) + struct zuc_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct zuc_session *)get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct zuc_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct zuc_session *)c_sess->_private; + sess = (struct zuc_session *)_sess_private_data; if (unlikely(zuc_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -172,18 +208,11 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[ZUC_MAX_BURST], *dst[ZUC_MAX_BURST]; - uint8_t *IV[ZUC_MAX_BURST]; + uint8_t *iv[ZUC_MAX_BURST]; uint32_t num_bytes[ZUC_MAX_BURST]; uint8_t *cipher_keys[ZUC_MAX_BURST]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (unlikely(ops[i]->sym->cipher.iv.length != ZUC_IV_KEY_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("iv"); - break; - } - if (((ops[i]->sym->cipher.data.length % BYTE_LEN) != 0) || ((ops[i]->sym->cipher.data.offset % BYTE_LEN) != 0)) { @@ -212,7 +241,8 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = ops[i]->sym->cipher.iv.data; + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; cipher_keys[i] = session->pKey_cipher; @@ -220,7 +250,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, processed_ops++; } - sso_zuc_eea3_n_buffer(cipher_keys, IV, src, dst, + sso_zuc_eea3_n_buffer(cipher_keys, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -237,20 +267,9 @@ process_zuc_hash_op(struct rte_crypto_op **ops, uint8_t *src; uint32_t *dst; uint32_t length_in_bits; + uint8_t *iv; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != ZUC_IV_KEY_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != ZUC_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -262,27 +281,29 @@ process_zuc_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); + iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->auth_iv_offset); if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { dst = (uint32_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + ZUC_DIGEST_LENGTH); sso_zuc_eia3_1_buffer(session->pKey_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + ZUC_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + ZUC_DIGEST_LENGTH); } else { dst = (uint32_t *)ops[i]->sym->auth.digest.data; sso_zuc_eia3_1_buffer(session->pKey_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); } processed_ops++; @@ -332,7 +353,11 @@ process_ops(struct rte_crypto_op **ops, struct zuc_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct zuc_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -448,28 +473,21 @@ cryptodev_zuc_create(const char *name, { struct rte_cryptodev *dev; struct zuc_private *internals; - uint64_t cpu_flags = 0; + uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; if (init_params->name[0] == '\0') snprintf(init_params->name, sizeof(init_params->name), "%s", name); - /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - ZUC_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct zuc_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct zuc_private), init_params->socket_id, + vdev); if (dev == NULL) { ZUC_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_ZUC_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_zuc_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -511,7 +529,7 @@ cryptodev_zuc_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -552,3 +570,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_zuc_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/zuc/rte_zuc_pmd_ops.c b/drivers/crypto/zuc/rte_zuc_pmd_ops.c index e793459c..52c6aed8 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_ops.c +++ b/drivers/crypto/zuc/rte_zuc_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities zuc_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { + .iv_size = { .min = 16, .max = 16, .increment = 0 @@ -80,7 +80,7 @@ static const struct rte_cryptodev_capabilities zuc_pmd_capabilities[] = { .min = 16, .max = 16, .increment = 0 - } + }, }, } }, } }, @@ -156,7 +156,7 @@ zuc_pmd_info_get(struct rte_cryptodev *dev, struct zuc_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -220,7 +220,7 @@ zuc_pmd_qp_create_processed_ops_ring(struct zuc_qp *qp, static int zuc_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct zuc_qp *qp = NULL; @@ -245,7 +245,7 @@ zuc_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -289,33 +289,56 @@ zuc_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a ZUC session from a crypto xform chain */ -static void * +static int zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { ZUC_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (zuc_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = zuc_set_session_parameters(sess_private_data, xform); + if (ret != 0) { ZUC_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -zuc_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +zuc_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct zuc_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct zuc_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops zuc_pmd_ops = { diff --git a/drivers/crypto/zuc/rte_zuc_pmd_private.h b/drivers/crypto/zuc/rte_zuc_pmd_private.h index 030f120b..b706e0aa 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_private.h +++ b/drivers/crypto/zuc/rte_zuc_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include <sso_zuc.h> +#define CRYPTODEV_NAME_ZUC_PMD crypto_zuc +/**< KASUMI PMD device name */ + #define ZUC_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \ @@ -92,6 +95,8 @@ struct zuc_session { enum rte_crypto_auth_operation auth_op; uint8_t pKey_cipher[ZUC_IV_KEY_LENGTH]; uint8_t pKey_hash[ZUC_IV_KEY_LENGTH]; + uint16_t cipher_iv_offset; + uint16_t auth_iv_offset; } __rte_cache_aligned; diff --git a/drivers/event/Makefile b/drivers/event/Makefile index 1cf389e8..3f6b8988 100644 --- a/drivers/event/Makefile +++ b/drivers/event/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 Cavium networks. All rights reserved. +# Copyright(c) 2016 Cavium, Inc. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -13,7 +13,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # @@ -39,5 +39,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += sw DEPDIRS-sw = $(core-libs) librte_kvargs librte_ring DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += octeontx DEPDIRS-octeontx = $(core-libs) +DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += dpaa2 +DEPDIRS-dpaa2 = $(core-libs) librte_bus_fslmc include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/drivers/event/dpaa2/Makefile b/drivers/event/dpaa2/Makefile new file mode 100644 index 00000000..3497d09d --- /dev/null +++ b/drivers/event/dpaa2/Makefile @@ -0,0 +1,60 @@ +# BSD LICENSE +# +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_dpaa2_event.a + +CFLAGS += $(WERROR_FLAGS) + +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/portal +CFLAGS += -I$(RTE_SDK)/drivers/mempool/dpaa2 +CFLAGS += -I$(RTE_SDK)/drivers/event/dpaa2 +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal + +# versioning export map +EXPORT_MAP := rte_pmd_dpaa2_event_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += dpaa2_hw_dpcon.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += dpaa2_eventdev.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c new file mode 100644 index 00000000..cf2d2741 --- /dev/null +++ b/drivers/event/dpaa2/dpaa2_eventdev.c @@ -0,0 +1,692 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <assert.h> +#include <stdio.h> +#include <stdbool.h> +#include <errno.h> +#include <stdint.h> +#include <string.h> +#include <sys/epoll.h> + +#include <rte_atomic.h> +#include <rte_byteorder.h> +#include <rte_common.h> +#include <rte_debug.h> +#include <rte_dev.h> +#include <rte_eal.h> +#include <rte_fslmc.h> +#include <rte_lcore.h> +#include <rte_log.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_memory.h> +#include <rte_memzone.h> +#include <rte_pci.h> +#include <rte_vdev.h> + +#include <fslmc_vfio.h> +#include <dpaa2_hw_pvt.h> +#include <dpaa2_hw_mempool.h> +#include <dpaa2_hw_dpio.h> +#include "dpaa2_eventdev.h" +#include <portal/dpaa2_hw_pvt.h> +#include <mc/fsl_dpci.h> + +/* Clarifications + * Evendev = SoC Instance + * Eventport = DPIO Instance + * Eventqueue = DPCON Instance + * 1 Eventdev can have N Eventqueue + * Soft Event Flow is DPCI Instance + */ + +static uint16_t +dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[], + uint16_t nb_events) +{ + struct rte_eventdev *ev_dev = + ((struct dpaa2_io_portal_t *)port)->eventdev; + struct dpaa2_eventdev *priv = ev_dev->data->dev_private; + uint32_t queue_id = ev[0].queue_id; + struct evq_info_t *evq_info = &priv->evq_info[queue_id]; + uint32_t fqid; + struct qbman_swp *swp; + struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; + uint32_t loop, frames_to_send; + struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; + uint16_t num_tx = 0; + int ret; + + RTE_SET_USED(port); + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + PMD_DRV_LOG(ERR, "Failure in affining portal\n"); + return 0; + } + } + + swp = DPAA2_PER_LCORE_PORTAL; + + while (nb_events) { + frames_to_send = (nb_events >> 3) ? + MAX_TX_RING_SLOTS : nb_events; + + for (loop = 0; loop < frames_to_send; loop++) { + const struct rte_event *event = &ev[num_tx + loop]; + + if (event->sched_type != RTE_SCHED_TYPE_ATOMIC) + fqid = evq_info->dpci->queue[ + DPAA2_EVENT_DPCI_PARALLEL_QUEUE].fqid; + else + fqid = evq_info->dpci->queue[ + DPAA2_EVENT_DPCI_ATOMIC_QUEUE].fqid; + + /* Prepare enqueue descriptor */ + qbman_eq_desc_clear(&eqdesc[loop]); + qbman_eq_desc_set_fq(&eqdesc[loop], fqid); + qbman_eq_desc_set_no_orp(&eqdesc[loop], 0); + qbman_eq_desc_set_response(&eqdesc[loop], 0, 0); + + if (event->impl_opaque) { + uint8_t dqrr_index = event->impl_opaque - 1; + + qbman_eq_desc_set_dca(&eqdesc[loop], 1, + dqrr_index, 0); + DPAA2_PER_LCORE_DPIO->dqrr_size--; + DPAA2_PER_LCORE_DPIO->dqrr_held &= + ~(1 << dqrr_index); + } + + memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); + + /* + * todo - need to align with hw context data + * to avoid copy + */ + struct rte_event *ev_temp = rte_malloc(NULL, + sizeof(struct rte_event), 0); + rte_memcpy(ev_temp, event, sizeof(struct rte_event)); + DPAA2_SET_FD_ADDR((&fd_arr[loop]), ev_temp); + DPAA2_SET_FD_LEN((&fd_arr[loop]), + sizeof(struct rte_event)); + } + loop = 0; + while (loop < frames_to_send) { + loop += qbman_swp_enqueue_multiple_eqdesc(swp, + &eqdesc[loop], &fd_arr[loop], + frames_to_send - loop); + } + num_tx += frames_to_send; + nb_events -= frames_to_send; + } + + return num_tx; +} + +static uint16_t +dpaa2_eventdev_enqueue(void *port, const struct rte_event *ev) +{ + return dpaa2_eventdev_enqueue_burst(port, ev, 1); +} + +static void dpaa2_eventdev_dequeue_wait(uint64_t timeout_ticks) +{ + struct epoll_event epoll_ev; + int ret, i = 0; + + qbman_swp_interrupt_clear_status(DPAA2_PER_LCORE_PORTAL, + QBMAN_SWP_INTERRUPT_DQRI); + +RETRY: + ret = epoll_wait(DPAA2_PER_LCORE_DPIO->epoll_fd, + &epoll_ev, 1, timeout_ticks); + if (ret < 1) { + /* sometimes due to some spurious interrupts epoll_wait fails + * with errno EINTR. so here we are retrying epoll_wait in such + * case to avoid the problem. + */ + if (errno == EINTR) { + PMD_DRV_LOG(DEBUG, "epoll_wait fails\n"); + if (i++ > 10) + PMD_DRV_LOG(DEBUG, "Dequeue burst Failed\n"); + goto RETRY; + } + } +} + +static void dpaa2_eventdev_process_parallel(struct qbman_swp *swp, + const struct qbman_fd *fd, + const struct qbman_result *dq, + struct rte_event *ev) +{ + struct rte_event *ev_temp = + (struct rte_event *)DPAA2_GET_FD_ADDR(fd); + rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); + rte_free(ev_temp); + + qbman_swp_dqrr_consume(swp, dq); +} + +static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp, + const struct qbman_fd *fd, + const struct qbman_result *dq, + struct rte_event *ev) +{ + struct rte_event *ev_temp = + (struct rte_event *)DPAA2_GET_FD_ADDR(fd); + uint8_t dqrr_index = qbman_get_dqrr_idx(dq); + + RTE_SET_USED(swp); + + rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); + rte_free(ev_temp); + ev->impl_opaque = dqrr_index + 1; + DPAA2_PER_LCORE_DPIO->dqrr_size++; + DPAA2_PER_LCORE_DPIO->dqrr_held |= 1 << dqrr_index; +} + +static uint16_t +dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[], + uint16_t nb_events, uint64_t timeout_ticks) +{ + const struct qbman_result *dq; + struct qbman_swp *swp; + const struct qbman_fd *fd; + struct dpaa2_queue *rxq; + int num_pkts = 0, ret, i = 0; + + RTE_SET_USED(port); + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + PMD_DRV_LOG(ERR, "Failure in affining portal\n"); + return 0; + } + } + + swp = DPAA2_PER_LCORE_PORTAL; + + /* Check if there are atomic contexts to be released */ + while (DPAA2_PER_LCORE_DPIO->dqrr_size) { + if (DPAA2_PER_LCORE_DPIO->dqrr_held & (1 << i)) { + dq = qbman_get_dqrr_from_idx(swp, i); + qbman_swp_dqrr_consume(swp, dq); + DPAA2_PER_LCORE_DPIO->dqrr_size--; + } + i++; + } + DPAA2_PER_LCORE_DPIO->dqrr_held = 0; + + do { + dq = qbman_swp_dqrr_next(swp); + if (!dq) { + if (!num_pkts && timeout_ticks) { + dpaa2_eventdev_dequeue_wait(timeout_ticks); + timeout_ticks = 0; + continue; + } + return num_pkts; + } + + fd = qbman_result_DQ_fd(dq); + + rxq = (struct dpaa2_queue *)qbman_result_DQ_fqd_ctx(dq); + if (rxq) { + rxq->cb(swp, fd, dq, &ev[num_pkts]); + } else { + qbman_swp_dqrr_consume(swp, dq); + PMD_DRV_LOG(ERR, "Null Return VQ received\n"); + return 0; + } + + num_pkts++; + } while (num_pkts < nb_events); + + return num_pkts; +} + +static uint16_t +dpaa2_eventdev_dequeue(void *port, struct rte_event *ev, + uint64_t timeout_ticks) +{ + return dpaa2_eventdev_dequeue_burst(port, ev, 1, timeout_ticks); +} + +static void +dpaa2_eventdev_info_get(struct rte_eventdev *dev, + struct rte_event_dev_info *dev_info) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + memset(dev_info, 0, sizeof(struct rte_event_dev_info)); + dev_info->min_dequeue_timeout_ns = + DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT; + dev_info->max_dequeue_timeout_ns = + DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT; + dev_info->dequeue_timeout_ns = + DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT; + dev_info->max_event_queues = priv->max_event_queues; + dev_info->max_event_queue_flows = + DPAA2_EVENT_MAX_QUEUE_FLOWS; + dev_info->max_event_queue_priority_levels = + DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS; + dev_info->max_event_priority_levels = + DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS; + dev_info->max_event_ports = RTE_MAX_LCORE; + dev_info->max_event_port_dequeue_depth = + DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; + dev_info->max_event_port_enqueue_depth = + DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; + dev_info->max_num_events = DPAA2_EVENT_MAX_NUM_EVENTS; + dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED | + RTE_EVENT_DEV_CAP_BURST_MODE; +} + +static int +dpaa2_eventdev_configure(const struct rte_eventdev *dev) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct rte_event_dev_config *conf = &dev->data->dev_conf; + + PMD_DRV_FUNC_TRACE(); + + priv->dequeue_timeout_ns = conf->dequeue_timeout_ns; + priv->nb_event_queues = conf->nb_event_queues; + priv->nb_event_ports = conf->nb_event_ports; + priv->nb_event_queue_flows = conf->nb_event_queue_flows; + priv->nb_event_port_dequeue_depth = conf->nb_event_port_dequeue_depth; + priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth; + priv->event_dev_cfg = conf->event_dev_cfg; + + PMD_DRV_LOG(DEBUG, "Configured eventdev devid=%d", dev->data->dev_id); + return 0; +} + +static int +dpaa2_eventdev_start(struct rte_eventdev *dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + return 0; +} + +static void +dpaa2_eventdev_stop(struct rte_eventdev *dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); +} + +static int +dpaa2_eventdev_close(struct rte_eventdev *dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + return 0; +} + +static void +dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, + struct rte_event_queue_conf *queue_conf) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); + RTE_SET_USED(queue_conf); + + queue_conf->nb_atomic_flows = DPAA2_EVENT_QUEUE_ATOMIC_FLOWS; + queue_conf->event_queue_cfg = RTE_EVENT_QUEUE_CFG_ATOMIC_ONLY | + RTE_EVENT_QUEUE_CFG_PARALLEL_ONLY; + queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL; +} + +static void +dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); +} + +static int +dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, + const struct rte_event_queue_conf *queue_conf) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct evq_info_t *evq_info = + &priv->evq_info[queue_id]; + + PMD_DRV_FUNC_TRACE(); + + evq_info->event_queue_cfg = queue_conf->event_queue_cfg; + + return 0; +} + +static void +dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, + struct rte_event_port_conf *port_conf) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(port_id); + RTE_SET_USED(port_conf); + + port_conf->new_event_threshold = + DPAA2_EVENT_MAX_NUM_EVENTS; + port_conf->dequeue_depth = + DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; + port_conf->enqueue_depth = + DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; +} + +static void +dpaa2_eventdev_port_release(void *port) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(port); +} + +static int +dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id, + const struct rte_event_port_conf *port_conf) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(port_conf); + + if (!dpaa2_io_portal[port_id].dpio_dev) { + dpaa2_io_portal[port_id].dpio_dev = + dpaa2_get_qbman_swp(port_id); + rte_atomic16_inc(&dpaa2_io_portal[port_id].dpio_dev->ref_count); + if (!dpaa2_io_portal[port_id].dpio_dev) + return -1; + } + + dpaa2_io_portal[port_id].eventdev = dev; + dev->data->ports[port_id] = &dpaa2_io_portal[port_id]; + return 0; +} + +static int +dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port, + uint8_t queues[], uint16_t nb_unlinks) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct dpaa2_io_portal_t *dpaa2_portal = port; + struct evq_info_t *evq_info; + int i; + + PMD_DRV_FUNC_TRACE(); + + for (i = 0; i < nb_unlinks; i++) { + evq_info = &priv->evq_info[queues[i]]; + qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal, + evq_info->dpcon->channel_index, 0); + dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio, + 0, dpaa2_portal->dpio_dev->token, + evq_info->dpcon->dpcon_id); + evq_info->link = 0; + } + + return (int)nb_unlinks; +} + +static int +dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, + const uint8_t queues[], const uint8_t priorities[], + uint16_t nb_links) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct dpaa2_io_portal_t *dpaa2_portal = port; + struct evq_info_t *evq_info; + uint8_t channel_index; + int ret, i, n; + + PMD_DRV_FUNC_TRACE(); + + for (i = 0; i < nb_links; i++) { + evq_info = &priv->evq_info[queues[i]]; + if (evq_info->link) + continue; + + ret = dpio_add_static_dequeue_channel( + dpaa2_portal->dpio_dev->dpio, + CMD_PRI_LOW, dpaa2_portal->dpio_dev->token, + evq_info->dpcon->dpcon_id, &channel_index); + if (ret < 0) { + PMD_DRV_ERR("Static dequeue cfg failed with ret: %d\n", + ret); + goto err; + } + + qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal, + channel_index, 1); + evq_info->dpcon->channel_index = channel_index; + evq_info->link = 1; + } + + RTE_SET_USED(priorities); + + return (int)nb_links; +err: + for (n = 0; n < i; n++) { + evq_info = &priv->evq_info[queues[n]]; + qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal, + evq_info->dpcon->channel_index, 0); + dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio, + 0, dpaa2_portal->dpio_dev->token, + evq_info->dpcon->dpcon_id); + evq_info->link = 0; + } + return ret; +} + +static int +dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns, + uint64_t *timeout_ticks) +{ + uint32_t scale = 1; + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + *timeout_ticks = ns * scale; + + return 0; +} + +static void +dpaa2_eventdev_dump(struct rte_eventdev *dev, FILE *f) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(f); +} + +static const struct rte_eventdev_ops dpaa2_eventdev_ops = { + .dev_infos_get = dpaa2_eventdev_info_get, + .dev_configure = dpaa2_eventdev_configure, + .dev_start = dpaa2_eventdev_start, + .dev_stop = dpaa2_eventdev_stop, + .dev_close = dpaa2_eventdev_close, + .queue_def_conf = dpaa2_eventdev_queue_def_conf, + .queue_setup = dpaa2_eventdev_queue_setup, + .queue_release = dpaa2_eventdev_queue_release, + .port_def_conf = dpaa2_eventdev_port_def_conf, + .port_setup = dpaa2_eventdev_port_setup, + .port_release = dpaa2_eventdev_port_release, + .port_link = dpaa2_eventdev_port_link, + .port_unlink = dpaa2_eventdev_port_unlink, + .timeout_ticks = dpaa2_eventdev_timeout_ticks, + .dump = dpaa2_eventdev_dump +}; + +static int +dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev, + struct dpaa2_dpcon_dev *dpcon_dev) +{ + struct dpci_rx_queue_cfg rx_queue_cfg; + int ret, i; + + /*Do settings to get the frame on a DPCON object*/ + rx_queue_cfg.options = DPCI_QUEUE_OPT_DEST | + DPCI_QUEUE_OPT_USER_CTX; + rx_queue_cfg.dest_cfg.dest_type = DPCI_DEST_DPCON; + rx_queue_cfg.dest_cfg.dest_id = dpcon_dev->dpcon_id; + rx_queue_cfg.dest_cfg.priority = DPAA2_EVENT_DEFAULT_DPCI_PRIO; + + dpci_dev->queue[DPAA2_EVENT_DPCI_PARALLEL_QUEUE].cb = + dpaa2_eventdev_process_parallel; + dpci_dev->queue[DPAA2_EVENT_DPCI_ATOMIC_QUEUE].cb = + dpaa2_eventdev_process_atomic; + + for (i = 0 ; i < DPAA2_EVENT_DPCI_MAX_QUEUES; i++) { + rx_queue_cfg.user_ctx = (uint64_t)(&dpci_dev->queue[i]); + ret = dpci_set_rx_queue(&dpci_dev->dpci, + CMD_PRI_LOW, + dpci_dev->token, i, + &rx_queue_cfg); + if (ret) { + PMD_DRV_LOG(ERR, + "set_rx_q failed with err code: %d", ret); + return ret; + } + } + return 0; +} + +static int +dpaa2_eventdev_create(const char *name) +{ + struct rte_eventdev *eventdev; + struct dpaa2_eventdev *priv; + struct dpaa2_dpcon_dev *dpcon_dev = NULL; + struct dpaa2_dpci_dev *dpci_dev = NULL; + int ret; + + eventdev = rte_event_pmd_vdev_init(name, + sizeof(struct dpaa2_eventdev), + rte_socket_id()); + if (eventdev == NULL) { + PMD_DRV_ERR("Failed to create eventdev vdev %s", name); + goto fail; + } + + eventdev->dev_ops = &dpaa2_eventdev_ops; + eventdev->schedule = NULL; + eventdev->enqueue = dpaa2_eventdev_enqueue; + eventdev->enqueue_burst = dpaa2_eventdev_enqueue_burst; + eventdev->enqueue_new_burst = dpaa2_eventdev_enqueue_burst; + eventdev->enqueue_forward_burst = dpaa2_eventdev_enqueue_burst; + eventdev->dequeue = dpaa2_eventdev_dequeue; + eventdev->dequeue_burst = dpaa2_eventdev_dequeue_burst; + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + priv = eventdev->data->dev_private; + priv->max_event_queues = 0; + + do { + dpcon_dev = rte_dpaa2_alloc_dpcon_dev(); + if (!dpcon_dev) + break; + priv->evq_info[priv->max_event_queues].dpcon = dpcon_dev; + + dpci_dev = rte_dpaa2_alloc_dpci_dev(); + if (!dpci_dev) { + rte_dpaa2_free_dpcon_dev(dpcon_dev); + break; + } + priv->evq_info[priv->max_event_queues].dpci = dpci_dev; + + ret = dpaa2_eventdev_setup_dpci(dpci_dev, dpcon_dev); + if (ret) { + PMD_DRV_LOG(ERR, + "dpci setup failed with err code: %d", ret); + return ret; + } + priv->max_event_queues++; + } while (dpcon_dev && dpci_dev); + + return 0; +fail: + return -EFAULT; +} + +static int +dpaa2_eventdev_probe(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + PMD_DRV_LOG(INFO, "Initializing %s", name); + return dpaa2_eventdev_create(name); +} + +static int +dpaa2_eventdev_remove(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + PMD_DRV_LOG(INFO, "Closing %s", name); + + return rte_event_pmd_vdev_uninit(name); +} + +static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = { + .probe = dpaa2_eventdev_probe, + .remove = dpaa2_eventdev_remove +}; + +RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd); diff --git a/drivers/event/dpaa2/dpaa2_eventdev.h b/drivers/event/dpaa2/dpaa2_eventdev.h new file mode 100644 index 00000000..f79f78aa --- /dev/null +++ b/drivers/event/dpaa2/dpaa2_eventdev.h @@ -0,0 +1,114 @@ +/* + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DPAA2_EVENTDEV_H__ +#define __DPAA2_EVENTDEV_H__ + +#include <rte_eventdev_pmd.h> +#include <rte_eventdev_pmd_vdev.h> +#include <rte_atomic.h> +#include <mc/fsl_dpcon.h> +#include <mc/fsl_mc_sys.h> + +#define EVENTDEV_NAME_DPAA2_PMD event_dpaa2 + +#ifdef RTE_LIBRTE_PMD_DPAA2_EVENTDEV_DEBUG +#define PMD_DRV_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#define PMD_DRV_FUNC_TRACE() PMD_DRV_LOG(DEBUG, ">>") +#else +#define PMD_DRV_LOG(level, fmt, args...) do { } while (0) +#define PMD_DRV_FUNC_TRACE() do { } while (0) +#endif + +#define PMD_DRV_ERR(fmt, args...) \ + RTE_LOG(ERR, PMD, "%s(): " fmt "\n", __func__, ## args) + +#define DPAA2_EVENT_DEFAULT_DPCI_PRIO 0 + +#define DPAA2_EVENT_MAX_QUEUES 16 +#define DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT 1 +#define DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT (UINT32_MAX - 1) +#define DPAA2_EVENT_MAX_QUEUE_FLOWS 2048 +#define DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS 8 +#define DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS 0 +#define DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH 8 +#define DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH 8 +#define DPAA2_EVENT_MAX_NUM_EVENTS (INT32_MAX - 1) + +#define DPAA2_EVENT_QUEUE_ATOMIC_FLOWS 2048 +#define DPAA2_EVENT_QUEUE_ORDER_SEQUENCES 2048 + +enum { + DPAA2_EVENT_DPCI_PARALLEL_QUEUE, + DPAA2_EVENT_DPCI_ATOMIC_QUEUE, + DPAA2_EVENT_DPCI_MAX_QUEUES +}; + +struct dpaa2_dpcon_dev { + TAILQ_ENTRY(dpaa2_dpcon_dev) next; + struct fsl_mc_io dpcon; + uint16_t token; + rte_atomic16_t in_use; + uint32_t dpcon_id; + uint16_t qbman_ch_id; + uint8_t num_priorities; + uint8_t channel_index; +}; + +struct evq_info_t { + /* DPcon device */ + struct dpaa2_dpcon_dev *dpcon; + /* Attached DPCI device */ + struct dpaa2_dpci_dev *dpci; + /* Configuration provided by the user */ + uint32_t event_queue_cfg; + uint8_t link; +}; + +struct dpaa2_eventdev { + struct evq_info_t evq_info[DPAA2_EVENT_MAX_QUEUES]; + uint32_t dequeue_timeout_ns; + uint8_t max_event_queues; + uint8_t nb_event_queues; + uint8_t nb_event_ports; + uint8_t resvd_1; + uint32_t nb_event_queue_flows; + uint32_t nb_event_port_dequeue_depth; + uint32_t nb_event_port_enqueue_depth; + uint32_t event_dev_cfg; +}; + +struct dpaa2_dpcon_dev *rte_dpaa2_alloc_dpcon_dev(void); +void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon); + +#endif /* __DPAA2_EVENTDEV_H__ */ diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c new file mode 100644 index 00000000..d3e73f90 --- /dev/null +++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c @@ -0,0 +1,139 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <unistd.h> +#include <stdio.h> +#include <sys/types.h> +#include <string.h> +#include <stdlib.h> +#include <fcntl.h> +#include <errno.h> + +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_string_fns.h> +#include <rte_cycles.h> +#include <rte_kvargs.h> +#include <rte_dev.h> +#include <rte_ethdev.h> + +#include <fslmc_vfio.h> +#include <mc/fsl_dpcon.h> +#include <portal/dpaa2_hw_pvt.h> +#include "dpaa2_eventdev.h" + +TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev); +static struct dpcon_dev_list dpcon_dev_list + = TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */ + +static int +rte_dpaa2_create_dpcon_device(struct fslmc_vfio_device *vdev __rte_unused, + struct vfio_device_info *obj_info __rte_unused, + int dpcon_id) +{ + struct dpaa2_dpcon_dev *dpcon_node; + struct dpcon_attr attr; + int ret; + + /* Allocate DPAA2 dpcon handle */ + dpcon_node = rte_malloc(NULL, sizeof(struct dpaa2_dpcon_dev), 0); + if (!dpcon_node) { + PMD_DRV_LOG(ERR, "Memory allocation failed for DPCON Device"); + return -1; + } + + /* Open the dpcon object */ + dpcon_node->dpcon.regs = rte_mcp_ptr_list[MC_PORTAL_INDEX]; + ret = dpcon_open(&dpcon_node->dpcon, + CMD_PRI_LOW, dpcon_id, &dpcon_node->token); + if (ret) { + PMD_DRV_LOG(ERR, "Resource alloc failure with err code: %d", + ret); + rte_free(dpcon_node); + return -1; + } + + /* Get the device attributes */ + ret = dpcon_get_attributes(&dpcon_node->dpcon, + CMD_PRI_LOW, dpcon_node->token, &attr); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Reading device failed with err code: %d", + ret); + rte_free(dpcon_node); + return -1; + } + + /* Updating device specific private information*/ + dpcon_node->qbman_ch_id = attr.qbman_ch_id; + dpcon_node->num_priorities = attr.num_priorities; + dpcon_node->dpcon_id = dpcon_id; + rte_atomic16_init(&dpcon_node->in_use); + + TAILQ_INSERT_TAIL(&dpcon_dev_list, dpcon_node, next); + + PMD_DRV_LOG(DEBUG, "DPAA2: Added [dpcon.%d]", dpcon_id); + + return 0; +} + +struct dpaa2_dpcon_dev *rte_dpaa2_alloc_dpcon_dev(void) +{ + struct dpaa2_dpcon_dev *dpcon_dev = NULL; + + /* Get DPCON dev handle from list using index */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { + if (dpcon_dev && rte_atomic16_test_and_set(&dpcon_dev->in_use)) + break; + } + + return dpcon_dev; +} + +void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon) +{ + struct dpaa2_dpcon_dev *dpcon_dev = NULL; + + /* Match DPCON handle and mark it free */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { + if (dpcon_dev == dpcon) { + rte_atomic16_dec(&dpcon_dev->in_use); + return; + } + } +} + +static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = { + .object_id = DPAA2_MC_DPCON_DEVID, + .create = rte_dpaa2_create_dpcon_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj); diff --git a/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map b/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map new file mode 100644 index 00000000..1c0b7559 --- /dev/null +++ b/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map @@ -0,0 +1,3 @@ +DPDK_17.08 { + local: *; +}; diff --git a/drivers/event/octeontx/Makefile b/drivers/event/octeontx/Makefile index aca3d095..e5661ca8 100644 --- a/drivers/event/octeontx/Makefile +++ b/drivers/event/octeontx/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2017 Cavium Networks. All rights reserved. +# Copyright(c) 2017 Cavium, Inc. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -13,7 +13,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium Networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # diff --git a/drivers/event/octeontx/rte_pmd_octeontx_ssovf.h b/drivers/event/octeontx/rte_pmd_octeontx_ssovf.h index 3da7cfdd..ba6d5142 100644 --- a/drivers/event/octeontx/rte_pmd_octeontx_ssovf.h +++ b/drivers/event/octeontx/rte_pmd_octeontx_ssovf.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c index c80a4437..d829b491 100644 --- a/drivers/event/octeontx/ssovf_evdev.c +++ b/drivers/event/octeontx/ssovf_evdev.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -30,6 +30,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <inttypes.h> + #include <rte_common.h> #include <rte_debug.h> #include <rte_dev.h> @@ -156,6 +158,8 @@ ssovf_fastpath_fns_set(struct rte_eventdev *dev) dev->schedule = NULL; dev->enqueue = ssows_enq; dev->enqueue_burst = ssows_enq_burst; + dev->enqueue_new_burst = ssows_enq_new_burst; + dev->enqueue_forward_burst = ssows_enq_fwd_burst; dev->dequeue = ssows_deq; dev->dequeue_burst = ssows_deq_burst; @@ -170,6 +174,7 @@ ssovf_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *dev_info) { struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + dev_info->driver_name = RTE_STR(EVENTDEV_NAME_OCTEONTX_PMD); dev_info->min_dequeue_timeout_ns = edev->min_deq_timeout_ns; dev_info->max_dequeue_timeout_ns = edev->max_deq_timeout_ns; dev_info->max_event_queues = edev->max_event_queues; @@ -194,6 +199,8 @@ ssovf_configure(const struct rte_eventdev *dev) ssovf_func_trace(); deq_tmo_ns = conf->dequeue_timeout_ns; + if (deq_tmo_ns == 0) + deq_tmo_ns = edev->min_deq_timeout_ns; if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { edev->is_timeout_deq = 1; diff --git a/drivers/event/octeontx/ssovf_evdev.h b/drivers/event/octeontx/ssovf_evdev.h index 6e0a3521..1cdc8104 100644 --- a/drivers/event/octeontx/ssovf_evdev.h +++ b/drivers/event/octeontx/ssovf_evdev.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -34,7 +34,7 @@ #define __SSOVF_EVDEV_H__ #include <rte_config.h> -#include <rte_eventdev_pmd.h> +#include <rte_eventdev_pmd_vdev.h> #include <rte_io.h> #include "rte_pmd_octeontx_ssovf.h" @@ -190,6 +190,10 @@ ssovf_pmd_priv(const struct rte_eventdev *eventdev) uint16_t ssows_enq(void *port, const struct rte_event *ev); uint16_t ssows_enq_burst(void *port, const struct rte_event ev[], uint16_t nb_events); +uint16_t ssows_enq_new_burst(void *port, + const struct rte_event ev[], uint16_t nb_events); +uint16_t ssows_enq_fwd_burst(void *port, + const struct rte_event ev[], uint16_t nb_events); uint16_t ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks); uint16_t ssows_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events, uint64_t timeout_ticks); diff --git a/drivers/event/octeontx/ssovf_mbox.c b/drivers/event/octeontx/ssovf_mbox.c index 7394a3a9..764414b5 100644 --- a/drivers/event/octeontx/ssovf_mbox.c +++ b/drivers/event/octeontx/ssovf_mbox.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -172,7 +172,7 @@ mbox_wait_response(struct mbox *m, struct octeontx_mbox_hdr *hdr, error: ssovf_log_err("Failed to send mbox(%d/%d) coproc=%d msg=%d ret=(%d,%d)", - m->tag_own, rx_hdr.tag, hdr->msg, hdr->coproc, res, + m->tag_own, rx_hdr.tag, hdr->coproc, hdr->msg, res, hdr->res_code); return res; } diff --git a/drivers/event/octeontx/ssovf_probe.c b/drivers/event/octeontx/ssovf_probe.c index b644ebde..e1c0c6d5 100644 --- a/drivers/event/octeontx/ssovf_probe.c +++ b/drivers/event/octeontx/ssovf_probe.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c index ad3fe684..5e17c7b8 100644 --- a/drivers/event/octeontx/ssovf_worker.c +++ b/drivers/event/octeontx/ssovf_worker.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -32,7 +32,7 @@ #include "ssovf_worker.h" -static force_inline void +static __rte_always_inline void ssows_new_event(struct ssows *ws, const struct rte_event *ev) { const uint64_t event_ptr = ev->u64; @@ -43,7 +43,7 @@ ssows_new_event(struct ssows *ws, const struct rte_event *ev) ssows_add_work(ws, event_ptr, tag, new_tt, grp); } -static force_inline void +static __rte_always_inline void ssows_fwd_swtag(struct ssows *ws, const struct rte_event *ev, const uint8_t grp) { const uint8_t cur_tt = ws->cur_tt; @@ -72,7 +72,7 @@ ssows_fwd_swtag(struct ssows *ws, const struct rte_event *ev, const uint8_t grp) #define OCT_EVENT_TYPE_GRP_FWD (RTE_EVENT_TYPE_MAX - 1) -static force_inline void +static __rte_always_inline void ssows_fwd_group(struct ssows *ws, const struct rte_event *ev, const uint8_t grp) { const uint64_t event_ptr = ev->u64; @@ -95,7 +95,7 @@ ssows_fwd_group(struct ssows *ws, const struct rte_event *ev, const uint8_t grp) ssows_add_work(ws, event_ptr, tag, new_tt, grp); } -static force_inline void +static __rte_always_inline void ssows_forward_event(struct ssows *ws, const struct rte_event *ev) { const uint8_t grp = ev->queue_id; @@ -112,39 +112,39 @@ ssows_forward_event(struct ssows *ws, const struct rte_event *ev) ssows_fwd_group(ws, ev, grp); } -static force_inline void +static __rte_always_inline void ssows_release_event(struct ssows *ws) { if (likely(ws->cur_tt != SSO_SYNC_UNTAGGED)) ssows_swtag_untag(ws); } -force_inline uint16_t __hot +__rte_always_inline uint16_t __hot ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks) { struct ssows *ws = port; RTE_SET_USED(timeout_ticks); - ssows_swtag_wait(ws); if (ws->swtag_req) { ws->swtag_req = 0; + ssows_swtag_wait(ws); return 1; } else { return ssows_get_work(ws, ev); } } -force_inline uint16_t __hot +__rte_always_inline uint16_t __hot ssows_deq_timeout(void *port, struct rte_event *ev, uint64_t timeout_ticks) { struct ssows *ws = port; uint64_t iter; uint16_t ret = 1; - ssows_swtag_wait(ws); if (ws->swtag_req) { ws->swtag_req = 0; + ssows_swtag_wait(ws); } else { ret = ssows_get_work(ws, ev); for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) @@ -171,7 +171,7 @@ ssows_deq_timeout_burst(void *port, struct rte_event ev[], uint16_t nb_events, return ssows_deq_timeout(port, ev, timeout_ticks); } -force_inline uint16_t __hot +__rte_always_inline uint16_t __hot ssows_enq(void *port, const struct rte_event *ev) { struct ssows *ws = port; @@ -179,6 +179,7 @@ ssows_enq(void *port, const struct rte_event *ev) switch (ev->op) { case RTE_EVENT_OP_NEW: + rte_smp_wmb(); ssows_new_event(ws, ev); break; case RTE_EVENT_OP_FORWARD: @@ -200,6 +201,30 @@ ssows_enq_burst(void *port, const struct rte_event ev[], uint16_t nb_events) return ssows_enq(port, ev); } +uint16_t __hot +ssows_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events) +{ + uint16_t i; + struct ssows *ws = port; + + rte_smp_wmb(); + for (i = 0; i < nb_events; i++) + ssows_new_event(ws, &ev[i]); + + return nb_events; +} + +uint16_t __hot +ssows_enq_fwd_burst(void *port, const struct rte_event ev[], uint16_t nb_events) +{ + struct ssows *ws = port; + RTE_SET_USED(nb_events); + + ssows_forward_event(ws, ev); + + return 1; +} + void ssows_flush_events(struct ssows *ws, uint8_t queue_id) { diff --git a/drivers/event/octeontx/ssovf_worker.h b/drivers/event/octeontx/ssovf_worker.h index 300dfae8..55f72555 100644 --- a/drivers/event/octeontx/ssovf_worker.h +++ b/drivers/event/octeontx/ssovf_worker.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -32,6 +32,7 @@ #include <rte_common.h> +#include <rte_branch_prediction.h> #include "ssovf_evdev.h" @@ -42,17 +43,13 @@ enum { SSO_SYNC_EMPTY }; -#ifndef force_inline -#define force_inline inline __attribute__((always_inline)) -#endif - #ifndef __hot #define __hot __attribute__((hot)) #endif /* SSO Operations */ -static force_inline uint16_t +static __rte_always_inline uint16_t ssows_get_work(struct ssows *ws, struct rte_event *ev) { uint64_t get_work0, get_work1; @@ -70,7 +67,7 @@ ssows_get_work(struct ssows *ws, struct rte_event *ev) return !!get_work1; } -static force_inline void +static __rte_always_inline void ssows_add_work(struct ssows *ws, const uint64_t event_ptr, const uint32_t tag, const uint8_t new_tt, const uint8_t grp) { @@ -80,7 +77,7 @@ ssows_add_work(struct ssows *ws, const uint64_t event_ptr, const uint32_t tag, ssovf_store_pair(add_work0, event_ptr, ws->grps[grp]); } -static force_inline void +static __rte_always_inline void ssows_swtag_full(struct ssows *ws, const uint64_t event_ptr, const uint32_t tag, const uint8_t new_tt, const uint8_t grp) { @@ -92,7 +89,7 @@ ssows_swtag_full(struct ssows *ws, const uint64_t event_ptr, const uint32_t tag, SSOW_VHWS_OP_SWTAG_FULL0)); } -static force_inline void +static __rte_always_inline void ssows_swtag_desched(struct ssows *ws, uint32_t tag, uint8_t new_tt, uint8_t grp) { uint64_t val; @@ -101,7 +98,7 @@ ssows_swtag_desched(struct ssows *ws, uint32_t tag, uint8_t new_tt, uint8_t grp) ssovf_write64(val, ws->base + SSOW_VHWS_OP_SWTAG_DESCHED); } -static force_inline void +static __rte_always_inline void ssows_swtag_norm(struct ssows *ws, uint32_t tag, uint8_t new_tt) { uint64_t val; @@ -110,27 +107,27 @@ ssows_swtag_norm(struct ssows *ws, uint32_t tag, uint8_t new_tt) ssovf_write64(val, ws->base + SSOW_VHWS_OP_SWTAG_NORM); } -static force_inline void +static __rte_always_inline void ssows_swtag_untag(struct ssows *ws) { ssovf_write64(0, ws->base + SSOW_VHWS_OP_SWTAG_UNTAG); ws->cur_tt = SSO_SYNC_UNTAGGED; } -static force_inline void +static __rte_always_inline void ssows_upd_wqp(struct ssows *ws, uint8_t grp, uint64_t event_ptr) { ssovf_store_pair((uint64_t)grp << 34, event_ptr, (ws->base + SSOW_VHWS_OP_UPD_WQP_GRP0)); } -static force_inline void +static __rte_always_inline void ssows_desched(struct ssows *ws) { ssovf_write64(0, ws->base + SSOW_VHWS_OP_DESCHED); } -static force_inline void +static __rte_always_inline void ssows_swtag_wait(struct ssows *ws) { /* Wait for the SWTAG/SWTAG_FULL operation */ diff --git a/drivers/event/skeleton/Makefile b/drivers/event/skeleton/Makefile index c2b2456b..e6d58711 100644 --- a/drivers/event/skeleton/Makefile +++ b/drivers/event/skeleton/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 Cavium Networks. All rights reserved. +# Copyright(c) 2016 Cavium, Inc. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -13,7 +13,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium Networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c index 800bd76e..bcd20556 100644 --- a/drivers/event/skeleton/skeleton_eventdev.c +++ b/drivers/event/skeleton/skeleton_eventdev.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -43,10 +43,9 @@ #include <rte_dev.h> #include <rte_eal.h> #include <rte_log.h> +#include <rte_malloc.h> #include <rte_memory.h> #include <rte_memzone.h> -#include <rte_malloc.h> -#include <rte_pci.h> #include <rte_lcore.h> #include <rte_vdev.h> @@ -130,6 +129,7 @@ skeleton_eventdev_info_get(struct rte_eventdev *dev, dev_info->max_event_port_enqueue_depth = 16; dev_info->max_num_events = (1ULL << 20); dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_QUEUE_QOS | + RTE_EVENT_DEV_CAP_BURST_MODE | RTE_EVENT_DEV_CAP_EVENT_QOS; } @@ -427,18 +427,28 @@ static const struct rte_pci_id pci_id_skeleton_map[] = { }, }; -static struct rte_eventdev_driver pci_eventdev_skeleton_pmd = { - .pci_drv = { - .id_table = pci_id_skeleton_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - .probe = rte_event_pmd_pci_probe, - .remove = rte_event_pmd_pci_remove, - }, - .eventdev_init = skeleton_eventdev_init, - .dev_private_size = sizeof(struct skeleton_eventdev), +static int +event_skeleton_pci_probe(struct rte_pci_driver *pci_drv, + struct rte_pci_device *pci_dev) +{ + return rte_event_pmd_pci_probe(pci_drv, pci_dev, + sizeof(struct skeleton_eventdev), skeleton_eventdev_init); +} + +static int +event_skeleton_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_event_pmd_pci_remove(pci_dev, NULL); +} + +static struct rte_pci_driver pci_eventdev_skeleton_pmd = { + .id_table = pci_id_skeleton_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = event_skeleton_pci_probe, + .remove = event_skeleton_pci_remove, }; -RTE_PMD_REGISTER_PCI(event_skeleton_pci, pci_eventdev_skeleton_pmd.pci_drv); +RTE_PMD_REGISTER_PCI(event_skeleton_pci, pci_eventdev_skeleton_pmd); RTE_PMD_REGISTER_PCI_TABLE(event_skeleton_pci, pci_id_skeleton_map); /* VDEV based event device */ diff --git a/drivers/event/skeleton/skeleton_eventdev.h b/drivers/event/skeleton/skeleton_eventdev.h index 1ce62da7..32064721 100644 --- a/drivers/event/skeleton/skeleton_eventdev.h +++ b/drivers/event/skeleton/skeleton_eventdev.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -33,7 +33,8 @@ #ifndef __SKELETON_EVENTDEV_H__ #define __SKELETON_EVENTDEV_H__ -#include <rte_eventdev_pmd.h> +#include <rte_eventdev_pmd_pci.h> +#include <rte_eventdev_pmd_vdev.h> #ifdef RTE_LIBRTE_PMD_SKELETON_EVENTDEV_DEBUG #define PMD_DRV_LOG(level, fmt, args...) \ diff --git a/drivers/event/sw/event_ring.h b/drivers/event/sw/event_ring.h index cdaee95d..734a3b4b 100644 --- a/drivers/event/sw/event_ring.h +++ b/drivers/event/sw/event_ring.h @@ -61,10 +61,6 @@ struct qe_ring { struct rte_event ring[0] __rte_cache_aligned; }; -#ifndef force_inline -#define force_inline inline __attribute__((always_inline)) -#endif - static inline struct qe_ring * qe_ring_create(const char *name, unsigned int size, unsigned int socket_id) { @@ -91,19 +87,19 @@ qe_ring_destroy(struct qe_ring *r) rte_free(r); } -static force_inline unsigned int +static __rte_always_inline unsigned int qe_ring_count(const struct qe_ring *r) { return r->write_idx - r->read_idx; } -static force_inline unsigned int +static __rte_always_inline unsigned int qe_ring_free_count(const struct qe_ring *r) { return r->size - qe_ring_count(r); } -static force_inline unsigned int +static __rte_always_inline unsigned int qe_ring_enqueue_burst(struct qe_ring *r, const struct rte_event *qes, unsigned int nb_qes, uint16_t *free_count) { @@ -130,7 +126,7 @@ qe_ring_enqueue_burst(struct qe_ring *r, const struct rte_event *qes, return nb_qes; } -static force_inline unsigned int +static __rte_always_inline unsigned int qe_ring_enqueue_burst_with_ops(struct qe_ring *r, const struct rte_event *qes, unsigned int nb_qes, uint8_t *ops) { @@ -157,7 +153,7 @@ qe_ring_enqueue_burst_with_ops(struct qe_ring *r, const struct rte_event *qes, return nb_qes; } -static force_inline unsigned int +static __rte_always_inline unsigned int qe_ring_dequeue_burst(struct qe_ring *r, struct rte_event *qes, unsigned int nb_qes) { diff --git a/drivers/event/sw/iq_ring.h b/drivers/event/sw/iq_ring.h index d480d156..64cf6784 100644 --- a/drivers/event/sw/iq_ring.h +++ b/drivers/event/sw/iq_ring.h @@ -56,10 +56,6 @@ struct iq_ring { struct rte_event ring[QID_IQ_DEPTH]; }; -#ifndef force_inline -#define force_inline inline __attribute__((always_inline)) -#endif - static inline struct iq_ring * iq_ring_create(const char *name, unsigned int socket_id) { @@ -81,19 +77,19 @@ iq_ring_destroy(struct iq_ring *r) rte_free(r); } -static force_inline uint16_t +static __rte_always_inline uint16_t iq_ring_count(const struct iq_ring *r) { return r->write_idx - r->read_idx; } -static force_inline uint16_t +static __rte_always_inline uint16_t iq_ring_free_count(const struct iq_ring *r) { return QID_IQ_MASK - iq_ring_count(r); } -static force_inline uint16_t +static __rte_always_inline uint16_t iq_ring_enqueue_burst(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) { const uint16_t read = r->read_idx; @@ -112,7 +108,7 @@ iq_ring_enqueue_burst(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) return nb_qes; } -static force_inline uint16_t +static __rte_always_inline uint16_t iq_ring_dequeue_burst(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) { uint16_t read = r->read_idx; @@ -132,7 +128,7 @@ iq_ring_dequeue_burst(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) } /* assumes there is space, from a previous dequeue_burst */ -static force_inline uint16_t +static __rte_always_inline uint16_t iq_ring_put_back(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) { uint16_t i, read = r->read_idx; @@ -144,19 +140,19 @@ iq_ring_put_back(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) return nb_qes; } -static force_inline const struct rte_event * +static __rte_always_inline const struct rte_event * iq_ring_peek(const struct iq_ring *r) { return &r->ring[r->read_idx & QID_IQ_MASK]; } -static force_inline void +static __rte_always_inline void iq_ring_pop(struct iq_ring *r) { r->read_idx++; } -static force_inline int +static __rte_always_inline int iq_ring_enqueue(struct iq_ring *r, const struct rte_event *qe) { const uint16_t read = r->read_idx; diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c index a31aaa66..9c534b7f 100644 --- a/drivers/event/sw/sw_evdev.c +++ b/drivers/event/sw/sw_evdev.c @@ -30,6 +30,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <inttypes.h> #include <string.h> #include <rte_vdev.h> @@ -37,10 +38,11 @@ #include <rte_kvargs.h> #include <rte_ring.h> #include <rte_errno.h> +#include <rte_event_ring.h> +#include <rte_service_component.h> #include "sw_evdev.h" #include "iq_ring.h" -#include "event_ring.h" #define EVENTDEV_NAME_SW_PMD event_sw #define NUMA_NODE_ARG "numa_node" @@ -90,7 +92,8 @@ sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[], } else if (q->type == RTE_SCHED_TYPE_ORDERED) { p->num_ordered_qids++; p->num_qids_mapped++; - } else if (q->type == RTE_SCHED_TYPE_ATOMIC) { + } else if (q->type == RTE_SCHED_TYPE_ATOMIC || + q->type == RTE_SCHED_TYPE_PARALLEL) { p->num_qids_mapped++; } @@ -138,7 +141,7 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id, { struct sw_evdev *sw = sw_pmd_priv(dev); struct sw_port *p = &sw->ports[port_id]; - char buf[QE_RING_NAMESIZE]; + char buf[RTE_RING_NAMESIZE]; unsigned int i; struct rte_event_dev_info info; @@ -159,10 +162,19 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id, p->id = port_id; p->sw = sw; - snprintf(buf, sizeof(buf), "sw%d_%s", dev->data->dev_id, - "rx_worker_ring"); - p->rx_worker_ring = qe_ring_create(buf, MAX_SW_PROD_Q_DEPTH, - dev->data->socket_id); + /* check to see if rings exists - port_setup() can be called multiple + * times legally (assuming device is stopped). If ring exists, free it + * to so it gets re-created with the correct size + */ + snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id, + port_id, "rx_worker_ring"); + struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf); + if (existing_ring) + rte_event_ring_free(existing_ring); + + p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH, + dev->data->socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); if (p->rx_worker_ring == NULL) { SW_LOG_ERR("Error creating RX worker ring for port %d\n", port_id); @@ -171,12 +183,18 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id, p->inflight_max = conf->new_event_threshold; - snprintf(buf, sizeof(buf), "sw%d_%s", dev->data->dev_id, - "cq_worker_ring"); - p->cq_worker_ring = qe_ring_create(buf, conf->dequeue_depth, - dev->data->socket_id); + /* check if ring exists, same as rx_worker above */ + snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id, + port_id, "cq_worker_ring"); + existing_ring = rte_event_ring_lookup(buf); + if (existing_ring) + rte_event_ring_free(existing_ring); + + p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth, + dev->data->socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); if (p->cq_worker_ring == NULL) { - qe_ring_destroy(p->rx_worker_ring); + rte_event_ring_free(p->rx_worker_ring); SW_LOG_ERR("Error creating CQ worker ring for port %d\n", port_id); return -1; @@ -202,8 +220,8 @@ sw_port_release(void *port) if (p == NULL) return; - qe_ring_destroy(p->rx_worker_ring); - qe_ring_destroy(p->cq_worker_ring); + rte_event_ring_free(p->rx_worker_ring); + rte_event_ring_free(p->cq_worker_ring); memset(p, 0, sizeof(*p)); } @@ -435,6 +453,7 @@ sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info) .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH, .max_num_events = SW_INFLIGHT_EVENTS_TOTAL, .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS | + RTE_EVENT_DEV_CAP_BURST_MODE | RTE_EVENT_DEV_CAP_EVENT_QOS), }; @@ -509,8 +528,9 @@ sw_dump(struct rte_eventdev *dev, FILE *f) fprintf(f, "\n"); if (p->rx_worker_ring) { - uint64_t used = qe_ring_count(p->rx_worker_ring); - uint64_t space = qe_ring_free_count(p->rx_worker_ring); + uint64_t used = rte_event_ring_count(p->rx_worker_ring); + uint64_t space = rte_event_ring_free_count( + p->rx_worker_ring); const char *col = (space == 0) ? COL_RED : COL_RESET; fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4" PRIu64 COL_RESET"\n", col, used, space); @@ -518,8 +538,9 @@ sw_dump(struct rte_eventdev *dev, FILE *f) fprintf(f, "\trx ring not initialized.\n"); if (p->cq_worker_ring) { - uint64_t used = qe_ring_count(p->cq_worker_ring); - uint64_t space = qe_ring_free_count(p->cq_worker_ring); + uint64_t used = rte_event_ring_count(p->cq_worker_ring); + uint64_t space = rte_event_ring_free_count( + p->cq_worker_ring); const char *col = (space == 0) ? COL_RED : COL_RESET; fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4" PRIu64 COL_RESET"\n", col, used, space); @@ -559,12 +580,13 @@ sw_dump(struct rte_eventdev *dev, FILE *f) inflights += qid->fids[flow].pcount; } - uint32_t cq; - fprintf(f, "\tInflights: %u\tFlows pinned per port: ", - inflights); - for (cq = 0; cq < sw->port_count; cq++) - fprintf(f, "%d ", affinities_per_port[cq]); - fprintf(f, "\n"); + uint32_t port; + fprintf(f, "\tPer Port Stats:\n"); + for (port = 0; port < sw->port_count; port++) { + fprintf(f, "\t Port %d: Pkts: %"PRIu64, port, + qid->to_port[port]); + fprintf(f, "\tFlows: %d\n", affinities_per_port[port]); + } uint32_t iq; uint32_t iq_printed = 0; @@ -593,6 +615,13 @@ sw_start(struct rte_eventdev *dev) { unsigned int i, j; struct sw_evdev *sw = sw_pmd_priv(dev); + + /* check a service core is mapped to this service */ + struct rte_service_spec *s = rte_service_get_by_name(sw->service_name); + if (!rte_service_is_running(s)) + SW_LOG_ERR("Warning: No Service core enabled on service %s\n", + s->name); + /* check all ports are set up */ for (i = 0; i < sw->port_count; i++) if (sw->ports[i].rx_worker_ring == NULL) { @@ -695,6 +724,14 @@ set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque) return 0; } + +static int32_t sw_sched_service_func(void *args) +{ + struct rte_eventdev *dev = args; + sw_event_schedule(dev); + return 0; +} + static int sw_probe(struct rte_vdev_device *vdev) { @@ -792,6 +829,8 @@ sw_probe(struct rte_vdev_device *vdev) dev->dev_ops = &evdev_sw_ops; dev->enqueue = sw_event_enqueue; dev->enqueue_burst = sw_event_enqueue_burst; + dev->enqueue_new_burst = sw_event_enqueue_burst; + dev->enqueue_forward_burst = sw_event_enqueue_burst; dev->dequeue = sw_event_dequeue; dev->dequeue_burst = sw_event_dequeue_burst; dev->schedule = sw_event_schedule; @@ -806,6 +845,22 @@ sw_probe(struct rte_vdev_device *vdev) sw->credit_update_quanta = credit_quanta; sw->sched_quanta = sched_quanta; + /* register service with EAL */ + struct rte_service_spec service; + memset(&service, 0, sizeof(struct rte_service_spec)); + snprintf(service.name, sizeof(service.name), "%s_service", name); + snprintf(sw->service_name, sizeof(sw->service_name), "%s_service", + name); + service.socket_id = socket_id; + service.callback = sw_sched_service_func; + service.callback_userdata = (void *)dev; + + int32_t ret = rte_service_register(&service); + if (ret) { + SW_LOG_ERR("service register() failed"); + return -ENOEXEC; + } + return 0; } diff --git a/drivers/event/sw/sw_evdev.h b/drivers/event/sw/sw_evdev.h index 61c671d6..71de3c14 100644 --- a/drivers/event/sw/sw_evdev.h +++ b/drivers/event/sw/sw_evdev.h @@ -34,7 +34,7 @@ #define _SW_EVDEV_H_ #include <rte_eventdev.h> -#include <rte_eventdev_pmd.h> +#include <rte_eventdev_pmd_vdev.h> #include <rte_atomic.h> #define SW_DEFAULT_CREDIT_QUANTA 32 @@ -59,6 +59,7 @@ #define EVENTDEV_NAME_SW_PMD event_sw #define SW_PMD_NAME RTE_STR(event_sw) +#define SW_PMD_NAME_MAX 64 #define SW_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1) @@ -149,6 +150,7 @@ struct sw_qid { uint32_t cq_num_mapped_cqs; uint32_t cq_next_tx; /* cq to write next (non-atomic) packet */ uint32_t cq_map[SW_PORTS_MAX]; + uint64_t to_port[SW_PORTS_MAX]; /* Track flow ids for atomic load balancing */ struct sw_fid_t fids[SW_QID_NUM_FIDS]; @@ -189,9 +191,9 @@ struct sw_port { int16_t num_ordered_qids; /** Ring and buffer for pulling events from workers for scheduling */ - struct qe_ring *rx_worker_ring __rte_cache_aligned; + struct rte_event_ring *rx_worker_ring __rte_cache_aligned; /** Ring and buffer for pushing packets to workers after scheduling */ - struct qe_ring *cq_worker_ring; + struct rte_event_ring *cq_worker_ring; /* hole */ @@ -275,6 +277,8 @@ struct sw_evdev { /* store num stats and offset of the stats for each queue */ uint16_t xstats_count_per_qid[RTE_EVENT_MAX_QUEUES_PER_DEV]; uint16_t xstats_offset_for_qid[RTE_EVENT_MAX_QUEUES_PER_DEV]; + + char service_name[SW_PMD_NAME_MAX]; }; static inline struct sw_evdev * diff --git a/drivers/event/sw/sw_evdev_scheduler.c b/drivers/event/sw/sw_evdev_scheduler.c index a333a6f0..8a2c9d4f 100644 --- a/drivers/event/sw/sw_evdev_scheduler.c +++ b/drivers/event/sw/sw_evdev_scheduler.c @@ -32,9 +32,9 @@ #include <rte_ring.h> #include <rte_hash_crc.h> +#include <rte_event_ring.h> #include "sw_evdev.h" #include "iq_ring.h" -#include "event_ring.h" #define SW_IQS_MASK (SW_IQS_MAX-1) @@ -119,11 +119,12 @@ sw_schedule_atomic_to_cq(struct sw_evdev *sw, struct sw_qid * const qid, p->stats.tx_pkts++; qid->stats.tx_pkts++; + qid->to_port[cq]++; /* if we just filled in the last slot, flush the buffer */ if (sw->cq_ring_space[cq] == 0) { - struct qe_ring *worker = p->cq_worker_ring; - qe_ring_enqueue_burst(worker, p->cq_buf, + struct rte_event_ring *worker = p->cq_worker_ring; + rte_event_ring_enqueue_burst(worker, p->cq_buf, p->cq_buf_count, &sw->cq_ring_space[cq]); p->cq_buf_count = 0; @@ -170,7 +171,8 @@ sw_schedule_parallel_to_cq(struct sw_evdev *sw, struct sw_qid * const qid, cq = qid->cq_map[cq_idx]; if (++cq_idx == qid->cq_num_mapped_cqs) cq_idx = 0; - } while (qe_ring_free_count(sw->ports[cq].cq_worker_ring) == 0 || + } while (rte_event_ring_free_count( + sw->ports[cq].cq_worker_ring) == 0 || sw->ports[cq].inflights == SW_PORT_HIST_LIST); struct sw_port *p = &sw->ports[cq]; @@ -362,17 +364,17 @@ sw_schedule_reorder(struct sw_evdev *sw, int qid_start, int qid_end) return pkts_iter; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void sw_refill_pp_buf(struct sw_evdev *sw, struct sw_port *port) { RTE_SET_USED(sw); - struct qe_ring *worker = port->rx_worker_ring; + struct rte_event_ring *worker = port->rx_worker_ring; port->pp_buf_start = 0; - port->pp_buf_count = qe_ring_dequeue_burst(worker, port->pp_buf, - RTE_DIM(port->pp_buf)); + port->pp_buf_count = rte_event_ring_dequeue_burst(worker, port->pp_buf, + RTE_DIM(port->pp_buf), NULL); } -static inline uint32_t __attribute__((always_inline)) +static __rte_always_inline uint32_t __pull_port_lb(struct sw_evdev *sw, uint32_t port_id, int allow_reorder) { static struct reorder_buffer_entry dummy_rob; @@ -585,8 +587,8 @@ sw_event_schedule(struct rte_eventdev *dev) * worker cores: aka, do the ring transfers batched. */ for (i = 0; i < sw->port_count; i++) { - struct qe_ring *worker = sw->ports[i].cq_worker_ring; - qe_ring_enqueue_burst(worker, sw->ports[i].cq_buf, + struct rte_event_ring *worker = sw->ports[i].cq_worker_ring; + rte_event_ring_enqueue_burst(worker, sw->ports[i].cq_buf, sw->ports[i].cq_buf_count, &sw->cq_ring_space[i]); sw->ports[i].cq_buf_count = 0; diff --git a/drivers/event/sw/sw_evdev_worker.c b/drivers/event/sw/sw_evdev_worker.c index 9cb6bef5..d76d3d5c 100644 --- a/drivers/event/sw/sw_evdev_worker.c +++ b/drivers/event/sw/sw_evdev_worker.c @@ -32,9 +32,9 @@ #include <rte_atomic.h> #include <rte_cycles.h> +#include <rte_event_ring.h> #include "sw_evdev.h" -#include "event_ring.h" #define PORT_ENQUEUE_MAX_BURST_SIZE 64 @@ -52,13 +52,31 @@ sw_event_release(struct sw_port *p, uint8_t index) ev.op = sw_qe_flag_map[RTE_EVENT_OP_RELEASE]; uint16_t free_count; - qe_ring_enqueue_burst(p->rx_worker_ring, &ev, 1, &free_count); + rte_event_ring_enqueue_burst(p->rx_worker_ring, &ev, 1, &free_count); /* each release returns one credit */ p->outstanding_releases--; p->inflight_credits++; } +/* + * special-case of rte_event_ring enqueue, with overriding the ops member on + * the events that get written to the ring. + */ +static inline unsigned int +enqueue_burst_with_ops(struct rte_event_ring *r, const struct rte_event *events, + unsigned int n, uint8_t *ops) +{ + struct rte_event tmp_evs[PORT_ENQUEUE_MAX_BURST_SIZE]; + unsigned int i; + + memcpy(tmp_evs, events, n * sizeof(events[0])); + for (i = 0; i < n; i++) + tmp_evs[i].op = ops[i]; + + return rte_event_ring_enqueue_burst(r, tmp_evs, n, NULL); +} + uint16_t sw_event_enqueue_burst(void *port, const struct rte_event ev[], uint16_t num) { @@ -87,6 +105,7 @@ sw_event_enqueue_burst(void *port, const struct rte_event ev[], uint16_t num) return 0; } + uint32_t forwards = 0; for (i = 0; i < num; i++) { int op = ev[i].op; int outstanding = p->outstanding_releases > 0; @@ -95,6 +114,7 @@ sw_event_enqueue_burst(void *port, const struct rte_event ev[], uint16_t num) p->inflight_credits -= (op == RTE_EVENT_OP_NEW); p->inflight_credits += (op == RTE_EVENT_OP_RELEASE) * outstanding; + forwards += (op == RTE_EVENT_OP_FORWARD); new_ops[i] = sw_qe_flag_map[op]; new_ops[i] &= ~(invalid_qid << QE_FLAG_VALID_SHIFT); @@ -113,8 +133,11 @@ sw_event_enqueue_burst(void *port, const struct rte_event ev[], uint16_t num) } } + /* handle directed port forward credits */ + p->inflight_credits -= forwards * p->is_directed; + /* returns number of events actually enqueued */ - uint32_t enq = qe_ring_enqueue_burst_with_ops(p->rx_worker_ring, ev, i, + uint32_t enq = enqueue_burst_with_ops(p->rx_worker_ring, ev, i, new_ops); if (p->outstanding_releases == 0 && p->last_dequeue_burst_sz != 0) { uint64_t burst_ticks = rte_get_timer_cycles() - @@ -141,7 +164,7 @@ sw_event_dequeue_burst(void *port, struct rte_event *ev, uint16_t num, RTE_SET_USED(wait); struct sw_port *p = (void *)port; struct sw_evdev *sw = (void *)p->sw; - struct qe_ring *ring = p->cq_worker_ring; + struct rte_event_ring *ring = p->cq_worker_ring; uint32_t credit_update_quanta = sw->credit_update_quanta; /* check that all previous dequeues have been released */ @@ -153,7 +176,7 @@ sw_event_dequeue_burst(void *port, struct rte_event *ev, uint16_t num, } /* returns number of events actually dequeued */ - uint16_t ndeq = qe_ring_dequeue_burst(ring, ev, num); + uint16_t ndeq = rte_event_ring_dequeue_burst(ring, ev, num, NULL); if (unlikely(ndeq == 0)) { p->outstanding_releases = 0; p->zero_polls++; diff --git a/drivers/event/sw/sw_evdev_xstats.c b/drivers/event/sw/sw_evdev_xstats.c index c7b1abe8..8cb6d88d 100644 --- a/drivers/event/sw/sw_evdev_xstats.c +++ b/drivers/event/sw/sw_evdev_xstats.c @@ -30,9 +30,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <rte_event_ring.h> #include "sw_evdev.h" #include "iq_ring.h" -#include "event_ring.h" enum xstats_type { /* common stats */ @@ -57,6 +57,7 @@ enum xstats_type { iq_used, /* qid port mapping specific */ pinned, + pkts, /* note: qid-to-port pkts */ }; typedef uint64_t (*xstats_fn)(const struct sw_evdev *dev, @@ -104,10 +105,10 @@ get_port_stat(const struct sw_evdev *sw, uint16_t obj_idx, case calls: return p->total_polls; case credits: return p->inflight_credits; case poll_return: return p->zero_polls; - case rx_used: return qe_ring_count(p->rx_worker_ring); - case rx_free: return qe_ring_free_count(p->rx_worker_ring); - case tx_used: return qe_ring_count(p->cq_worker_ring); - case tx_free: return qe_ring_free_count(p->cq_worker_ring); + case rx_used: return rte_event_ring_count(p->rx_worker_ring); + case rx_free: return rte_event_ring_free_count(p->rx_worker_ring); + case tx_used: return rte_event_ring_count(p->cq_worker_ring); + case tx_free: return rte_event_ring_free_count(p->cq_worker_ring); default: return -1; } } @@ -179,6 +180,8 @@ get_qid_port_stat(const struct sw_evdev *sw, uint16_t obj_idx, return pin; } while (0); break; + case pkts: + return qid->to_port[port]; default: return -1; } } @@ -246,8 +249,11 @@ sw_xstats_init(struct sw_evdev *sw) static const enum xstats_type qid_iq_types[] = { iq_used }; /* reset allowed */ - static const char * const qid_port_stats[] = { "pinned_flows" }; - static const enum xstats_type qid_port_types[] = { pinned }; + static const char * const qid_port_stats[] = { "pinned_flows", + "packets" + }; + static const enum xstats_type qid_port_types[] = { pinned, pkts }; + static const uint8_t qid_port_reset_allowed[] = {0, 1}; /* reset allowed */ /* ---- end of stat definitions ---- */ @@ -312,8 +318,9 @@ sw_xstats_init(struct sw_evdev *sw) port, port_stats[i]); } - for (bkt = 0; bkt < (sw->ports[port].cq_worker_ring->size >> - SW_DEQ_STAT_BUCKET_SHIFT) + 1; bkt++) { + for (bkt = 0; bkt < (rte_event_ring_get_capacity( + sw->ports[port].cq_worker_ring) >> + SW_DEQ_STAT_BUCKET_SHIFT) + 1; bkt++) { for (i = 0; i < RTE_DIM(port_bucket_stats); i++) { sw->xstats[stat] = (struct sw_xstats_entry){ .fn = get_port_bucket_stat, @@ -376,7 +383,8 @@ sw_xstats_init(struct sw_evdev *sw) .stat = qid_port_types[i], .mode = RTE_EVENT_DEV_XSTATS_QUEUE, .extra_arg = port, - .reset_allowed = 0, + .reset_allowed = + qid_port_reset_allowed[i], }; snprintf(sname, sizeof(sname), "qid_%u_port_%u_%s", diff --git a/drivers/mempool/Makefile b/drivers/mempool/Makefile index 8fd40e1b..efd55f23 100644 --- a/drivers/mempool/Makefile +++ b/drivers/mempool/Makefile @@ -1,7 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2017 NXP. All rights reserved. -# All rights reserved. +# Copyright 2017 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/drivers/mempool/dpaa2/Makefile b/drivers/mempool/dpaa2/Makefile index 26ccebde..1a174968 100644 --- a/drivers/mempool/dpaa2/Makefile +++ b/drivers/mempool/dpaa2/Makefile @@ -1,7 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 NXP. All rights reserved. -# All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c index 5a5d6aaa..6df203fc 100644 --- a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c +++ b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -47,7 +47,6 @@ #include <rte_cycles.h> #include <rte_kvargs.h> #include <rte_dev.h> -#include <rte_ethdev.h> #include <fslmc_logs.h> #include <mc/fsl_dpbp.h> @@ -63,6 +62,7 @@ rte_hw_mbuf_create_pool(struct rte_mempool *mp) { struct dpaa2_bp_list *bp_list; struct dpaa2_dpbp_dev *avail_dpbp; + struct dpaa2_bp_info *bp_info; struct dpbp_attr dpbp_attr; uint32_t bpid; int ret, p_ret; @@ -127,7 +127,12 @@ rte_hw_mbuf_create_pool(struct rte_mempool *mp) rte_dpaa2_bpid_info[bpid].bp_list = bp_list; rte_dpaa2_bpid_info[bpid].bpid = bpid; - mp->pool_data = (void *)&rte_dpaa2_bpid_info[bpid]; + bp_info = rte_malloc(NULL, + sizeof(struct dpaa2_bp_info), + RTE_CACHE_LINE_SIZE); + rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid], + sizeof(struct dpaa2_bp_info)); + mp->pool_data = (void *)bp_info; PMD_INIT_LOG(DEBUG, "BP List created for bpid =%d", dpbp_attr.bpid); @@ -161,7 +166,7 @@ rte_hw_mbuf_free_pool(struct rte_mempool *mp) while (temp) { if (temp == bp) { prev->next = temp->next; - free(bp); + rte_free(bp); break; } prev = temp; @@ -169,6 +174,7 @@ rte_hw_mbuf_free_pool(struct rte_mempool *mp) } } + rte_free(mp->pool_data); dpaa2_free_dpbp_dev(dpbp_node); } @@ -188,7 +194,7 @@ rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused, if (unlikely(!DPAA2_PER_LCORE_DPIO)) { ret = dpaa2_affine_qbman_swp(); if (ret != 0) { - RTE_LOG(ERR, PMD, "Failed to allocate IO portal"); + RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n"); return; } } @@ -267,7 +273,7 @@ rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, if (unlikely(!DPAA2_PER_LCORE_DPIO)) { ret = dpaa2_affine_qbman_swp(); if (ret != 0) { - RTE_LOG(ERR, PMD, "Failed to allocate IO portal"); + RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n"); return ret; } } @@ -294,7 +300,7 @@ rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, /* Releasing all buffers allocated */ rte_dpaa2_mbuf_release(pool, obj_table, bpid, bp_info->meta_data_size, n); - return ret; + return -ENOBUFS; } /* assigning mbuf from the acquired objects */ for (i = 0; (i < ret) && bufs[i]; i++) { @@ -323,7 +329,7 @@ rte_hw_mbuf_free_bulk(struct rte_mempool *pool, bp_info = mempool_to_bpinfo(pool); if (!(bp_info->bp_list)) { - RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured"); + RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); return -ENOENT; } rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid, @@ -341,7 +347,7 @@ rte_hw_mbuf_get_count(const struct rte_mempool *mp) struct dpaa2_dpbp_dev *dpbp_node; if (!mp || !mp->pool_data) { - RTE_LOG(ERR, PMD, "Invalid mempool provided"); + RTE_LOG(ERR, PMD, "Invalid mempool provided\n"); return 0; } @@ -351,12 +357,12 @@ rte_hw_mbuf_get_count(const struct rte_mempool *mp) ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token, &num_of_bufs); if (ret) { - RTE_LOG(ERR, PMD, "Unable to obtain free buf count (err=%d)", + RTE_LOG(ERR, PMD, "Unable to obtain free buf count (err=%d)\n", ret); return 0; } - RTE_LOG(DEBUG, PMD, "Free bufs = %u", num_of_bufs); + RTE_LOG(DEBUG, PMD, "Free bufs = %u\n", num_of_bufs); return num_of_bufs; } diff --git a/drivers/mempool/dpaa2/dpaa2_hw_mempool.h b/drivers/mempool/dpaa2/dpaa2_hw_mempool.h index c4d7fc02..56b71bed 100644 --- a/drivers/mempool/dpaa2/dpaa2_hw_mempool.h +++ b/drivers/mempool/dpaa2/dpaa2_hw_mempool.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/drivers/mempool/ring/Makefile b/drivers/mempool/ring/Makefile index 54630d99..b339d907 100644 --- a/drivers/mempool/ring/Makefile +++ b/drivers/mempool/ring/Makefile @@ -1,7 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2017 NXP. -# All rights reserved. +# Copyright 2017 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/drivers/mempool/stack/Makefile b/drivers/mempool/stack/Makefile index 8f3125c7..7577b23c 100644 --- a/drivers/mempool/stack/Makefile +++ b/drivers/mempool/stack/Makefile @@ -1,7 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2017 NXP. -# All rights reserved. +# Copyright 2017 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/drivers/net/Makefile b/drivers/net/Makefile index 35ed8135..d33c9590 100644 --- a/drivers/net/Makefile +++ b/drivers/net/Makefile @@ -59,6 +59,8 @@ DIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena DEPDIRS-ena = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic DEPDIRS-enic = $(core-libs) librte_hash +DIRS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe +DEPDIRS-failsafe = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k DEPDIRS-fm10k = $(core-libs) librte_hash DIRS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e diff --git a/drivers/net/af_packet/rte_eth_af_packet.c b/drivers/net/af_packet/rte_eth_af_packet.c index 9ccb7af9..9a47852c 100644 --- a/drivers/net/af_packet/rte_eth_af_packet.c +++ b/drivers/net/af_packet/rte_eth_af_packet.c @@ -413,7 +413,7 @@ eth_rx_queue_setup(struct rte_eth_dev *dev, if (data_size > buf_size) { RTE_LOG(ERR, PMD, "%s: %d bytes will not fit in mbuf (%d bytes)\n", - dev->data->name, data_size, buf_size); + dev->device->name, data_size, buf_size); return -ENOMEM; } @@ -568,7 +568,12 @@ rte_pmd_init_internals(struct rte_vdev_device *dev, int rc, tpver, discard; int qsockfd = -1; unsigned int i, q, rdsize; - int fanout_arg __rte_unused, bypass __rte_unused; +#if defined(PACKET_FANOUT) + int fanout_arg; +#endif +#if defined(PACKET_QDISC_BYPASS) + int bypass; +#endif for (k_idx = 0; k_idx < kvlist->count; k_idx++) { pair = &kvlist->pairs[k_idx]; diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c index 017817e2..6db362b0 100644 --- a/drivers/net/ark/ark_ethdev.c +++ b/drivers/net/ark/ark_ethdev.c @@ -77,6 +77,7 @@ static int eth_ark_macaddr_add(struct rte_eth_dev *dev, uint32_t pool); static void eth_ark_macaddr_remove(struct rte_eth_dev *dev, uint32_t index); +static int eth_ark_set_mtu(struct rte_eth_dev *dev, uint16_t size); /* * The packet generator is a functional block used to generate packet @@ -179,6 +180,8 @@ static const struct eth_dev_ops ark_eth_dev_ops = { .mac_addr_add = eth_ark_macaddr_add, .mac_addr_remove = eth_ark_macaddr_remove, .mac_addr_set = eth_ark_set_default_mac_addr, + + .mtu_set = eth_ark_set_mtu, }; static int @@ -256,6 +259,10 @@ check_for_ext(struct ark_adapter *ark) (void (*)(struct rte_eth_dev *, struct ether_addr *, void *)) dlsym(ark->d_handle, "mac_addr_set"); + ark->user_ext.set_mtu = + (int (*)(struct rte_eth_dev *, uint16_t, + void *)) + dlsym(ark->d_handle, "set_mtu"); return found; } @@ -278,7 +285,7 @@ eth_ark_dev_init(struct rte_eth_dev *dev) ret = check_for_ext(ark); if (ret) return ret; - pci_dev = ARK_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); rte_eth_copy_pci_info(dev, pci_dev); /* Use dummy function until setup */ @@ -346,8 +353,9 @@ eth_ark_dev_init(struct rte_eth_dev *dev) } if (ark->user_ext.dev_init) { - ark->user_data = ark->user_ext.dev_init(dev, ark->a_bar, 0); - if (!ark->user_data) { + ark->user_data[dev->data->port_id] = + ark->user_ext.dev_init(dev, ark->a_bar, 0); + if (!ark->user_data[dev->data->port_id]) { PMD_DRV_LOG(INFO, "Failed to initialize PMD extension!" " continuing without it\n"); @@ -369,7 +377,8 @@ eth_ark_dev_init(struct rte_eth_dev *dev) */ if (ark->user_ext.dev_get_port_count) port_count = - ark->user_ext.dev_get_port_count(dev, ark->user_data); + ark->user_ext.dev_get_port_count(dev, + ark->user_data[dev->data->port_id]); ark->num_ports = port_count; for (p = 0; p < port_count; p++) { @@ -410,9 +419,10 @@ eth_ark_dev_init(struct rte_eth_dev *dev) goto error; } - if (ark->user_ext.dev_init) - ark->user_data = + if (ark->user_ext.dev_init) { + ark->user_data[eth_dev->data->port_id] = ark->user_ext.dev_init(dev, ark->a_bar, p); + } } return ret; @@ -508,7 +518,8 @@ eth_ark_dev_uninit(struct rte_eth_dev *dev) return 0; if (ark->user_ext.dev_uninit) - ark->user_ext.dev_uninit(dev, ark->user_data); + ark->user_ext.dev_uninit(dev, + ark->user_data[dev->data->port_id]); ark_pktgen_uninit(ark->pg); ark_pktchkr_uninit(ark->pc); @@ -529,7 +540,8 @@ eth_ark_dev_configure(struct rte_eth_dev *dev) eth_ark_dev_set_link_up(dev); if (ark->user_ext.dev_configure) - return ark->user_ext.dev_configure(dev, ark->user_data); + return ark->user_ext.dev_configure(dev, + ark->user_data[dev->data->port_id]); return 0; } @@ -592,7 +604,8 @@ eth_ark_dev_start(struct rte_eth_dev *dev) } if (ark->user_ext.dev_start) - ark->user_ext.dev_start(dev, ark->user_data); + ark->user_ext.dev_start(dev, + ark->user_data[dev->data->port_id]); return 0; } @@ -614,7 +627,8 @@ eth_ark_dev_stop(struct rte_eth_dev *dev) /* Stop the extension first */ if (ark->user_ext.dev_stop) - ark->user_ext.dev_stop(dev, ark->user_data); + ark->user_ext.dev_stop(dev, + ark->user_data[dev->data->port_id]); /* Stop the packet generator */ if (ark->start_pg) @@ -636,7 +650,7 @@ eth_ark_dev_stop(struct rte_eth_dev *dev) } /* Stop DDM */ - /* Wait up to 0.1 second. each stop is upto 1000 * 10 useconds */ + /* Wait up to 0.1 second. each stop is up to 1000 * 10 useconds */ for (i = 0; i < 10; i++) { status = ark_ddm_stop(ark->ddm.v, 1); if (status == 0) @@ -697,7 +711,8 @@ eth_ark_dev_close(struct rte_eth_dev *dev) uint16_t i; if (ark->user_ext.dev_close) - ark->user_ext.dev_close(dev, ark->user_data); + ark->user_ext.dev_close(dev, + ark->user_data[dev->data->port_id]); eth_ark_dev_stop(dev); eth_ark_udm_force_close(dev); @@ -751,7 +766,7 @@ eth_ark_dev_info_get(struct rte_eth_dev *dev, ETH_LINK_SPEED_40G | ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G); - dev_info->pci_dev = ARK_DEV_TO_PCI(dev); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); } static int @@ -765,7 +780,7 @@ eth_ark_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) if (ark->user_ext.link_update) { return ark->user_ext.link_update (dev, wait_to_complete, - ark->user_data); + ark->user_data[dev->data->port_id]); } return 0; } @@ -778,7 +793,8 @@ eth_ark_dev_set_link_up(struct rte_eth_dev *dev) (struct ark_adapter *)dev->data->dev_private; if (ark->user_ext.dev_set_link_up) - return ark->user_ext.dev_set_link_up(dev, ark->user_data); + return ark->user_ext.dev_set_link_up(dev, + ark->user_data[dev->data->port_id]); return 0; } @@ -790,7 +806,8 @@ eth_ark_dev_set_link_down(struct rte_eth_dev *dev) (struct ark_adapter *)dev->data->dev_private; if (ark->user_ext.dev_set_link_down) - return ark->user_ext.dev_set_link_down(dev, ark->user_data); + return ark->user_ext.dev_set_link_down(dev, + ark->user_data[dev->data->port_id]); return 0; } @@ -813,7 +830,8 @@ eth_ark_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) for (i = 0; i < dev->data->nb_rx_queues; i++) eth_rx_queue_stats_get(dev->data->rx_queues[i], stats); if (ark->user_ext.stats_get) - ark->user_ext.stats_get(dev, stats, ark->user_data); + ark->user_ext.stats_get(dev, stats, + ark->user_data[dev->data->port_id]); } static void @@ -824,11 +842,12 @@ eth_ark_dev_stats_reset(struct rte_eth_dev *dev) (struct ark_adapter *)dev->data->dev_private; for (i = 0; i < dev->data->nb_tx_queues; i++) - eth_tx_queue_stats_reset(dev->data->rx_queues[i]); + eth_tx_queue_stats_reset(dev->data->tx_queues[i]); for (i = 0; i < dev->data->nb_rx_queues; i++) eth_rx_queue_stats_reset(dev->data->rx_queues[i]); if (ark->user_ext.stats_reset) - ark->user_ext.stats_reset(dev, ark->user_data); + ark->user_ext.stats_reset(dev, + ark->user_data[dev->data->port_id]); } static int @@ -845,7 +864,7 @@ eth_ark_macaddr_add(struct rte_eth_dev *dev, mac_addr, index, pool, - ark->user_data); + ark->user_data[dev->data->port_id]); return 0; } return -ENOTSUP; @@ -858,7 +877,8 @@ eth_ark_macaddr_remove(struct rte_eth_dev *dev, uint32_t index) (struct ark_adapter *)dev->data->dev_private; if (ark->user_ext.mac_addr_remove) - ark->user_ext.mac_addr_remove(dev, index, ark->user_data); + ark->user_ext.mac_addr_remove(dev, index, + ark->user_data[dev->data->port_id]); } static void @@ -869,7 +889,21 @@ eth_ark_set_default_mac_addr(struct rte_eth_dev *dev, (struct ark_adapter *)dev->data->dev_private; if (ark->user_ext.mac_addr_set) - ark->user_ext.mac_addr_set(dev, mac_addr, ark->user_data); + ark->user_ext.mac_addr_set(dev, mac_addr, + ark->user_data[dev->data->port_id]); +} + +static int +eth_ark_set_mtu(struct rte_eth_dev *dev, uint16_t size) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.set_mtu) + return ark->user_ext.set_mtu(dev, size, + ark->user_data[dev->data->port_id]); + + return -ENOTSUP; } static inline int diff --git a/drivers/net/ark/ark_ethdev.h b/drivers/net/ark/ark_ethdev.h index 9f8d32fc..df5547bf 100644 --- a/drivers/net/ark/ark_ethdev.h +++ b/drivers/net/ark/ark_ethdev.h @@ -34,8 +34,4 @@ #ifndef _ARK_ETHDEV_H_ #define _ARK_ETHDEV_H_ -#define ARK_DEV_TO_PCI(eth_dev) \ - RTE_DEV_TO_PCI((eth_dev)->device) - - #endif diff --git a/drivers/net/ark/ark_ext.h b/drivers/net/ark/ark_ext.h index f805f64f..63b7a261 100644 --- a/drivers/net/ark/ark_ext.h +++ b/drivers/net/ark/ark_ext.h @@ -112,4 +112,8 @@ void mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, void *user_data); +int set_mtu(struct rte_eth_dev *dev, + uint16_t size, + void *user_data); + #endif diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h index a2e9e8ff..2a6375fe 100644 --- a/drivers/net/ark/ark_global.h +++ b/drivers/net/ark/ark_global.h @@ -64,7 +64,7 @@ #define ARK_RCPACING_BASE 0xb0000 #define ARK_EXTERNAL_BASE 0x100000 #define ARK_MPU_QOFFSET 0x00100 -#define ARK_MAX_PORTS 8 +#define ARK_MAX_PORTS RTE_MAX_ETHPORTS #define offset8(n) n #define offset16(n) ((n) / 2) @@ -106,11 +106,12 @@ struct ark_user_ext { void *); void (*mac_addr_remove)(struct rte_eth_dev *, uint32_t, void *); void (*mac_addr_set)(struct rte_eth_dev *, struct ether_addr *, void *); + int (*set_mtu)(struct rte_eth_dev *, uint16_t, void *); }; struct ark_adapter { /* User extension private data */ - void *user_data; + void *user_data[ARK_MAX_PORTS]; /* Pointers to packet generator and checker */ int start_pg; diff --git a/drivers/net/avp/avp_ethdev.c b/drivers/net/avp/avp_ethdev.c index fe6849f5..c746a0e2 100644 --- a/drivers/net/avp/avp_ethdev.c +++ b/drivers/net/avp/avp_ethdev.c @@ -71,8 +71,7 @@ static void avp_dev_close(struct rte_eth_dev *dev); static void avp_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static void avp_vlan_offload_set(struct rte_eth_dev *dev, int mask); -static int avp_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); +static int avp_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete); static void avp_dev_promiscuous_enable(struct rte_eth_dev *dev); static void avp_dev_promiscuous_disable(struct rte_eth_dev *dev); @@ -113,9 +112,6 @@ static void avp_dev_stats_get(struct rte_eth_dev *dev, static void avp_dev_stats_reset(struct rte_eth_dev *dev); -#define AVP_DEV_TO_PCI(eth_dev) RTE_DEV_TO_PCI((eth_dev)->device) - - #define AVP_MAX_RX_BURST 64 #define AVP_MAX_TX_BURST 64 #define AVP_MAX_MAC_ADDRS 1 @@ -393,7 +389,7 @@ static void * avp_dev_translate_address(struct rte_eth_dev *eth_dev, phys_addr_t host_phys_addr) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_mem_resource *resource; struct rte_avp_memmap_info *info; struct rte_avp_memmap *map; @@ -446,7 +442,7 @@ avp_dev_version_check(uint32_t version) static int avp_dev_check_regions(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_avp_memmap_info *memmap; struct rte_avp_device_info *info; struct rte_mem_resource *resource; @@ -582,7 +578,7 @@ _avp_set_rx_queue_mappings(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) static void _avp_set_queue_counts(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); struct rte_avp_device_info *host_info; void *addr; @@ -642,7 +638,7 @@ avp_dev_attach(struct rte_eth_dev *eth_dev) * re-run the device create utility which will parse the new host info * and setup the AVP device queue pointers. */ - ret = avp_dev_create(AVP_DEV_TO_PCI(eth_dev), eth_dev); + ret = avp_dev_create(RTE_ETH_DEV_TO_PCI(eth_dev), eth_dev); if (ret < 0) { PMD_DRV_LOG(ERR, "Failed to re-create AVP device, ret=%d\n", ret); @@ -696,7 +692,7 @@ static void avp_dev_interrupt_handler(void *data) { struct rte_eth_dev *eth_dev = data; - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; uint32_t status, value; int ret; @@ -755,7 +751,7 @@ avp_dev_interrupt_handler(void *data) static int avp_dev_enable_interrupts(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; int ret; @@ -780,7 +776,7 @@ avp_dev_enable_interrupts(struct rte_eth_dev *eth_dev) static int avp_dev_disable_interrupts(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; int ret; @@ -805,7 +801,7 @@ avp_dev_disable_interrupts(struct rte_eth_dev *eth_dev) static int avp_dev_setup_interrupts(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); int ret; /* register a callback handler with UIO for interrupt notifications */ @@ -825,7 +821,7 @@ avp_dev_setup_interrupts(struct rte_eth_dev *eth_dev) static int avp_dev_migration_pending(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; uint32_t value; @@ -986,7 +982,7 @@ eth_avp_dev_init(struct rte_eth_dev *eth_dev) struct rte_pci_device *pci_dev; int ret; - pci_dev = AVP_DEV_TO_PCI(eth_dev); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); eth_dev->dev_ops = &avp_eth_dev_ops; eth_dev->rx_pkt_burst = &avp_recv_pkts; eth_dev->tx_pkt_burst = &avp_xmit_pkts; @@ -2011,7 +2007,7 @@ avp_dev_tx_queue_release(void *tx_queue) static int avp_dev_configure(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); struct rte_avp_device_info *host_info; struct rte_avp_device_config config; @@ -2206,8 +2202,7 @@ avp_dev_info_get(struct rte_eth_dev *eth_dev, { struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); - dev_info->driver_name = "rte_avp_pmd"; - dev_info->pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); dev_info->max_rx_queues = avp->max_rx_queues; dev_info->max_tx_queues = avp->max_tx_queues; dev_info->min_rx_bufsize = AVP_MIN_RX_BUFSIZE; diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c index 1a7e1c8e..06733d15 100644 --- a/drivers/net/bnx2x/bnx2x.c +++ b/drivers/net/bnx2x/bnx2x.c @@ -120,7 +120,7 @@ static int bnx2x_alloc_mem(struct bnx2x_softc *sc); static void bnx2x_free_mem(struct bnx2x_softc *sc); static int bnx2x_alloc_fw_stats_mem(struct bnx2x_softc *sc); static void bnx2x_free_fw_stats_mem(struct bnx2x_softc *sc); -static __attribute__ ((noinline)) +static __rte_noinline int bnx2x_nic_load(struct bnx2x_softc *sc); static int bnx2x_handle_sp_tq(struct bnx2x_softc *sc); @@ -887,7 +887,7 @@ storm_memset_eq_prod(struct bnx2x_softc *sc, uint16_t eq_prod, uint16_t pfid) /* * Post a slowpath command. * - * A slowpath command is used to propogate a configuration change through + * A slowpath command is used to propagate a configuration change through * the controller in a controlled manner, allowing each STORM processor and * other H/W blocks to phase in the change. The commands sent on the * slowpath are referred to as ramrods. Depending on the ramrod used the @@ -1962,7 +1962,7 @@ static void bnx2x_squeeze_objects(struct bnx2x_softc *sc) } /* stop the controller */ -__attribute__ ((noinline)) +__rte_noinline int bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link) { @@ -2002,7 +2002,7 @@ bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link /* * Nothing to do during unload if previous bnx2x_nic_load() - * did not completed succesfully - all resourses are released. + * did not completed successfully - all resourses are released. */ if ((sc->state == BNX2X_STATE_CLOSED) || (sc->state == BNX2X_STATE_ERROR)) { return 0; @@ -3931,7 +3931,7 @@ static void bnx2x_attn_int_deasserted2(struct bnx2x_softc *sc, uint32_t attn) mask1 = REG_RD(sc, PXP2_REG_PXP2_INT_MASK_1); val0 = REG_RD(sc, PXP2_REG_PXP2_INT_STS_0); /* - * If the olny PXP2_EOP_ERROR_BIT is set in + * If the only PXP2_EOP_ERROR_BIT is set in * STS0 and STS1 - clear it * * probably we lose additional attentions between @@ -5910,7 +5910,7 @@ static void bnx2x_set_234_gates(struct bnx2x_softc *sc, uint8_t close) (val | HC_CONFIG_0_REG_BLOCK_DISABLE_0)); } else { -/* Prevent incomming interrupts in IGU */ +/* Prevent incoming interrupts in IGU */ val = REG_RD(sc, IGU_REG_BLOCK_CONFIGURATION); if (close) @@ -7124,7 +7124,7 @@ void bnx2x_periodic_callout(struct bnx2x_softc *sc) } /* start the controller */ -static __attribute__ ((noinline)) +static __rte_noinline int bnx2x_nic_load(struct bnx2x_softc *sc) { uint32_t val; diff --git a/drivers/net/bnx2x/bnx2x_ethdev.c b/drivers/net/bnx2x/bnx2x_ethdev.c index 6d7c002f..6f62a37f 100644 --- a/drivers/net/bnx2x/bnx2x_ethdev.c +++ b/drivers/net/bnx2x/bnx2x_ethdev.c @@ -119,7 +119,7 @@ bnx2x_interrupt_action(struct rte_eth_dev *dev) bnx2x_link_update(dev); } -static __rte_unused void +static void bnx2x_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; @@ -439,10 +439,10 @@ bnx2x_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, } static void -bnx2x_dev_infos_get(struct rte_eth_dev *dev, __rte_unused struct rte_eth_dev_info *dev_info) +bnx2x_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct bnx2x_softc *sc = dev->data->dev_private; - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_queues = sc->max_rx_queues; dev_info->max_tx_queues = sc->max_tx_queues; dev_info->min_rx_bufsize = BNX2X_MIN_RX_BUF_SIZE; @@ -531,7 +531,7 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf) PMD_INIT_FUNC_TRACE(); eth_dev->dev_ops = is_vf ? &bnx2xvf_eth_dev_ops : &bnx2x_eth_dev_ops; - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); diff --git a/drivers/net/bnx2x/bnx2x_stats.c b/drivers/net/bnx2x/bnx2x_stats.c index c489cbee..6223cfef 100644 --- a/drivers/net/bnx2x/bnx2x_stats.c +++ b/drivers/net/bnx2x/bnx2x_stats.c @@ -1371,9 +1371,9 @@ bnx2x_prep_fw_stats_req(struct bnx2x_softc *sc) cur_query_entry = &sc->fw_stats_req->query[BNX2X_PORT_QUERY_IDX]; cur_query_entry->kind = STATS_TYPE_PORT; - /* For port query index is a DONT CARE */ + /* For port query index is a DON'T CARE */ cur_query_entry->index = SC_PORT(sc); - /* For port query funcID is a DONT CARE */ + /* For port query funcID is a DON'T CARE */ cur_query_entry->funcID = htole16(SC_FUNC(sc)); cur_query_entry->address.hi = htole32(U64_HI(cur_data_offset)); cur_query_entry->address.lo = htole32(U64_LO(cur_data_offset)); @@ -1385,7 +1385,7 @@ bnx2x_prep_fw_stats_req(struct bnx2x_softc *sc) cur_query_entry = &sc->fw_stats_req->query[BNX2X_PF_QUERY_IDX]; cur_query_entry->kind = STATS_TYPE_PF; - /* For PF query index is a DONT CARE */ + /* For PF query index is a DON'T CARE */ cur_query_entry->index = SC_PORT(sc); cur_query_entry->funcID = htole16(SC_FUNC(sc)); cur_query_entry->address.hi = htole32(U64_HI(cur_data_offset)); @@ -1450,7 +1450,7 @@ void bnx2x_memset_stats(struct bnx2x_softc *sc) if (sc->port.pmf && sc->port.port_stx) bnx2x_port_stats_base_init(sc); - /* mark the end of statistics initializiation */ + /* mark the end of statistics initialization */ sc->stats_init = false; } @@ -1536,7 +1536,7 @@ bnx2x_stats_init(struct bnx2x_softc *sc) bnx2x_port_stats_base_init(sc); } - /* mark the end of statistics initializiation */ + /* mark the end of statistics initialization */ sc->stats_init = FALSE; } diff --git a/drivers/net/bnx2x/bnx2x_vfpf.h b/drivers/net/bnx2x/bnx2x_vfpf.h index 955ea982..d7cc11be 100644 --- a/drivers/net/bnx2x/bnx2x_vfpf.h +++ b/drivers/net/bnx2x/bnx2x_vfpf.h @@ -282,7 +282,7 @@ struct bnx2x_vf_bulletin { uint16_t version; uint16_t length; - uint64_t valid_bitmap; /* bitmap indicating wich fields + uint64_t valid_bitmap; /* bitmap indicating which fields * hold valid values */ diff --git a/drivers/net/bnx2x/ecore_hsi.h b/drivers/net/bnx2x/ecore_hsi.h index 5808e1ae..5cce6647 100644 --- a/drivers/net/bnx2x/ecore_hsi.h +++ b/drivers/net/bnx2x/ecore_hsi.h @@ -2499,7 +2499,7 @@ struct shmem2_region { * SHMEM_EEE_XXX_ADV definitions (where XXX is replaced by speed). * bit 28 when 1'b1 EEE was requested. * bit 29 when 1'b1 tx lpi was requested. - * bit 30 when 1'b1 EEE was negotiated. Tx lpi will be asserted iff + * bit 30 when 1'b1 EEE was negotiated. Tx lpi will be asserted if * 30:29 are 2'b11. * bit 31 when 1'b0 bits 15:0 contain a PORT_FEAT_CFG_EEE_ define as * value. When 1'b1 those bits contains a value times 16 microseconds. @@ -3911,7 +3911,7 @@ struct client_init_rx_data uint16_t bd_pause_thr_high /* number of remaining bds above which, we send un-pause message */; uint16_t sge_pause_thr_low /* number of remaining sges under which, we send pause message */; uint16_t sge_pause_thr_high /* number of remaining sges above which, we send un-pause message */; - uint16_t rx_cos_mask /* the bits that will be set on pfc/ safc paket whith will be genratet when this ring is full. for regular flow control set this to 1 */; + uint16_t rx_cos_mask /* the bits that will be set on pfc/ safc paket with will be genratet when this ring is full. for regular flow control set this to 1 */; uint16_t silent_vlan_value /* The vlan to compare, in case, silent vlan is set */; uint16_t silent_vlan_mask /* The vlan mask, in case, silent vlan is set */; uint32_t reserved6[2]; @@ -4903,7 +4903,7 @@ enum eth_tx_vlan_type */ enum eth_vlan_filter_mode { - ETH_VLAN_FILTER_ANY_VLAN /* Dont filter by vlan */, + ETH_VLAN_FILTER_ANY_VLAN /* Don't filter by vlan */, ETH_VLAN_FILTER_SPECIFIC_VLAN /* Only the vlan_id is allowed */, ETH_VLAN_FILTER_CLASSIFY /* Vlan will be added to CAM for classification */, MAX_ETH_VLAN_FILTER_MODE}; @@ -4937,7 +4937,7 @@ struct mac_configuration_entry #define MAC_CONFIGURATION_ENTRY_RDMA_MAC_SHIFT 1 #define MAC_CONFIGURATION_ENTRY_VLAN_FILTERING_MODE (0x3<<2) /* BitField flags (use enum eth_vlan_filter_mode) */ #define MAC_CONFIGURATION_ENTRY_VLAN_FILTERING_MODE_SHIFT 2 -#define MAC_CONFIGURATION_ENTRY_OVERRIDE_VLAN_REMOVAL (0x1<<4) /* BitField flags BitField flags 0 - cant remove vlan 1 - can remove vlan. relevant only to everest1 */ +#define MAC_CONFIGURATION_ENTRY_OVERRIDE_VLAN_REMOVAL (0x1<<4) /* BitField flags BitField flags 0 - can't remove vlan 1 - can remove vlan. relevant only to everest1 */ #define MAC_CONFIGURATION_ENTRY_OVERRIDE_VLAN_REMOVAL_SHIFT 4 #define MAC_CONFIGURATION_ENTRY_BROADCAST (0x1<<5) /* BitField flags BitField flags 0 - not broadcast 1 - broadcast. relevant only to everest1 */ #define MAC_CONFIGURATION_ENTRY_BROADCAST_SHIFT 5 @@ -5024,7 +5024,7 @@ struct tstorm_eth_function_common_config #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_RSS_IPV6_TCP_CAPABILITY_SHIFT 3 #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_RSS_MODE (0x7<<4) /* BitField config_flagsGeneral configuration flags RSS mode of operation (use enum eth_rss_mode) */ #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_RSS_MODE_SHIFT 4 -#define TSTORM_ETH_FUNCTION_COMMON_CONFIG_VLAN_FILTERING_ENABLE (0x1<<7) /* BitField config_flagsGeneral configuration flags 0 - Dont filter by vlan, 1 - Filter according to the vlans specificied in mac_filter_config */ +#define TSTORM_ETH_FUNCTION_COMMON_CONFIG_VLAN_FILTERING_ENABLE (0x1<<7) /* BitField config_flagsGeneral configuration flags 0 - Don't filter by vlan, 1 - Filter according to the vlans specificied in mac_filter_config */ #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_VLAN_FILTERING_ENABLE_SHIFT 7 #define __TSTORM_ETH_FUNCTION_COMMON_CONFIG_RESERVED0 (0xFF<<8) /* BitField config_flagsGeneral configuration flags */ #define __TSTORM_ETH_FUNCTION_COMMON_CONFIG_RESERVED0_SHIFT 8 @@ -5634,7 +5634,7 @@ struct flow_control_configuration struct function_start_data { uint8_t function_mode /* the function mode */; - uint8_t allow_npar_tx_switching /* If set, inter-pf tx switching is allowed in Switch Independant function mode. (E2/E3 Only) */; + uint8_t allow_npar_tx_switching /* If set, inter-pf tx switching is allowed in Switch Independent function mode. (E2/E3 Only) */; uint16_t sd_vlan_tag /* value of Vlan in case of switch depended multi-function mode */; uint16_t vif_id /* value of VIF id in case of NIV multi-function mode */; uint8_t path_id; diff --git a/drivers/net/bnx2x/ecore_init.h b/drivers/net/bnx2x/ecore_init.h index d25e2803..4576c565 100644 --- a/drivers/net/bnx2x/ecore_init.h +++ b/drivers/net/bnx2x/ecore_init.h @@ -304,7 +304,7 @@ static inline void ecore_dcb_config_qm(struct bnx2x_softc *sc, enum cos_mode mod /* - * congestion managment port init api description + * congestion management port init api description * the api works as follows: * the driver should pass the cmng_init_input struct, the port_init function * will prepare the required internal ram structure which will be passed back diff --git a/drivers/net/bnx2x/ecore_sp.c b/drivers/net/bnx2x/ecore_sp.c index 22f2dc95..ef7f9fea 100644 --- a/drivers/net/bnx2x/ecore_sp.c +++ b/drivers/net/bnx2x/ecore_sp.c @@ -2676,7 +2676,7 @@ static void ecore_mcast_hdl_del(struct bnx2x_softc *sc, * @cmd: * @start_cnt: first line in the ramrod data that may be used * - * This function is called iff there is enough place for the current command in + * This function is called if there is enough place for the current command in * the ramrod data. * Returns number of lines filled in the ramrod data in total. */ @@ -2834,7 +2834,7 @@ static int ecore_mcast_setup_e2(struct bnx2x_softc *sc, if (ECORE_LIST_IS_EMPTY(&o->pending_cmds_head)) o->clear_sched(o); - /* The below may be TRUE iff there was enough room in ramrod + /* The below may be TRUE if there was enough room in ramrod * data for all pending commands and for the current * command. Otherwise the current command would have been added * to the pending commands and p->mcast_list_len would have been diff --git a/drivers/net/bnx2x/ecore_sp.h b/drivers/net/bnx2x/ecore_sp.h index 9c1f55df..e7ec96e9 100644 --- a/drivers/net/bnx2x/ecore_sp.h +++ b/drivers/net/bnx2x/ecore_sp.h @@ -1116,10 +1116,10 @@ struct ecore_config_rss_params { /* RSS hash values */ uint32_t rss_key[10]; - /* valid only iff ECORE_RSS_UPDATE_TOE is set */ + /* valid only if ECORE_RSS_UPDATE_TOE is set */ uint16_t toe_rss_bitmap; - /* valid iff ECORE_RSS_TUNNELING is set */ + /* valid if ECORE_RSS_TUNNELING is set */ uint16_t tunnel_value; uint16_t tunnel_mask; }; @@ -1286,14 +1286,14 @@ struct rxq_pause_params { uint16_t bd_th_hi; uint16_t rcq_th_lo; uint16_t rcq_th_hi; - uint16_t sge_th_lo; /* valid iff ECORE_Q_FLG_TPA */ - uint16_t sge_th_hi; /* valid iff ECORE_Q_FLG_TPA */ + uint16_t sge_th_lo; /* valid if ECORE_Q_FLG_TPA */ + uint16_t sge_th_hi; /* valid if ECORE_Q_FLG_TPA */ uint16_t pri_map; }; /* general */ struct ecore_general_setup_params { - /* valid iff ECORE_Q_FLG_STATS */ + /* valid if ECORE_Q_FLG_STATS */ uint8_t stat_id; uint8_t spcl_id; @@ -1312,19 +1312,19 @@ struct ecore_rxq_setup_params { uint8_t fw_sb_id; uint8_t cl_qzone_id; - /* valid iff ECORE_Q_FLG_TPA */ + /* valid if ECORE_Q_FLG_TPA */ uint16_t tpa_agg_sz; uint8_t max_tpa_queues; uint8_t rss_engine_id; - /* valid iff ECORE_Q_FLG_MCAST */ + /* valid if ECORE_Q_FLG_MCAST */ uint8_t mcast_engine_id; uint8_t cache_line_log; uint8_t sb_cq_index; - /* valid iff BXN2X_Q_FLG_SILENT_VLAN_REM */ + /* valid if BXN2X_Q_FLG_SILENT_VLAN_REM */ uint16_t silent_removal_value; uint16_t silent_removal_mask; }; @@ -1335,12 +1335,12 @@ struct ecore_txq_setup_params { uint8_t fw_sb_id; uint8_t sb_cq_index; - uint8_t cos; /* valid iff ECORE_Q_FLG_COS */ + uint8_t cos; /* valid if ECORE_Q_FLG_COS */ uint16_t traffic_type; /* equals to the leading rss client id, used for TX classification*/ uint8_t tss_leading_cl_id; - /* valid iff ECORE_Q_FLG_DEF_VLAN */ + /* valid if ECORE_Q_FLG_DEF_VLAN */ uint16_t default_vlan; }; @@ -1733,7 +1733,7 @@ void ecore_init_mcast_obj(struct bnx2x_softc *sc, * the current command will be enqueued to the tail of the * pending commands list. * - * Return: 0 is operation was successfull and there are no pending completions, + * Return: 0 is operation was successful and there are no pending completions, * negative if there were errors, positive if there are pending * completions. */ diff --git a/drivers/net/bnx2x/elink.c b/drivers/net/bnx2x/elink.c index 9ffa7dc6..9d0f3136 100644 --- a/drivers/net/bnx2x/elink.c +++ b/drivers/net/bnx2x/elink.c @@ -2702,7 +2702,7 @@ static elink_status_t elink_eee_initial_config(struct elink_params *params, { vars->eee_status |= ((uint32_t) mode) << SHMEM_EEE_SUPPORTED_SHIFT; - /* Propogate params' bits --> vars (for migration exposure) */ + /* Propagate params' bits --> vars (for migration exposure) */ if (params->eee_mode & ELINK_EEE_MODE_ENABLE_LPI) vars->eee_status |= SHMEM_EEE_LPI_REQUESTED_BIT; else @@ -3450,7 +3450,7 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy, {MDIO_WC_DEVAD, MDIO_WC_REG_CL72_USERB0_CL72_TX_FIR_TAP, 0}, }; PMD_DRV_LOG(DEBUG, "Enable Auto Negotiation for KR"); - /* Set to default registers that may be overriden by 10G force */ + /* Set to default registers that may be overridden by 10G force */ for (i = 0; i < ARRAY_SIZE(reg_set); i++) elink_cl45_write(sc, phy, reg_set[i].devad, reg_set[i].reg, reg_set[i].val); @@ -4363,14 +4363,14 @@ static void elink_sync_link(struct elink_params *params, switch (vars->link_status & LINK_STATUS_SPEED_AND_DUPLEX_MASK) { case ELINK_LINK_10THD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_10TFD: vars->line_speed = ELINK_SPEED_10; break; case ELINK_LINK_100TXHD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_100T4: case ELINK_LINK_100TXFD: vars->line_speed = ELINK_SPEED_100; @@ -4378,14 +4378,14 @@ static void elink_sync_link(struct elink_params *params, case ELINK_LINK_1000THD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_1000TFD: vars->line_speed = ELINK_SPEED_1000; break; case ELINK_LINK_2500THD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_2500TFD: vars->line_speed = ELINK_SPEED_2500; break; @@ -6341,7 +6341,7 @@ elink_status_t elink_link_update(struct elink_params * params, * hence its link is expected to be down * - SECOND_PHY means that first phy should not be able * to link up by itself (using configuration) - * - DEFAULT should be overriden during initialiazation + * - DEFAULT should be overridden during initialization */ PMD_DRV_LOG(DEBUG, "Invalid link indication" "mpc=0x%x. DISABLING LINK !!!", @@ -12680,7 +12680,7 @@ static void elink_check_over_curr(struct elink_params *params, vars->phy_flags &= ~PHY_OVER_CURRENT_FLAG; } -/* Returns 0 if no change occured since last check; 1 otherwise. */ +/* Returns 0 if no change occurred since last check; 1 otherwise. */ static uint8_t elink_analyze_link_error(struct elink_params *params, struct elink_vars *vars, uint32_t status, uint32_t phy_flag, diff --git a/drivers/net/bnxt/Makefile b/drivers/net/bnxt/Makefile index 0fffe356..b03f65dc 100644 --- a/drivers/net/bnxt/Makefile +++ b/drivers/net/bnxt/Makefile @@ -38,6 +38,8 @@ include $(RTE_SDK)/mk/rte.vars.mk # LIB = librte_pmd_bnxt.a +EXPORT_MAP := rte_pmd_bnxt_version.map + LIBABIVER := 1 CFLAGS += -O3 @@ -60,10 +62,12 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_txq.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_txr.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_vnic.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_irq.c +SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += rte_pmd_bnxt.c # # Export include files # SYMLINK-y-include += +SYMLINK-$(CONFIG_RTE_LIBRTE_BNXT_PMD)-include := rte_pmd_bnxt.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h index 4418c7fd..405d94de 100644 --- a/drivers/net/bnxt/bnxt.h +++ b/drivers/net/bnxt/bnxt.h @@ -35,8 +35,10 @@ #define _BNXT_H_ #include <inttypes.h> +#include <stdbool.h> #include <sys/queue.h> +#include <rte_pci.h> #include <rte_ethdev.h> #include <rte_memory.h> #include <rte_lcore.h> @@ -44,8 +46,44 @@ #include "bnxt_cpr.h" -#define BNXT_MAX_MTU 9000 +#define BNXT_MAX_MTU 9500 #define VLAN_TAG_SIZE 4 +#define BNXT_MAX_LED 4 + +struct bnxt_led_info { + uint8_t led_id; + uint8_t led_type; + uint8_t led_group_id; + uint8_t unused; + uint16_t led_state_caps; +#define BNXT_LED_ALT_BLINK_CAP(x) ((x) & \ + rte_cpu_to_le_16(HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINKALT)) + + uint16_t led_color_caps; +}; + +struct bnxt_led_cfg { + uint8_t led_id; + uint8_t led_state; + uint8_t led_color; + uint8_t unused; + uint16_t led_blink_on; + uint16_t led_blink_off; + uint8_t led_group_id; + uint8_t rsvd; +}; + +#define BNXT_LED_DFLT_ENA \ + (HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_ID | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_STATE | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_ON | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_OFF | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_GROUP_ID) + +#define BNXT_LED_DFLT_ENA_SHIFT 6 + +#define BNXT_LED_DFLT_ENABLES(x) \ + rte_cpu_to_le_32(BNXT_LED_DFLT_ENA << (BNXT_LED_DFLT_ENA_SHIFT * (x))) enum bnxt_hw_context { HW_CONTEXT_NONE = 0, @@ -54,17 +92,32 @@ enum bnxt_hw_context { HW_CONTEXT_IS_LB = 3, }; -struct bnxt_vf_info { - uint16_t fw_fid; - uint8_t mac_addr[ETHER_ADDR_LEN]; - uint16_t max_rsscos_ctx; - uint16_t max_cp_rings; - uint16_t max_tx_rings; - uint16_t max_rx_rings; - uint16_t max_l2_ctx; - uint16_t max_vnics; - uint16_t vlan; - struct bnxt_pf_info *pf; +struct bnxt_vlan_table_entry { + uint16_t tpid; + uint16_t vid; +} __attribute__((packed)); + +struct bnxt_vlan_antispoof_table_entry { + uint16_t tpid; + uint16_t vid; + uint16_t mask; +} __attribute__((packed)); + +struct bnxt_child_vf_info { + void *req_buf; + struct bnxt_vlan_table_entry *vlan_table; + struct bnxt_vlan_antispoof_table_entry *vlan_as_table; + STAILQ_HEAD(, bnxt_filter_info) filter; + uint32_t func_cfg_flags; + uint32_t l2_rx_mask; + uint16_t fid; + uint16_t max_tx_rate; + uint16_t dflt_vlan; + uint16_t vlan_count; + uint8_t mac_spoof_en; + uint8_t vlan_spoof_en; + bool random_mac; + bool persist_stats; }; struct bnxt_pf_info { @@ -73,22 +126,20 @@ struct bnxt_pf_info { #define BNXT_FIRST_VF_FID 128 #define BNXT_PF_RINGS_USED(bp) bnxt_get_num_queues(bp) #define BNXT_PF_RINGS_AVAIL(bp) (bp->pf.max_cp_rings - BNXT_PF_RINGS_USED(bp)) - uint32_t fw_fid; uint8_t port_id; - uint8_t mac_addr[ETHER_ADDR_LEN]; - uint16_t max_rsscos_ctx; - uint16_t max_cp_rings; - uint16_t max_tx_rings; - uint16_t max_rx_rings; - uint16_t max_l2_ctx; - uint16_t max_vnics; uint16_t first_vf_id; uint16_t active_vfs; uint16_t max_vfs; + uint32_t func_cfg_flags; void *vf_req_buf; phys_addr_t vf_req_buf_dma_addr; uint32_t vf_req_fwd[8]; - struct bnxt_vf_info *vf; + uint16_t total_vnics; + struct bnxt_child_vf_info *vf_info; +#define BNXT_EVB_MODE_NONE 0 +#define BNXT_EVB_MODE_VEB 1 +#define BNXT_EVB_MODE_VEPA 2 + uint8_t evb_mode; }; /* Max wait time is 10 * 100ms = 1s */ @@ -120,6 +171,7 @@ struct bnxt_cos_queue_info { uint8_t profile; }; +#define BNXT_HWRM_SHORT_REQ_LEN sizeof(struct hwrm_short_input) struct bnxt { void *bar0; @@ -129,6 +181,9 @@ struct bnxt { uint32_t flags; #define BNXT_FLAG_REGISTERED (1 << 0) #define BNXT_FLAG_VF (1 << 1) +#define BNXT_FLAG_PORT_STATS (1 << 2) +#define BNXT_FLAG_JUMBO (1 << 3) +#define BNXT_FLAG_SHORT_CMD (1 << 4) #define BNXT_PF(bp) (!((bp)->flags & BNXT_FLAG_VF)) #define BNXT_VF(bp) ((bp)->flags & BNXT_FLAG_VF) #define BNXT_NPAR_ENABLED(bp) ((bp)->port_partition_type) @@ -137,10 +192,16 @@ struct bnxt { unsigned int rx_nr_rings; unsigned int rx_cp_nr_rings; struct bnxt_rx_queue **rx_queues; + const void *rx_mem_zone; + struct rx_port_stats *hw_rx_port_stats; + phys_addr_t hw_rx_port_stats_map; unsigned int tx_nr_rings; unsigned int tx_cp_nr_rings; struct bnxt_tx_queue **tx_queues; + const void *tx_mem_zone; + struct tx_port_stats *hw_tx_port_stats; + phys_addr_t hw_tx_port_stats_map; /* Default completion ring */ struct bnxt_cp_ring_info *def_cp_ring; @@ -167,6 +228,8 @@ struct bnxt { uint16_t hwrm_cmd_seq; void *hwrm_cmd_resp_addr; phys_addr_t hwrm_cmd_resp_dma_addr; + void *hwrm_short_cmd_req_addr; + phys_addr_t hwrm_short_cmd_req_dma_addr; rte_spinlock_t hwrm_lock; uint16_t max_req_len; uint16_t max_resp_len; @@ -174,12 +237,36 @@ struct bnxt { struct bnxt_link_info link_info; struct bnxt_cos_queue_info cos_queue[BNXT_COS_QUEUE_COUNT]; + uint16_t fw_fid; + uint8_t dflt_mac_addr[ETHER_ADDR_LEN]; + uint16_t max_rsscos_ctx; + uint16_t max_cp_rings; + uint16_t max_tx_rings; + uint16_t max_rx_rings; + uint16_t max_l2_ctx; + uint16_t max_vnics; + uint16_t max_stat_ctx; + uint16_t vlan; struct bnxt_pf_info pf; - struct bnxt_vf_info vf; uint8_t port_partition_type; uint8_t dev_stopped; + uint8_t vxlan_port_cnt; + uint8_t geneve_port_cnt; + uint16_t vxlan_port; + uint16_t geneve_port; + uint16_t vxlan_fw_dst_port_id; + uint16_t geneve_fw_dst_port_id; + uint32_t fw_ver; + rte_atomic64_t rx_mbuf_alloc_fail; + + struct bnxt_led_info leds[BNXT_MAX_LED]; + uint8_t num_leds; }; int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete); +int bnxt_rcv_msg_from_vf(struct bnxt *bp, uint16_t vf_id, void *msg); + +#define RX_PROD_AGG_BD_TYPE_RX_PROD_AGG 0x6 +bool is_bnxt_supported(struct rte_eth_dev *dev); #endif diff --git a/drivers/net/bnxt/bnxt_cpr.c b/drivers/net/bnxt/bnxt_cpr.c index 3aedcb8d..68979bc4 100644 --- a/drivers/net/bnxt/bnxt_cpr.c +++ b/drivers/net/bnxt/bnxt_cpr.c @@ -58,63 +58,123 @@ void bnxt_handle_async_event(struct bnxt *bp, bnxt_link_update_op(bp->eth_dev, 0); break; default: - RTE_LOG(ERR, PMD, "handle_async_event id = 0x%x\n", event_id); + RTE_LOG(DEBUG, PMD, "handle_async_event id = 0x%x\n", event_id); break; } } void bnxt_handle_fwd_req(struct bnxt *bp, struct cmpl_base *cmpl) { + struct hwrm_exec_fwd_resp_input *fwreq; struct hwrm_fwd_req_cmpl *fwd_cmpl = (struct hwrm_fwd_req_cmpl *)cmpl; struct input *fwd_cmd; - uint16_t logical_vf_id, error_code; + uint16_t fw_vf_id; + uint16_t vf_id; + uint16_t req_len; + int rc; - /* Qualify the fwd request */ - if (fwd_cmpl->source_id < bp->pf.first_vf_id) { - RTE_LOG(ERR, PMD, - "FWD req's source_id 0x%x > first_vf_id 0x%x\n", - fwd_cmpl->source_id, bp->pf.first_vf_id); - error_code = HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED; - goto reject; - } else if (fwd_cmpl->req_len_type >> HWRM_FWD_REQ_CMPL_REQ_LEN_SFT > - 128 - sizeof(struct input)) { - RTE_LOG(ERR, PMD, - "FWD req's cmd len 0x%x > 108 bytes allowed\n", - fwd_cmpl->req_len_type >> HWRM_FWD_REQ_CMPL_REQ_LEN_SFT); - error_code = HWRM_ERR_CODE_INVALID_PARAMS; - goto reject; + if (bp->pf.active_vfs <= 0) { + RTE_LOG(ERR, PMD, "Forwarded VF with no active VFs\n"); + return; } + /* Qualify the fwd request */ + fw_vf_id = rte_le_to_cpu_16(fwd_cmpl->source_id); + vf_id = fw_vf_id - bp->pf.first_vf_id; + + req_len = (rte_le_to_cpu_16(fwd_cmpl->req_len_type) & + HWRM_FWD_REQ_CMPL_REQ_LEN_MASK) >> + HWRM_FWD_REQ_CMPL_REQ_LEN_SFT; + if (req_len > sizeof(fwreq->encap_request)) + req_len = sizeof(fwreq->encap_request); + /* Locate VF's forwarded command */ - logical_vf_id = fwd_cmpl->source_id - bp->pf.first_vf_id; - fwd_cmd = (struct input *)((uint8_t *)bp->pf.vf_req_buf + - (logical_vf_id * 128)); - - /* Provision the request */ - switch (fwd_cmd->req_type) { - case HWRM_CFA_L2_FILTER_ALLOC: - case HWRM_CFA_L2_FILTER_FREE: - case HWRM_CFA_L2_FILTER_CFG: - case HWRM_CFA_L2_SET_RX_MASK: - break; - default: - error_code = HWRM_ERR_CODE_INVALID_PARAMS; + fwd_cmd = (struct input *)bp->pf.vf_info[vf_id].req_buf; + + if (fw_vf_id < bp->pf.first_vf_id || + fw_vf_id >= (bp->pf.first_vf_id) + bp->pf.active_vfs) { + RTE_LOG(ERR, PMD, + "FWD req's source_id 0x%x out of range 0x%x - 0x%x (%d %d)\n", + fw_vf_id, bp->pf.first_vf_id, + (bp->pf.first_vf_id) + bp->pf.active_vfs - 1, + bp->pf.first_vf_id, bp->pf.active_vfs); goto reject; } - /* Forward */ - fwd_cmd->target_id = fwd_cmpl->source_id; - bnxt_hwrm_exec_fwd_resp(bp, fwd_cmd); - return; + if (bnxt_rcv_msg_from_vf(bp, vf_id, fwd_cmd) == true) { + /* + * In older firmware versions, the MAC had to be all zeros for + * the VF to set it's MAC via hwrm_func_vf_cfg. Set to all + * zeros if it's being configured and has been ok'd by caller. + */ + if (fwd_cmd->req_type == HWRM_FUNC_VF_CFG) { + struct hwrm_func_vf_cfg_input *vfc = (void *)fwd_cmd; + + if (vfc->enables & + HWRM_FUNC_VF_CFG_INPUT_ENABLES_DFLT_MAC_ADDR) { + bnxt_hwrm_func_vf_mac(bp, vf_id, + (const uint8_t *)"\x00\x00\x00\x00\x00"); + } + } + if (fwd_cmd->req_type == HWRM_CFA_L2_SET_RX_MASK) { + struct hwrm_cfa_l2_set_rx_mask_input *srm = + (void *)fwd_cmd; + + srm->vlan_tag_tbl_addr = rte_cpu_to_le_64(0); + srm->num_vlan_tags = rte_cpu_to_le_32(0); + srm->mask &= ~rte_cpu_to_le_32( + HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLANONLY | + HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN | + HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN); + } + /* Forward */ + rc = bnxt_hwrm_exec_fwd_resp(bp, fw_vf_id, fwd_cmd, req_len); + if (rc) { + RTE_LOG(ERR, PMD, + "Failed to send FWD req VF 0x%x, type 0x%x.\n", + fw_vf_id - bp->pf.first_vf_id, + rte_le_to_cpu_16(fwd_cmd->req_type)); + } + return; + } reject: - /* TODO: Encap the reject error resp into the hwrm_err_iput? */ - /* Use the error_code for the reject cmd */ - RTE_LOG(ERR, PMD, - "Error 0x%x found in the forward request\n", error_code); + rc = bnxt_hwrm_reject_fwd_resp(bp, fw_vf_id, fwd_cmd, req_len); + if (rc) { + RTE_LOG(ERR, PMD, + "Failed to send REJECT req VF 0x%x, type 0x%x.\n", + fw_vf_id - bp->pf.first_vf_id, + rte_le_to_cpu_16(fwd_cmd->req_type)); + } + + return; } /* For the default completion ring only */ +int bnxt_alloc_def_cp_ring(struct bnxt *bp) +{ + struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; + struct bnxt_ring *cp_ring = cpr->cp_ring_struct; + int rc; + + rc = bnxt_hwrm_ring_alloc(bp, cp_ring, + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, + 0, HWRM_NA_SIGNATURE, + HWRM_NA_SIGNATURE); + if (rc) + goto err_out; + cpr->cp_doorbell = bp->pdev->mem_resource[2].addr; + B_CP_DIS_DB(cpr, cpr->cp_raw_cons); + bp->grp_info[0].cp_fw_ring_id = cp_ring->fw_ring_id; + if (BNXT_PF(bp)) + rc = bnxt_hwrm_func_cfg_def_cp(bp); + else + rc = bnxt_hwrm_vf_func_cfg_def_cp(bp); + +err_out: + return rc; +} + void bnxt_free_def_cp_ring(struct bnxt *bp) { struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; diff --git a/drivers/net/bnxt/bnxt_cpr.h b/drivers/net/bnxt/bnxt_cpr.h index 83e53761..a6e87858 100644 --- a/drivers/net/bnxt/bnxt_cpr.h +++ b/drivers/net/bnxt/bnxt_cpr.h @@ -33,6 +33,7 @@ #ifndef _BNXT_CPR_H_ #define _BNXT_CPR_H_ +#include <stdbool.h> #include <rte_io.h> @@ -56,6 +57,19 @@ RING_CMP(((cpr)->cp_ring_struct), raw_cons)), \ ((cpr)->cp_doorbell)) +#define B_CP_DB_ARM(cpr) rte_write32((DB_KEY_CP), ((cpr)->cp_doorbell)) +#define B_CP_DB_DISARM(cpr) (*(uint32_t *)((cpr)->cp_doorbell) = \ + DB_KEY_CP | DB_IRQ_DIS) + +#define B_CP_DB_IDX_ARM(cpr, cons) \ + (*(uint32_t *)((cpr)->cp_doorbell) = (DB_CP_REARM_FLAGS | \ + (cons))) + +#define B_CP_DB_IDX_DISARM(cpr, cons) do { \ + rte_smp_wmb(); \ + (*(uint32_t *)((cpr)->cp_doorbell) = (DB_CP_FLAGS | \ + (cons)); \ +} while (0) #define B_CP_DIS_DB(cpr, raw_cons) \ rte_write32((DB_CP_FLAGS | \ RING_CMP(((cpr)->cp_ring_struct), raw_cons)), \ @@ -75,6 +89,8 @@ struct bnxt_cp_ring_info { uint32_t hw_stats_ctx_id; struct bnxt_ring *cp_ring_struct; + uint16_t cp_cons; + bool v; }; #define RX_CMP_L2_ERRORS \ @@ -82,6 +98,7 @@ struct bnxt_cp_ring_info { struct bnxt; +int bnxt_alloc_def_cp_ring(struct bnxt *bp); void bnxt_free_def_cp_ring(struct bnxt *bp); int bnxt_init_def_ring_struct(struct bnxt *bp, unsigned int socket_id); void bnxt_handle_async_event(struct bnxt *bp, struct cmpl_base *cmp); diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 81711e48..c9d11228 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -60,6 +60,7 @@ static const char bnxt_version[] = #define PCI_VENDOR_ID_BROADCOM 0x14E4 +#define BROADCOM_DEV_ID_STRATUS_NIC_VF 0x1609 #define BROADCOM_DEV_ID_STRATUS_NIC 0x1614 #define BROADCOM_DEV_ID_57414_VF 0x16c1 #define BROADCOM_DEV_ID_57301 0x16c8 @@ -96,6 +97,8 @@ static const char bnxt_version[] = #define BROADCOM_DEV_ID_57416_MF 0x16ee static const struct rte_pci_id bnxt_pci_id_map[] = { + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, + BROADCOM_DEV_ID_STRATUS_NIC_VF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_STRATUS_NIC) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_VF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57301) }, @@ -141,6 +144,8 @@ static const struct rte_pci_id bnxt_pci_id_map[] = { ETH_RSS_NONFRAG_IPV6_TCP | \ ETH_RSS_NONFRAG_IPV6_UDP) +static void bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask); + /***********************/ /* @@ -198,6 +203,14 @@ static int bnxt_init_chip(struct bnxt *bp) struct rte_eth_link new; int rc; + if (bp->eth_dev->data->mtu > ETHER_MTU) { + bp->eth_dev->data->dev_conf.rxmode.jumbo_frame = 1; + bp->flags |= BNXT_FLAG_JUMBO; + } else { + bp->eth_dev->data->dev_conf.rxmode.jumbo_frame = 0; + bp->flags &= ~BNXT_FLAG_JUMBO; + } + rc = bnxt_alloc_all_hwrm_stat_ctxs(bp); if (rc) { RTE_LOG(ERR, PMD, "HWRM stat ctx alloc failure rc: %x\n", rc); @@ -228,28 +241,31 @@ static int bnxt_init_chip(struct bnxt *bp) rc = bnxt_hwrm_vnic_alloc(bp, vnic); if (rc) { - RTE_LOG(ERR, PMD, "HWRM vnic alloc failure rc: %x\n", - rc); + RTE_LOG(ERR, PMD, "HWRM vnic %d alloc failure rc: %x\n", + i, rc); goto err_out; } rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic); if (rc) { RTE_LOG(ERR, PMD, - "HWRM vnic ctx alloc failure rc: %x\n", rc); + "HWRM vnic %d ctx alloc failure rc: %x\n", + i, rc); goto err_out; } rc = bnxt_hwrm_vnic_cfg(bp, vnic); if (rc) { - RTE_LOG(ERR, PMD, "HWRM vnic cfg failure rc: %x\n", rc); + RTE_LOG(ERR, PMD, "HWRM vnic %d cfg failure rc: %x\n", + i, rc); goto err_out; } rc = bnxt_set_hwrm_vnic_filters(bp, vnic); if (rc) { - RTE_LOG(ERR, PMD, "HWRM vnic filter failure rc: %x\n", - rc); + RTE_LOG(ERR, PMD, + "HWRM vnic %d filter failure rc: %x\n", + i, rc); goto err_out; } if (vnic->rss_table && vnic->hash_type) { @@ -269,13 +285,20 @@ static int bnxt_init_chip(struct bnxt *bp) rc = bnxt_hwrm_vnic_rss_cfg(bp, vnic); if (rc) { RTE_LOG(ERR, PMD, - "HWRM vnic set RSS failure rc: %x\n", - rc); + "HWRM vnic %d set RSS failure rc: %x\n", + i, rc); goto err_out; } } + + bnxt_hwrm_vnic_plcmode_cfg(bp, vnic); + + if (bp->eth_dev->data->dev_conf.rxmode.enable_lro) + bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 1); + else + bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 0); } - rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0]); + rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0], 0, NULL); if (rc) { RTE_LOG(ERR, PMD, "HWRM cfa l2 rx mask failure rc: %x\n", rc); @@ -338,25 +361,19 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; uint16_t max_vnics, i, j, vpool, vrxq; - dev_info->pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); /* MAC Specifics */ dev_info->max_mac_addrs = MAX_NUM_MAC_ADDR; dev_info->max_hash_mac_addrs = 0; /* PF/VF specifics */ - if (BNXT_PF(bp)) { - dev_info->max_rx_queues = bp->pf.max_rx_rings; - dev_info->max_tx_queues = bp->pf.max_tx_rings; - dev_info->max_vfs = bp->pf.active_vfs; - dev_info->reta_size = bp->pf.max_rsscos_ctx; - max_vnics = bp->pf.max_vnics; - } else { - dev_info->max_rx_queues = bp->vf.max_rx_rings; - dev_info->max_tx_queues = bp->vf.max_tx_rings; - dev_info->reta_size = bp->vf.max_rsscos_ctx; - max_vnics = bp->vf.max_vnics; - } + if (BNXT_PF(bp)) + dev_info->max_vfs = bp->pdev->max_vfs; + dev_info->max_rx_queues = bp->max_rx_rings; + dev_info->max_tx_queues = bp->max_tx_rings; + dev_info->reta_size = bp->max_rsscos_ctx; + max_vnics = bp->max_vnics; /* Fast path specifics */ dev_info->min_rx_bufsize = 1; @@ -366,7 +383,12 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, dev_info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_TSO; + DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO | + DEV_TX_OFFLOAD_IPIP_TNL_TSO | + DEV_TX_OFFLOAD_GENEVE_TNL_TSO; /* *INDENT-OFF* */ dev_info->default_rxconf = (struct rte_eth_rxconf) { @@ -485,44 +507,29 @@ static int bnxt_dev_lsc_intr_setup(struct rte_eth_dev *eth_dev) static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev) { struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + int vlan_mask = 0; int rc; bp->dev_stopped = 0; - rc = bnxt_hwrm_func_reset(bp); - if (rc) { - RTE_LOG(ERR, PMD, "hwrm chip reset failure rc: %x\n", rc); - rc = -1; - goto error; - } - - rc = bnxt_setup_int(bp); - if (rc) - goto error; - - rc = bnxt_alloc_mem(bp); - if (rc) - goto error; - - rc = bnxt_request_int(bp); - if (rc) - goto error; rc = bnxt_init_nic(bp); if (rc) goto error; - bnxt_enable_int(bp); - bnxt_link_update_op(eth_dev, 0); + + if (eth_dev->data->dev_conf.rxmode.hw_vlan_filter) + vlan_mask |= ETH_VLAN_FILTER_MASK; + if (eth_dev->data->dev_conf.rxmode.hw_vlan_strip) + vlan_mask |= ETH_VLAN_STRIP_MASK; + bnxt_vlan_offload_set_op(eth_dev, vlan_mask); + return 0; error: bnxt_shutdown_nic(bp); - bnxt_disable_int(bp); - bnxt_free_int(bp); bnxt_free_tx_mbufs(bp); bnxt_free_rx_mbufs(bp); - bnxt_free_mem(bp); return rc; } @@ -554,8 +561,7 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev) eth_dev->data->dev_link.link_status = 0; } bnxt_set_hwrm_link_config(bp, false); - bnxt_disable_int(bp); - bnxt_free_int(bp); + bnxt_hwrm_port_clr_stats(bp); bnxt_shutdown_nic(bp); bp->dev_stopped = 1; } @@ -651,7 +657,7 @@ static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev, STAILQ_INSERT_TAIL(&vnic->filter, filter, next); filter->mac_index = index; memcpy(filter->l2_addr, mac_addr, ETHER_ADDR_LEN); - return bnxt_hwrm_set_filter(bp, vnic, filter); + return bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id, filter); } int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete) @@ -669,7 +675,7 @@ int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete) new.link_speed = ETH_LINK_SPEED_100M; new.link_duplex = ETH_LINK_FULL_DUPLEX; RTE_LOG(ERR, PMD, - "Failed to retrieve link rc = 0x%x!", rc); + "Failed to retrieve link rc = 0x%x!\n", rc); goto out; } rte_delay_ms(BNXT_LINK_WAIT_INTERVAL); @@ -700,7 +706,7 @@ static void bnxt_promiscuous_enable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags |= BNXT_VNIC_INFO_PROMISC; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static void bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev) @@ -714,7 +720,7 @@ static void bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags &= ~BNXT_VNIC_INFO_PROMISC; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static void bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev) @@ -728,7 +734,7 @@ static void bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags |= BNXT_VNIC_INFO_ALLMULTI; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static void bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev) @@ -742,7 +748,7 @@ static void bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static int bnxt_reta_update_op(struct rte_eth_dev *eth_dev, @@ -918,7 +924,7 @@ static int bnxt_rss_hash_conf_get_op(struct rte_eth_dev *eth_dev, } static int bnxt_flow_ctrl_get_op(struct rte_eth_dev *dev, - struct rte_eth_fc_conf *fc_conf __rte_unused) + struct rte_eth_fc_conf *fc_conf) { struct bnxt *bp = (struct bnxt *)dev->data->dev_private; struct rte_eth_link link_info; @@ -1003,6 +1009,514 @@ static int bnxt_flow_ctrl_set_op(struct rte_eth_dev *dev, return bnxt_set_hwrm_link_config(bp, true); } +/* Add UDP tunneling port */ +static int +bnxt_udp_tunnel_port_add_op(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tunnel) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + uint16_t tunnel_type = 0; + int rc = 0; + + switch (udp_tunnel->prot_type) { + case RTE_TUNNEL_TYPE_VXLAN: + if (bp->vxlan_port_cnt) { + RTE_LOG(ERR, PMD, "Tunnel Port %d already programmed\n", + udp_tunnel->udp_port); + if (bp->vxlan_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Only one port allowed\n"); + return -ENOSPC; + } + bp->vxlan_port_cnt++; + return 0; + } + tunnel_type = + HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN; + bp->vxlan_port_cnt++; + break; + case RTE_TUNNEL_TYPE_GENEVE: + if (bp->geneve_port_cnt) { + RTE_LOG(ERR, PMD, "Tunnel Port %d already programmed\n", + udp_tunnel->udp_port); + if (bp->geneve_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Only one port allowed\n"); + return -ENOSPC; + } + bp->geneve_port_cnt++; + return 0; + } + tunnel_type = + HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE; + bp->geneve_port_cnt++; + break; + default: + RTE_LOG(ERR, PMD, "Tunnel type is not supported\n"); + return -ENOTSUP; + } + rc = bnxt_hwrm_tunnel_dst_port_alloc(bp, udp_tunnel->udp_port, + tunnel_type); + return rc; +} + +static int +bnxt_udp_tunnel_port_del_op(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tunnel) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + uint16_t tunnel_type = 0; + uint16_t port = 0; + int rc = 0; + + switch (udp_tunnel->prot_type) { + case RTE_TUNNEL_TYPE_VXLAN: + if (!bp->vxlan_port_cnt) { + RTE_LOG(ERR, PMD, "No Tunnel port configured yet\n"); + return -EINVAL; + } + if (bp->vxlan_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Req Port: %d. Configured port: %d\n", + udp_tunnel->udp_port, bp->vxlan_port); + return -EINVAL; + } + if (--bp->vxlan_port_cnt) + return 0; + + tunnel_type = + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN; + port = bp->vxlan_fw_dst_port_id; + break; + case RTE_TUNNEL_TYPE_GENEVE: + if (!bp->geneve_port_cnt) { + RTE_LOG(ERR, PMD, "No Tunnel port configured yet\n"); + return -EINVAL; + } + if (bp->geneve_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Req Port: %d. Configured port: %d\n", + udp_tunnel->udp_port, bp->geneve_port); + return -EINVAL; + } + if (--bp->geneve_port_cnt) + return 0; + + tunnel_type = + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE; + port = bp->geneve_fw_dst_port_id; + break; + default: + RTE_LOG(ERR, PMD, "Tunnel type is not supported\n"); + return -ENOTSUP; + } + + rc = bnxt_hwrm_tunnel_dst_port_free(bp, port, tunnel_type); + if (!rc) { + if (tunnel_type == + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN) + bp->vxlan_port = 0; + if (tunnel_type == + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE) + bp->geneve_port = 0; + } + return rc; +} + +static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id) +{ + struct bnxt_filter_info *filter, *temp_filter, *new_filter; + struct bnxt_vnic_info *vnic; + unsigned int i; + int rc = 0; + uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN; + + /* Cycle through all VNICs */ + for (i = 0; i < bp->nr_vnics; i++) { + /* + * For each VNIC and each associated filter(s) + * if VLAN exists && VLAN matches vlan_id + * remove the MAC+VLAN filter + * add a new MAC only filter + * else + * VLAN filter doesn't exist, just skip and continue + */ + STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) { + filter = STAILQ_FIRST(&vnic->filter); + while (filter) { + temp_filter = STAILQ_NEXT(filter, next); + + if (filter->enables & chk && + filter->l2_ovlan == vlan_id) { + /* Must delete the filter */ + STAILQ_REMOVE(&vnic->filter, filter, + bnxt_filter_info, next); + bnxt_hwrm_clear_filter(bp, filter); + STAILQ_INSERT_TAIL( + &bp->free_filter_list, + filter, next); + + /* + * Need to examine to see if the MAC + * filter already existed or not before + * allocating a new one + */ + + new_filter = bnxt_alloc_filter(bp); + if (!new_filter) { + RTE_LOG(ERR, PMD, + "MAC/VLAN filter alloc failed\n"); + rc = -ENOMEM; + goto exit; + } + STAILQ_INSERT_TAIL(&vnic->filter, + new_filter, next); + /* Inherit MAC from previous filter */ + new_filter->mac_index = + filter->mac_index; + memcpy(new_filter->l2_addr, + filter->l2_addr, ETHER_ADDR_LEN); + /* MAC only filter */ + rc = bnxt_hwrm_set_filter(bp, + vnic->fw_vnic_id, + new_filter); + if (rc) + goto exit; + RTE_LOG(INFO, PMD, + "Del Vlan filter for %d\n", + vlan_id); + } + filter = temp_filter; + } + } + } +exit: + return rc; +} + +static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id) +{ + struct bnxt_filter_info *filter, *temp_filter, *new_filter; + struct bnxt_vnic_info *vnic; + unsigned int i; + int rc = 0; + uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK; + uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN; + + /* Cycle through all VNICs */ + for (i = 0; i < bp->nr_vnics; i++) { + /* + * For each VNIC and each associated filter(s) + * if VLAN exists: + * if VLAN matches vlan_id + * VLAN filter already exists, just skip and continue + * else + * add a new MAC+VLAN filter + * else + * Remove the old MAC only filter + * Add a new MAC+VLAN filter + */ + STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) { + filter = STAILQ_FIRST(&vnic->filter); + while (filter) { + temp_filter = STAILQ_NEXT(filter, next); + + if (filter->enables & chk) { + if (filter->l2_ovlan == vlan_id) + goto cont; + } else { + /* Must delete the MAC filter */ + STAILQ_REMOVE(&vnic->filter, filter, + bnxt_filter_info, next); + bnxt_hwrm_clear_filter(bp, filter); + filter->l2_ovlan = 0; + STAILQ_INSERT_TAIL( + &bp->free_filter_list, + filter, next); + } + new_filter = bnxt_alloc_filter(bp); + if (!new_filter) { + RTE_LOG(ERR, PMD, + "MAC/VLAN filter alloc failed\n"); + rc = -ENOMEM; + goto exit; + } + STAILQ_INSERT_TAIL(&vnic->filter, new_filter, + next); + /* Inherit MAC from the previous filter */ + new_filter->mac_index = filter->mac_index; + memcpy(new_filter->l2_addr, filter->l2_addr, + ETHER_ADDR_LEN); + /* MAC + VLAN ID filter */ + new_filter->l2_ovlan = vlan_id; + new_filter->l2_ovlan_mask = 0xF000; + new_filter->enables |= en; + rc = bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id, + new_filter); + if (rc) + goto exit; + RTE_LOG(INFO, PMD, + "Added Vlan filter for %d\n", vlan_id); +cont: + filter = temp_filter; + } + } + } +exit: + return rc; +} + +static int bnxt_vlan_filter_set_op(struct rte_eth_dev *eth_dev, + uint16_t vlan_id, int on) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + + /* These operations apply to ALL existing MAC/VLAN filters */ + if (on) + return bnxt_add_vlan_filter(bp, vlan_id); + else + return bnxt_del_vlan_filter(bp, vlan_id); +} + +static void +bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + unsigned int i; + + if (mask & ETH_VLAN_FILTER_MASK) { + if (!dev->data->dev_conf.rxmode.hw_vlan_filter) { + /* Remove any VLAN filters programmed */ + for (i = 0; i < 4095; i++) + bnxt_del_vlan_filter(bp, i); + } + RTE_LOG(INFO, PMD, "VLAN Filtering: %d\n", + dev->data->dev_conf.rxmode.hw_vlan_filter); + } + + if (mask & ETH_VLAN_STRIP_MASK) { + /* Enable or disable VLAN stripping */ + for (i = 0; i < bp->nr_vnics; i++) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + if (dev->data->dev_conf.rxmode.hw_vlan_strip) + vnic->vlan_strip = true; + else + vnic->vlan_strip = false; + bnxt_hwrm_vnic_cfg(bp, vnic); + } + RTE_LOG(INFO, PMD, "VLAN Strip Offload: %d\n", + dev->data->dev_conf.rxmode.hw_vlan_strip); + } + + if (mask & ETH_VLAN_EXTEND_MASK) + RTE_LOG(ERR, PMD, "Extend VLAN Not supported\n"); +} + +static void +bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev, struct ether_addr *addr) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + /* Default Filter is tied to VNIC 0 */ + struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; + struct bnxt_filter_info *filter; + int rc; + + if (BNXT_VF(bp)) + return; + + memcpy(bp->mac_addr, addr, sizeof(bp->mac_addr)); + memcpy(&dev->data->mac_addrs[0], bp->mac_addr, ETHER_ADDR_LEN); + + STAILQ_FOREACH(filter, &vnic->filter, next) { + /* Default Filter is at Index 0 */ + if (filter->mac_index != 0) + continue; + rc = bnxt_hwrm_clear_filter(bp, filter); + if (rc) + break; + memcpy(filter->l2_addr, bp->mac_addr, ETHER_ADDR_LEN); + memset(filter->l2_addr_mask, 0xff, ETHER_ADDR_LEN); + filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX; + filter->enables |= + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK; + rc = bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id, filter); + if (rc) + break; + filter->mac_index = 0; + RTE_LOG(DEBUG, PMD, "Set MAC addr\n"); + } +} + +static int +bnxt_dev_set_mc_addr_list_op(struct rte_eth_dev *eth_dev, + struct ether_addr *mc_addr_set, + uint32_t nb_mc_addr) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + char *mc_addr_list = (char *)mc_addr_set; + struct bnxt_vnic_info *vnic; + uint32_t off = 0, i = 0; + + vnic = &bp->vnic_info[0]; + + if (nb_mc_addr > BNXT_MAX_MC_ADDRS) { + vnic->flags |= BNXT_VNIC_INFO_ALLMULTI; + goto allmulti; + } + + /* TODO Check for Duplicate mcast addresses */ + vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI; + for (i = 0; i < nb_mc_addr; i++) { + memcpy(vnic->mc_list + off, &mc_addr_list[i], ETHER_ADDR_LEN); + off += ETHER_ADDR_LEN; + } + + vnic->mc_addr_cnt = i; + +allmulti: + return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); +} + +static int +bnxt_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + uint8_t fw_major = (bp->fw_ver >> 24) & 0xff; + uint8_t fw_minor = (bp->fw_ver >> 16) & 0xff; + uint8_t fw_updt = (bp->fw_ver >> 8) & 0xff; + int ret; + + ret = snprintf(fw_version, fw_size, "%d.%d.%d", + fw_major, fw_minor, fw_updt); + + ret += 1; /* add the size of '\0' */ + if (fw_size < (uint32_t)ret) + return ret; + else + return 0; +} + +static void +bnxt_rxq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_info *qinfo) +{ + struct bnxt_rx_queue *rxq; + + rxq = dev->data->rx_queues[queue_id]; + + qinfo->mp = rxq->mb_pool; + qinfo->scattered_rx = dev->data->scattered_rx; + qinfo->nb_desc = rxq->nb_rx_desc; + + qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; + qinfo->conf.rx_drop_en = 0; + qinfo->conf.rx_deferred_start = 0; +} + +static void +bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_txq_info *qinfo) +{ + struct bnxt_tx_queue *txq; + + txq = dev->data->tx_queues[queue_id]; + + qinfo->nb_desc = txq->nb_tx_desc; + + qinfo->conf.tx_thresh.pthresh = txq->pthresh; + qinfo->conf.tx_thresh.hthresh = txq->hthresh; + qinfo->conf.tx_thresh.wthresh = txq->wthresh; + + qinfo->conf.tx_free_thresh = txq->tx_free_thresh; + qinfo->conf.tx_rs_thresh = 0; + qinfo->conf.txq_flags = txq->txq_flags; + qinfo->conf.tx_deferred_start = txq->tx_deferred_start; +} + +static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) +{ + struct bnxt *bp = eth_dev->data->dev_private; + struct rte_eth_dev_info dev_info; + uint32_t max_dev_mtu; + uint32_t rc = 0; + uint32_t i; + + bnxt_dev_info_get_op(eth_dev, &dev_info); + max_dev_mtu = dev_info.max_rx_pktlen - + ETHER_HDR_LEN - ETHER_CRC_LEN - VLAN_TAG_SIZE * 2; + + if (new_mtu < ETHER_MIN_MTU || new_mtu > max_dev_mtu) { + RTE_LOG(ERR, PMD, "MTU requested must be within (%d, %d)\n", + ETHER_MIN_MTU, max_dev_mtu); + return -EINVAL; + } + + + if (new_mtu > ETHER_MTU) { + bp->flags |= BNXT_FLAG_JUMBO; + eth_dev->data->dev_conf.rxmode.jumbo_frame = 1; + } else { + eth_dev->data->dev_conf.rxmode.jumbo_frame = 0; + bp->flags &= ~BNXT_FLAG_JUMBO; + } + + eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = + new_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + VLAN_TAG_SIZE * 2; + + eth_dev->data->mtu = new_mtu; + RTE_LOG(INFO, PMD, "New MTU is %d\n", eth_dev->data->mtu); + + for (i = 0; i < bp->nr_vnics; i++) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + + vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE * 2; + rc = bnxt_hwrm_vnic_cfg(bp, vnic); + if (rc) + break; + + rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic); + if (rc) + return rc; + } + + return rc; +} + +static int +bnxt_vlan_pvid_set_op(struct rte_eth_dev *dev, uint16_t pvid, int on) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + uint16_t vlan = bp->vlan; + int rc; + + if (BNXT_NPAR_PF(bp) || BNXT_VF(bp)) { + RTE_LOG(ERR, PMD, + "PVID cannot be modified for this function\n"); + return -ENOTSUP; + } + bp->vlan = on ? pvid : 0; + + rc = bnxt_hwrm_set_default_vlan(bp, 0, 0); + if (rc) + bp->vlan = vlan; + return rc; +} + +static int +bnxt_dev_led_on_op(struct rte_eth_dev *dev) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + + return bnxt_hwrm_port_led_cfg(bp, true); +} + +static int +bnxt_dev_led_off_op(struct rte_eth_dev *dev) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + + return bnxt_hwrm_port_led_cfg(bp, false); +} + /* * Initialization */ @@ -1034,6 +1548,22 @@ static const struct eth_dev_ops bnxt_dev_ops = { .mac_addr_remove = bnxt_mac_addr_remove_op, .flow_ctrl_get = bnxt_flow_ctrl_get_op, .flow_ctrl_set = bnxt_flow_ctrl_set_op, + .udp_tunnel_port_add = bnxt_udp_tunnel_port_add_op, + .udp_tunnel_port_del = bnxt_udp_tunnel_port_del_op, + .vlan_filter_set = bnxt_vlan_filter_set_op, + .vlan_offload_set = bnxt_vlan_offload_set_op, + .vlan_pvid_set = bnxt_vlan_pvid_set_op, + .mtu_set = bnxt_mtu_set_op, + .mac_addr_set = bnxt_set_default_mac_addr_op, + .xstats_get = bnxt_dev_xstats_get_op, + .xstats_get_names = bnxt_dev_xstats_get_names_op, + .xstats_reset = bnxt_dev_xstats_reset_op, + .fw_version_get = bnxt_fw_version_get, + .set_mc_addr_list = bnxt_dev_set_mc_addr_list_op, + .rxq_info_get = bnxt_rxq_info_get_op, + .txq_info_get = bnxt_txq_info_get_op, + .dev_led_on = bnxt_dev_led_on_op, + .dev_led_off = bnxt_dev_led_off_op, }; static bool bnxt_vf_pciid(uint16_t id) @@ -1041,7 +1571,9 @@ static bool bnxt_vf_pciid(uint16_t id) if (id == BROADCOM_DEV_ID_57304_VF || id == BROADCOM_DEV_ID_57406_VF || id == BROADCOM_DEV_ID_5731X_VF || - id == BROADCOM_DEV_ID_5741X_VF) + id == BROADCOM_DEV_ID_5741X_VF || + id == BROADCOM_DEV_ID_57414_VF || + id == BROADCOM_DEV_ID_STRATUS_NIC_VF) return true; return false; } @@ -1049,7 +1581,7 @@ static bool bnxt_vf_pciid(uint16_t id) static int bnxt_init_board(struct rte_eth_dev *eth_dev) { struct bnxt *bp = eth_dev->data->dev_private; - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); int rc; /* enable device (incl. PCI PM wakeup), and bus-mastering */ @@ -1082,22 +1614,35 @@ init_err_disable: static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev); +#define ALLOW_FUNC(x) \ + { \ + typeof(x) arg = (x); \ + bp->pf.vf_req_fwd[((arg) >> 5)] &= \ + ~rte_cpu_to_le_32(1 << ((arg) & 0x1f)); \ + } static int bnxt_dev_init(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + char mz_name[RTE_MEMZONE_NAMESIZE]; + const struct rte_memzone *mz = NULL; static int version_printed; + uint32_t total_alloc_len; + phys_addr_t mz_phys_addr; struct bnxt *bp; int rc; if (version_printed++ == 0) - RTE_LOG(INFO, PMD, "%s", bnxt_version); + RTE_LOG(INFO, PMD, "%s\n", bnxt_version); rte_eth_copy_pci_info(eth_dev, pci_dev); eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; bp = eth_dev->data->dev_private; + rte_atomic64_init(&bp->rx_mbuf_alloc_fail); + bp->dev_stopped = 1; + if (bnxt_vf_pciid(pci_dev->id.device_id)) bp->flags |= BNXT_FLAG_VF; @@ -1111,6 +1656,80 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) eth_dev->rx_pkt_burst = &bnxt_recv_pkts; eth_dev->tx_pkt_burst = &bnxt_xmit_pkts; + if (BNXT_PF(bp) && pci_dev->id.device_id != BROADCOM_DEV_ID_NS2) { + snprintf(mz_name, RTE_MEMZONE_NAMESIZE, + "bnxt_%04x:%02x:%02x:%02x-%s", pci_dev->addr.domain, + pci_dev->addr.bus, pci_dev->addr.devid, + pci_dev->addr.function, "rx_port_stats"); + mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0; + mz = rte_memzone_lookup(mz_name); + total_alloc_len = RTE_CACHE_LINE_ROUNDUP( + sizeof(struct rx_port_stats) + 512); + if (!mz) { + mz = rte_memzone_reserve(mz_name, total_alloc_len, + SOCKET_ID_ANY, + RTE_MEMZONE_2MB | + RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) + return -ENOMEM; + } + memset(mz->addr, 0, mz->len); + mz_phys_addr = mz->phys_addr; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2phy()\n"); + mz_phys_addr = rte_mem_virt2phy(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map address to physical memory\n"); + return -ENOMEM; + } + } + + bp->rx_mem_zone = (const void *)mz; + bp->hw_rx_port_stats = mz->addr; + bp->hw_rx_port_stats_map = mz_phys_addr; + + snprintf(mz_name, RTE_MEMZONE_NAMESIZE, + "bnxt_%04x:%02x:%02x:%02x-%s", pci_dev->addr.domain, + pci_dev->addr.bus, pci_dev->addr.devid, + pci_dev->addr.function, "tx_port_stats"); + mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0; + mz = rte_memzone_lookup(mz_name); + total_alloc_len = RTE_CACHE_LINE_ROUNDUP( + sizeof(struct tx_port_stats) + 512); + if (!mz) { + mz = rte_memzone_reserve(mz_name, total_alloc_len, + SOCKET_ID_ANY, + RTE_MEMZONE_2MB | + RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) + return -ENOMEM; + } + memset(mz->addr, 0, mz->len); + mz_phys_addr = mz->phys_addr; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2phy()\n"); + mz_phys_addr = rte_mem_virt2phy(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map address to physical memory\n"); + return -ENOMEM; + } + } + + bp->tx_mem_zone = (const void *)mz; + bp->hw_tx_port_stats = mz->addr; + bp->hw_tx_port_stats_map = mz_phys_addr; + + bp->flags |= BNXT_FLAG_PORT_STATS; + } + rc = bnxt_alloc_hwrm_resources(bp); if (rc) { RTE_LOG(ERR, PMD, @@ -1130,6 +1749,11 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) RTE_LOG(ERR, PMD, "hwrm query capability failure rc: %x\n", rc); goto error_free; } + if (bp->max_tx_rings == 0) { + RTE_LOG(ERR, PMD, "No TX rings available!\n"); + rc = -EBUSY; + goto error_free; + } eth_dev->data->mac_addrs = rte_zmalloc("bnxt_mac_addr_tbl", ETHER_ADDR_LEN * MAX_NUM_MAC_ADDR, 0); if (eth_dev->data->mac_addrs == NULL) { @@ -1140,10 +1764,7 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) goto error_free; } /* Copy the permanent MAC from the qcap response address now. */ - if (BNXT_PF(bp)) - memcpy(bp->mac_addr, bp->pf.mac_addr, sizeof(bp->mac_addr)); - else - memcpy(bp->mac_addr, bp->vf.mac_addr, sizeof(bp->mac_addr)); + memcpy(bp->mac_addr, bp->dflt_mac_addr, sizeof(bp->mac_addr)); memcpy(ð_dev->data->mac_addrs[0], bp->mac_addr, ETHER_ADDR_LEN); bp->grp_info = rte_zmalloc("bnxt_grp_info", sizeof(*bp->grp_info) * bp->max_ring_grps, 0); @@ -1155,8 +1776,29 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) goto error_free; } - rc = bnxt_hwrm_func_driver_register(bp, 0, - bp->pf.vf_req_fwd); + /* Forward all requests if firmware is new enough */ + if (((bp->fw_ver >= ((20 << 24) | (6 << 16) | (100 << 8))) && + (bp->fw_ver < ((20 << 24) | (7 << 16)))) || + ((bp->fw_ver >= ((20 << 24) | (8 << 16))))) { + memset(bp->pf.vf_req_fwd, 0xff, sizeof(bp->pf.vf_req_fwd)); + } else { + RTE_LOG(WARNING, PMD, + "Firmware too old for VF mailbox functionality\n"); + memset(bp->pf.vf_req_fwd, 0, sizeof(bp->pf.vf_req_fwd)); + } + + /* + * The following are used for driver cleanup. If we disallow these, + * VF drivers can't clean up cleanly. + */ + ALLOW_FUNC(HWRM_FUNC_DRV_UNRGTR); + ALLOW_FUNC(HWRM_VNIC_FREE); + ALLOW_FUNC(HWRM_RING_FREE); + ALLOW_FUNC(HWRM_RING_GRP_FREE); + ALLOW_FUNC(HWRM_VNIC_RSS_COS_LB_CTX_FREE); + ALLOW_FUNC(HWRM_CFA_L2_FILTER_FREE); + ALLOW_FUNC(HWRM_STAT_CTX_FREE); + rc = bnxt_hwrm_func_driver_register(bp); if (rc) { RTE_LOG(ERR, PMD, "Failed to register driver"); @@ -1169,10 +1811,61 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) pci_dev->mem_resource[0].phys_addr, pci_dev->mem_resource[0].addr); - bp->dev_stopped = 0; + rc = bnxt_hwrm_func_reset(bp); + if (rc) { + RTE_LOG(ERR, PMD, "hwrm chip reset failure rc: %x\n", rc); + rc = -1; + goto error_free; + } + + if (BNXT_PF(bp)) { + //if (bp->pf.active_vfs) { + // TODO: Deallocate VF resources? + //} + if (bp->pdev->max_vfs) { + rc = bnxt_hwrm_allocate_vfs(bp, bp->pdev->max_vfs); + if (rc) { + RTE_LOG(ERR, PMD, "Failed to allocate VFs\n"); + goto error_free; + } + } else { + rc = bnxt_hwrm_allocate_pf_only(bp); + if (rc) { + RTE_LOG(ERR, PMD, + "Failed to allocate PF resources\n"); + goto error_free; + } + } + } + + bnxt_hwrm_port_led_qcaps(bp); + + rc = bnxt_setup_int(bp); + if (rc) + goto error_free; + + rc = bnxt_alloc_mem(bp); + if (rc) + goto error_free_int; + + rc = bnxt_request_int(bp); + if (rc) + goto error_free_int; + + rc = bnxt_alloc_def_cp_ring(bp); + if (rc) + goto error_free_int; + + bnxt_enable_int(bp); return 0; +error_free_int: + bnxt_disable_int(bp); + bnxt_free_def_cp_ring(bp); + bnxt_hwrm_func_buf_unrgtr(bp); + bnxt_free_int(bp); + bnxt_free_mem(bp); error_free: bnxt_dev_uninit(eth_dev); error: @@ -1184,6 +1877,9 @@ bnxt_dev_uninit(struct rte_eth_dev *eth_dev) { struct bnxt *bp = eth_dev->data->dev_private; int rc; + bnxt_disable_int(bp); + bnxt_free_int(bp); + bnxt_free_mem(bp); if (eth_dev->data->mac_addrs != NULL) { rte_free(eth_dev->data->mac_addrs); eth_dev->data->mac_addrs = NULL; @@ -1194,8 +1890,12 @@ bnxt_dev_uninit(struct rte_eth_dev *eth_dev) { } rc = bnxt_hwrm_func_driver_unregister(bp, 0); bnxt_free_hwrm_resources(bp); + rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone); + rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone); if (bp->dev_stopped == 0) bnxt_dev_close_op(eth_dev); + if (bp->pf.vf_info) + rte_free(bp->pf.vf_info); eth_dev->dev_ops = NULL; eth_dev->rx_pkt_burst = NULL; eth_dev->tx_pkt_burst = NULL; @@ -1223,6 +1923,20 @@ static struct rte_pci_driver bnxt_rte_pmd = { .remove = bnxt_pci_remove, }; +static bool +is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) +{ + if (strcmp(dev->device->driver->name, drv->driver.name)) + return false; + + return true; +} + +bool is_bnxt_supported(struct rte_eth_dev *dev) +{ + return is_device_supported(dev, &bnxt_rte_pmd); +} + RTE_PMD_REGISTER_PCI(net_bnxt, bnxt_rte_pmd); RTE_PMD_REGISTER_PCI_TABLE(net_bnxt, bnxt_pci_id_map); RTE_PMD_REGISTER_KMOD_DEP(net_bnxt, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/drivers/net/bnxt/bnxt_filter.c b/drivers/net/bnxt/bnxt_filter.c index df1042cf..e9aac271 100644 --- a/drivers/net/bnxt/bnxt_filter.c +++ b/drivers/net/bnxt/bnxt_filter.c @@ -68,20 +68,28 @@ struct bnxt_filter_info *bnxt_alloc_filter(struct bnxt *bp) return filter; } -void bnxt_init_filters(struct bnxt *bp) +struct bnxt_filter_info *bnxt_alloc_vf_filter(struct bnxt *bp, uint16_t vf) { struct bnxt_filter_info *filter; - int i, max_filters; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; + filter = rte_zmalloc("bnxt_vf_filter_info", sizeof(*filter), 0); + if (!filter) { + RTE_LOG(ERR, PMD, "Failed to alloc memory for VF %hu filters\n", + vf); + return NULL; + } - max_filters = pf->max_l2_ctx; - } else { - struct bnxt_vf_info *vf = &bp->vf; + filter->fw_l2_filter_id = UINT64_MAX; + STAILQ_INSERT_TAIL(&bp->pf.vf_info[vf].filter, filter, next); + return filter; +} - max_filters = vf->max_l2_ctx; - } +void bnxt_init_filters(struct bnxt *bp) +{ + struct bnxt_filter_info *filter; + int i, max_filters; + + max_filters = bp->max_l2_ctx; STAILQ_INIT(&bp->free_filter_list); for (i = 0; i < max_filters; i++) { filter = &bp->filter_info[i]; @@ -110,6 +118,12 @@ void bnxt_free_all_filters(struct bnxt *bp) STAILQ_INIT(&vnic->filter); } } + + for (i = 0; i < bp->pf.max_vfs; i++) { + STAILQ_FOREACH(filter, &bp->pf.vf_info[i].filter, next) { + bnxt_hwrm_clear_filter(bp, filter); + } + } } void bnxt_free_filter_mem(struct bnxt *bp) @@ -122,15 +136,7 @@ void bnxt_free_filter_mem(struct bnxt *bp) return; /* Ensure that all filters are freed */ - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_filters = pf->max_l2_ctx; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_filters = vf->max_l2_ctx; - } + max_filters = bp->max_l2_ctx; for (i = 0; i < max_filters; i++) { filter = &bp->filter_info[i]; if (filter->fw_l2_filter_id != ((uint64_t)-1)) { @@ -142,7 +148,7 @@ void bnxt_free_filter_mem(struct bnxt *bp) "HWRM filter cannot be freed rc = %d\n", rc); } - filter->fw_l2_filter_id = -1; + filter->fw_l2_filter_id = UINT64_MAX; } STAILQ_INIT(&bp->free_filter_list); @@ -155,15 +161,7 @@ int bnxt_alloc_filter_mem(struct bnxt *bp) struct bnxt_filter_info *filter_mem; uint16_t max_filters; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_filters = pf->max_l2_ctx; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_filters = vf->max_l2_ctx; - } + max_filters = bp->max_l2_ctx; /* Allocate memory for VNIC pool and filter pool */ filter_mem = rte_zmalloc("bnxt_filter_info", max_filters * sizeof(struct bnxt_filter_info), diff --git a/drivers/net/bnxt/bnxt_filter.h b/drivers/net/bnxt/bnxt_filter.h index 06fe134a..613b2eea 100644 --- a/drivers/net/bnxt/bnxt_filter.h +++ b/drivers/net/bnxt/bnxt_filter.h @@ -63,9 +63,12 @@ struct bnxt_filter_info { uint32_t vni; uint8_t pri_hint; uint64_t l2_filter_id_hint; + uint32_t src_id; + uint8_t src_type; }; struct bnxt_filter_info *bnxt_alloc_filter(struct bnxt *bp); +struct bnxt_filter_info *bnxt_alloc_vf_filter(struct bnxt *bp, uint16_t vf); void bnxt_init_filters(struct bnxt *bp); void bnxt_free_all_filters(struct bnxt *bp); void bnxt_free_filter_mem(struct bnxt *bp); diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c index d8987234..e710e636 100644 --- a/drivers/net/bnxt/bnxt_hwrm.c +++ b/drivers/net/bnxt/bnxt_hwrm.c @@ -31,6 +31,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <unistd.h> + #include <rte_byteorder.h> #include <rte_common.h> #include <rte_cycles.h> @@ -54,6 +56,38 @@ #define HWRM_CMD_TIMEOUT 2000 +struct bnxt_plcmodes_cfg { + uint32_t flags; + uint16_t jumbo_thresh; + uint16_t hds_offset; + uint16_t hds_threshold; +}; + +static int page_getenum(size_t size) +{ + if (size <= 1 << 4) + return 4; + if (size <= 1 << 12) + return 12; + if (size <= 1 << 13) + return 13; + if (size <= 1 << 16) + return 16; + if (size <= 1 << 21) + return 21; + if (size <= 1 << 22) + return 22; + if (size <= 1 << 30) + return 30; + RTE_LOG(ERR, PMD, "Page size %zu out of range\n", size); + return sizeof(void *) * 8 - 1; +} + +static int page_roundup(size_t size) +{ + return 1 << page_getenum(size); +} + /* * HWRM Functions (sent to HWRM) * These are named bnxt_hwrm_*() and return -1 if bnxt_hwrm_send_message() @@ -70,6 +104,30 @@ static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg, uint32_t *data = msg; uint8_t *bar; uint8_t *valid; + uint16_t max_req_len = bp->max_req_len; + struct hwrm_short_input short_input = { 0 }; + + if (bp->flags & BNXT_FLAG_SHORT_CMD) { + void *short_cmd_req = bp->hwrm_short_cmd_req_addr; + + memset(short_cmd_req, 0, bp->max_req_len); + memcpy(short_cmd_req, req, msg_len); + + short_input.req_type = rte_cpu_to_le_16(req->req_type); + short_input.signature = rte_cpu_to_le_16( + HWRM_SHORT_REQ_SIGNATURE_SHORT_CMD); + short_input.size = rte_cpu_to_le_16(msg_len); + short_input.req_addr = + rte_cpu_to_le_64(bp->hwrm_short_cmd_req_dma_addr); + + data = (uint32_t *)&short_input; + msg_len = sizeof(short_input); + + /* Sync memory write before updating doorbell */ + rte_wmb(); + + max_req_len = BNXT_HWRM_SHORT_REQ_LEN; + } /* Write request msg to hwrm channel */ for (i = 0; i < msg_len; i += 4) { @@ -79,7 +137,7 @@ static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg, } /* Zero the rest of the request space */ - for (; i < bp->max_req_len; i += 4) { + for (; i < max_req_len; i += 4) { bar = (uint8_t *)bp->bar0 + i; rte_write32(0, bar); } @@ -103,7 +161,7 @@ static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg, } if (i >= HWRM_CMD_TIMEOUT) { - RTE_LOG(ERR, PMD, "Error sending msg %x\n", + RTE_LOG(ERR, PMD, "Error sending msg 0x%04x\n", req->req_type); goto err_ret; } @@ -140,7 +198,22 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, uint32_t msg_len) } \ if (resp->error_code) { \ rc = rte_le_to_cpu_16(resp->error_code); \ - RTE_LOG(ERR, PMD, "%s error %d\n", __func__, rc); \ + if (resp->resp_len >= 16) { \ + struct hwrm_err_output *tmp_hwrm_err_op = \ + (void *)resp; \ + RTE_LOG(ERR, PMD, \ + "%s error %d:%d:%08x:%04x\n", \ + __func__, \ + rc, tmp_hwrm_err_op->cmd_err, \ + rte_le_to_cpu_32(\ + tmp_hwrm_err_op->opaque_0), \ + rte_le_to_cpu_16(\ + tmp_hwrm_err_op->opaque_1)); \ + } \ + else { \ + RTE_LOG(ERR, PMD, \ + "%s error %d\n", __func__, rc); \ + } \ return rc; \ } \ } @@ -162,7 +235,10 @@ int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic) return rc; } -int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic) +int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, + struct bnxt_vnic_info *vnic, + uint16_t vlan_count, + struct bnxt_vlan_table_entry *vlan_table) { int rc = 0; struct hwrm_cfa_l2_set_rx_mask_input req = {.req_type = 0 }; @@ -175,10 +251,28 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic) /* FIXME add multicast flag, when multicast adding options is supported * by ethtool. */ + if (vnic->flags & BNXT_VNIC_INFO_BCAST) + mask = HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST; + if (vnic->flags & BNXT_VNIC_INFO_UNTAGGED) + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN; if (vnic->flags & BNXT_VNIC_INFO_PROMISC) - mask = HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS; + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS; if (vnic->flags & BNXT_VNIC_INFO_ALLMULTI) - mask = HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; + if (vnic->flags & BNXT_VNIC_INFO_MCAST) + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST; + if (vnic->mc_addr_cnt) { + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST; + req.num_mc_entries = rte_cpu_to_le_32(vnic->mc_addr_cnt); + req.mc_tbl_addr = rte_cpu_to_le_64(vnic->mc_list_dma_addr); + } + if (vlan_table) { + if (!(mask & HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN)) + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLANONLY; + req.vlan_tag_tbl_addr = rte_cpu_to_le_16( + rte_mem_virt2phy(vlan_table)); + req.num_vlan_tags = rte_cpu_to_le_32((uint32_t)vlan_count); + } req.mask = rte_cpu_to_le_32(HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST | mask); @@ -189,6 +283,44 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic) return rc; } +int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid, + uint16_t vlan_count, + struct bnxt_vlan_antispoof_table_entry *vlan_table) +{ + int rc = 0; + struct hwrm_cfa_vlan_antispoof_cfg_input req = {.req_type = 0 }; + struct hwrm_cfa_vlan_antispoof_cfg_output *resp = + bp->hwrm_cmd_resp_addr; + + /* + * Older HWRM versions did not support this command, and the set_rx_mask + * list was used for anti-spoof. In 1.8.0, the TX path configuration was + * removed from set_rx_mask call, and this command was added. + * + * This command is also present from 1.7.8.11 and higher, + * as well as 1.7.8.0 + */ + if (bp->fw_ver < ((1 << 24) | (8 << 16))) { + if (bp->fw_ver != ((1 << 24) | (7 << 16) | (8 << 8))) { + if (bp->fw_ver < ((1 << 24) | (7 << 16) | (8 << 8) | + (11))) + return 0; + } + } + HWRM_PREP(req, CFA_VLAN_ANTISPOOF_CFG, -1, resp); + req.fid = rte_cpu_to_le_16(fid); + + req.vlan_tag_mask_tbl_addr = + rte_cpu_to_le_64(rte_mem_virt2phy(vlan_table)); + req.num_vlan_entries = rte_cpu_to_le_32((uint32_t)vlan_count); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + int bnxt_hwrm_clear_filter(struct bnxt *bp, struct bnxt_filter_info *filter) { @@ -196,6 +328,9 @@ int bnxt_hwrm_clear_filter(struct bnxt *bp, struct hwrm_cfa_l2_filter_free_input req = {.req_type = 0 }; struct hwrm_cfa_l2_filter_free_output *resp = bp->hwrm_cmd_resp_addr; + if (filter->fw_l2_filter_id == UINT64_MAX) + return 0; + HWRM_PREP(req, CFA_L2_FILTER_FREE, -1, resp); req.l2_filter_id = rte_cpu_to_le_64(filter->fw_l2_filter_id); @@ -210,7 +345,7 @@ int bnxt_hwrm_clear_filter(struct bnxt *bp, } int bnxt_hwrm_set_filter(struct bnxt *bp, - struct bnxt_vnic_info *vnic, + uint16_t dst_id, struct bnxt_filter_info *filter) { int rc = 0; @@ -218,13 +353,16 @@ int bnxt_hwrm_set_filter(struct bnxt *bp, struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; uint32_t enables = 0; + if (filter->fw_l2_filter_id != UINT64_MAX) + bnxt_hwrm_clear_filter(bp, filter); + HWRM_PREP(req, CFA_L2_FILTER_ALLOC, -1, resp); req.flags = rte_cpu_to_le_32(filter->flags); enables = filter->enables | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID; - req.dst_id = rte_cpu_to_le_16(vnic->fw_vnic_id); + req.dst_id = rte_cpu_to_le_16(dst_id); if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR) @@ -240,6 +378,10 @@ int bnxt_hwrm_set_filter(struct bnxt *bp, if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK) req.l2_ovlan_mask = filter->l2_ovlan_mask; + if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID) + req.src_id = rte_cpu_to_le_32(filter->src_id); + if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_TYPE) + req.src_type = filter->src_type; req.enables = rte_cpu_to_le_32(enables); @@ -252,29 +394,13 @@ int bnxt_hwrm_set_filter(struct bnxt *bp, return rc; } -int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, void *fwd_cmd) -{ - int rc; - struct hwrm_exec_fwd_resp_input req = {.req_type = 0 }; - struct hwrm_exec_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr; - - HWRM_PREP(req, EXEC_FWD_RESP, -1, resp); - - memcpy(req.encap_request, fwd_cmd, - sizeof(req.encap_request)); - - rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - - HWRM_CHECK_RESULT; - - return rc; -} - int bnxt_hwrm_func_qcaps(struct bnxt *bp) { int rc = 0; struct hwrm_func_qcaps_input req = {.req_type = 0 }; struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; + uint16_t new_max_vfs; + int i; HWRM_PREP(req, FUNC_QCAPS, -1, resp); @@ -286,31 +412,63 @@ int bnxt_hwrm_func_qcaps(struct bnxt *bp) bp->max_ring_grps = rte_le_to_cpu_32(resp->max_hw_ring_grps); if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - pf->fw_fid = rte_le_to_cpu_32(resp->fid); - pf->port_id = resp->port_id; - memcpy(pf->mac_addr, resp->mac_address, ETHER_ADDR_LEN); - pf->max_rsscos_ctx = rte_le_to_cpu_16(resp->max_rsscos_ctx); - pf->max_cp_rings = rte_le_to_cpu_16(resp->max_cmpl_rings); - pf->max_tx_rings = rte_le_to_cpu_16(resp->max_tx_rings); - pf->max_rx_rings = rte_le_to_cpu_16(resp->max_rx_rings); - pf->max_l2_ctx = rte_le_to_cpu_16(resp->max_l2_ctxs); - pf->max_vnics = rte_le_to_cpu_16(resp->max_vnics); - pf->first_vf_id = rte_le_to_cpu_16(resp->first_vf_id); - pf->max_vfs = rte_le_to_cpu_16(resp->max_vfs); - } else { - struct bnxt_vf_info *vf = &bp->vf; + bp->pf.port_id = resp->port_id; + bp->pf.first_vf_id = rte_le_to_cpu_16(resp->first_vf_id); + new_max_vfs = bp->pdev->max_vfs; + if (new_max_vfs != bp->pf.max_vfs) { + if (bp->pf.vf_info) + rte_free(bp->pf.vf_info); + bp->pf.vf_info = rte_malloc("bnxt_vf_info", + sizeof(bp->pf.vf_info[0]) * new_max_vfs, 0); + bp->pf.max_vfs = new_max_vfs; + for (i = 0; i < new_max_vfs; i++) { + bp->pf.vf_info[i].fid = bp->pf.first_vf_id + i; + bp->pf.vf_info[i].vlan_table = + rte_zmalloc("VF VLAN table", + getpagesize(), + getpagesize()); + if (bp->pf.vf_info[i].vlan_table == NULL) + RTE_LOG(ERR, PMD, + "Fail to alloc VLAN table for VF %d\n", + i); + else + rte_mem_lock_page( + bp->pf.vf_info[i].vlan_table); + bp->pf.vf_info[i].vlan_as_table = + rte_zmalloc("VF VLAN AS table", + getpagesize(), + getpagesize()); + if (bp->pf.vf_info[i].vlan_as_table == NULL) + RTE_LOG(ERR, PMD, + "Alloc VLAN AS table for VF %d fail\n", + i); + else + rte_mem_lock_page( + bp->pf.vf_info[i].vlan_as_table); + STAILQ_INIT(&bp->pf.vf_info[i].filter); + } + } + } - vf->fw_fid = rte_le_to_cpu_32(resp->fid); - memcpy(vf->mac_addr, &resp->mac_address, ETHER_ADDR_LEN); - vf->max_rsscos_ctx = rte_le_to_cpu_16(resp->max_rsscos_ctx); - vf->max_cp_rings = rte_le_to_cpu_16(resp->max_cmpl_rings); - vf->max_tx_rings = rte_le_to_cpu_16(resp->max_tx_rings); - vf->max_rx_rings = rte_le_to_cpu_16(resp->max_rx_rings); - vf->max_l2_ctx = rte_le_to_cpu_16(resp->max_l2_ctxs); - vf->max_vnics = rte_le_to_cpu_16(resp->max_vnics); + bp->fw_fid = rte_le_to_cpu_32(resp->fid); + memcpy(bp->dflt_mac_addr, &resp->mac_address, ETHER_ADDR_LEN); + bp->max_rsscos_ctx = rte_le_to_cpu_16(resp->max_rsscos_ctx); + bp->max_cp_rings = rte_le_to_cpu_16(resp->max_cmpl_rings); + bp->max_tx_rings = rte_le_to_cpu_16(resp->max_tx_rings); + bp->max_rx_rings = rte_le_to_cpu_16(resp->max_rx_rings); + bp->max_l2_ctx = rte_le_to_cpu_16(resp->max_l2_ctxs); + /* TODO: For now, do not support VMDq/RFS on VFs. */ + if (BNXT_PF(bp)) { + if (bp->pf.max_vfs) + bp->max_vnics = 1; + else + bp->max_vnics = rte_le_to_cpu_16(resp->max_vnics); + } else { + bp->max_vnics = 1; } + bp->max_stat_ctx = rte_le_to_cpu_16(resp->max_stat_ctx); + if (BNXT_PF(bp)) + bp->pf.total_vnics = rte_le_to_cpu_16(resp->max_vnics); return rc; } @@ -332,8 +490,7 @@ int bnxt_hwrm_func_reset(struct bnxt *bp) return rc; } -int bnxt_hwrm_func_driver_register(struct bnxt *bp, uint32_t flags, - uint32_t *vf_req_fwd) +int bnxt_hwrm_func_driver_register(struct bnxt *bp) { int rc; struct hwrm_func_drv_rgtr_input req = {.req_type = 0 }; @@ -343,16 +500,22 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp, uint32_t flags, return 0; HWRM_PREP(req, FUNC_DRV_RGTR, -1, resp); - req.flags = flags; - req.enables = HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER | - HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD; + req.enables = rte_cpu_to_le_32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER | + HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD); req.ver_maj = RTE_VER_YEAR; req.ver_min = RTE_VER_MONTH; req.ver_upd = RTE_VER_MINOR; - memcpy(req.vf_req_fwd, vf_req_fwd, sizeof(req.vf_req_fwd)); + if (BNXT_PF(bp)) { + req.enables |= rte_cpu_to_le_32( + HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VF_INPUT_FWD); + memcpy(req.vf_req_fwd, bp->pf.vf_req_fwd, + RTE_MIN(sizeof(req.vf_req_fwd), + sizeof(bp->pf.vf_req_fwd))); + } req.async_event_fwd[0] |= rte_cpu_to_le_32(0x1); /* TODO: Use MACRO */ + memset(req.async_event_fwd, 0xff, sizeof(req.async_event_fwd)); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -372,7 +535,9 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) uint32_t fw_version; uint16_t max_resp_len; char type[RTE_MEMZONE_NAMESIZE]; + uint32_t dev_caps_cfg; + bp->max_req_len = HWRM_MAX_REQ_LEN; HWRM_PREP(req, VER_GET, -1, resp); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; @@ -391,6 +556,8 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd, resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld); + bp->fw_ver = (resp->hwrm_fw_maj << 24) | (resp->hwrm_fw_min << 16) | + (resp->hwrm_fw_bld << 8) | resp->hwrm_fw_rsvd; RTE_LOG(INFO, PMD, "Driver HWRM version: %d.%d.%d\n", HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, HWRM_VERSION_UPDATE); @@ -427,8 +594,10 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) RTE_LOG(ERR, PMD, "Unsupported request length\n"); rc = -EINVAL; } - bp->max_req_len = resp->max_req_win_len; + bp->max_req_len = rte_le_to_cpu_16(resp->max_req_win_len); max_resp_len = resp->max_resp_len; + dev_caps_cfg = rte_le_to_cpu_32(resp->dev_caps_cfg); + if (bp->max_resp_len != max_resp_len) { sprintf(type, "bnxt_hwrm_%04x:%02x:%02x:%02x", bp->pdev->addr.domain, bp->pdev->addr.bus, @@ -441,11 +610,46 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) rc = -ENOMEM; goto error; } + rte_mem_lock_page(bp->hwrm_cmd_resp_addr); bp->hwrm_cmd_resp_dma_addr = - rte_malloc_virt2phy(bp->hwrm_cmd_resp_addr); + rte_mem_virt2phy(bp->hwrm_cmd_resp_addr); + if (bp->hwrm_cmd_resp_dma_addr == 0) { + RTE_LOG(ERR, PMD, + "Unable to map response buffer to physical memory.\n"); + rc = -ENOMEM; + goto error; + } bp->max_resp_len = max_resp_len; } + if ((dev_caps_cfg & + HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && + (dev_caps_cfg & + HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_INPUTUIRED)) { + RTE_LOG(DEBUG, PMD, "Short command supported\n"); + + rte_free(bp->hwrm_short_cmd_req_addr); + + bp->hwrm_short_cmd_req_addr = rte_malloc(type, + bp->max_req_len, 0); + if (bp->hwrm_short_cmd_req_addr == NULL) { + rc = -ENOMEM; + goto error; + } + rte_mem_lock_page(bp->hwrm_short_cmd_req_addr); + bp->hwrm_short_cmd_req_dma_addr = + rte_mem_virt2phy(bp->hwrm_short_cmd_req_addr); + if (bp->hwrm_short_cmd_req_dma_addr == 0) { + rte_free(bp->hwrm_short_cmd_req_addr); + RTE_LOG(ERR, PMD, + "Unable to map buffer to physical memory.\n"); + rc = -ENOMEM; + goto error; + } + + bp->flags |= BNXT_FLAG_SHORT_CMD; + } + error: rte_spinlock_unlock(&bp->hwrm_lock); return rc; @@ -478,6 +682,8 @@ static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf) struct hwrm_port_phy_cfg_input req = {0}; struct hwrm_port_phy_cfg_output *resp = bp->hwrm_cmd_resp_addr; uint32_t enables = 0; + uint32_t link_speed_mask = + HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK; HWRM_PREP(req, PORT_PHY_CFG, -1, resp); @@ -489,14 +695,20 @@ static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf) * any auto mode, even "none". */ if (!conf->link_speed) { - req.auto_mode |= conf->auto_mode; - enables = HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE; - req.auto_link_speed_mask = conf->auto_link_speed_mask; - enables |= - HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK; - req.auto_link_speed = bp->link_info.auto_link_speed; - enables |= + req.auto_mode = conf->auto_mode; + enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE; + if (conf->auto_mode == + HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_SPEED_MASK) { + req.auto_link_speed_mask = + conf->auto_link_speed_mask; + enables |= link_speed_mask; + } + if (bp->link_info.auto_link_speed) { + req.auto_link_speed = + bp->link_info.auto_link_speed; + enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED; + } } req.auto_duplex = conf->duplex; enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_DUPLEX; @@ -511,7 +723,7 @@ static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf) req.enables = rte_cpu_to_le_32(enables); } else { req.flags = - rte_cpu_to_le_32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DOWN); + rte_cpu_to_le_32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DWN); RTE_LOG(INFO, PMD, "Force Link Down\n"); } @@ -536,13 +748,10 @@ static int bnxt_hwrm_port_phy_qcfg(struct bnxt *bp, HWRM_CHECK_RESULT; link_info->phy_link_status = resp->link; - if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) { - link_info->link_up = 1; - link_info->link_speed = rte_le_to_cpu_16(resp->link_speed); - } else { - link_info->link_up = 0; - link_info->link_speed = 0; - } + link_info->link_up = + (link_info->phy_link_status == + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) ? 1 : 0; + link_info->link_speed = rte_le_to_cpu_16(resp->link_speed); link_info->duplex = resp->duplex; link_info->pause = resp->pause; link_info->auto_pause = resp->auto_pause; @@ -590,20 +799,20 @@ int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) int bnxt_hwrm_ring_alloc(struct bnxt *bp, struct bnxt_ring *ring, uint32_t ring_type, uint32_t map_index, - uint32_t stats_ctx_id) + uint32_t stats_ctx_id, uint32_t cmpl_ring_id) { int rc = 0; + uint32_t enables = 0; struct hwrm_ring_alloc_input req = {.req_type = 0 }; struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; HWRM_PREP(req, RING_ALLOC, -1, resp); - req.enables = rte_cpu_to_le_32(0); - req.page_tbl_addr = rte_cpu_to_le_64(ring->bd_dma); req.fbo = rte_cpu_to_le_32(0); /* Association of ring index with doorbell index */ req.logical_id = rte_cpu_to_le_16(map_index); + req.length = rte_cpu_to_le_32(ring->ring_size); switch (ring_type) { case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX: @@ -611,27 +820,26 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp, /* FALLTHROUGH */ case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX: req.ring_type = ring_type; - req.cmpl_ring_id = - rte_cpu_to_le_16(bp->grp_info[map_index].cp_fw_ring_id); - req.length = rte_cpu_to_le_32(ring->ring_size); + req.cmpl_ring_id = rte_cpu_to_le_16(cmpl_ring_id); req.stat_ctx_id = rte_cpu_to_le_16(stats_ctx_id); - req.enables = rte_cpu_to_le_32(rte_le_to_cpu_32(req.enables) | - HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); + if (stats_ctx_id != INVALID_STATS_CTX_ID) + enables |= + HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID; break; - case HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL: + case HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL: req.ring_type = ring_type; /* * TODO: Some HWRM versions crash with * HWRM_RING_ALLOC_INPUT_INT_MODE_POLL */ req.int_mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; - req.length = rte_cpu_to_le_32(ring->ring_size); break; default: RTE_LOG(ERR, PMD, "hwrm alloc invalid ring type %d\n", ring_type); return -1; } + req.enables = rte_cpu_to_le_32(enables); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -639,7 +847,7 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp, if (rc == 0 && resp->error_code) rc = rte_le_to_cpu_16(resp->error_code); switch (ring_type) { - case HWRM_RING_FREE_INPUT_RING_TYPE_CMPL: + case HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL: RTE_LOG(ERR, PMD, "hwrm_ring_alloc cp failed. rc:%d\n", rc); return rc; @@ -680,7 +888,7 @@ int bnxt_hwrm_ring_free(struct bnxt *bp, rc = rte_le_to_cpu_16(resp->error_code); switch (ring_type) { - case HWRM_RING_FREE_INPUT_RING_TYPE_CMPL: + case HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL: RTE_LOG(ERR, PMD, "hwrm_ring_free cp failed. rc:%d\n", rc); return rc; @@ -747,13 +955,12 @@ int bnxt_hwrm_stat_clear(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) struct hwrm_stat_ctx_clr_stats_input req = {.req_type = 0 }; struct hwrm_stat_ctx_clr_stats_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, STAT_CTX_CLR_STATS, -1, resp); - if (cpr->hw_stats_ctx_id == (uint32_t)HWRM_NA_SIGNATURE) return rc; + HWRM_PREP(req, STAT_CTX_CLR_STATS, -1, resp); + req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id); - req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -762,8 +969,8 @@ int bnxt_hwrm_stat_clear(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) return rc; } -int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr, unsigned int idx) +int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, + unsigned int idx __rte_unused) { int rc; struct hwrm_stat_ctx_alloc_input req = {.req_type = 0 }; @@ -771,9 +978,8 @@ int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, HWRM_PREP(req, STAT_CTX_ALLOC, -1, resp); - req.update_period_ms = rte_cpu_to_le_32(1000); + req.update_period_ms = rte_cpu_to_le_32(0); - req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); req.stats_dma_addr = rte_cpu_to_le_64(cpr->hw_stats_map); @@ -782,13 +988,12 @@ int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, HWRM_CHECK_RESULT; cpr->hw_stats_ctx_id = rte_le_to_cpu_16(resp->stat_ctx_id); - bp->grp_info[idx].fw_stats_ctx = cpr->hw_stats_ctx_id; return rc; } -int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr, unsigned int idx) +int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, + unsigned int idx __rte_unused) { int rc; struct hwrm_stat_ctx_free_input req = {.req_type = 0 }; @@ -797,15 +1002,11 @@ int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, HWRM_PREP(req, STAT_CTX_FREE, -1, resp); req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id); - req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); HWRM_CHECK_RESULT; - cpr->hw_stats_ctx_id = HWRM_NA_SIGNATURE; - bp->grp_info[idx].fw_stats_ctx = cpr->hw_stats_ctx_id; - return rc; } @@ -816,26 +1017,80 @@ int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic) struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; /* map ring groups to this vnic */ - for (i = vnic->start_grp_id, j = 0; i <= vnic->end_grp_id; i++, j++) { - if (bp->grp_info[i].fw_grp_id == (uint16_t)HWRM_NA_SIGNATURE) { - RTE_LOG(ERR, PMD, - "Not enough ring groups avail:%x req:%x\n", j, - (vnic->end_grp_id - vnic->start_grp_id) + 1); - break; - } + RTE_LOG(DEBUG, PMD, "Alloc VNIC. Start %x, End %x\n", + vnic->start_grp_id, vnic->end_grp_id); + for (i = vnic->start_grp_id, j = 0; i <= vnic->end_grp_id; i++, j++) vnic->fw_grp_ids[j] = bp->grp_info[i].fw_grp_id; - } - - vnic->fw_rss_cos_lb_ctx = (uint16_t)HWRM_NA_SIGNATURE; - vnic->ctx_is_rss_cos_lb = HW_CONTEXT_NONE; - + vnic->dflt_ring_grp = bp->grp_info[vnic->start_grp_id].fw_grp_id; + vnic->rss_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->cos_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->lb_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE; HWRM_PREP(req, VNIC_ALLOC, -1, resp); + if (vnic->func_default) + req.flags = HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT; rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); HWRM_CHECK_RESULT; vnic->fw_vnic_id = rte_le_to_cpu_16(resp->vnic_id); + RTE_LOG(DEBUG, PMD, "VNIC ID %x\n", vnic->fw_vnic_id); + return rc; +} + +static int bnxt_hwrm_vnic_plcmodes_qcfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, + struct bnxt_plcmodes_cfg *pmode) +{ + int rc = 0; + struct hwrm_vnic_plcmodes_qcfg_input req = {.req_type = 0 }; + struct hwrm_vnic_plcmodes_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, VNIC_PLCMODES_QCFG, -1, resp); + + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + pmode->flags = rte_le_to_cpu_32(resp->flags); + /* dflt_vnic bit doesn't exist in the _cfg command */ + pmode->flags &= ~(HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_DFLT_VNIC); + pmode->jumbo_thresh = rte_le_to_cpu_16(resp->jumbo_thresh); + pmode->hds_offset = rte_le_to_cpu_16(resp->hds_offset); + pmode->hds_threshold = rte_le_to_cpu_16(resp->hds_threshold); + + return rc; +} + +static int bnxt_hwrm_vnic_plcmodes_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, + struct bnxt_plcmodes_cfg *pmode) +{ + int rc = 0; + struct hwrm_vnic_plcmodes_cfg_input req = {.req_type = 0 }; + struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, VNIC_PLCMODES_CFG, -1, resp); + + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + req.flags = rte_cpu_to_le_32(pmode->flags); + req.jumbo_thresh = rte_cpu_to_le_16(pmode->jumbo_thresh); + req.hds_offset = rte_cpu_to_le_16(pmode->hds_offset); + req.hds_threshold = rte_cpu_to_le_16(pmode->hds_threshold); + req.enables = rte_cpu_to_le_32( + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_THRESHOLD_VALID | + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_OFFSET_VALID | + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_JUMBO_THRESH_VALID + ); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + return rc; } @@ -844,32 +1099,105 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic) int rc = 0; struct hwrm_vnic_cfg_input req = {.req_type = 0 }; struct hwrm_vnic_cfg_output *resp = bp->hwrm_cmd_resp_addr; + uint32_t ctx_enable_flag = HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE; + struct bnxt_plcmodes_cfg pmodes; + + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) { + RTE_LOG(DEBUG, PMD, "VNIC ID %x\n", vnic->fw_vnic_id); + return rc; + } + + rc = bnxt_hwrm_vnic_plcmodes_qcfg(bp, vnic, &pmodes); + if (rc) + return rc; HWRM_PREP(req, VNIC_CFG, -1, resp); /* Only RSS support for now TBD: COS & LB */ req.enables = rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_ENABLES_DFLT_RING_GRP | - HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE | HWRM_VNIC_CFG_INPUT_ENABLES_MRU); + if (vnic->lb_rule != 0xffff) + ctx_enable_flag = HWRM_VNIC_CFG_INPUT_ENABLES_LB_RULE; + if (vnic->cos_rule != 0xffff) + ctx_enable_flag = HWRM_VNIC_CFG_INPUT_ENABLES_COS_RULE; + if (vnic->rss_rule != 0xffff) + ctx_enable_flag = HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE; + req.enables |= rte_cpu_to_le_32(ctx_enable_flag); req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); - req.dflt_ring_grp = - rte_cpu_to_le_16(bp->grp_info[vnic->start_grp_id].fw_grp_id); - req.rss_rule = rte_cpu_to_le_16(vnic->fw_rss_cos_lb_ctx); - req.cos_rule = rte_cpu_to_le_16(0xffff); - req.lb_rule = rte_cpu_to_le_16(0xffff); - req.mru = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + - ETHER_CRC_LEN + VLAN_TAG_SIZE); + req.dflt_ring_grp = rte_cpu_to_le_16(vnic->dflt_ring_grp); + req.rss_rule = rte_cpu_to_le_16(vnic->rss_rule); + req.cos_rule = rte_cpu_to_le_16(vnic->cos_rule); + req.lb_rule = rte_cpu_to_le_16(vnic->lb_rule); + req.mru = rte_cpu_to_le_16(vnic->mru); if (vnic->func_default) - req.flags = 1; + req.flags |= + rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_DEFAULT); if (vnic->vlan_strip) req.flags |= rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_VLAN_STRIP_MODE); + if (vnic->bd_stall) + req.flags |= + rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_BD_STALL_MODE); + if (vnic->roce_dual) + req.flags |= rte_cpu_to_le_32( + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_DUAL_VNIC_MODE); + if (vnic->roce_only) + req.flags |= rte_cpu_to_le_32( + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_ONLY_VNIC_MODE); + if (vnic->rss_dflt_cr) + req.flags |= rte_cpu_to_le_32( + HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + rc = bnxt_hwrm_vnic_plcmodes_cfg(bp, vnic, &pmodes); + + return rc; +} + +int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic, + int16_t fw_vf_id) +{ + int rc = 0; + struct hwrm_vnic_qcfg_input req = {.req_type = 0 }; + struct hwrm_vnic_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) { + RTE_LOG(DEBUG, PMD, "VNIC QCFG ID %d\n", vnic->fw_vnic_id); + return rc; + } + HWRM_PREP(req, VNIC_QCFG, -1, resp); + + req.enables = + rte_cpu_to_le_32(HWRM_VNIC_QCFG_INPUT_ENABLES_VF_ID_VALID); + req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); + req.vf_id = rte_cpu_to_le_16(fw_vf_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); HWRM_CHECK_RESULT; + vnic->dflt_ring_grp = rte_le_to_cpu_16(resp->dflt_ring_grp); + vnic->rss_rule = rte_le_to_cpu_16(resp->rss_rule); + vnic->cos_rule = rte_le_to_cpu_16(resp->cos_rule); + vnic->lb_rule = rte_le_to_cpu_16(resp->lb_rule); + vnic->mru = rte_le_to_cpu_16(resp->mru); + vnic->func_default = rte_le_to_cpu_32( + resp->flags) & HWRM_VNIC_QCFG_OUTPUT_FLAGS_DEFAULT; + vnic->vlan_strip = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_VLAN_STRIP_MODE; + vnic->bd_stall = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_BD_STALL_MODE; + vnic->roce_dual = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_DUAL_VNIC_MODE; + vnic->roce_only = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_ONLY_VNIC_MODE; + vnic->rss_dflt_cr = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE; + return rc; } @@ -886,7 +1214,8 @@ int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic) HWRM_CHECK_RESULT; - vnic->fw_rss_cos_lb_ctx = rte_le_to_cpu_16(resp->rss_cos_lb_ctx_id); + vnic->rss_rule = rte_le_to_cpu_16(resp->rss_cos_lb_ctx_id); + RTE_LOG(DEBUG, PMD, "VNIC RSS Rule %x\n", vnic->rss_rule); return rc; } @@ -898,15 +1227,19 @@ int bnxt_hwrm_vnic_ctx_free(struct bnxt *bp, struct bnxt_vnic_info *vnic) struct hwrm_vnic_rss_cos_lb_ctx_free_output *resp = bp->hwrm_cmd_resp_addr; + if (vnic->rss_rule == 0xffff) { + RTE_LOG(DEBUG, PMD, "VNIC RSS Rule %x\n", vnic->rss_rule); + return rc; + } HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_FREE, -1, resp); - req.rss_cos_lb_ctx_id = rte_cpu_to_le_16(vnic->fw_rss_cos_lb_ctx); + req.rss_cos_lb_ctx_id = rte_cpu_to_le_16(vnic->rss_rule); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); HWRM_CHECK_RESULT; - vnic->fw_rss_cos_lb_ctx = INVALID_HW_RING_ID; + vnic->rss_rule = INVALID_HW_RING_ID; return rc; } @@ -917,8 +1250,10 @@ int bnxt_hwrm_vnic_free(struct bnxt *bp, struct bnxt_vnic_info *vnic) struct hwrm_vnic_free_input req = {.req_type = 0 }; struct hwrm_vnic_free_output *resp = bp->hwrm_cmd_resp_addr; - if (vnic->fw_vnic_id == INVALID_HW_RING_ID) + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) { + RTE_LOG(DEBUG, PMD, "VNIC FREE ID %x\n", vnic->fw_vnic_id); return rc; + } HWRM_PREP(req, VNIC_FREE, -1, resp); @@ -947,7 +1282,168 @@ int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp, rte_cpu_to_le_64(vnic->rss_table_dma_addr); req.hash_key_tbl_addr = rte_cpu_to_le_64(vnic->rss_hash_key_dma_addr); - req.rss_ctx_idx = rte_cpu_to_le_16(vnic->fw_rss_cos_lb_ctx); + req.rss_ctx_idx = rte_cpu_to_le_16(vnic->rss_rule); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic) +{ + int rc = 0; + struct hwrm_vnic_plcmodes_cfg_input req = {.req_type = 0 }; + struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr; + uint16_t size; + + HWRM_PREP(req, VNIC_PLCMODES_CFG, -1, resp); + + req.flags = rte_cpu_to_le_32( + HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_JUMBO_PLACEMENT); + + req.enables = rte_cpu_to_le_32( + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_JUMBO_THRESH_VALID); + + size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool); + size -= RTE_PKTMBUF_HEADROOM; + + req.jumbo_thresh = rte_cpu_to_le_16(size); + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, bool enable) +{ + int rc = 0; + struct hwrm_vnic_tpa_cfg_input req = {.req_type = 0 }; + struct hwrm_vnic_tpa_cfg_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, VNIC_TPA_CFG, -1, resp); + + if (enable) { + req.enables = rte_cpu_to_le_32( + HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS | + HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS | + HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN); + req.flags = rte_cpu_to_le_32( + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ); + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + req.max_agg_segs = rte_cpu_to_le_16(5); + req.max_aggs = + rte_cpu_to_le_16(HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX); + req.min_agg_len = rte_cpu_to_le_32(512); + } + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_func_vf_mac(struct bnxt *bp, uint16_t vf, const uint8_t *mac_addr) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags); + req.enables = rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + memcpy(req.dflt_mac_addr, mac_addr, sizeof(req.dflt_mac_addr)); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + + HWRM_PREP(req, FUNC_CFG, -1, resp); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + bp->pf.vf_info[vf].random_mac = false; + + return rc; +} + +int bnxt_hwrm_func_qstats_tx_drop(struct bnxt *bp, uint16_t fid, + uint64_t *dropped) +{ + int rc = 0; + struct hwrm_func_qstats_input req = {.req_type = 0}; + struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_QSTATS, -1, resp); + + req.fid = rte_cpu_to_le_16(fid); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + if (dropped) + *dropped = rte_le_to_cpu_64(resp->tx_drop_pkts); + + return rc; +} + +int bnxt_hwrm_func_qstats(struct bnxt *bp, uint16_t fid, + struct rte_eth_stats *stats) +{ + int rc = 0; + struct hwrm_func_qstats_input req = {.req_type = 0}; + struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_QSTATS, -1, resp); + + req.fid = rte_cpu_to_le_16(fid); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + stats->ipackets = rte_le_to_cpu_64(resp->rx_ucast_pkts); + stats->ipackets += rte_le_to_cpu_64(resp->rx_mcast_pkts); + stats->ipackets += rte_le_to_cpu_64(resp->rx_bcast_pkts); + stats->ibytes = rte_le_to_cpu_64(resp->rx_ucast_bytes); + stats->ibytes += rte_le_to_cpu_64(resp->rx_mcast_bytes); + stats->ibytes += rte_le_to_cpu_64(resp->rx_bcast_bytes); + + stats->opackets = rte_le_to_cpu_64(resp->tx_ucast_pkts); + stats->opackets += rte_le_to_cpu_64(resp->tx_mcast_pkts); + stats->opackets += rte_le_to_cpu_64(resp->tx_bcast_pkts); + stats->obytes = rte_le_to_cpu_64(resp->tx_ucast_bytes); + stats->obytes += rte_le_to_cpu_64(resp->tx_mcast_bytes); + stats->obytes += rte_le_to_cpu_64(resp->tx_bcast_bytes); + + stats->ierrors = rte_le_to_cpu_64(resp->rx_err_pkts); + stats->oerrors = rte_le_to_cpu_64(resp->tx_err_pkts); + + stats->imissed = rte_le_to_cpu_64(resp->rx_drop_pkts); + + return rc; +} + +int bnxt_hwrm_func_clr_stats(struct bnxt *bp, uint16_t fid) +{ + int rc = 0; + struct hwrm_func_clr_stats_input req = {.req_type = 0}; + struct hwrm_func_clr_stats_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_CLR_STATS, -1, resp); + + req.fid = rte_cpu_to_le_16(fid); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -992,14 +1488,20 @@ int bnxt_free_all_hwrm_stat_ctxs(struct bnxt *bp) struct bnxt_cp_ring_info *cpr; for (i = 0; i < bp->rx_cp_nr_rings + bp->tx_cp_nr_rings; i++) { - unsigned int idx = i + 1; if (i >= bp->rx_cp_nr_rings) cpr = bp->tx_queues[i - bp->rx_cp_nr_rings]->cp_ring; else cpr = bp->rx_queues[i]->cp_ring; if (cpr->hw_stats_ctx_id != HWRM_NA_SIGNATURE) { - rc = bnxt_hwrm_stat_ctx_free(bp, cpr, idx); + rc = bnxt_hwrm_stat_ctx_free(bp, cpr, i); + cpr->hw_stats_ctx_id = HWRM_NA_SIGNATURE; + /* + * TODO. Need a better way to reset grp_info.stats_ctx + * for Rx rings only. stats_ctx is not saved for Tx + * in grp_info. + */ + bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; if (rc) return rc; } @@ -1016,7 +1518,6 @@ int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp) struct bnxt_tx_queue *txq; struct bnxt_rx_queue *rxq; struct bnxt_cp_ring_info *cpr; - unsigned int idx = i + 1; if (i >= bp->rx_cp_nr_rings) { txq = bp->tx_queues[i - bp->rx_cp_nr_rings]; @@ -1026,7 +1527,7 @@ int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp) cpr = rxq->cp_ring; } - rc = bnxt_hwrm_stat_ctx_alloc(bp, cpr, idx); + rc = bnxt_hwrm_stat_ctx_alloc(bp, cpr, i); if (rc) return rc; @@ -1036,11 +1537,10 @@ int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp) int bnxt_free_all_hwrm_ring_grps(struct bnxt *bp) { - uint16_t i; + uint16_t idx; uint32_t rc = 0; - for (i = 0; i < bp->rx_cp_nr_rings; i++) { - unsigned int idx = i + 1; + for (idx = 0; idx < bp->rx_cp_nr_rings; idx++) { if (bp->grp_info[idx].fw_grp_id == INVALID_HW_RING_ID) { RTE_LOG(ERR, PMD, @@ -1057,13 +1557,13 @@ int bnxt_free_all_hwrm_ring_grps(struct bnxt *bp) return rc; } -static void bnxt_free_cp_ring(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr, unsigned int idx) +static void bnxt_free_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, + unsigned int idx __rte_unused) { struct bnxt_ring *cp_ring = cpr->cp_ring_struct; bnxt_hwrm_ring_free(bp, cp_ring, - HWRM_RING_FREE_INPUT_RING_TYPE_CMPL); + HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL); cp_ring->fw_ring_id = INVALID_HW_RING_ID; bp->grp_info[idx].cp_fw_ring_id = INVALID_HW_RING_ID; memset(cpr->cp_desc_ring, 0, cpr->cp_ring_struct->ring_size * @@ -1096,8 +1596,10 @@ int bnxt_free_all_hwrm_rings(struct bnxt *bp) txr->tx_prod = 0; txr->tx_cons = 0; } - if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) + if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) { bnxt_free_cp_ring(bp, cpr, idx); + cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID; + } } for (i = 0; i < bp->rx_cp_nr_rings; i++) { @@ -1119,17 +1621,26 @@ int bnxt_free_all_hwrm_rings(struct bnxt *bp) rxr->rx_ring_struct->ring_size * sizeof(*rxr->rx_buf_ring)); rxr->rx_prod = 0; + memset(rxr->ag_buf_ring, 0, + rxr->ag_ring_struct->ring_size * + sizeof(*rxr->ag_buf_ring)); + rxr->ag_prod = 0; } - if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) + if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) { bnxt_free_cp_ring(bp, cpr, idx); + bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; + cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID; + } } /* Default completion ring */ { struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; - if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) + if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) { bnxt_free_cp_ring(bp, cpr, 0); + cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID; + } } return rc; @@ -1141,14 +1652,7 @@ int bnxt_alloc_all_hwrm_ring_grps(struct bnxt *bp) uint32_t rc = 0; for (i = 0; i < bp->rx_cp_nr_rings; i++) { - unsigned int idx = i + 1; - - if (bp->grp_info[idx].cp_fw_ring_id == INVALID_HW_RING_ID || - bp->grp_info[idx].rx_fw_ring_id == INVALID_HW_RING_ID) - continue; - - rc = bnxt_hwrm_ring_grp_alloc(bp, idx); - + rc = bnxt_hwrm_ring_grp_alloc(bp, i); if (rc) return rc; } @@ -1159,8 +1663,11 @@ void bnxt_free_hwrm_resources(struct bnxt *bp) { /* Release memzone */ rte_free(bp->hwrm_cmd_resp_addr); + rte_free(bp->hwrm_short_cmd_req_addr); bp->hwrm_cmd_resp_addr = NULL; + bp->hwrm_short_cmd_req_addr = NULL; bp->hwrm_cmd_resp_dma_addr = 0; + bp->hwrm_short_cmd_req_dma_addr = 0; } int bnxt_alloc_hwrm_resources(struct bnxt *bp) @@ -1170,13 +1677,18 @@ int bnxt_alloc_hwrm_resources(struct bnxt *bp) sprintf(type, "bnxt_hwrm_%04x:%02x:%02x:%02x", pdev->addr.domain, pdev->addr.bus, pdev->addr.devid, pdev->addr.function); - bp->max_req_len = HWRM_MAX_REQ_LEN; bp->max_resp_len = HWRM_MAX_RESP_LEN; bp->hwrm_cmd_resp_addr = rte_malloc(type, bp->max_resp_len, 0); + rte_mem_lock_page(bp->hwrm_cmd_resp_addr); if (bp->hwrm_cmd_resp_addr == NULL) return -ENOMEM; bp->hwrm_cmd_resp_dma_addr = - rte_malloc_virt2phy(bp->hwrm_cmd_resp_addr); + rte_mem_virt2phy(bp->hwrm_cmd_resp_addr); + if (bp->hwrm_cmd_resp_dma_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map response address to physical memory\n"); + return -ENOMEM; + } rte_spinlock_init(&bp->hwrm_lock); return 0; @@ -1201,13 +1713,25 @@ int bnxt_set_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic) int rc = 0; STAILQ_FOREACH(filter, &vnic->filter, next) { - rc = bnxt_hwrm_set_filter(bp, vnic, filter); + rc = bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id, filter); if (rc) break; } return rc; } +void bnxt_free_tunnel_ports(struct bnxt *bp) +{ + if (bp->vxlan_port_cnt) + bnxt_hwrm_tunnel_dst_port_free(bp, bp->vxlan_fw_dst_port_id, + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN); + bp->vxlan_port = 0; + if (bp->geneve_port_cnt) + bnxt_hwrm_tunnel_dst_port_free(bp, bp->geneve_fw_dst_port_id, + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE); + bp->geneve_port = 0; +} + void bnxt_free_all_hwrm_resources(struct bnxt *bp) { struct bnxt_vnic_info *vnic; @@ -1217,7 +1741,8 @@ void bnxt_free_all_hwrm_resources(struct bnxt *bp) return; vnic = &bp->vnic_info[0]; - bnxt_hwrm_cfa_l2_clear_rx_mask(bp, vnic); + if (BNXT_PF(bp)) + bnxt_hwrm_cfa_l2_clear_rx_mask(bp, vnic); /* VNIC resources */ for (i = 0; i < bp->nr_vnics; i++) { @@ -1226,12 +1751,16 @@ void bnxt_free_all_hwrm_resources(struct bnxt *bp) bnxt_clear_hwrm_vnic_filters(bp, vnic); bnxt_hwrm_vnic_ctx_free(bp, vnic); + + bnxt_hwrm_vnic_tpa_cfg(bp, vnic, false); + bnxt_hwrm_vnic_free(bp, vnic); } /* Ring resources */ bnxt_free_all_hwrm_rings(bp); bnxt_free_all_hwrm_ring_grps(bp); bnxt_free_all_hwrm_stat_ctxs(bp); + bnxt_free_tunnel_ports(bp); } static uint16_t bnxt_parse_eth_link_duplex(uint32_t conf_link_speed) @@ -1337,12 +1866,16 @@ static int bnxt_valid_link_speed(uint32_t link_speed, uint8_t port_id) return 0; } -static uint16_t bnxt_parse_eth_link_speed_mask(uint32_t link_speed) +static uint16_t +bnxt_parse_eth_link_speed_mask(struct bnxt *bp, uint32_t link_speed) { uint16_t ret = 0; - if (link_speed == ETH_LINK_SPEED_AUTONEG) + if (link_speed == ETH_LINK_SPEED_AUTONEG) { + if (bp->link_info.support_speeds) + return bp->link_info.support_speeds; link_speed = BNXT_SUPPORTED_SPEEDS; + } if (link_speed & ETH_LINK_SPEED_100M) ret |= HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MB; @@ -1434,7 +1967,7 @@ int bnxt_get_hwrm_link_config(struct bnxt *bp, struct rte_eth_link *link) "Get link config failed with rc %d\n", rc); goto exit; } - if (link_info->link_up) + if (link_info->link_speed) link->link_speed = bnxt_parse_hw_link_speed(link_info->link_speed); else @@ -1443,7 +1976,7 @@ int bnxt_get_hwrm_link_config(struct bnxt *bp, struct rte_eth_link *link) link->link_status = link_info->link_up; link->link_autoneg = link_info->auto_mode == HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE ? - ETH_LINK_SPEED_FIXED : ETH_LINK_SPEED_AUTONEG; + ETH_LINK_FIXED : ETH_LINK_AUTONEG; exit: return rc; } @@ -1476,7 +2009,8 @@ int bnxt_set_hwrm_link_config(struct bnxt *bp, bool link_up) link_req.auto_mode = HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_SPEED_MASK; link_req.auto_link_speed_mask = - bnxt_parse_eth_link_speed_mask(dev_conf->link_speeds); + bnxt_parse_eth_link_speed_mask(bp, + dev_conf->link_speeds); } else { link_req.phy_flags |= HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE; link_req.link_speed = speed; @@ -1493,7 +2027,6 @@ port_phy_cfg: "Set link config failed with rc %d\n", rc); } - rte_delay_ms(BNXT_LINK_WAIT_INTERVAL); error: return rc; } @@ -1512,12 +2045,8 @@ int bnxt_hwrm_func_qcfg(struct bnxt *bp) HWRM_CHECK_RESULT; - if (BNXT_VF(bp)) { - struct bnxt_vf_info *vf = &bp->vf; - - /* Hard Coded.. 0xfff VLAN ID mask */ - vf->vlan = rte_le_to_cpu_16(resp->vlan) & 0xfff; - } + /* Hard Coded.. 0xfff VLAN ID mask */ + bp->vlan = rte_le_to_cpu_16(resp->vlan) & 0xfff; switch (resp->port_partition_type) { case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0: @@ -1532,3 +2061,946 @@ int bnxt_hwrm_func_qcfg(struct bnxt *bp) return rc; } + +static void copy_func_cfg_to_qcaps(struct hwrm_func_cfg_input *fcfg, + struct hwrm_func_qcaps_output *qcaps) +{ + qcaps->max_rsscos_ctx = fcfg->num_rsscos_ctxs; + memcpy(qcaps->mac_address, fcfg->dflt_mac_addr, + sizeof(qcaps->mac_address)); + qcaps->max_l2_ctxs = fcfg->num_l2_ctxs; + qcaps->max_rx_rings = fcfg->num_rx_rings; + qcaps->max_tx_rings = fcfg->num_tx_rings; + qcaps->max_cmpl_rings = fcfg->num_cmpl_rings; + qcaps->max_stat_ctx = fcfg->num_stat_ctxs; + qcaps->max_vfs = 0; + qcaps->first_vf_id = 0; + qcaps->max_vnics = fcfg->num_vnics; + qcaps->max_decap_records = 0; + qcaps->max_encap_records = 0; + qcaps->max_tx_wm_flows = 0; + qcaps->max_tx_em_flows = 0; + qcaps->max_rx_wm_flows = 0; + qcaps->max_rx_em_flows = 0; + qcaps->max_flow_id = 0; + qcaps->max_mcast_filters = fcfg->num_mcast_filters; + qcaps->max_sp_tx_rings = 0; + qcaps->max_hw_ring_grps = fcfg->num_hw_ring_grps; +} + +static int bnxt_hwrm_pf_func_cfg(struct bnxt *bp, int tx_rings) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + req.enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_MTU | + HWRM_FUNC_CFG_INPUT_ENABLES_MRU | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_STAT_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_CMPL_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_TX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_L2_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_VNICS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS); + req.flags = rte_cpu_to_le_32(bp->pf.func_cfg_flags); + req.mtu = rte_cpu_to_le_16(BNXT_MAX_MTU); + req.mru = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE); + req.num_rsscos_ctxs = rte_cpu_to_le_16(bp->max_rsscos_ctx); + req.num_stat_ctxs = rte_cpu_to_le_16(bp->max_stat_ctx); + req.num_cmpl_rings = rte_cpu_to_le_16(bp->max_cp_rings); + req.num_tx_rings = rte_cpu_to_le_16(tx_rings); + req.num_rx_rings = rte_cpu_to_le_16(bp->max_rx_rings); + req.num_l2_ctxs = rte_cpu_to_le_16(bp->max_l2_ctx); + req.num_vnics = rte_cpu_to_le_16(bp->max_vnics); + req.num_hw_ring_grps = rte_cpu_to_le_16(bp->max_ring_grps); + req.fid = rte_cpu_to_le_16(0xffff); + + HWRM_PREP(req, FUNC_CFG, -1, resp); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +static void populate_vf_func_cfg_req(struct bnxt *bp, + struct hwrm_func_cfg_input *req, + int num_vfs) +{ + req->enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_MTU | + HWRM_FUNC_CFG_INPUT_ENABLES_MRU | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_STAT_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_CMPL_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_TX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_L2_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_VNICS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS); + + req->mtu = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE); + req->mru = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE); + req->num_rsscos_ctxs = rte_cpu_to_le_16(bp->max_rsscos_ctx / + (num_vfs + 1)); + req->num_stat_ctxs = rte_cpu_to_le_16(bp->max_stat_ctx / (num_vfs + 1)); + req->num_cmpl_rings = rte_cpu_to_le_16(bp->max_cp_rings / + (num_vfs + 1)); + req->num_tx_rings = rte_cpu_to_le_16(bp->max_tx_rings / (num_vfs + 1)); + req->num_rx_rings = rte_cpu_to_le_16(bp->max_rx_rings / (num_vfs + 1)); + req->num_l2_ctxs = rte_cpu_to_le_16(bp->max_l2_ctx / (num_vfs + 1)); + /* TODO: For now, do not support VMDq/RFS on VFs. */ + req->num_vnics = rte_cpu_to_le_16(1); + req->num_hw_ring_grps = rte_cpu_to_le_16(bp->max_ring_grps / + (num_vfs + 1)); +} + +static void add_random_mac_if_needed(struct bnxt *bp, + struct hwrm_func_cfg_input *cfg_req, + int vf) +{ + struct ether_addr mac; + + if (bnxt_hwrm_func_qcfg_vf_default_mac(bp, vf, &mac)) + return; + + if (memcmp(mac.addr_bytes, "\x00\x00\x00\x00\x00", 6) == 0) { + cfg_req->enables |= + rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + eth_random_addr(cfg_req->dflt_mac_addr); + bp->pf.vf_info[vf].random_mac = true; + } else { + memcpy(cfg_req->dflt_mac_addr, mac.addr_bytes, ETHER_ADDR_LEN); + } +} + +static void reserve_resources_from_vf(struct bnxt *bp, + struct hwrm_func_cfg_input *cfg_req, + int vf) +{ + struct hwrm_func_qcaps_input req = {0}; + struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + /* Get the actual allocated values now */ + HWRM_PREP(req, FUNC_QCAPS, -1, resp); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + if (rc) { + RTE_LOG(ERR, PMD, "hwrm_func_qcaps failed rc:%d\n", rc); + copy_func_cfg_to_qcaps(cfg_req, resp); + } else if (resp->error_code) { + rc = rte_le_to_cpu_16(resp->error_code); + RTE_LOG(ERR, PMD, "hwrm_func_qcaps error %d\n", rc); + copy_func_cfg_to_qcaps(cfg_req, resp); + } + + bp->max_rsscos_ctx -= rte_le_to_cpu_16(resp->max_rsscos_ctx); + bp->max_stat_ctx -= rte_le_to_cpu_16(resp->max_stat_ctx); + bp->max_cp_rings -= rte_le_to_cpu_16(resp->max_cmpl_rings); + bp->max_tx_rings -= rte_le_to_cpu_16(resp->max_tx_rings); + bp->max_rx_rings -= rte_le_to_cpu_16(resp->max_rx_rings); + bp->max_l2_ctx -= rte_le_to_cpu_16(resp->max_l2_ctxs); + /* + * TODO: While not supporting VMDq with VFs, max_vnics is always + * forced to 1 in this case + */ + //bp->max_vnics -= rte_le_to_cpu_16(esp->max_vnics); + bp->max_ring_grps -= rte_le_to_cpu_16(resp->max_hw_ring_grps); +} + +int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf) +{ + struct hwrm_func_qcfg_input req = {0}; + struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + /* Check for zero MAC address */ + HWRM_PREP(req, FUNC_QCFG, -1, resp); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + if (rc) { + RTE_LOG(ERR, PMD, "hwrm_func_qcfg failed rc:%d\n", rc); + return -1; + } else if (resp->error_code) { + rc = rte_le_to_cpu_16(resp->error_code); + RTE_LOG(ERR, PMD, "hwrm_func_qcfg error %d\n", rc); + return -1; + } + return rte_le_to_cpu_16(resp->vlan); +} + +static int update_pf_resource_max(struct bnxt *bp) +{ + struct hwrm_func_qcfg_input req = {0}; + struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + /* And copy the allocated numbers into the pf struct */ + HWRM_PREP(req, FUNC_QCFG, -1, resp); + req.fid = rte_cpu_to_le_16(0xffff); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + /* Only TX ring value reflects actual allocation? TODO */ + bp->max_tx_rings = rte_le_to_cpu_16(resp->alloc_tx_rings); + bp->pf.evb_mode = resp->evb_mode; + + return rc; +} + +int bnxt_hwrm_allocate_pf_only(struct bnxt *bp) +{ + int rc; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, "Attempt to allcoate VFs on a VF!\n"); + return -1; + } + + rc = bnxt_hwrm_func_qcaps(bp); + if (rc) + return rc; + + bp->pf.func_cfg_flags &= + ~(HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE | + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE); + bp->pf.func_cfg_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE; + rc = bnxt_hwrm_pf_func_cfg(bp, bp->max_tx_rings); + return rc; +} + +int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int i; + size_t sz; + int rc = 0; + size_t req_buf_sz; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, "Attempt to allcoate VFs on a VF!\n"); + return -1; + } + + rc = bnxt_hwrm_func_qcaps(bp); + + if (rc) + return rc; + + bp->pf.active_vfs = num_vfs; + + /* + * First, configure the PF to only use one TX ring. This ensures that + * there are enough rings for all VFs. + * + * If we don't do this, when we call func_alloc() later, we will lock + * extra rings to the PF that won't be available during func_cfg() of + * the VFs. + * + * This has been fixed with firmware versions above 20.6.54 + */ + bp->pf.func_cfg_flags &= + ~(HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE | + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE); + bp->pf.func_cfg_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE; + rc = bnxt_hwrm_pf_func_cfg(bp, 1); + if (rc) + return rc; + + /* + * Now, create and register a buffer to hold forwarded VF requests + */ + req_buf_sz = num_vfs * HWRM_MAX_REQ_LEN; + bp->pf.vf_req_buf = rte_malloc("bnxt_vf_fwd", req_buf_sz, + page_roundup(num_vfs * HWRM_MAX_REQ_LEN)); + if (bp->pf.vf_req_buf == NULL) { + rc = -ENOMEM; + goto error_free; + } + for (sz = 0; sz < req_buf_sz; sz += getpagesize()) + rte_mem_lock_page(((char *)bp->pf.vf_req_buf) + sz); + for (i = 0; i < num_vfs; i++) + bp->pf.vf_info[i].req_buf = ((char *)bp->pf.vf_req_buf) + + (i * HWRM_MAX_REQ_LEN); + + rc = bnxt_hwrm_func_buf_rgtr(bp); + if (rc) + goto error_free; + + populate_vf_func_cfg_req(bp, &req, num_vfs); + + bp->pf.active_vfs = 0; + for (i = 0; i < num_vfs; i++) { + add_random_mac_if_needed(bp, &req, i); + + HWRM_PREP(req, FUNC_CFG, -1, resp); + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[i].func_cfg_flags); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[i].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + /* Clear enable flag for next pass */ + req.enables &= ~rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + + if (rc || resp->error_code) { + RTE_LOG(ERR, PMD, + "Failed to initizlie VF %d\n", i); + RTE_LOG(ERR, PMD, + "Not all VFs available. (%d, %d)\n", + rc, resp->error_code); + break; + } + + reserve_resources_from_vf(bp, &req, i); + bp->pf.active_vfs++; + bnxt_hwrm_func_clr_stats(bp, bp->pf.vf_info[i].fid); + } + + /* + * Now configure the PF to use "the rest" of the resources + * We're using STD_TX_RING_MODE here though which will limit the TX + * rings. This will allow QoS to function properly. Not setting this + * will cause PF rings to break bandwidth settings. + */ + rc = bnxt_hwrm_pf_func_cfg(bp, bp->max_tx_rings); + if (rc) + goto error_free; + + rc = update_pf_resource_max(bp); + if (rc) + goto error_free; + + return rc; + +error_free: + bnxt_hwrm_func_buf_unrgtr(bp); + return rc; +} + +int bnxt_hwrm_pf_evb_mode(struct bnxt *bp) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + HWRM_PREP(req, FUNC_CFG, -1, resp); + + req.fid = rte_cpu_to_le_16(0xffff); + req.enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_EVB_MODE); + req.evb_mode = bp->pf.evb_mode; + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type) +{ + struct hwrm_tunnel_dst_port_alloc_input req = {0}; + struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; + int rc = 0; + + HWRM_PREP(req, TUNNEL_DST_PORT_ALLOC, -1, resp); + req.tunnel_type = tunnel_type; + req.tunnel_dst_port_val = port; + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + switch (tunnel_type) { + case HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN: + bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id; + bp->vxlan_port = port; + break; + case HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE: + bp->geneve_fw_dst_port_id = resp->tunnel_dst_port_id; + bp->geneve_port = port; + break; + default: + break; + } + return rc; +} + +int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type) +{ + struct hwrm_tunnel_dst_port_free_input req = {0}; + struct hwrm_tunnel_dst_port_free_output *resp = bp->hwrm_cmd_resp_addr; + int rc = 0; + + HWRM_PREP(req, TUNNEL_DST_PORT_FREE, -1, resp); + req.tunnel_type = tunnel_type; + req.tunnel_dst_port_id = rte_cpu_to_be_16(port); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_func_cfg_vf_set_flags(struct bnxt *bp, uint16_t vf, + uint32_t flags) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG, -1, resp); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.flags = rte_cpu_to_le_32(flags); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +void vf_vnic_set_rxmask_cb(struct bnxt_vnic_info *vnic, void *flagp) +{ + uint32_t *flag = flagp; + + vnic->flags = *flag; +} + +int bnxt_set_rx_mask_no_vlan(struct bnxt *bp, struct bnxt_vnic_info *vnic) +{ + return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); +} + +int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp) +{ + int rc = 0; + struct hwrm_func_buf_rgtr_input req = {.req_type = 0 }; + struct hwrm_func_buf_rgtr_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_BUF_RGTR, -1, resp); + + req.req_buf_num_pages = rte_cpu_to_le_16(1); + req.req_buf_page_size = rte_cpu_to_le_16( + page_getenum(bp->pf.active_vfs * HWRM_MAX_REQ_LEN)); + req.req_buf_len = rte_cpu_to_le_16(HWRM_MAX_REQ_LEN); + req.req_buf_page_addr[0] = + rte_cpu_to_le_64(rte_mem_virt2phy(bp->pf.vf_req_buf)); + if (req.req_buf_page_addr[0] == 0) { + RTE_LOG(ERR, PMD, + "unable to map buffer address to physical memory\n"); + return -ENOMEM; + } + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_func_buf_unrgtr(struct bnxt *bp) +{ + int rc = 0; + struct hwrm_func_buf_unrgtr_input req = {.req_type = 0 }; + struct hwrm_func_buf_unrgtr_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_BUF_UNRGTR, -1, resp); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG, -1, resp); + req.fid = rte_cpu_to_le_16(0xffff); + req.flags = rte_cpu_to_le_32(bp->pf.func_cfg_flags); + req.enables = rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR); + req.async_event_cr = rte_cpu_to_le_16( + bp->def_cp_ring->cp_ring_struct->fw_ring_id); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_vf_func_cfg_def_cp(struct bnxt *bp) +{ + struct hwrm_func_vf_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_vf_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_VF_CFG, -1, resp); + req.enables = rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR); + req.async_event_cr = rte_cpu_to_le_16( + bp->def_cp_ring->cp_ring_struct->fw_ring_id); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + uint16_t dflt_vlan, fid; + uint32_t func_cfg_flags; + int rc = 0; + + HWRM_PREP(req, FUNC_CFG, -1, resp); + + if (is_vf) { + dflt_vlan = bp->pf.vf_info[vf].dflt_vlan; + fid = bp->pf.vf_info[vf].fid; + func_cfg_flags = bp->pf.vf_info[vf].func_cfg_flags; + } else { + fid = rte_cpu_to_le_16(0xffff); + func_cfg_flags = bp->pf.func_cfg_flags; + dflt_vlan = bp->vlan; + } + + req.flags = rte_cpu_to_le_32(func_cfg_flags); + req.fid = rte_cpu_to_le_16(fid); + req.enables |= rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN); + req.dflt_vlan = rte_cpu_to_le_16(dflt_vlan); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_func_bw_cfg(struct bnxt *bp, uint16_t vf, + uint16_t max_bw, uint16_t enables) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG, -1, resp); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.enables |= rte_cpu_to_le_32(enables); + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags); + req.max_bw = rte_cpu_to_le_32(max_bw); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_set_vf_vlan(struct bnxt *bp, int vf) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc = 0; + + HWRM_PREP(req, FUNC_CFG, -1, resp); + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.enables |= rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN); + req.dflt_vlan = rte_cpu_to_le_16(bp->pf.vf_info[vf].dflt_vlan); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_reject_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size) +{ + int rc = 0; + struct hwrm_reject_fwd_resp_input req = {.req_type = 0}; + struct hwrm_reject_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr; + + if (ec_size > sizeof(req.encap_request)) + return -1; + + HWRM_PREP(req, REJECT_FWD_RESP, -1, resp); + + req.encap_resp_target_id = rte_cpu_to_le_16(target_id); + memcpy(req.encap_request, encaped, ec_size); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_func_qcfg_vf_default_mac(struct bnxt *bp, uint16_t vf, + struct ether_addr *mac) +{ + struct hwrm_func_qcfg_input req = {0}; + struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + HWRM_PREP(req, FUNC_QCFG, -1, resp); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + memcpy(mac->addr_bytes, resp->mac_address, ETHER_ADDR_LEN); + return rc; +} + +int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size) +{ + int rc = 0; + struct hwrm_exec_fwd_resp_input req = {.req_type = 0}; + struct hwrm_exec_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr; + + if (ec_size > sizeof(req.encap_request)) + return -1; + + HWRM_PREP(req, EXEC_FWD_RESP, -1, resp); + + req.encap_resp_target_id = rte_cpu_to_le_16(target_id); + memcpy(req.encap_request, encaped, ec_size); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_ctx_qstats(struct bnxt *bp, uint32_t cid, int idx, + struct rte_eth_stats *stats) +{ + int rc = 0; + struct hwrm_stat_ctx_query_input req = {.req_type = 0}; + struct hwrm_stat_ctx_query_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, STAT_CTX_QUERY, -1, resp); + + req.stat_ctx_id = rte_cpu_to_le_32(cid); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT; + + stats->q_ipackets[idx] = rte_le_to_cpu_64(resp->rx_ucast_pkts); + stats->q_ipackets[idx] += rte_le_to_cpu_64(resp->rx_mcast_pkts); + stats->q_ipackets[idx] += rte_le_to_cpu_64(resp->rx_bcast_pkts); + stats->q_ibytes[idx] = rte_le_to_cpu_64(resp->rx_ucast_bytes); + stats->q_ibytes[idx] += rte_le_to_cpu_64(resp->rx_mcast_bytes); + stats->q_ibytes[idx] += rte_le_to_cpu_64(resp->rx_bcast_bytes); + + stats->q_opackets[idx] = rte_le_to_cpu_64(resp->tx_ucast_pkts); + stats->q_opackets[idx] += rte_le_to_cpu_64(resp->tx_mcast_pkts); + stats->q_opackets[idx] += rte_le_to_cpu_64(resp->tx_bcast_pkts); + stats->q_obytes[idx] = rte_le_to_cpu_64(resp->tx_ucast_bytes); + stats->q_obytes[idx] += rte_le_to_cpu_64(resp->tx_mcast_bytes); + stats->q_obytes[idx] += rte_le_to_cpu_64(resp->tx_bcast_bytes); + + stats->q_errors[idx] = rte_le_to_cpu_64(resp->rx_err_pkts); + stats->q_errors[idx] += rte_le_to_cpu_64(resp->tx_err_pkts); + stats->q_errors[idx] += rte_le_to_cpu_64(resp->rx_drop_pkts); + + return rc; +} + +int bnxt_hwrm_port_qstats(struct bnxt *bp) +{ + struct hwrm_port_qstats_input req = {0}; + struct hwrm_port_qstats_output *resp = bp->hwrm_cmd_resp_addr; + struct bnxt_pf_info *pf = &bp->pf; + int rc; + + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) + return 0; + + HWRM_PREP(req, PORT_QSTATS, -1, resp); + req.port_id = rte_cpu_to_le_16(pf->port_id); + req.tx_stat_host_addr = rte_cpu_to_le_64(bp->hw_tx_port_stats_map); + req.rx_stat_host_addr = rte_cpu_to_le_64(bp->hw_rx_port_stats_map); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + return rc; +} + +int bnxt_hwrm_port_clr_stats(struct bnxt *bp) +{ + struct hwrm_port_clr_stats_input req = {0}; + struct hwrm_port_clr_stats_output *resp = bp->hwrm_cmd_resp_addr; + struct bnxt_pf_info *pf = &bp->pf; + int rc; + + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) + return 0; + + HWRM_PREP(req, PORT_CLR_STATS, -1, resp); + req.port_id = rte_cpu_to_le_16(pf->port_id); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + return rc; +} + +int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) +{ + struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_port_led_qcaps_input req = {0}; + int rc; + + if (BNXT_VF(bp)) + return 0; + + HWRM_PREP(req, PORT_LED_QCAPS, -1, resp); + req.port_id = bp->pf.port_id; + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { + unsigned int i; + + bp->num_leds = resp->num_leds; + memcpy(bp->leds, &resp->led0_id, + sizeof(bp->leds[0]) * bp->num_leds); + for (i = 0; i < bp->num_leds; i++) { + struct bnxt_led_info *led = &bp->leds[i]; + + uint16_t caps = led->led_state_caps; + + if (!led->led_group_id || + !BNXT_LED_ALT_BLINK_CAP(caps)) { + bp->num_leds = 0; + break; + } + } + } + return rc; +} + +int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on) +{ + struct hwrm_port_led_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_port_led_cfg_input req = {0}; + struct bnxt_led_cfg *led_cfg; + uint8_t led_state = HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_DEFAULT; + uint16_t duration = 0; + int rc, i; + + if (!bp->num_leds || BNXT_VF(bp)) + return -EOPNOTSUPP; + + HWRM_PREP(req, PORT_LED_CFG, -1, resp); + if (led_on) { + led_state = HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINKALT; + duration = rte_cpu_to_le_16(500); + } + req.port_id = bp->pf.port_id; + req.num_leds = bp->num_leds; + led_cfg = (struct bnxt_led_cfg *)&req.led0_id; + for (i = 0; i < bp->num_leds; i++, led_cfg++) { + req.enables |= BNXT_LED_DFLT_ENABLES(i); + led_cfg->led_id = bp->leds[i].led_id; + led_cfg->led_state = led_state; + led_cfg->led_blink_on = duration; + led_cfg->led_blink_off = duration; + led_cfg->led_group_id = bp->leds[i].led_group_id; + } + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +static void +bnxt_vnic_count(struct bnxt_vnic_info *vnic __rte_unused, void *cbdata) +{ + uint32_t *count = cbdata; + + *count = *count + 1; +} + +static int bnxt_vnic_count_hwrm_stub(struct bnxt *bp __rte_unused, + struct bnxt_vnic_info *vnic __rte_unused) +{ + return 0; +} + +int bnxt_vf_vnic_count(struct bnxt *bp, uint16_t vf) +{ + uint32_t count = 0; + + bnxt_hwrm_func_vf_vnic_query_and_config(bp, vf, bnxt_vnic_count, + &count, bnxt_vnic_count_hwrm_stub); + + return count; +} + +static int bnxt_hwrm_func_vf_vnic_query(struct bnxt *bp, uint16_t vf, + uint16_t *vnic_ids) +{ + struct hwrm_func_vf_vnic_ids_query_input req = {0}; + struct hwrm_func_vf_vnic_ids_query_output *resp = + bp->hwrm_cmd_resp_addr; + int rc; + + /* First query all VNIC ids */ + HWRM_PREP(req, FUNC_VF_VNIC_IDS_QUERY, -1, resp_vf_vnic_ids); + + req.vf_id = rte_cpu_to_le_16(bp->pf.first_vf_id + vf); + req.max_vnic_id_cnt = rte_cpu_to_le_32(bp->pf.total_vnics); + req.vnic_id_tbl_addr = rte_cpu_to_le_64(rte_mem_virt2phy(vnic_ids)); + + if (req.vnic_id_tbl_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map VNIC ID table address to physical memory\n"); + return -ENOMEM; + } + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + if (rc) { + RTE_LOG(ERR, PMD, "hwrm_func_vf_vnic_query failed rc:%d\n", rc); + return -1; + } else if (resp->error_code) { + rc = rte_le_to_cpu_16(resp->error_code); + RTE_LOG(ERR, PMD, "hwrm_func_vf_vnic_query error %d\n", rc); + return -1; + } + + return rte_le_to_cpu_32(resp->vnic_id_cnt); +} + +/* + * This function queries the VNIC IDs for a specified VF. It then calls + * the vnic_cb to update the necessary field in vnic_info with cbdata. + * Then it calls the hwrm_cb function to program this new vnic configuration. + */ +int bnxt_hwrm_func_vf_vnic_query_and_config(struct bnxt *bp, uint16_t vf, + void (*vnic_cb)(struct bnxt_vnic_info *, void *), void *cbdata, + int (*hwrm_cb)(struct bnxt *bp, struct bnxt_vnic_info *vnic)) +{ + struct bnxt_vnic_info vnic; + int rc = 0; + int i, num_vnic_ids; + uint16_t *vnic_ids; + size_t vnic_id_sz; + size_t sz; + + /* First query all VNIC ids */ + vnic_id_sz = bp->pf.total_vnics * sizeof(*vnic_ids); + vnic_ids = rte_malloc("bnxt_hwrm_vf_vnic_ids_query", vnic_id_sz, + RTE_CACHE_LINE_SIZE); + if (vnic_ids == NULL) { + rc = -ENOMEM; + return rc; + } + for (sz = 0; sz < vnic_id_sz; sz += getpagesize()) + rte_mem_lock_page(((char *)vnic_ids) + sz); + + num_vnic_ids = bnxt_hwrm_func_vf_vnic_query(bp, vf, vnic_ids); + + if (num_vnic_ids < 0) + return num_vnic_ids; + + /* Retrieve VNIC, update bd_stall then update */ + + for (i = 0; i < num_vnic_ids; i++) { + memset(&vnic, 0, sizeof(struct bnxt_vnic_info)); + vnic.fw_vnic_id = rte_le_to_cpu_16(vnic_ids[i]); + rc = bnxt_hwrm_vnic_qcfg(bp, &vnic, bp->pf.first_vf_id + vf); + if (rc) + break; + if (vnic.mru <= 4) /* Indicates unallocated */ + continue; + + vnic_cb(&vnic, cbdata); + + rc = hwrm_cb(bp, &vnic); + if (rc) + break; + } + + rte_free(vnic_ids); + + return rc; +} + +int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf, + bool on) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG, -1, resp); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.enables |= rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_VLAN_ANTISPOOF_MODE); + req.vlan_antispoof_mode = on ? + HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN : + HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_NOCHECK; + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT; + + return rc; +} + +int bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(struct bnxt *bp, int vf) +{ + struct bnxt_vnic_info vnic; + uint16_t *vnic_ids; + size_t vnic_id_sz; + int num_vnic_ids, i; + size_t sz; + int rc; + + vnic_id_sz = bp->pf.total_vnics * sizeof(*vnic_ids); + vnic_ids = rte_malloc("bnxt_hwrm_vf_vnic_ids_query", vnic_id_sz, + RTE_CACHE_LINE_SIZE); + if (vnic_ids == NULL) { + rc = -ENOMEM; + return rc; + } + + for (sz = 0; sz < vnic_id_sz; sz += getpagesize()) + rte_mem_lock_page(((char *)vnic_ids) + sz); + + rc = bnxt_hwrm_func_vf_vnic_query(bp, vf, vnic_ids); + if (rc <= 0) + goto exit; + num_vnic_ids = rc; + + /* + * Loop through to find the default VNIC ID. + * TODO: The easier way would be to obtain the resp->dflt_vnic_id + * by sending the hwrm_func_qcfg command to the firmware. + */ + for (i = 0; i < num_vnic_ids; i++) { + memset(&vnic, 0, sizeof(struct bnxt_vnic_info)); + vnic.fw_vnic_id = rte_le_to_cpu_16(vnic_ids[i]); + rc = bnxt_hwrm_vnic_qcfg(bp, &vnic, + bp->pf.first_vf_id + vf); + if (rc) + goto exit; + if (vnic.func_default) { + rte_free(vnic_ids); + return vnic.fw_vnic_id; + } + } + /* Could not find a default VNIC. */ + RTE_LOG(ERR, PMD, "No default VNIC\n"); +exit: + rte_free(vnic_ids); + return -1; +} diff --git a/drivers/net/bnxt/bnxt_hwrm.h b/drivers/net/bnxt/bnxt_hwrm.h index 6519ef21..51cd0dd4 100644 --- a/drivers/net/bnxt/bnxt_hwrm.h +++ b/drivers/net/bnxt/bnxt_hwrm.h @@ -45,27 +45,42 @@ struct bnxt_cp_ring_info; int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic); -int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic); +int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic, + uint16_t vlan_count, + struct bnxt_vlan_table_entry *vlan_table); +int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid, + uint16_t vlan_count, + struct bnxt_vlan_antispoof_table_entry *vlan_table); int bnxt_hwrm_clear_filter(struct bnxt *bp, struct bnxt_filter_info *filter); int bnxt_hwrm_set_filter(struct bnxt *bp, - struct bnxt_vnic_info *vnic, + uint16_t dst_id, struct bnxt_filter_info *filter); +int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size); +int bnxt_hwrm_reject_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size); -int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, void *fwd_cmd); - -int bnxt_hwrm_func_driver_register(struct bnxt *bp, uint32_t flags, - uint32_t *vf_req_fwd); +int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp); +int bnxt_hwrm_func_buf_unrgtr(struct bnxt *bp); +int bnxt_hwrm_func_driver_register(struct bnxt *bp); int bnxt_hwrm_func_qcaps(struct bnxt *bp); int bnxt_hwrm_func_reset(struct bnxt *bp); int bnxt_hwrm_func_driver_unregister(struct bnxt *bp, uint32_t flags); +int bnxt_hwrm_func_qstats(struct bnxt *bp, uint16_t fid, + struct rte_eth_stats *stats); +int bnxt_hwrm_func_qstats_tx_drop(struct bnxt *bp, uint16_t fid, + uint64_t *dropped); +int bnxt_hwrm_func_clr_stats(struct bnxt *bp, uint16_t fid); +int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp); +int bnxt_hwrm_vf_func_cfg_def_cp(struct bnxt *bp); int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); int bnxt_hwrm_ring_alloc(struct bnxt *bp, struct bnxt_ring *ring, uint32_t ring_type, uint32_t map_index, - uint32_t stats_ctx_id); + uint32_t stats_ctx_id, uint32_t cmpl_ring_id); int bnxt_hwrm_ring_free(struct bnxt *bp, struct bnxt_ring *ring, uint32_t ring_type); int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp, unsigned int idx); @@ -76,16 +91,24 @@ int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, unsigned int idx); int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, unsigned int idx); +int bnxt_hwrm_ctx_qstats(struct bnxt *bp, uint32_t cid, int idx, + struct rte_eth_stats *stats); int bnxt_hwrm_ver_get(struct bnxt *bp); int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic); +int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic, + int16_t fw_vf_id); int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_ctx_free(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_free(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic); +int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic); +int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, bool enable); int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp); int bnxt_clear_all_hwrm_stat_ctxs(struct bnxt *bp); @@ -101,5 +124,36 @@ int bnxt_alloc_hwrm_resources(struct bnxt *bp); int bnxt_get_hwrm_link_config(struct bnxt *bp, struct rte_eth_link *link); int bnxt_set_hwrm_link_config(struct bnxt *bp, bool link_up); int bnxt_hwrm_func_qcfg(struct bnxt *bp); - +int bnxt_hwrm_allocate_pf_only(struct bnxt *bp); +int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs); +int bnxt_hwrm_func_vf_mac(struct bnxt *bp, uint16_t vf, + const uint8_t *mac_addr); +int bnxt_hwrm_pf_evb_mode(struct bnxt *bp); +int bnxt_hwrm_func_bw_cfg(struct bnxt *bp, uint16_t vf, + uint16_t max_bw, uint16_t enables); +int bnxt_hwrm_set_vf_vlan(struct bnxt *bp, int vf); +int bnxt_hwrm_func_qcfg_vf_default_mac(struct bnxt *bp, uint16_t vf, + struct ether_addr *mac); +int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf); +int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type); +int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type); +void bnxt_free_tunnel_ports(struct bnxt *bp); +int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf); +int bnxt_hwrm_port_qstats(struct bnxt *bp); +int bnxt_hwrm_port_clr_stats(struct bnxt *bp); +int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on); +int bnxt_hwrm_port_led_qcaps(struct bnxt *bp); +int bnxt_hwrm_func_cfg_vf_set_flags(struct bnxt *bp, uint16_t vf, + uint32_t flags); +void vf_vnic_set_rxmask_cb(struct bnxt_vnic_info *vnic, void *flagp); +int bnxt_set_rx_mask_no_vlan(struct bnxt *bp, struct bnxt_vnic_info *vnic); +int bnxt_vf_vnic_count(struct bnxt *bp, uint16_t vf); +int bnxt_hwrm_func_vf_vnic_query_and_config(struct bnxt *bp, uint16_t vf, + void (*vnic_cb)(struct bnxt_vnic_info *, void *), void *cbdata, + int (*hwrm_cb)(struct bnxt *bp, struct bnxt_vnic_info *vnic)); +int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf, + bool on); +int bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(struct bnxt *bp, int vf); #endif diff --git a/drivers/net/bnxt/bnxt_irq.c b/drivers/net/bnxt/bnxt_irq.c index 20e17ff5..47cda7e5 100644 --- a/drivers/net/bnxt/bnxt_irq.c +++ b/drivers/net/bnxt/bnxt_irq.c @@ -66,16 +66,26 @@ static void bnxt_int_handler(void *param) /* Handle any async event */ bnxt_handle_async_event(bp, cmp); break; - case CMPL_BASE_TYPE_HWRM_FWD_RESP: + case CMPL_BASE_TYPE_HWRM_FWD_REQ: /* Handle HWRM forwarded responses */ bnxt_handle_fwd_req(bp, cmp); break; default: /* Ignore any other events */ + if (cmp->type & rte_cpu_to_le_16(0x01)) { + if (!CMP_VALID(cmp, raw_cons, + cpr->cp_ring_struct)) + goto no_more; + } + RTE_LOG(INFO, PMD, + "Ignoring %02x completion\n", CMP_TYPE(cmp)); break; } raw_cons = NEXT_RAW_CMP(raw_cons); - } + + }; +no_more: + cpr->cp_raw_cons = raw_cons; B_CP_DB_REARM(cpr, cpr->cp_raw_cons); } @@ -102,14 +112,15 @@ void bnxt_disable_int(struct bnxt *bp) struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; /* Only the default completion ring */ - B_CP_DIS_DB(cpr, cpr->cp_raw_cons); + if (cpr != NULL && cpr->cp_doorbell != NULL) + B_CP_DB_DISARM(cpr); } void bnxt_enable_int(struct bnxt *bp) { struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; - B_CP_DB_REARM(cpr, cpr->cp_raw_cons); + B_CP_DB_ARM(cpr); } int bnxt_setup_int(struct bnxt *bp) @@ -126,7 +137,7 @@ int bnxt_setup_int(struct bnxt *bp) for (i = 0; i < total_vecs; i++) { bp->irq_tbl[i].vector = i; snprintf(bp->irq_tbl[i].name, len, - "%s-%d", bp->eth_dev->data->name, i); + "%s-%d", bp->eth_dev->device->name, i); bp->irq_tbl[i].handler = bnxt_int_handler; } } else { @@ -136,7 +147,7 @@ int bnxt_setup_int(struct bnxt *bp) return 0; setup_exit: - RTE_LOG(ERR, PMD, "bnxt_irq_tbl setup failed"); + RTE_LOG(ERR, PMD, "bnxt_irq_tbl setup failed\n"); return rc; } diff --git a/drivers/net/bnxt/bnxt_ring.c b/drivers/net/bnxt/bnxt_ring.c index 0fafa13f..9d0ae277 100644 --- a/drivers/net/bnxt/bnxt_ring.c +++ b/drivers/net/bnxt/bnxt_ring.c @@ -31,7 +31,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <rte_bitmap.h> #include <rte_memzone.h> +#include <unistd.h> #include "bnxt.h" #include "bnxt_cpr.h" @@ -96,6 +98,8 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, struct rte_pci_device *pdev = bp->pdev; const struct rte_memzone *mz = NULL; char mz_name[RTE_MEMZONE_NAMESIZE]; + phys_addr_t mz_phys_addr; + int sz; int stats_len = (tx_ring_info || rx_ring_info) ? RTE_CACHE_LINE_ROUNDUP(sizeof(struct ctx_hw_stats64)) : 0; @@ -112,8 +116,15 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, int rx_vmem_len = rx_ring_info ? RTE_CACHE_LINE_ROUNDUP(rx_ring_info-> rx_ring_struct->vmem_size) : 0; + int ag_vmem_start = 0; + int ag_vmem_len = 0; + int cp_ring_start = 0; + + ag_vmem_start = rx_vmem_start + rx_vmem_len; + ag_vmem_len = rx_ring_info ? RTE_CACHE_LINE_ROUNDUP( + rx_ring_info->ag_ring_struct->vmem_size) : 0; + cp_ring_start = ag_vmem_start + ag_vmem_len; - int cp_ring_start = rx_vmem_start + rx_vmem_len; int cp_ring_len = RTE_CACHE_LINE_ROUNDUP(cp_ring->ring_size * sizeof(struct cmpl_base)); @@ -127,7 +138,23 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, RTE_CACHE_LINE_ROUNDUP(rx_ring_info->rx_ring_struct->ring_size * sizeof(struct rx_prod_pkt_bd)) : 0; - int total_alloc_len = rx_ring_start + rx_ring_len; + int ag_ring_start = rx_ring_start + rx_ring_len; + int ag_ring_len = rx_ring_len * AGG_RING_SIZE_FACTOR; + + int ag_bitmap_start = ag_ring_start + ag_ring_len; + int ag_bitmap_len = rx_ring_info ? + RTE_CACHE_LINE_ROUNDUP(rte_bitmap_get_memory_footprint( + rx_ring_info->rx_ring_struct->ring_size * + AGG_RING_SIZE_FACTOR)) : 0; + + int tpa_info_start = ag_bitmap_start + ag_bitmap_len; + int tpa_info_len = rx_ring_info ? + RTE_CACHE_LINE_ROUNDUP(BNXT_TPA_MAX * + sizeof(struct bnxt_tpa_info)) : 0; + + int total_alloc_len = tpa_info_start; + if (bp->eth_dev->data->dev_conf.rxmode.enable_lro) + total_alloc_len += tpa_info_len; snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_%04x:%02x:%02x:%02x-%04x_%s", pdev->addr.domain, @@ -136,21 +163,37 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0; mz = rte_memzone_lookup(mz_name); if (!mz) { - mz = rte_memzone_reserve(mz_name, total_alloc_len, + mz = rte_memzone_reserve_aligned(mz_name, total_alloc_len, SOCKET_ID_ANY, RTE_MEMZONE_2MB | - RTE_MEMZONE_SIZE_HINT_ONLY); + RTE_MEMZONE_SIZE_HINT_ONLY, + getpagesize()); if (mz == NULL) return -ENOMEM; } memset(mz->addr, 0, mz->len); + mz_phys_addr = mz->phys_addr; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2phy()\n"); + for (sz = 0; sz < total_alloc_len; sz += getpagesize()) + rte_mem_lock_page(((char *)mz->addr) + sz); + mz_phys_addr = rte_mem_virt2phy(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map ring address to physical memory\n"); + return -ENOMEM; + } + } if (tx_ring_info) { tx_ring = tx_ring_info->tx_ring_struct; tx_ring->bd = ((char *)mz->addr + tx_ring_start); tx_ring_info->tx_desc_ring = (struct tx_bd_long *)tx_ring->bd; - tx_ring->bd_dma = mz->phys_addr + tx_ring_start; + tx_ring->bd_dma = mz_phys_addr + tx_ring_start; tx_ring_info->tx_desc_mapping = tx_ring->bd_dma; tx_ring->mem_zone = (const void *)mz; @@ -170,7 +213,7 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, rx_ring->bd = ((char *)mz->addr + rx_ring_start); rx_ring_info->rx_desc_ring = (struct rx_prod_pkt_bd *)rx_ring->bd; - rx_ring->bd_dma = mz->phys_addr + rx_ring_start; + rx_ring->bd_dma = mz_phys_addr + rx_ring_start; rx_ring_info->rx_desc_mapping = rx_ring->bd_dma; rx_ring->mem_zone = (const void *)mz; @@ -182,10 +225,39 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, rx_ring_info->rx_buf_ring = (struct bnxt_sw_rx_bd *)rx_ring->vmem; } + + rx_ring = rx_ring_info->ag_ring_struct; + + rx_ring->bd = ((char *)mz->addr + ag_ring_start); + rx_ring_info->ag_desc_ring = + (struct rx_prod_pkt_bd *)rx_ring->bd; + rx_ring->bd_dma = mz->phys_addr + ag_ring_start; + rx_ring_info->ag_desc_mapping = rx_ring->bd_dma; + rx_ring->mem_zone = (const void *)mz; + + if (!rx_ring->bd) + return -ENOMEM; + if (rx_ring->vmem_size) { + rx_ring->vmem = + (void **)((char *)mz->addr + ag_vmem_start); + rx_ring_info->ag_buf_ring = + (struct bnxt_sw_rx_bd *)rx_ring->vmem; + } + + rx_ring_info->ag_bitmap = + rte_bitmap_init(rx_ring_info->rx_ring_struct->ring_size * + AGG_RING_SIZE_FACTOR, (uint8_t *)mz->addr + + ag_bitmap_start, ag_bitmap_len); + + /* TPA info */ + if (bp->eth_dev->data->dev_conf.rxmode.enable_lro) + rx_ring_info->tpa_info = + ((struct bnxt_tpa_info *)((char *)mz->addr + + tpa_info_start)); } cp_ring->bd = ((char *)mz->addr + cp_ring_start); - cp_ring->bd_dma = mz->phys_addr + cp_ring_start; + cp_ring->bd_dma = mz_phys_addr + cp_ring_start; cp_ring_info->cp_desc_ring = cp_ring->bd; cp_ring_info->cp_desc_mapping = cp_ring->bd_dma; cp_ring->mem_zone = (const void *)mz; @@ -196,7 +268,7 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, *cp_ring->vmem = ((char *)mz->addr + stats_len); if (stats_len) { cp_ring_info->hw_stats = mz->addr; - cp_ring_info->hw_stats_map = mz->phys_addr; + cp_ring_info->hw_stats_map = mz_phys_addr; } cp_ring_info->hw_stats_ctx_id = HWRM_NA_SIGNATURE; return 0; @@ -213,21 +285,6 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp) unsigned int i; int rc = 0; - /* Default completion ring */ - { - struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; - struct bnxt_ring *cp_ring = cpr->cp_ring_struct; - - rc = bnxt_hwrm_ring_alloc(bp, cp_ring, - HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, - 0, HWRM_NA_SIGNATURE); - if (rc) - goto err_out; - cpr->cp_doorbell = pci_dev->mem_resource[2].addr; - B_CP_DIS_DB(cpr, cpr->cp_raw_cons); - bp->grp_info[0].cp_fw_ring_id = cp_ring->fw_ring_id; - } - for (i = 0; i < bp->rx_cp_nr_rings; i++) { struct bnxt_rx_queue *rxq = bp->rx_queues[i]; struct bnxt_cp_ring_info *cpr = rxq->cp_ring; @@ -235,35 +292,64 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp) struct bnxt_rx_ring_info *rxr = rxq->rx_ring; struct bnxt_ring *ring = rxr->rx_ring_struct; unsigned int idx = i + 1; + unsigned int map_idx = idx + bp->rx_cp_nr_rings; + + bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; /* Rx cmpl */ rc = bnxt_hwrm_ring_alloc(bp, cp_ring, - HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, - idx, HWRM_NA_SIGNATURE); + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, + idx, HWRM_NA_SIGNATURE, + HWRM_NA_SIGNATURE); if (rc) goto err_out; cpr->cp_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; - bp->grp_info[idx].cp_fw_ring_id = cp_ring->fw_ring_id; + bp->grp_info[i].cp_fw_ring_id = cp_ring->fw_ring_id; B_CP_DIS_DB(cpr, cpr->cp_raw_cons); /* Rx ring */ rc = bnxt_hwrm_ring_alloc(bp, ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, - idx, cpr->hw_stats_ctx_id); + idx, cpr->hw_stats_ctx_id, + cp_ring->fw_ring_id); if (rc) goto err_out; rxr->rx_prod = 0; rxr->rx_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; - bp->grp_info[idx].rx_fw_ring_id = ring->fw_ring_id; + bp->grp_info[i].rx_fw_ring_id = ring->fw_ring_id; B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + + ring = rxr->ag_ring_struct; + /* Agg ring */ + if (ring == NULL) + RTE_LOG(ERR, PMD, "Alloc AGG Ring is NULL!\n"); + + rc = bnxt_hwrm_ring_alloc(bp, ring, + HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, + map_idx, HWRM_NA_SIGNATURE, + cp_ring->fw_ring_id); + if (rc) + goto err_out; + RTE_LOG(DEBUG, PMD, "Alloc AGG Done!\n"); + rxr->ag_prod = 0; + rxr->ag_doorbell = + (char *)pci_dev->mem_resource[2].addr + + map_idx * 0x80; + bp->grp_info[i].ag_fw_ring_id = ring->fw_ring_id; + B_RX_DB(rxr->ag_doorbell, rxr->ag_prod); + + rxq->rx_buf_use_size = BNXT_MAX_MTU + ETHER_HDR_LEN + + ETHER_CRC_LEN + (2 * VLAN_TAG_SIZE); if (bnxt_init_one_rx_ring(rxq)) { - RTE_LOG(ERR, PMD, "bnxt_init_one_rx_ring failed!"); + RTE_LOG(ERR, PMD, "bnxt_init_one_rx_ring failed!\n"); bnxt_rx_queue_release_op(rxq); return -ENOMEM; } B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + B_RX_DB(rxr->ag_doorbell, rxr->ag_prod); + rxq->index = idx; } for (i = 0; i < bp->tx_cp_nr_rings; i++) { @@ -272,29 +358,34 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp) struct bnxt_ring *cp_ring = cpr->cp_ring_struct; struct bnxt_tx_ring_info *txr = txq->tx_ring; struct bnxt_ring *ring = txr->tx_ring_struct; - unsigned int idx = 1 + bp->rx_cp_nr_rings + i; + unsigned int idx = i + 1 + bp->rx_cp_nr_rings; + + /* Account for AGG Rings. AGG ring cnt = Rx Cmpl ring cnt */ + idx += bp->rx_cp_nr_rings; /* Tx cmpl */ rc = bnxt_hwrm_ring_alloc(bp, cp_ring, - HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, - idx, HWRM_NA_SIGNATURE); + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, + idx, HWRM_NA_SIGNATURE, + HWRM_NA_SIGNATURE); if (rc) goto err_out; cpr->cp_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; - bp->grp_info[idx].cp_fw_ring_id = cp_ring->fw_ring_id; B_CP_DIS_DB(cpr, cpr->cp_raw_cons); /* Tx ring */ rc = bnxt_hwrm_ring_alloc(bp, ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, - idx, cpr->hw_stats_ctx_id); + idx, cpr->hw_stats_ctx_id, + cp_ring->fw_ring_id); if (rc) goto err_out; txr->tx_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; + txq->index = idx; } err_out: diff --git a/drivers/net/bnxt/bnxt_ring.h b/drivers/net/bnxt/bnxt_ring.h index 8656549a..6d1eb588 100644 --- a/drivers/net/bnxt/bnxt_ring.h +++ b/drivers/net/bnxt/bnxt_ring.h @@ -57,7 +57,8 @@ #define DEFAULT_RX_RING_SIZE 256 #define DEFAULT_TX_RING_SIZE 256 -#define MAX_TPA 128 +#define BNXT_TPA_MAX 64 +#define AGG_RING_SIZE_FACTOR 2 /* These assume 4k pages */ #define MAX_RX_DESC_CNT (8 * 1024) @@ -65,6 +66,7 @@ #define MAX_CP_DESC_CNT (16 * 1024) #define INVALID_HW_RING_ID ((uint16_t)-1) +#define INVALID_STATS_CTX_ID ((uint16_t)-1) struct bnxt_ring { void *bd; diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c index cddf17d5..0793820b 100644 --- a/drivers/net/bnxt/bnxt_rxq.c +++ b/drivers/net/bnxt/bnxt_rxq.c @@ -76,6 +76,7 @@ int bnxt_mq_rx_configure(struct bnxt *bp) rc = -ENOMEM; goto err_out; } + vnic->flags |= BNXT_VNIC_INFO_BCAST; STAILQ_INSERT_TAIL(&bp->ff_pool[0], vnic, next); bp->nr_vnics++; @@ -84,9 +85,8 @@ int bnxt_mq_rx_configure(struct bnxt *bp) vnic->func_default = true; vnic->ff_pool_idx = 0; - vnic->start_grp_id = 1; - vnic->end_grp_id = vnic->start_grp_id + - bp->rx_cp_nr_rings - 1; + vnic->start_grp_id = 0; + vnic->end_grp_id = vnic->start_grp_id; filter = bnxt_alloc_filter(bp); if (!filter) { RTE_LOG(ERR, PMD, "L2 filter alloc failed\n"); @@ -121,13 +121,16 @@ int bnxt_mq_rx_configure(struct bnxt *bp) } /* For each pool, allocate MACVLAN CFA rule & VNIC */ if (!pools) { + pools = RTE_MIN(bp->max_vnics, + RTE_MIN(bp->max_l2_ctx, + RTE_MIN(bp->max_rsscos_ctx, ETH_64_POOLS))); RTE_LOG(ERR, PMD, "VMDq pool not set, defaulted to 64\n"); pools = ETH_64_POOLS; } nb_q_per_grp = bp->rx_cp_nr_rings / pools; - start_grp_id = 1; - end_grp_id = start_grp_id + nb_q_per_grp - 1; + start_grp_id = 0; + end_grp_id = nb_q_per_grp; ring_idx = 0; for (i = 0; i < pools; i++) { @@ -138,6 +141,7 @@ int bnxt_mq_rx_configure(struct bnxt *bp) rc = -ENOMEM; goto err_out; } + vnic->flags |= BNXT_VNIC_INFO_BCAST; STAILQ_INSERT_TAIL(&bp->ff_pool[i], vnic, next); bp->nr_vnics++; @@ -178,6 +182,7 @@ int bnxt_mq_rx_configure(struct bnxt *bp) rc = -ENOMEM; goto err_out; } + vnic->flags |= BNXT_VNIC_INFO_BCAST; /* Partition the rx queues for the single pool */ for (i = 0; i < bp->rx_cp_nr_rings; i++) { rxq = bp->eth_dev->data->rx_queues[i]; @@ -188,9 +193,8 @@ int bnxt_mq_rx_configure(struct bnxt *bp) vnic->func_default = true; vnic->ff_pool_idx = 0; - vnic->start_grp_id = 1; - vnic->end_grp_id = vnic->start_grp_id + - bp->rx_cp_nr_rings - 1; + vnic->start_grp_id = 0; + vnic->end_grp_id = bp->rx_cp_nr_rings; filter = bnxt_alloc_filter(bp); if (!filter) { RTE_LOG(ERR, PMD, "L2 filter alloc failed\n"); @@ -213,9 +217,10 @@ err_out: return rc; } -static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq __rte_unused) +static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) { struct bnxt_sw_rx_bd *sw_ring; + struct bnxt_tpa_info *tpa_info; uint16_t i; if (rxq) { @@ -228,6 +233,27 @@ static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq __rte_unused) } } } + /* Free up mbufs in Agg ring */ + sw_ring = rxq->rx_ring->ag_buf_ring; + if (sw_ring) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (sw_ring[i].mbuf) { + rte_pktmbuf_free_seg(sw_ring[i].mbuf); + sw_ring[i].mbuf = NULL; + } + } + } + + /* Free up mbufs in TPA */ + tpa_info = rxq->rx_ring->tpa_info; + if (tpa_info) { + for (i = 0; i < BNXT_TPA_MAX; i++) { + if (tpa_info[i].mbuf) { + rte_pktmbuf_free_seg(tpa_info[i].mbuf); + tpa_info[i].mbuf = NULL; + } + } + } } } @@ -251,6 +277,8 @@ void bnxt_rx_queue_release_op(void *rx_queue) /* Free RX ring hardware descriptors */ bnxt_free_ring(rxq->rx_ring->rx_ring_struct); + /* Free RX Agg ring hardware descriptors */ + bnxt_free_ring(rxq->rx_ring->ag_ring_struct); /* Free RX completion ring hardware descriptors */ bnxt_free_ring(rxq->cp_ring->cp_ring_struct); @@ -273,7 +301,7 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, int rc = 0; if (!nb_desc || nb_desc > MAX_RX_DESC_CNT) { - RTE_LOG(ERR, PMD, "nb_desc %d is invalid", nb_desc); + RTE_LOG(ERR, PMD, "nb_desc %d is invalid\n", nb_desc); rc = -EINVAL; goto out; } @@ -286,7 +314,7 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, rxq = rte_zmalloc_socket("bnxt_rx_queue", sizeof(struct bnxt_rx_queue), RTE_CACHE_LINE_SIZE, socket_id); if (!rxq) { - RTE_LOG(ERR, PMD, "bnxt_rx_queue allocation failed!"); + RTE_LOG(ERR, PMD, "bnxt_rx_queue allocation failed!\n"); rc = -ENOMEM; goto out; } @@ -295,6 +323,9 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, rxq->nb_rx_desc = nb_desc; rxq->rx_free_thresh = rx_conf->rx_free_thresh; + RTE_LOG(DEBUG, PMD, "RX Buf size is %d\n", rxq->rx_buf_use_size); + RTE_LOG(DEBUG, PMD, "RX Buf MTU %d\n", eth_dev->data->mtu); + rc = bnxt_init_rx_ring_struct(rxq, socket_id); if (rc) goto out; @@ -308,7 +339,8 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, /* Allocate RX ring hardware descriptors */ if (bnxt_alloc_rings(bp, queue_idx, NULL, rxq->rx_ring, rxq->cp_ring, "rxr")) { - RTE_LOG(ERR, PMD, "ring_dma_zone_reserve for rx_ring failed!"); + RTE_LOG(ERR, PMD, + "ring_dma_zone_reserve for rx_ring failed!\n"); bnxt_rx_queue_release_op(rxq); rc = -ENOMEM; goto out; diff --git a/drivers/net/bnxt/bnxt_rxq.h b/drivers/net/bnxt/bnxt_rxq.h index 95543298..01aaa007 100644 --- a/drivers/net/bnxt/bnxt_rxq.h +++ b/drivers/net/bnxt/bnxt_rxq.h @@ -52,12 +52,15 @@ struct bnxt_rx_queue { uint8_t crc_len; /* 0 if CRC stripped, 4 otherwise */ struct bnxt *bp; + int index; struct bnxt_vnic_info *vnic; uint32_t rx_buf_size; uint32_t rx_buf_use_size; /* useable size */ struct bnxt_rx_ring_info *rx_ring; struct bnxt_cp_ring_info *cp_ring; + + struct bnxt_tpa_info *rx_tpa; }; void bnxt_free_rxq_stats(struct bnxt_rx_queue *rxq); diff --git a/drivers/net/bnxt/bnxt_rxr.c b/drivers/net/bnxt/bnxt_rxr.c index 5d93de26..bee67d33 100644 --- a/drivers/net/bnxt/bnxt_rxr.c +++ b/drivers/net/bnxt/bnxt_rxr.c @@ -34,6 +34,7 @@ #include <inttypes.h> #include <stdbool.h> +#include <rte_bitmap.h> #include <rte_byteorder.h> #include <rte_malloc.h> #include <rte_memory.h> @@ -67,8 +68,37 @@ static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, struct rte_mbuf *data; data = __bnxt_alloc_rx_data(rxq->mb_pool); - if (!data) + if (!data) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); return -ENOMEM; + } + + rx_buf->mbuf = data; + + rxbd->addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(rx_buf->mbuf)); + + return 0; +} + +static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq, + struct bnxt_rx_ring_info *rxr, + uint16_t prod) +{ + struct rx_prod_pkt_bd *rxbd = &rxr->ag_desc_ring[prod]; + struct bnxt_sw_rx_bd *rx_buf = &rxr->ag_buf_ring[prod]; + struct rte_mbuf *data; + + data = __bnxt_alloc_rx_data(rxq->mb_pool); + if (!data) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); + return -ENOMEM; + } + + if (rxbd == NULL) + RTE_LOG(ERR, PMD, "Jumbo Frame. rxbd is NULL\n"); + if (rx_buf == NULL) + RTE_LOG(ERR, PMD, "Jumbo Frame. rx_buf is NULL\n"); + rx_buf->mbuf = data; @@ -77,24 +107,232 @@ static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, return 0; } -static void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, uint16_t cons, +static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, struct rte_mbuf *mbuf) { - uint16_t prod = rxr->rx_prod; + uint16_t prod = RING_NEXT(rxr->rx_ring_struct, rxr->rx_prod); struct bnxt_sw_rx_bd *prod_rx_buf; - struct rx_prod_pkt_bd *prod_bd, *cons_bd; + struct rx_prod_pkt_bd *prod_bd; prod_rx_buf = &rxr->rx_buf_ring[prod]; + RTE_ASSERT(prod_rx_buf->mbuf == NULL); + RTE_ASSERT(mbuf != NULL); + prod_rx_buf->mbuf = mbuf; prod_bd = &rxr->rx_desc_ring[prod]; - cons_bd = &rxr->rx_desc_ring[cons]; + + prod_bd->addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(mbuf)); + + rxr->rx_prod = prod; +} + +#ifdef BNXT_DEBUG +static void bnxt_reuse_ag_mbuf(struct bnxt_rx_ring_info *rxr, uint16_t cons, + struct rte_mbuf *mbuf) +{ + uint16_t prod = rxr->ag_prod; + struct bnxt_sw_rx_bd *prod_rx_buf; + struct rx_prod_pkt_bd *prod_bd, *cons_bd; + + prod_rx_buf = &rxr->ag_buf_ring[prod]; + + prod_rx_buf->mbuf = mbuf; + + prod_bd = &rxr->ag_desc_ring[prod]; + cons_bd = &rxr->ag_desc_ring[cons]; prod_bd->addr = cons_bd->addr; } +#endif + +static inline +struct rte_mbuf *bnxt_consume_rx_buf(struct bnxt_rx_ring_info *rxr, + uint16_t cons) +{ + struct bnxt_sw_rx_bd *cons_rx_buf; + struct rte_mbuf *mbuf; + + cons_rx_buf = &rxr->rx_buf_ring[cons]; + RTE_ASSERT(cons_rx_buf->mbuf != NULL); + mbuf = cons_rx_buf->mbuf; + cons_rx_buf->mbuf = NULL; + return mbuf; +} + +static void bnxt_tpa_start(struct bnxt_rx_queue *rxq, + struct rx_tpa_start_cmpl *tpa_start, + struct rx_tpa_start_cmpl_hi *tpa_start1) +{ + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint8_t agg_id = rte_le_to_cpu_32(tpa_start->agg_id & + RX_TPA_START_CMPL_AGG_ID_MASK) >> RX_TPA_START_CMPL_AGG_ID_SFT; + uint16_t data_cons; + struct bnxt_tpa_info *tpa_info; + struct rte_mbuf *mbuf; + + data_cons = tpa_start->opaque; + tpa_info = &rxr->tpa_info[agg_id]; + + mbuf = bnxt_consume_rx_buf(rxr, data_cons); + + bnxt_reuse_rx_mbuf(rxr, tpa_info->mbuf); + + tpa_info->mbuf = mbuf; + tpa_info->len = rte_le_to_cpu_32(tpa_start->len); + + mbuf->nb_segs = 1; + mbuf->next = NULL; + mbuf->pkt_len = rte_le_to_cpu_32(tpa_start->len); + mbuf->data_len = mbuf->pkt_len; + mbuf->port = rxq->port_id; + mbuf->ol_flags = PKT_RX_LRO; + if (likely(tpa_start->flags_type & + rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) { + mbuf->hash.rss = rte_le_to_cpu_32(tpa_start->rss_hash); + mbuf->ol_flags |= PKT_RX_RSS_HASH; + } else { + mbuf->hash.fdir.id = rte_le_to_cpu_16(tpa_start1->cfa_code); + mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; + } + if (tpa_start1->flags2 & + rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) { + mbuf->vlan_tci = rte_le_to_cpu_32(tpa_start1->metadata); + mbuf->ol_flags |= PKT_RX_VLAN_PKT; + } + if (likely(tpa_start1->flags2 & + rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC))) + mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + + /* recycle next mbuf */ + data_cons = RING_NEXT(rxr->rx_ring_struct, data_cons); + bnxt_reuse_rx_mbuf(rxr, bnxt_consume_rx_buf(rxr, data_cons)); +} + +static int bnxt_agg_bufs_valid(struct bnxt_cp_ring_info *cpr, + uint8_t agg_bufs, uint32_t raw_cp_cons) +{ + uint16_t last_cp_cons; + struct rx_pkt_cmpl *agg_cmpl; + + raw_cp_cons = ADV_RAW_CMP(raw_cp_cons, agg_bufs); + last_cp_cons = RING_CMP(cpr->cp_ring_struct, raw_cp_cons); + agg_cmpl = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[last_cp_cons]; + return CMP_VALID(agg_cmpl, raw_cp_cons, cpr->cp_ring_struct); +} + +/* TPA consume agg buffer out of order, allocate connected data only */ +static int bnxt_prod_ag_mbuf(struct bnxt_rx_queue *rxq) +{ + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint16_t next = RING_NEXT(rxr->ag_ring_struct, rxr->ag_prod); + + /* TODO batch allocation for better performance */ + while (rte_bitmap_get(rxr->ag_bitmap, next)) { + if (unlikely(bnxt_alloc_ag_data(rxq, rxr, next))) { + RTE_LOG(ERR, PMD, + "agg mbuf alloc failed: prod=0x%x\n", next); + break; + } + rte_bitmap_clear(rxr->ag_bitmap, next); + rxr->ag_prod = next; + next = RING_NEXT(rxr->ag_ring_struct, next); + } + + return 0; +} + +static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, + struct rte_mbuf *mbuf, uint32_t *tmp_raw_cons, + uint8_t agg_buf) +{ + struct bnxt_cp_ring_info *cpr = rxq->cp_ring; + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + int i; + uint16_t cp_cons, ag_cons; + struct rx_pkt_cmpl *rxcmp; + struct rte_mbuf *last = mbuf; + + for (i = 0; i < agg_buf; i++) { + struct bnxt_sw_rx_bd *ag_buf; + struct rte_mbuf *ag_mbuf; + *tmp_raw_cons = NEXT_RAW_CMP(*tmp_raw_cons); + cp_cons = RING_CMP(cpr->cp_ring_struct, *tmp_raw_cons); + rxcmp = (struct rx_pkt_cmpl *) + &cpr->cp_desc_ring[cp_cons]; + +#ifdef BNXT_DEBUG + bnxt_dump_cmpl(cp_cons, rxcmp); +#endif + + ag_cons = rxcmp->opaque; + RTE_ASSERT(ag_cons <= rxr->ag_ring_struct->ring_mask); + ag_buf = &rxr->ag_buf_ring[ag_cons]; + ag_mbuf = ag_buf->mbuf; + RTE_ASSERT(ag_mbuf != NULL); -static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, + ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len); + + mbuf->nb_segs++; + mbuf->pkt_len += ag_mbuf->data_len; + + last->next = ag_mbuf; + last = ag_mbuf; + + ag_buf->mbuf = NULL; + + /* + * As aggregation buffer consumed out of order in TPA module, + * use bitmap to track freed slots to be allocated and notified + * to NIC + */ + rte_bitmap_set(rxr->ag_bitmap, ag_cons); + } + bnxt_prod_ag_mbuf(rxq); + return 0; +} + +static inline struct rte_mbuf *bnxt_tpa_end( + struct bnxt_rx_queue *rxq, + uint32_t *raw_cp_cons, + struct rx_tpa_end_cmpl *tpa_end, + struct rx_tpa_end_cmpl_hi *tpa_end1 __rte_unused) +{ + struct bnxt_cp_ring_info *cpr = rxq->cp_ring; + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint8_t agg_id = (tpa_end->agg_id & RX_TPA_END_CMPL_AGG_ID_MASK) + >> RX_TPA_END_CMPL_AGG_ID_SFT; + struct rte_mbuf *mbuf; + uint8_t agg_bufs; + struct bnxt_tpa_info *tpa_info; + + tpa_info = &rxr->tpa_info[agg_id]; + mbuf = tpa_info->mbuf; + RTE_ASSERT(mbuf != NULL); + + rte_prefetch0(mbuf); + agg_bufs = (rte_le_to_cpu_32(tpa_end->agg_bufs_v1) & + RX_TPA_END_CMPL_AGG_BUFS_MASK) >> RX_TPA_END_CMPL_AGG_BUFS_SFT; + if (agg_bufs) { + if (!bnxt_agg_bufs_valid(cpr, agg_bufs, *raw_cp_cons)) + return NULL; + bnxt_rx_pages(rxq, mbuf, raw_cp_cons, agg_bufs); + } + mbuf->l4_len = tpa_end->payload_offset; + + struct rte_mbuf *new_data = __bnxt_alloc_rx_data(rxq->mb_pool); + RTE_ASSERT(new_data != NULL); + if (!new_data) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); + return NULL; + } + tpa_info->mbuf = new_data; + + return mbuf; +} + +static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, struct bnxt_rx_queue *rxq, uint32_t *raw_cons) { struct bnxt_cp_ring_info *cpr = rxq->cp_ring; @@ -104,9 +342,13 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, uint32_t tmp_raw_cons = *raw_cons; uint16_t cons, prod, cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); - struct bnxt_sw_rx_bd *rx_buf; +#ifdef BNXT_DEBUG + uint16_t ag_cons; +#endif struct rte_mbuf *mbuf; int rc = 0; + uint8_t agg_buf = 0; + uint16_t cmp_type; rxcmp = (struct rx_pkt_cmpl *) &cpr->cp_desc_ring[cp_cons]; @@ -118,14 +360,39 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, if (!CMP_VALID(rxcmp1, tmp_raw_cons, cpr->cp_ring_struct)) return -EBUSY; + cmp_type = CMP_TYPE(rxcmp); + if (cmp_type == RX_PKT_CMPL_TYPE_RX_L2_TPA_START) { + bnxt_tpa_start(rxq, (struct rx_tpa_start_cmpl *)rxcmp, + (struct rx_tpa_start_cmpl_hi *)rxcmp1); + rc = -EINVAL; /* Continue w/o new mbuf */ + goto next_rx; + } else if (cmp_type == RX_PKT_CMPL_TYPE_RX_L2_TPA_END) { + mbuf = bnxt_tpa_end(rxq, &tmp_raw_cons, + (struct rx_tpa_end_cmpl *)rxcmp, + (struct rx_tpa_end_cmpl_hi *)rxcmp1); + if (unlikely(!mbuf)) + return -EBUSY; + *rx_pkt = mbuf; + goto next_rx; + } else if (cmp_type != 0x11) { + rc = -EINVAL; + goto next_rx; + } + + agg_buf = (rxcmp->agg_bufs_v1 & RX_PKT_CMPL_AGG_BUFS_MASK) + >> RX_PKT_CMPL_AGG_BUFS_SFT; + if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons)) + return -EBUSY; + prod = rxr->rx_prod; - /* EW - GRO deferred to phase 3 */ cons = rxcmp->opaque; - rx_buf = &rxr->rx_buf_ring[cons]; - mbuf = rx_buf->mbuf; + mbuf = bnxt_consume_rx_buf(rxr, cons); rte_prefetch0(mbuf); + if (mbuf == NULL) + return -ENOMEM; + mbuf->nb_segs = 1; mbuf->next = NULL; mbuf->pkt_len = rxcmp->len; @@ -139,6 +406,10 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, mbuf->hash.fdir.id = rxcmp1->cfa_code; mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; } + + if (agg_buf) + bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf); + if (rxcmp1->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { mbuf->vlan_tci = rxcmp1->metadata & (RX_PKT_CMPL_METADATA_VID_MASK | @@ -147,14 +418,17 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, mbuf->ol_flags |= PKT_RX_VLAN_PKT; } - rx_buf->mbuf = NULL; +#ifdef BNXT_DEBUG if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) { /* Re-install the mbuf back to the rx ring */ bnxt_reuse_rx_mbuf(rxr, cons, mbuf); + if (agg_buf) + bnxt_reuse_ag_mbuf(rxr, ag_cons, mbuf); rc = -EIO; goto next_rx; } +#endif /* * TODO: Redesign this.... * If the allocation fails, the packet does not get received. @@ -170,24 +444,20 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, * calls in favour of a tight loop with the same function being called * in it. */ + prod = RING_NEXT(rxr->rx_ring_struct, prod); if (bnxt_alloc_rx_data(rxq, rxr, prod)) { RTE_LOG(ERR, PMD, "mbuf alloc failed with prod=0x%x\n", prod); rc = -ENOMEM; - goto next_rx; } - + rxr->rx_prod = prod; /* * All MBUFs are allocated with the same size under DPDK, * no optimization for rx_copy_thresh */ - /* AGG buf operation is deferred */ - - /* EW - VLAN reception. Must compare against the ol_flags */ - *rx_pkt = mbuf; + next_rx: - rxr->rx_prod = RING_NEXT(rxr->rx_ring_struct, prod); *raw_cons = tmp_raw_cons; @@ -203,8 +473,9 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t raw_cons = cpr->cp_raw_cons; uint32_t cons; int nb_rx_pkts = 0; - bool rx_event = false; struct rx_pkt_cmpl *rxcmp; + uint16_t prod = rxr->rx_prod; + uint16_t ag_prod = rxr->ag_prod; /* Handle RX burst request */ while (1) { @@ -222,26 +493,27 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons); if (likely(!rc)) nb_rx_pkts++; - else if (rc == -EBUSY) /* partial completion */ + if (rc == -EBUSY) /* partial completion */ break; - rx_event = true; } raw_cons = NEXT_RAW_CMP(raw_cons); if (nb_rx_pkts == nb_pkts) break; } - if (raw_cons == cpr->cp_raw_cons) { + + cpr->cp_raw_cons = raw_cons; + if (prod == rxr->rx_prod && ag_prod == rxr->ag_prod) { /* * For PMD, there is no need to keep on pushing to REARM * the doorbell if there are no new completions */ return nb_rx_pkts; } - cpr->cp_raw_cons = raw_cons; B_CP_DIS_DB(cpr, cpr->cp_raw_cons); - if (rx_event) - B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + /* Ring the AGG ring DB */ + B_RX_DB(rxr->ag_doorbell, rxr->ag_prod); return nb_rx_pkts; } @@ -257,6 +529,12 @@ void bnxt_free_rx_rings(struct bnxt *bp) bnxt_free_ring(rxq->rx_ring->rx_ring_struct); rte_free(rxq->rx_ring->rx_ring_struct); + + /* Free the Aggregator ring */ + bnxt_free_ring(rxq->rx_ring->ag_ring_struct); + rte_free(rxq->rx_ring->ag_ring_struct); + rxq->rx_ring->ag_ring_struct = NULL; + rte_free(rxq->rx_ring); bnxt_free_ring(rxq->cp_ring->cp_ring_struct); @@ -270,13 +548,11 @@ void bnxt_free_rx_rings(struct bnxt *bp) int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) { - struct bnxt *bp = rxq->bp; struct bnxt_cp_ring_info *cpr; struct bnxt_rx_ring_info *rxr; struct bnxt_ring *ring; - rxq->rx_buf_use_size = bp->eth_dev->data->mtu + - ETHER_HDR_LEN + ETHER_CRC_LEN + + rxq->rx_buf_use_size = BNXT_MAX_MTU + ETHER_HDR_LEN + ETHER_CRC_LEN + (2 * VLAN_TAG_SIZE); rxq->rx_buf_size = rxq->rx_buf_use_size + sizeof(struct rte_mbuf); @@ -313,13 +589,29 @@ int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) if (ring == NULL) return -ENOMEM; cpr->cp_ring_struct = ring; - ring->ring_size = rxr->rx_ring_struct->ring_size * 2; + ring->ring_size = rte_align32pow2(rxr->rx_ring_struct->ring_size * + (2 + AGG_RING_SIZE_FACTOR)); ring->ring_mask = ring->ring_size - 1; ring->bd = (void *)cpr->cp_desc_ring; ring->bd_dma = cpr->cp_desc_mapping; ring->vmem_size = 0; ring->vmem = NULL; + /* Allocate Aggregator rings */ + ring = rte_zmalloc_socket("bnxt_rx_ring_struct", + sizeof(struct bnxt_ring), + RTE_CACHE_LINE_SIZE, socket_id); + if (ring == NULL) + return -ENOMEM; + rxr->ag_ring_struct = ring; + ring->ring_size = rte_align32pow2(rxq->nb_rx_desc * + AGG_RING_SIZE_FACTOR); + ring->ring_mask = ring->ring_size - 1; + ring->bd = (void *)rxr->ag_desc_ring; + ring->bd_dma = rxr->ag_desc_mapping; + ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); + ring->vmem = (void **)&rxr->ag_buf_ring; + return 0; } @@ -332,8 +624,8 @@ static void bnxt_init_rxbds(struct bnxt_ring *ring, uint32_t type, if (!rx_bd_ring) return; for (j = 0; j < ring->ring_size; j++) { - rx_bd_ring[j].flags_type = type; - rx_bd_ring[j].len = len; + rx_bd_ring[j].flags_type = rte_cpu_to_le_16(type); + rx_bd_ring[j].len = rte_cpu_to_le_16(len); rx_bd_ring[j].opaque = j; } } @@ -344,12 +636,17 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) struct bnxt_ring *ring; uint32_t prod, type; unsigned int i; + uint16_t size; + + size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; + if (rxq->rx_buf_use_size <= size) + size = rxq->rx_buf_use_size; - type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT | RX_PROD_PKT_BD_FLAGS_EOP_PAD; + type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT; rxr = rxq->rx_ring; ring = rxr->rx_ring_struct; - bnxt_init_rxbds(ring, type, rxq->rx_buf_use_size); + bnxt_init_rxbds(ring, type, size); prod = rxr->rx_prod; for (i = 0; i < ring->ring_size; i++) { @@ -362,6 +659,36 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) rxr->rx_prod = prod; prod = RING_NEXT(rxr->rx_ring_struct, prod); } + RTE_LOG(DEBUG, PMD, "%s\n", __func__); + + ring = rxr->ag_ring_struct; + type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG; + bnxt_init_rxbds(ring, type, size); + prod = rxr->ag_prod; + + for (i = 0; i < ring->ring_size; i++) { + if (bnxt_alloc_ag_data(rxq, rxr, prod) != 0) { + RTE_LOG(WARNING, PMD, + "init'ed AG ring %d with %d/%d mbufs only\n", + rxq->queue_id, i, ring->ring_size); + break; + } + rxr->ag_prod = prod; + prod = RING_NEXT(rxr->ag_ring_struct, prod); + } + RTE_LOG(DEBUG, PMD, "%s AGG Done!\n", __func__); + + if (rxr->tpa_info) { + for (i = 0; i < BNXT_TPA_MAX; i++) { + rxr->tpa_info[i].mbuf = + __bnxt_alloc_rx_data(rxq->mb_pool); + if (!rxr->tpa_info[i].mbuf) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); + return -ENOMEM; + } + } + } + RTE_LOG(DEBUG, PMD, "%s TPA alloc Done!\n", __func__); return 0; } diff --git a/drivers/net/bnxt/bnxt_rxr.h b/drivers/net/bnxt/bnxt_rxr.h index f766b26c..f8d6dc80 100644 --- a/drivers/net/bnxt/bnxt_rxr.h +++ b/drivers/net/bnxt/bnxt_rxr.h @@ -37,20 +37,66 @@ #define B_RX_DB(db, prod) \ (*(uint32_t *)db = (DB_KEY_RX | prod)) +#define BNXT_TPA_L4_SIZE(x) \ + { \ + typeof(x) hdr_info = (x); \ + (((hdr_info) & 0xf8000000) ? ((hdr_info) >> 27) : 32) \ + } + +#define BNXT_TPA_INNER_L3_OFF(hdr_info) \ + (((hdr_info) >> 18) & 0x1ff) + +#define BNXT_TPA_INNER_L2_OFF(hdr_info) \ + (((hdr_info) >> 9) & 0x1ff) + +#define BNXT_TPA_OUTER_L3_OFF(hdr_info) \ + ((hdr_info) & 0x1ff) + +enum pkt_hash_types { + PKT_HASH_TYPE_NONE, /* Undefined type */ + PKT_HASH_TYPE_L2, /* Input: src_MAC, dest_MAC */ + PKT_HASH_TYPE_L3, /* Input: src_IP, dst_IP */ + PKT_HASH_TYPE_L4, /* Input: src_IP, dst_IP, src_port, dst_port */ +}; + +struct bnxt_tpa_info { + struct rte_mbuf *mbuf; + uint16_t len; + unsigned short gso_type; + uint32_t flags2; + uint32_t metadata; + enum pkt_hash_types hash_type; + uint32_t rss_hash; + uint32_t hdr_info; +}; + struct bnxt_sw_rx_bd { struct rte_mbuf *mbuf; /* data associated with RX descriptor */ }; struct bnxt_rx_ring_info { uint16_t rx_prod; + uint16_t ag_prod; void *rx_doorbell; + void *ag_doorbell; struct rx_prod_pkt_bd *rx_desc_ring; + struct rx_prod_pkt_bd *ag_desc_ring; struct bnxt_sw_rx_bd *rx_buf_ring; /* sw ring */ + struct bnxt_sw_rx_bd *ag_buf_ring; /* sw ring */ phys_addr_t rx_desc_mapping; + phys_addr_t ag_desc_mapping; struct bnxt_ring *rx_ring_struct; + struct bnxt_ring *ag_ring_struct; + + /* + * To deal with out of order return from TPA, use free buffer indicator + */ + struct rte_bitmap *ag_bitmap; + + struct bnxt_tpa_info *tpa_info; }; uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, diff --git a/drivers/net/bnxt/bnxt_stats.c b/drivers/net/bnxt/bnxt_stats.c index 40c9cac1..d7d0e35c 100644 --- a/drivers/net/bnxt/bnxt_stats.c +++ b/drivers/net/bnxt/bnxt_stats.c @@ -43,6 +43,171 @@ #include "bnxt_txq.h" #include "hsi_struct_def_dpdk.h" +static const struct bnxt_xstats_name_off bnxt_rx_stats_strings[] = { + {"rx_64b_frames", offsetof(struct rx_port_stats, + rx_64b_frames)}, + {"rx_65b_127b_frames", offsetof(struct rx_port_stats, + rx_65b_127b_frames)}, + {"rx_128b_255b_frames", offsetof(struct rx_port_stats, + rx_128b_255b_frames)}, + {"rx_256b_511b_frames", offsetof(struct rx_port_stats, + rx_256b_511b_frames)}, + {"rx_512b_1023b_frames", offsetof(struct rx_port_stats, + rx_512b_1023b_frames)}, + {"rx_1024b_1518_frames", offsetof(struct rx_port_stats, + rx_1024b_1518_frames)}, + {"rx_good_vlan_frames", offsetof(struct rx_port_stats, + rx_good_vlan_frames)}, + {"rx_1519b_2047b_frames", offsetof(struct rx_port_stats, + rx_1519b_2047b_frames)}, + {"rx_2048b_4095b_frames", offsetof(struct rx_port_stats, + rx_2048b_4095b_frames)}, + {"rx_4096b_9216b_frames", offsetof(struct rx_port_stats, + rx_4096b_9216b_frames)}, + {"rx_9217b_16383b_frames", offsetof(struct rx_port_stats, + rx_9217b_16383b_frames)}, + {"rx_total_frames", offsetof(struct rx_port_stats, + rx_total_frames)}, + {"rx_ucast_frames", offsetof(struct rx_port_stats, + rx_ucast_frames)}, + {"rx_mcast_frames", offsetof(struct rx_port_stats, + rx_mcast_frames)}, + {"rx_bcast_frames", offsetof(struct rx_port_stats, + rx_bcast_frames)}, + {"rx_fcs_err_frames", offsetof(struct rx_port_stats, + rx_fcs_err_frames)}, + {"rx_ctrl_frames", offsetof(struct rx_port_stats, + rx_ctrl_frames)}, + {"rx_pause_frames", offsetof(struct rx_port_stats, + rx_pause_frames)}, + {"rx_pfc_frames", offsetof(struct rx_port_stats, + rx_pfc_frames)}, + {"rx_align_err_frames", offsetof(struct rx_port_stats, + rx_align_err_frames)}, + {"rx_ovrsz_frames", offsetof(struct rx_port_stats, + rx_ovrsz_frames)}, + {"rx_jbr_frames", offsetof(struct rx_port_stats, + rx_jbr_frames)}, + {"rx_mtu_err_frames", offsetof(struct rx_port_stats, + rx_mtu_err_frames)}, + {"rx_tagged_frames", offsetof(struct rx_port_stats, + rx_tagged_frames)}, + {"rx_double_tagged_frames", offsetof(struct rx_port_stats, + rx_double_tagged_frames)}, + {"rx_good_frames", offsetof(struct rx_port_stats, + rx_good_frames)}, + {"rx_undrsz_frames", offsetof(struct rx_port_stats, + rx_undrsz_frames)}, + {"rx_eee_lpi_events", offsetof(struct rx_port_stats, + rx_eee_lpi_events)}, + {"rx_eee_lpi_duration", offsetof(struct rx_port_stats, + rx_eee_lpi_duration)}, + {"rx_bytes", offsetof(struct rx_port_stats, + rx_bytes)}, + {"rx_runt_bytes", offsetof(struct rx_port_stats, + rx_runt_bytes)}, + {"rx_runt_frames", offsetof(struct rx_port_stats, + rx_runt_frames)}, +}; + +static const struct bnxt_xstats_name_off bnxt_tx_stats_strings[] = { + {"tx_64b_frames", offsetof(struct tx_port_stats, + tx_64b_frames)}, + {"tx_65b_127b_frames", offsetof(struct tx_port_stats, + tx_65b_127b_frames)}, + {"tx_128b_255b_frames", offsetof(struct tx_port_stats, + tx_128b_255b_frames)}, + {"tx_256b_511b_frames", offsetof(struct tx_port_stats, + tx_256b_511b_frames)}, + {"tx_512b_1023b_frames", offsetof(struct tx_port_stats, + tx_512b_1023b_frames)}, + {"tx_1024b_1518_frames", offsetof(struct tx_port_stats, + tx_1024b_1518_frames)}, + {"tx_good_vlan_frames", offsetof(struct tx_port_stats, + tx_good_vlan_frames)}, + {"tx_1519b_2047_frames", offsetof(struct tx_port_stats, + tx_1519b_2047_frames)}, + {"tx_2048b_4095b_frames", offsetof(struct tx_port_stats, + tx_2048b_4095b_frames)}, + {"tx_4096b_9216b_frames", offsetof(struct tx_port_stats, + tx_4096b_9216b_frames)}, + {"tx_9217b_16383b_frames", offsetof(struct tx_port_stats, + tx_9217b_16383b_frames)}, + {"tx_good_frames", offsetof(struct tx_port_stats, + tx_good_frames)}, + {"tx_total_frames", offsetof(struct tx_port_stats, + tx_total_frames)}, + {"tx_ucast_frames", offsetof(struct tx_port_stats, + tx_ucast_frames)}, + {"tx_mcast_frames", offsetof(struct tx_port_stats, + tx_mcast_frames)}, + {"tx_bcast_frames", offsetof(struct tx_port_stats, + tx_bcast_frames)}, + {"tx_pause_frames", offsetof(struct tx_port_stats, + tx_pause_frames)}, + {"tx_pfc_frames", offsetof(struct tx_port_stats, + tx_pfc_frames)}, + {"tx_jabber_frames", offsetof(struct tx_port_stats, + tx_jabber_frames)}, + {"tx_fcs_err_frames", offsetof(struct tx_port_stats, + tx_fcs_err_frames)}, + {"tx_err", offsetof(struct tx_port_stats, + tx_err)}, + {"tx_fifo_underruns", offsetof(struct tx_port_stats, + tx_fifo_underruns)}, + {"tx_eee_lpi_events", offsetof(struct tx_port_stats, + tx_eee_lpi_events)}, + {"tx_eee_lpi_duration", offsetof(struct tx_port_stats, + tx_eee_lpi_duration)}, + {"tx_total_collisions", offsetof(struct tx_port_stats, + tx_total_collisions)}, + {"tx_bytes", offsetof(struct tx_port_stats, + tx_bytes)}, +}; + +static const struct bnxt_xstats_name_off bnxt_func_stats_strings[] = { + {"tx_ucast_pkts", offsetof(struct hwrm_func_qstats_output, + tx_ucast_pkts)}, + {"tx_mcast_pkts", offsetof(struct hwrm_func_qstats_output, + tx_mcast_pkts)}, + {"tx_bcast_pkts", offsetof(struct hwrm_func_qstats_output, + tx_bcast_pkts)}, + {"tx_err_pkts", offsetof(struct hwrm_func_qstats_output, + tx_err_pkts)}, + {"tx_drop_pkts", offsetof(struct hwrm_func_qstats_output, + tx_drop_pkts)}, + {"tx_ucast_bytes", offsetof(struct hwrm_func_qstats_output, + tx_ucast_bytes)}, + {"tx_mcast_bytes", offsetof(struct hwrm_func_qstats_output, + tx_mcast_bytes)}, + {"tx_bcast_bytes", offsetof(struct hwrm_func_qstats_output, + tx_bcast_bytes)}, + {"rx_ucast_pkts", offsetof(struct hwrm_func_qstats_output, + rx_ucast_pkts)}, + {"rx_mcast_pkts", offsetof(struct hwrm_func_qstats_output, + rx_mcast_pkts)}, + {"rx_bcast_pkts", offsetof(struct hwrm_func_qstats_output, + rx_bcast_pkts)}, + {"rx_err_pkts", offsetof(struct hwrm_func_qstats_output, + rx_err_pkts)}, + {"rx_drop_pkts", offsetof(struct hwrm_func_qstats_output, + rx_drop_pkts)}, + {"rx_ucast_bytes", offsetof(struct hwrm_func_qstats_output, + rx_ucast_bytes)}, + {"rx_mcast_bytes", offsetof(struct hwrm_func_qstats_output, + rx_mcast_bytes)}, + {"rx_bcast_bytes", offsetof(struct hwrm_func_qstats_output, + rx_bcast_bytes)}, + {"rx_agg_pkts", offsetof(struct hwrm_func_qstats_output, + rx_agg_pkts)}, + {"rx_agg_bytes", offsetof(struct hwrm_func_qstats_output, + rx_agg_bytes)}, + {"rx_agg_events", offsetof(struct hwrm_func_qstats_output, + rx_agg_events)}, + {"rx_agg_aborts", offsetof(struct hwrm_func_qstats_output, + rx_agg_aborts)}, +}; + /* * Statistics functions */ @@ -74,64 +239,18 @@ void bnxt_stats_get_op(struct rte_eth_dev *eth_dev, for (i = 0; i < bp->rx_cp_nr_rings; i++) { struct bnxt_rx_queue *rxq = bp->rx_queues[i]; struct bnxt_cp_ring_info *cpr = rxq->cp_ring; - struct ctx_hw_stats64 *hw_stats = - (struct ctx_hw_stats64 *)cpr->hw_stats; - - bnxt_stats->q_ipackets[i] += - rte_le_to_cpu_64(hw_stats->rx_ucast_pkts); - bnxt_stats->q_ipackets[i] += - rte_le_to_cpu_64(hw_stats->rx_mcast_pkts); - bnxt_stats->q_ipackets[i] += - rte_le_to_cpu_64(hw_stats->rx_bcast_pkts); - - bnxt_stats->q_ibytes[i] += - rte_le_to_cpu_64(hw_stats->rx_ucast_bytes); - bnxt_stats->q_ibytes[i] += - rte_le_to_cpu_64(hw_stats->rx_mcast_bytes); - bnxt_stats->q_ibytes[i] += - rte_le_to_cpu_64(hw_stats->rx_bcast_bytes); - - /* - * TBD: No clear mapping to this... we don't seem - * to have a stat specifically for dropped due to - * insufficient mbufs. - */ - bnxt_stats->q_errors[i] = 0; - - /* These get replaced once the *_QSTATS commands work */ - bnxt_stats->ipackets += bnxt_stats->q_ipackets[i]; - bnxt_stats->ibytes += bnxt_stats->q_ibytes[i]; - bnxt_stats->imissed += bnxt_stats->q_errors[i]; - bnxt_stats->ierrors += - rte_le_to_cpu_64(hw_stats->rx_discard_pkts); + + bnxt_hwrm_ctx_qstats(bp, cpr->hw_stats_ctx_id, i, bnxt_stats); } for (i = 0; i < bp->tx_cp_nr_rings; i++) { struct bnxt_tx_queue *txq = bp->tx_queues[i]; struct bnxt_cp_ring_info *cpr = txq->cp_ring; - struct ctx_hw_stats64 *hw_stats = - (struct ctx_hw_stats64 *)cpr->hw_stats; - - bnxt_stats->q_opackets[i] += - rte_le_to_cpu_64(hw_stats->tx_ucast_pkts); - bnxt_stats->q_opackets[i] += - rte_le_to_cpu_64(hw_stats->tx_mcast_pkts); - bnxt_stats->q_opackets[i] += - rte_le_to_cpu_64(hw_stats->tx_bcast_pkts); - - bnxt_stats->q_obytes[i] += - rte_le_to_cpu_64(hw_stats->tx_ucast_bytes); - bnxt_stats->q_obytes[i] += - rte_le_to_cpu_64(hw_stats->tx_mcast_bytes); - bnxt_stats->q_obytes[i] += - rte_le_to_cpu_64(hw_stats->tx_bcast_bytes); - - /* These get replaced once the *_QSTATS commands work */ - bnxt_stats->opackets += bnxt_stats->q_opackets[i]; - bnxt_stats->obytes += bnxt_stats->q_obytes[i]; - bnxt_stats->oerrors += rte_le_to_cpu_64(hw_stats->tx_drop_pkts); - bnxt_stats->oerrors += rte_le_to_cpu_64(hw_stats->tx_discard_pkts); + + bnxt_hwrm_ctx_qstats(bp, cpr->hw_stats_ctx_id, i, bnxt_stats); } + bnxt_hwrm_func_qstats(bp, 0xffff, bnxt_stats); + bnxt_stats->rx_nombuf = rte_atomic64_read(&bp->rx_mbuf_alloc_fail); } void bnxt_stats_reset_op(struct rte_eth_dev *eth_dev) @@ -139,4 +258,103 @@ void bnxt_stats_reset_op(struct rte_eth_dev *eth_dev) struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; bnxt_clear_all_hwrm_stat_ctxs(bp); + rte_atomic64_clear(&bp->rx_mbuf_alloc_fail); +} + +int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev, + struct rte_eth_xstat *xstats, unsigned int n) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + + unsigned int count, i; + uint64_t tx_drop_pkts; + + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) { + RTE_LOG(ERR, PMD, "xstats not supported for VF\n"); + return 0; + } + + bnxt_hwrm_port_qstats(bp); + bnxt_hwrm_func_qstats_tx_drop(bp, 0xffff, &tx_drop_pkts); + + count = RTE_DIM(bnxt_rx_stats_strings) + + RTE_DIM(bnxt_tx_stats_strings) + 1; /* For tx_drop_pkts */ + + if (n < count) + return count; + + count = 0; + for (i = 0; i < RTE_DIM(bnxt_rx_stats_strings); i++) { + uint64_t *rx_stats = (uint64_t *)bp->hw_rx_port_stats; + xstats[count].value = rte_le_to_cpu_64( + *(uint64_t *)((char *)rx_stats + + bnxt_rx_stats_strings[i].offset)); + count++; + } + + for (i = 0; i < RTE_DIM(bnxt_tx_stats_strings); i++) { + uint64_t *tx_stats = (uint64_t *)bp->hw_tx_port_stats; + xstats[count].value = rte_le_to_cpu_64( + *(uint64_t *)((char *)tx_stats + + bnxt_tx_stats_strings[i].offset)); + count++; + } + + /* The Tx drop pkts aka the Anti spoof coounter */ + xstats[count].value = rte_le_to_cpu_64(tx_drop_pkts); + count++; + + return count; +} + +int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned int limit) +{ + /* Account for the Tx drop pkts aka the Anti spoof counter */ + const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) + + RTE_DIM(bnxt_tx_stats_strings) + 1; + unsigned int i, count; + + if (xstats_names != NULL) { + count = 0; + + for (i = 0; i < RTE_DIM(bnxt_rx_stats_strings); i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + bnxt_rx_stats_strings[i].name); + count++; + } + + for (i = 0; i < RTE_DIM(bnxt_tx_stats_strings); i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + bnxt_tx_stats_strings[i].name); + count++; + } + + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + bnxt_func_stats_strings[4].name); + count++; + } + return stat_cnt; +} + +void bnxt_dev_xstats_reset_op(struct rte_eth_dev *eth_dev) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + + if (bp->flags & BNXT_FLAG_PORT_STATS && !BNXT_NPAR_PF(bp)) + bnxt_hwrm_port_clr_stats(bp); + + if (BNXT_VF(bp)) + RTE_LOG(ERR, PMD, "Operation not supported on a VF device\n"); + if (BNXT_NPAR_PF(bp)) + RTE_LOG(ERR, PMD, "Operation not supported on a MF device\n"); + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) + RTE_LOG(ERR, PMD, "Operation not supported\n"); } diff --git a/drivers/net/bnxt/bnxt_stats.h b/drivers/net/bnxt/bnxt_stats.h index 65408a44..b6d133ef 100644 --- a/drivers/net/bnxt/bnxt_stats.h +++ b/drivers/net/bnxt/bnxt_stats.h @@ -40,5 +40,15 @@ void bnxt_free_stats(struct bnxt *bp); void bnxt_stats_get_op(struct rte_eth_dev *eth_dev, struct rte_eth_stats *bnxt_stats); void bnxt_stats_reset_op(struct rte_eth_dev *eth_dev); +int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned int limit); +int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev, + struct rte_eth_xstat *xstats, unsigned int n); +void bnxt_dev_xstats_reset_op(struct rte_eth_dev *eth_dev); +struct bnxt_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + uint64_t offset; +}; #endif diff --git a/drivers/net/bnxt/bnxt_txr.c b/drivers/net/bnxt/bnxt_txr.c index 0d15bb1e..6870b16d 100644 --- a/drivers/net/bnxt/bnxt_txr.c +++ b/drivers/net/bnxt/bnxt_txr.c @@ -213,7 +213,8 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt, /* TSO */ txbd1->lflags = TX_BD_LONG_LFLAGS_LSO; txbd1->hdr_size = tx_pkt->l2_len + tx_pkt->l3_len + - tx_pkt->l4_len; + tx_pkt->l4_len + tx_pkt->outer_l2_len + + tx_pkt->outer_l3_len; txbd1->mss = tx_pkt->tso_segsz; } else if (tx_pkt->ol_flags & (PKT_TX_TCP_CKSUM | diff --git a/drivers/net/bnxt/bnxt_vnic.c b/drivers/net/bnxt/bnxt_vnic.c index 33fdde2f..db9fb079 100644 --- a/drivers/net/bnxt/bnxt_vnic.c +++ b/drivers/net/bnxt/bnxt_vnic.c @@ -69,21 +69,14 @@ void bnxt_init_vnics(struct bnxt *bp) uint16_t max_vnics; int i, j; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; STAILQ_INIT(&bp->free_vnic_list); for (i = 0; i < max_vnics; i++) { vnic = &bp->vnic_info[i]; vnic->fw_vnic_id = (uint16_t)HWRM_NA_SIGNATURE; - vnic->fw_rss_cos_lb_ctx = (uint16_t)HWRM_NA_SIGNATURE; - vnic->ctx_is_rss_cos_lb = HW_CONTEXT_NONE; + vnic->rss_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->cos_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->lb_rule = (uint16_t)HWRM_NA_SIGNATURE; for (j = 0; j < MAX_QUEUES_PER_VNIC; j++) vnic->fw_grp_ids[j] = (uint16_t)HWRM_NA_SIGNATURE; @@ -177,19 +170,13 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp) char mz_name[RTE_MEMZONE_NAMESIZE]; uint32_t entry_length = RTE_CACHE_LINE_ROUNDUP( HW_HASH_INDEX_SIZE * sizeof(*vnic->rss_table) + - HW_HASH_KEY_SIZE); + HW_HASH_KEY_SIZE + + BNXT_MAX_MC_ADDRS * ETHER_ADDR_LEN); uint16_t max_vnics; int i; + phys_addr_t mz_phys_addr; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_%04x:%02x:%02x:%02x_vnicattr", pdev->addr.domain, pdev->addr.bus, pdev->addr.devid, pdev->addr.function); @@ -204,6 +191,19 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp) if (!mz) return -ENOMEM; } + mz_phys_addr = mz->phys_addr; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2phy()\n"); + mz_phys_addr = rte_mem_virt2phy(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map vnic address to physical memory\n"); + return -ENOMEM; + } + } for (i = 0; i < max_vnics; i++) { vnic = &bp->vnic_info[i]; @@ -213,12 +213,16 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp) (void *)((char *)mz->addr + (entry_length * i)); memset(vnic->rss_table, -1, entry_length); - vnic->rss_table_dma_addr = mz->phys_addr + (entry_length * i); + vnic->rss_table_dma_addr = mz_phys_addr + (entry_length * i); vnic->rss_hash_key = (void *)((char *)vnic->rss_table + HW_HASH_INDEX_SIZE * sizeof(*vnic->rss_table)); vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + HW_HASH_INDEX_SIZE * sizeof(*vnic->rss_table); + vnic->mc_list = (void *)((char *)vnic->rss_hash_key + + HW_HASH_KEY_SIZE); + vnic->mc_list_dma_addr = vnic->rss_hash_key_dma_addr + + HW_HASH_KEY_SIZE; } return 0; @@ -232,15 +236,7 @@ void bnxt_free_vnic_mem(struct bnxt *bp) if (bp->vnic_info == NULL) return; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; for (i = 0; i < max_vnics; i++) { vnic = &bp->vnic_info[i]; if (vnic->fw_vnic_id != (uint16_t)HWRM_NA_SIGNATURE) { @@ -258,15 +254,7 @@ int bnxt_alloc_vnic_mem(struct bnxt *bp) struct bnxt_vnic_info *vnic_mem; uint16_t max_vnics; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; /* Allocate memory for VNIC pool and filter pool */ vnic_mem = rte_zmalloc("bnxt_vnic_info", max_vnics * sizeof(struct bnxt_vnic_info), 0); diff --git a/drivers/net/bnxt/bnxt_vnic.h b/drivers/net/bnxt/bnxt_vnic.h index 9671ba42..993f2212 100644 --- a/drivers/net/bnxt/bnxt_vnic.h +++ b/drivers/net/bnxt/bnxt_vnic.h @@ -42,8 +42,7 @@ struct bnxt_vnic_info { uint8_t ff_pool_idx; uint16_t fw_vnic_id; /* returned by Chimp during alloc */ - uint16_t fw_rss_cos_lb_ctx; - uint16_t ctx_is_rss_cos_lb; + uint16_t rss_rule; #define MAX_NUM_TRAFFIC_CLASSES 8 #define MAX_NUM_RSS_QUEUES_PER_VNIC 16 #define MAX_QUEUES_PER_VNIC (MAX_NUM_RSS_QUEUES_PER_VNIC + \ @@ -51,17 +50,34 @@ struct bnxt_vnic_info { uint16_t start_grp_id; uint16_t end_grp_id; uint16_t fw_grp_ids[MAX_QUEUES_PER_VNIC]; + uint16_t dflt_ring_grp; + uint16_t mru; uint16_t hash_type; phys_addr_t rss_table_dma_addr; uint16_t *rss_table; phys_addr_t rss_hash_key_dma_addr; void *rss_hash_key; + phys_addr_t mc_list_dma_addr; + char *mc_list; + uint32_t mc_addr_cnt; +#define BNXT_MAX_MC_ADDRS 16 uint32_t flags; #define BNXT_VNIC_INFO_PROMISC (1 << 0) #define BNXT_VNIC_INFO_ALLMULTI (1 << 1) +#define BNXT_VNIC_INFO_BCAST (1 << 2) +#define BNXT_VNIC_INFO_UCAST (1 << 3) +#define BNXT_VNIC_INFO_MCAST (1 << 4) +#define BNXT_VNIC_INFO_TAGGED (1 << 5) +#define BNXT_VNIC_INFO_UNTAGGED (1 << 6) + uint16_t cos_rule; + uint16_t lb_rule; bool vlan_strip; bool func_default; + bool bd_stall; + bool roce_dual; + bool roce_only; + bool rss_dflt_cr; STAILQ_HEAD(, bnxt_filter_info) filter; }; diff --git a/drivers/net/bnxt/hsi_struct_def_dpdk.h b/drivers/net/bnxt/hsi_struct_def_dpdk.h index f0248377..cb8660af 100644 --- a/drivers/net/bnxt/hsi_struct_def_dpdk.h +++ b/drivers/net/bnxt/hsi_struct_def_dpdk.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) Broadcom Limited. + * Copyright(c) 2001-2017 Broadcom Limited. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,74 +31,89 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _HSI_STRUCT_DEF_EXTERNAL_H_ -#define _HSI_STRUCT_DEF_EXTERNAL_H_ - -/* - * per-context HW statistics -- chip view - */ - -struct ctx_hw_stats64 { - uint64_t rx_ucast_pkts; - uint64_t rx_mcast_pkts; - uint64_t rx_bcast_pkts; - uint64_t rx_drop_pkts; - uint64_t rx_discard_pkts; - uint64_t rx_ucast_bytes; - uint64_t rx_mcast_bytes; - uint64_t rx_bcast_bytes; - - uint64_t tx_ucast_pkts; - uint64_t tx_mcast_pkts; - uint64_t tx_bcast_pkts; - uint64_t tx_drop_pkts; - uint64_t tx_discard_pkts; - uint64_t tx_ucast_bytes; - uint64_t tx_mcast_bytes; - uint64_t tx_bcast_bytes; - - uint64_t tpa_pkts; - uint64_t tpa_bytes; - uint64_t tpa_events; - uint64_t tpa_aborts; -} __attribute__((packed)); - -/* HW Resource Manager Specification 1.5.1 */ +#ifndef _HSI_STRUCT_DEF_DPDK_ +#define _HSI_STRUCT_DEF_DPDK_ +/* HSI and HWRM Specification 1.7.7 */ #define HWRM_VERSION_MAJOR 1 -#define HWRM_VERSION_MINOR 5 -#define HWRM_VERSION_UPDATE 1 - -#define HWRM_VERSION_STR "1.5.1" +#define HWRM_VERSION_MINOR 7 +#define HWRM_VERSION_UPDATE 7 +#define HWRM_VERSION_STR "1.7.7" /* * Following is the signature for HWRM message field that indicates not - * applicable (All F's). Need to cast it the size of the field if needed. + * applicable (All F's). Need to cast it the size of the field if needed. */ -#define HWRM_NA_SIGNATURE ((uint32_t)(-1)) +#define HWRM_NA_SIGNATURE ((uint32_t)(-1)) #define HWRM_MAX_REQ_LEN (128) /* hwrm_func_buf_rgtr */ -#define HWRM_MAX_RESP_LEN (176) /* hwrm_func_qstats */ -#define HW_HASH_INDEX_SIZE 0x80 /* 7 bit indirection table index. */ -#define HW_HASH_KEY_SIZE 40 +#define HWRM_MAX_RESP_LEN (248) /* hwrm_selftest_qlist */ +#define HW_HASH_INDEX_SIZE 0x80 /* 7 bit indirection table index. */ +#define HW_HASH_KEY_SIZE 40 #define HWRM_RESP_VALID_KEY 1 /* valid key for HWRM response */ +#define HWRM_ROCE_SP_HSI_VERSION_MAJOR 1 +#define HWRM_ROCE_SP_HSI_VERSION_MINOR 7 +#define HWRM_ROCE_SP_HSI_VERSION_UPDATE 4 /* * Request types */ #define HWRM_VER_GET (UINT32_C(0x0)) +#define HWRM_FUNC_BUF_UNRGTR (UINT32_C(0xe)) +#define HWRM_FUNC_VF_CFG (UINT32_C(0xf)) + /* Reserved for future use */ +#define RESERVED1 (UINT32_C(0x10)) #define HWRM_FUNC_RESET (UINT32_C(0x11)) +#define HWRM_FUNC_GETFID (UINT32_C(0x12)) +#define HWRM_FUNC_VF_ALLOC (UINT32_C(0x13)) +#define HWRM_FUNC_VF_FREE (UINT32_C(0x14)) #define HWRM_FUNC_QCAPS (UINT32_C(0x15)) #define HWRM_FUNC_QCFG (UINT32_C(0x16)) +#define HWRM_FUNC_CFG (UINT32_C(0x17)) +#define HWRM_FUNC_QSTATS (UINT32_C(0x18)) +#define HWRM_FUNC_CLR_STATS (UINT32_C(0x19)) #define HWRM_FUNC_DRV_UNRGTR (UINT32_C(0x1a)) +#define HWRM_FUNC_VF_RESC_FREE (UINT32_C(0x1b)) +#define HWRM_FUNC_VF_VNIC_IDS_QUERY (UINT32_C(0x1c)) #define HWRM_FUNC_DRV_RGTR (UINT32_C(0x1d)) +#define HWRM_FUNC_DRV_QVER (UINT32_C(0x1e)) +#define HWRM_FUNC_BUF_RGTR (UINT32_C(0x1f)) #define HWRM_PORT_PHY_CFG (UINT32_C(0x20)) +#define HWRM_PORT_MAC_CFG (UINT32_C(0x21)) +#define HWRM_PORT_QSTATS (UINT32_C(0x23)) +#define HWRM_PORT_LPBK_QSTATS (UINT32_C(0x24)) +#define HWRM_PORT_CLR_STATS (UINT32_C(0x25)) #define HWRM_PORT_PHY_QCFG (UINT32_C(0x27)) +#define HWRM_PORT_MAC_QCFG (UINT32_C(0x28)) +#define HWRM_PORT_PHY_QCAPS (UINT32_C(0x2a)) +#define HWRM_PORT_LED_CFG (UINT32_C(0x2d)) +#define HWRM_PORT_LED_QCFG (UINT32_C(0x2e)) +#define HWRM_PORT_LED_QCAPS (UINT32_C(0x2f)) #define HWRM_QUEUE_QPORTCFG (UINT32_C(0x30)) +#define HWRM_QUEUE_QCFG (UINT32_C(0x31)) +#define HWRM_QUEUE_CFG (UINT32_C(0x32)) +#define HWRM_FUNC_VLAN_CFG (UINT32_C(0x33)) +#define HWRM_FUNC_VLAN_QCFG (UINT32_C(0x34)) +#define HWRM_QUEUE_PFCENABLE_QCFG (UINT32_C(0x35)) +#define HWRM_QUEUE_PFCENABLE_CFG (UINT32_C(0x36)) +#define HWRM_QUEUE_PRI2COS_QCFG (UINT32_C(0x37)) +#define HWRM_QUEUE_PRI2COS_CFG (UINT32_C(0x38)) +#define HWRM_QUEUE_COS2BW_QCFG (UINT32_C(0x39)) +#define HWRM_QUEUE_COS2BW_CFG (UINT32_C(0x3a)) +#define HWRM_VNIC_ALLOC (UINT32_C(0x40)) #define HWRM_VNIC_ALLOC (UINT32_C(0x40)) #define HWRM_VNIC_FREE (UINT32_C(0x41)) #define HWRM_VNIC_CFG (UINT32_C(0x42)) +#define HWRM_VNIC_QCFG (UINT32_C(0x43)) +#define HWRM_VNIC_TPA_CFG (UINT32_C(0x44)) #define HWRM_VNIC_RSS_CFG (UINT32_C(0x46)) +#define HWRM_VNIC_RSS_QCFG (UINT32_C(0x47)) +#define HWRM_VNIC_PLCMODES_CFG (UINT32_C(0x48)) +#define HWRM_VNIC_PLCMODES_QCFG (UINT32_C(0x49)) +#define HWRM_VNIC_QCAPS (UINT32_C(0x4a)) #define HWRM_RING_ALLOC (UINT32_C(0x50)) #define HWRM_RING_FREE (UINT32_C(0x51)) +#define HWRM_RING_CMPL_RING_QAGGINT_PARAMS (UINT32_C(0x52)) +#define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS (UINT32_C(0x53)) +#define HWRM_RING_RESET (UINT32_C(0x5e)) #define HWRM_RING_GRP_ALLOC (UINT32_C(0x60)) #define HWRM_RING_GRP_FREE (UINT32_C(0x61)) #define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC (UINT32_C(0x70)) @@ -107,16 +122,71 @@ struct ctx_hw_stats64 { #define HWRM_CFA_L2_FILTER_FREE (UINT32_C(0x91)) #define HWRM_CFA_L2_FILTER_CFG (UINT32_C(0x92)) #define HWRM_CFA_L2_SET_RX_MASK (UINT32_C(0x93)) + /* Reserved for future use */ +#define HWRM_CFA_VLAN_ANTISPOOF_CFG (UINT32_C(0x94)) +#define HWRM_CFA_TUNNEL_FILTER_ALLOC (UINT32_C(0x95)) +#define HWRM_CFA_TUNNEL_FILTER_FREE (UINT32_C(0x96)) +#define HWRM_CFA_NTUPLE_FILTER_ALLOC (UINT32_C(0x99)) +#define HWRM_CFA_NTUPLE_FILTER_FREE (UINT32_C(0x9a)) +#define HWRM_CFA_NTUPLE_FILTER_CFG (UINT32_C(0x9b)) +#define HWRM_TUNNEL_DST_PORT_QUERY (UINT32_C(0xa0)) +#define HWRM_TUNNEL_DST_PORT_ALLOC (UINT32_C(0xa1)) +#define HWRM_TUNNEL_DST_PORT_FREE (UINT32_C(0xa2)) #define HWRM_STAT_CTX_ALLOC (UINT32_C(0xb0)) #define HWRM_STAT_CTX_FREE (UINT32_C(0xb1)) +#define HWRM_STAT_CTX_QUERY (UINT32_C(0xb2)) #define HWRM_STAT_CTX_CLR_STATS (UINT32_C(0xb3)) +#define HWRM_FW_RESET (UINT32_C(0xc0)) +#define HWRM_FW_QSTATUS (UINT32_C(0xc1)) #define HWRM_EXEC_FWD_RESP (UINT32_C(0xd0)) +#define HWRM_REJECT_FWD_RESP (UINT32_C(0xd1)) +#define HWRM_FWD_RESP (UINT32_C(0xd2)) +#define HWRM_FWD_ASYNC_EVENT_CMPL (UINT32_C(0xd3)) +#define HWRM_TEMP_MONITOR_QUERY (UINT32_C(0xe0)) +#define HWRM_WOL_FILTER_ALLOC (UINT32_C(0xf0)) +#define HWRM_WOL_FILTER_FREE (UINT32_C(0xf1)) +#define HWRM_WOL_FILTER_QCFG (UINT32_C(0xf2)) +#define HWRM_WOL_REASON_QCFG (UINT32_C(0xf3)) +#define HWRM_DBG_DUMP (UINT32_C(0xff14)) +#define HWRM_NVM_VALIDATE_OPTION (UINT32_C(0xffef)) +#define HWRM_NVM_FLUSH (UINT32_C(0xfff0)) +#define HWRM_NVM_GET_VARIABLE (UINT32_C(0xfff1)) +#define HWRM_NVM_SET_VARIABLE (UINT32_C(0xfff2)) +#define HWRM_NVM_INSTALL_UPDATE (UINT32_C(0xfff3)) +#define HWRM_NVM_MODIFY (UINT32_C(0xfff4)) +#define HWRM_NVM_VERIFY_UPDATE (UINT32_C(0xfff5)) +#define HWRM_NVM_GET_DEV_INFO (UINT32_C(0xfff6)) +#define HWRM_NVM_ERASE_DIR_ENTRY (UINT32_C(0xfff7)) +#define HWRM_NVM_MOD_DIR_ENTRY (UINT32_C(0xfff8)) +#define HWRM_NVM_FIND_DIR_ENTRY (UINT32_C(0xfff9)) +#define HWRM_NVM_GET_DIR_ENTRIES (UINT32_C(0xfffa)) +#define HWRM_NVM_GET_DIR_INFO (UINT32_C(0xfffb)) +#define HWRM_NVM_RAW_DUMP (UINT32_C(0xfffc)) +#define HWRM_NVM_READ (UINT32_C(0xfffd)) +#define HWRM_NVM_WRITE (UINT32_C(0xfffe)) +#define HWRM_NVM_RAW_WRITE_BLK (UINT32_C(0xffff)) -/* Return Codes */ -#define HWRM_ERR_CODE_INVALID_PARAMS (UINT32_C(0x2)) -#define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED (UINT32_C(0x3)) - -/* Short TX BD (16 bytes) */ +/* + * Note: The Host Software Interface (HSI) and Hardware Resource Manager (HWRM) + * specification describes the data structures used in Ethernet packet or RDMA + * message data transfers as well as an abstract interface for managing Ethernet + * NIC hardware resources. + */ +/* Ethernet Data path Host Structures */ +/* + * Description: The following three sections document the host structures used + * between device and software drivers for communicating Ethernet packets. + */ +/* BD Ring Structures */ +/* + * Description: This structure is used to inform the NIC of a location for and + * an aggregation buffer that will be used for packet data that is received. An + * aggregation buffer creates a different kind of completion operation for a + * packet where a variable number of BDs may be used to place the packet in the + * host. RX Rings that have aggregation buffers are known as aggregation rings + * and must contain only aggregation buffers. + */ +/* Short TX BD (16 bytes) */ struct tx_bd_short { uint16_t flags_type; /* @@ -149,10 +219,10 @@ struct tx_bd_short { /* * This value indicates how many 16B BD locations are consumed * in the ring by this packet. A value of 1 indicates that this - * BD is the only BD (and that the it is a short BD). A value of + * BD is the only BD (and that the it is a short BD). A value of * 3 indicates either 3 short BDs or 1 long BD and one short BD * in the packet. A value of 0 indicates that there are 32 BD - * locations in the packet (the maximum). This field is valid + * locations in the packet (the maximum). This field is valid * only on the first BD of a packet. */ #define TX_BD_SHORT_FLAGS_BD_CNT_MASK UINT32_C(0x1f00) @@ -173,7 +243,8 @@ struct tx_bd_short { #define TX_BD_SHORT_FLAGS_LHINT_LT2K (UINT32_C(0x2) << 13) /* indicates packet length >= 2KB */ #define TX_BD_SHORT_FLAGS_LHINT_GTE2K (UINT32_C(0x3) << 13) - #define TX_BD_SHORT_FLAGS_LHINT_LAST TX_BD_SHORT_FLAGS_LHINT_GTE2K + #define TX_BD_SHORT_FLAGS_LHINT_LAST \ + TX_BD_SHORT_FLAGS_LHINT_GTE2K /* * If set to 1, the device immediately updates the Send Consumer * Index after the buffer associated with this descriptor has @@ -213,7 +284,7 @@ struct tx_bd_short { */ } __attribute__((packed)); -/* Long TX BD (32 bytes split to 2 16-byte struct) */ +/* Long TX BD (32 bytes split to 2 16-byte struct) */ struct tx_bd_long { uint16_t flags_type; /* @@ -246,10 +317,10 @@ struct tx_bd_long { /* * This value indicates how many 16B BD locations are consumed * in the ring by this packet. A value of 1 indicates that this - * BD is the only BD (and that the it is a short BD). A value of + * BD is the only BD (and that the it is a short BD). A value of * 3 indicates either 3 short BDs or 1 long BD and one short BD * in the packet. A value of 0 indicates that there are 32 BD - * locations in the packet (the maximum). This field is valid + * locations in the packet (the maximum). This field is valid * only on the first BD of a packet. */ #define TX_BD_LONG_FLAGS_BD_CNT_MASK UINT32_C(0x1f00) @@ -270,7 +341,8 @@ struct tx_bd_long { #define TX_BD_LONG_FLAGS_LHINT_LT2K (UINT32_C(0x2) << 13) /* indicates packet length >= 2KB */ #define TX_BD_LONG_FLAGS_LHINT_GTE2K (UINT32_C(0x3) << 13) - #define TX_BD_LONG_FLAGS_LHINT_LAST TX_BD_LONG_FLAGS_LHINT_GTE2K + #define TX_BD_LONG_FLAGS_LHINT_LAST \ + TX_BD_LONG_FLAGS_LHINT_GTE2K /* * If set to 1, the device immediately updates the Send Consumer * Index after the buffer associated with this descriptor has @@ -360,7 +432,7 @@ struct tx_bd_long_hi { * bit is set, outer UDP checksum will be calculated for the * following cases: 1. Packets with tcp_udp_chksum flag set to * offload checksum for inner packet AND the inner packet is - * TCP/UDP. If the inner packet is ICMP for example (non- + * TCP/UDP. If the inner packet is ICMP for example (non- * TCP/UDP), even if the tcp_udp_chksum is set, the outer UDP * checksum will not be calculated. 2. Packets with lso flag set * which implies inner TCP checksum calculation as part of LSO @@ -392,7 +464,7 @@ struct tx_bd_long_hi { * to one when LSO is '1', then the IPID of the tunnel IP header * will be incremented for each subsequent segment of an LSO * operation. The flag is ignored if the LSO packet is a normal - * (non-tunneled) TCP packet. + * (non-tunneled) TCP packet. */ #define TX_BD_LONG_LFLAGS_T_IPID UINT32_C(0x80) /* @@ -460,7 +532,7 @@ struct tx_bd_long_hi { #define TX_BD_LONG_CFA_META_VLAN_TPID_TPID9300 (UINT32_C(0x4) << 16) /* Value programmed in CFA VLANTPID register. */ #define TX_BD_LONG_CFA_META_VLAN_TPID_TPIDCFG (UINT32_C(0x5) << 16) - #define TX_BD_LONG_CFA_META_VLAN_TPID_LAST \ + #define TX_BD_LONG_CFA_META_VLAN_TPID_LAST \ TX_BD_LONG_CFA_META_VLAN_TPID_TPIDCFG /* When key=1, This is the VLAN tag TPID select value. */ #define TX_BD_LONG_CFA_META_VLAN_RESERVED_MASK UINT32_C(0xff80000) @@ -474,15 +546,16 @@ struct tx_bd_long_hi { /* No editing */ #define TX_BD_LONG_CFA_META_KEY_NONE (UINT32_C(0x0) << 28) /* - * - meta[17:16] - TPID select value (0 = + * - meta[17:16] - TPID select value (0 = * 0x8100). - meta[15:12] - PRI/DE value. - * meta[11:0] - VID value. */ #define TX_BD_LONG_CFA_META_KEY_VLAN_TAG (UINT32_C(0x1) << 28) - #define TX_BD_LONG_CFA_META_KEY_LAST TX_BD_LONG_CFA_META_KEY_VLAN_TAG + #define TX_BD_LONG_CFA_META_KEY_LAST \ + TX_BD_LONG_CFA_META_KEY_VLAN_TAG } __attribute__((packed)); -/* RX Producer Packet BD (16 bytes) */ +/* RX Producer Packet BD (16 bytes) */ struct rx_prod_pkt_bd { uint16_t flags_type; /* This value identifies the type of buffer descriptor. */ @@ -490,7 +563,7 @@ struct rx_prod_pkt_bd { #define RX_PROD_PKT_BD_TYPE_SFT 0 /* * Indicates that this BD is 16B long and is an - * RX Producer (ie. empty) buffer descriptor. + * RX Producer (ie. empty) buffer descriptor. */ #define RX_PROD_PKT_BD_TYPE_RX_PROD_PKT UINT32_C(0x4) /* @@ -558,7 +631,7 @@ struct rx_prod_pkt_bd { /* Completion Ring Structures */ /* Note: This structure is used by the HWRM to communicate HWRM Error. */ -/* Base Completion Record (16 bytes) */ +/* Base Completion Record (16 bytes) */ struct cmpl_base { uint16_t type; /* unused is 10 b */ @@ -637,7 +710,7 @@ struct cmpl_base { /* info4 is 32 b */ } __attribute__((packed)); -/* TX Completion Record (16 bytes) */ +/* TX Completion Record (16 bytes) */ struct tx_cmpl { uint16_t flags_type; /* @@ -689,7 +762,7 @@ struct tx_cmpl { #define TX_CMPL_ERRORS_BUFFER_ERROR_NO_ERROR (UINT32_C(0x0) << 1) /* Bad Format: BDs were not formatted correctly. */ #define TX_CMPL_ERRORS_BUFFER_ERROR_BAD_FMT (UINT32_C(0x2) << 1) - #define TX_CMPL_ERRORS_BUFFER_ERROR_LAST \ + #define TX_CMPL_ERRORS_BUFFER_ERROR_LAST \ TX_CMPL_ERRORS_BUFFER_ERROR_BAD_FMT /* * When this bit is '1', it indicates that the length of the @@ -726,7 +799,7 @@ struct tx_cmpl { /* unused3 is 32 b */ } __attribute__((packed)); -/* RX Packet Completion Record (32 bytes split to 2 16-byte struct) */ +/* RX Packet Completion Record (32 bytes split to 2 16-byte struct) */ struct rx_pkt_cmpl { uint16_t flags_type; /* @@ -741,7 +814,9 @@ struct rx_pkt_cmpl { * RX L2 completion: Completion of and L2 RX * packet. Length = 32B */ - #define RX_PKT_CMPL_TYPE_RX_L2 UINT32_C(0x11) + #define RX_PKT_CMPL_TYPE_RX_L2 UINT32_C(0x11) + #define RX_PKT_CMPL_TYPE_RX_L2_TPA_START UINT32_C(0x13) + #define RX_PKT_CMPL_TYPE_RX_L2_TPA_END UINT32_C(0x15) /* * When this bit is '1', it indicates a packet that has an error * of some type. Type of error is indicated in error_flags. @@ -761,10 +836,12 @@ struct rx_pkt_cmpl { * field. */ #define RX_PKT_CMPL_FLAGS_PLACEMENT_HDS (UINT32_C(0x2) << 7) - #define RX_PKT_CMPL_FLAGS_PLACEMENT_LAST RX_PKT_CMPL_FLAGS_PLACEMENT_HDS + #define RX_PKT_CMPL_FLAGS_PLACEMENT_LAST \ + RX_PKT_CMPL_FLAGS_PLACEMENT_HDS /* This bit is '1' if the RSS field in this completion is valid. */ #define RX_PKT_CMPL_FLAGS_RSS_VALID UINT32_C(0x400) /* unused is 1 b */ + #define RX_PKT_CMPL_FLAGS_UNUSED UINT32_C(0x800) /* * This value indicates what the inner packet determined for the * packet was. @@ -820,7 +897,8 @@ struct rx_pkt_cmpl { * that a timestamp was taken for the packet. */ #define RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP (UINT32_C(0x9) << 12) - #define RX_PKT_CMPL_FLAGS_ITYPE_LAST RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP + #define RX_PKT_CMPL_FLAGS_ITYPE_LAST \ + RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP #define RX_PKT_CMPL_FLAGS_MASK UINT32_C(0xffc0) #define RX_PKT_CMPL_FLAGS_SFT 6 uint16_t len; @@ -938,7 +1016,7 @@ struct rx_pkt_cmpl_hi { * the vlan TPID value. */ #define RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN (UINT32_C(0x1) << 4) - #define RX_PKT_CMPL_FLAGS2_META_FORMAT_LAST \ + #define RX_PKT_CMPL_FLAGS2_META_FORMAT_LAST \ RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN /* * This field indicates the IP type for the inner-most IP @@ -988,15 +1066,18 @@ struct rx_pkt_cmpl_hi { * means that the packet could not be placed * into 7 physical buffers or less. */ - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_DID_NOT_FIT (UINT32_C(0x1) << 1) + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_DID_NOT_FIT \ + (UINT32_C(0x1) << 1) /* * Not On Chip: All BDs needed for the packet * were not on-chip when the packet arrived. */ - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_NOT_ON_CHIP (UINT32_C(0x2) << 1) + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_NOT_ON_CHIP \ + (UINT32_C(0x2) << 1) /* Bad Format: BDs were not formatted correctly. */ - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT (UINT32_C(0x3) << 1) - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_LAST \ + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT \ + (UINT32_C(0x3) << 1) + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_LAST \ RX_PKT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT /* This indicates that there was an error in the IP header checksum. */ #define RX_PKT_CMPL_ERRORS_IP_CS_ERROR UINT32_C(0x10) @@ -1037,39 +1118,45 @@ struct rx_pkt_cmpl_hi { * match expectation from L2 Ethertype for IPv4 * and IPv6 in the tunnel header. */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION (UINT32_C(0x1) << 9) + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \ + (UINT32_C(0x1) << 9) /* * Indicates that header length is out of range * in the tunnel header. Valid for IPv4. */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN (UINT32_C(0x2) << 9) + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \ + (UINT32_C(0x2) << 9) /* * Indicates that the physical packet is shorter * than that claimed by the PPPoE header length * for a tunnel PPPoE packet. */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR (UINT32_C(0x3) << 9) + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \ + (UINT32_C(0x3) << 9) /* * Indicates that physical packet is shorter * than that claimed by the tunnel l3 header * length. Valid for IPv4, or IPv6 tunnel packet * packets. */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR (UINT32_C(0x4) << 9) + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \ + (UINT32_C(0x4) << 9) /* * Indicates that the physical packet is shorter * than that claimed by the tunnel UDP header * length for a tunnel UDP packet that is not * fragmented. */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR (UINT32_C(0x5) << 9) + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \ + (UINT32_C(0x5) << 9) /* * indicates that the IPv4 TTL or IPv6 hop limit - * check have failed (e.g. TTL = 0) in the + * check have failed (e.g. TTL = 0) in the * tunnel header. Valid for IPv4, and IPv6. */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL (UINT32_C(0x6) << 9) - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_LAST \ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \ + (UINT32_C(0x6) << 9) + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_LAST \ RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL /* * This indicates that there was an error in the inner portion @@ -1089,15 +1176,17 @@ struct rx_pkt_cmpl_hi { * and IPv6 or that option other than VFT was * parsed on FCoE packet. */ - #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_VERSION (UINT32_C(0x1) << 12) + #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_VERSION \ + (UINT32_C(0x1) << 12) /* * indicates that header length is out of range. * Valid for IPv4 and RoCE */ - #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN (UINT32_C(0x2) << 12) + #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \ + (UINT32_C(0x2) << 12) /* * indicates that the IPv4 TTL or IPv6 hop limit - * check have failed (e.g. TTL = 0). Valid for + * check have failed (e.g. TTL = 0). Valid for * IPv4, and IPv6 */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_TTL (UINT32_C(0x3) << 12) @@ -1106,18 +1195,21 @@ struct rx_pkt_cmpl_hi { * than that claimed by the l3 header length. * Valid for IPv4, IPv6 packet or RoCE packets. */ - #define RX_PKT_CMPL_ERRORS_PKT_ERROR_IP_TOTAL_ERROR (UINT32_C(0x4) << 12) + #define RX_PKT_CMPL_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \ + (UINT32_C(0x4) << 12) /* * Indicates that the physical packet is shorter * than that claimed by the UDP header length * for a UDP packet that is not fragmented. */ - #define RX_PKT_CMPL_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR (UINT32_C(0x5) << 12) + #define RX_PKT_CMPL_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \ + (UINT32_C(0x5) << 12) /* * Indicates that TCP header length > IP * payload. Valid for TCP packets only. */ - #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN (UINT32_C(0x6) << 12) + #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \ + (UINT32_C(0x6) << 12) /* Indicates that TCP header length < 5. Valid for TCP. */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL \ (UINT32_C(0x7) << 12) @@ -1126,9 +1218,9 @@ struct rx_pkt_cmpl_hi { * TCP header size that does not match data * offset in TCP header. Valid for TCP. */ - #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \ + #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \ (UINT32_C(0x8) << 12) - #define RX_PKT_CMPL_ERRORS_PKT_ERROR_LAST \ + #define RX_PKT_CMPL_ERRORS_PKT_ERROR_LAST \ RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN #define RX_PKT_CMPL_ERRORS_MASK UINT32_C(0xfffe) #define RX_PKT_CMPL_ERRORS_SFT 1 @@ -1149,7 +1241,474 @@ struct rx_pkt_cmpl_hi { #define RX_PKT_CMPL_REORDER_SFT 0 } __attribute__((packed)); -/* HWRM Forwarded Request (16 bytes) */ +/* RX L2 TPA Start Completion Record (32 bytes split to 2 16-byte struct) */ +struct rx_tpa_start_cmpl { + uint16_t flags_type; + /* + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. + */ + #define RX_TPA_START_CMPL_TYPE_MASK UINT32_C(0x3f) + #define RX_TPA_START_CMPL_TYPE_SFT 0 + /* + * RX L2 TPA Start Completion: Completion at the + * beginning of a TPA operation. Length = 32B + */ + #define RX_TPA_START_CMPL_TYPE_RX_TPA_START UINT32_C(0x13) + /* This bit will always be '0' for TPA start completions. */ + #define RX_TPA_START_CMPL_FLAGS_ERROR UINT32_C(0x40) + /* This field indicates how the packet was placed in the buffer. */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_MASK UINT32_C(0x380) + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_SFT 7 + /* + * Jumbo: TPA Packet was placed using jumbo + * algorithm. This means that the first buffer + * will be filled with data before moving to + * aggregation buffers. Each aggregation buffer + * will be filled before moving to the next + * aggregation buffer. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_JUMBO (UINT32_C(0x1) << 7) + /* + * Header/Data Separation: Packet was placed + * using Header/Data separation algorithm. The + * separation location is indicated by the itype + * field. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_HDS (UINT32_C(0x2) << 7) + /* + * GRO/Jumbo: Packet will be placed using + * GRO/Jumbo where the first packet is filled + * with data. Subsequent packets will be placed + * such that any one packet does not span two + * aggregation buffers unless it starts at the + * beginning of an aggregation buffer. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_GRO_JUMBO \ + (UINT32_C(0x5) << 7) + /* + * GRO/Header-Data Separation: Packet will be + * placed using GRO/HDS where the header is in + * the first packet. Payload of each packet will + * be placed such that any one packet does not + * span two aggregation buffers unless it starts + * at the beginning of an aggregation buffer. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_GRO_HDS (UINT32_C(0x6) << 7) + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_LAST \ + RX_TPA_START_CMPL_FLAGS_PLACEMENT_GRO_HDS + /* This bit is '1' if the RSS field in this completion is valid. */ + #define RX_TPA_START_CMPL_FLAGS_RSS_VALID UINT32_C(0x400) + /* unused is 1 b */ + #define RX_TPA_START_CMPL_FLAGS_UNUSED UINT32_C(0x800) + /* + * This value indicates what the inner packet determined for the + * packet was. + */ + #define RX_TPA_START_CMPL_FLAGS_ITYPE_MASK UINT32_C(0xf000) + #define RX_TPA_START_CMPL_FLAGS_ITYPE_SFT 12 + /* TCP Packet: Indicates that the packet was IP and TCP. */ + #define RX_TPA_START_CMPL_FLAGS_ITYPE_TCP (UINT32_C(0x2) << 12) + #define RX_TPA_START_CMPL_FLAGS_ITYPE_LAST \ + RX_TPA_START_CMPL_FLAGS_ITYPE_TCP + #define RX_TPA_START_CMPL_FLAGS_MASK UINT32_C(0xffc0) + #define RX_TPA_START_CMPL_FLAGS_SFT 6 + uint16_t len; + /* + * This value indicates the amount of packet data written to the + * buffer the opaque field in this completion corresponds to. + */ + uint32_t opaque; + /* + * This is a copy of the opaque field from the RX BD this + * completion corresponds to. + */ + uint8_t v1; + /* unused1 is 7 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_START_CMPL_V1 UINT32_C(0x1) + /* unused1 is 7 b */ + uint8_t rss_hash_type; + /* + * This is the RSS hash type for the packet. The value is packed + * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]} + * . The value of tuple_extrac_op provides the information about + * what fields the hash was computed on. * 0: The RSS hash was + * computed over source IP address, destination IP address, + * source port, and destination port of inner IP and TCP or UDP + * headers. Note: For non-tunneled packets, the packet headers + * are considered inner packet headers for the RSS hash + * computation purpose. * 1: The RSS hash was computed over + * source IP address and destination IP address of inner IP + * header. Note: For non-tunneled packets, the packet headers + * are considered inner packet headers for the RSS hash + * computation purpose. * 2: The RSS hash was computed over + * source IP address, destination IP address, source port, and + * destination port of IP and TCP or UDP headers of outer tunnel + * headers. Note: For non-tunneled packets, this value is not + * applicable. * 3: The RSS hash was computed over source IP + * address and destination IP address of IP header of outer + * tunnel headers. Note: For non-tunneled packets, this value is + * not applicable. Note that 4-tuples values listed above are + * applicable for layer 4 protocols supported and enabled for + * RSS in the hardware, HWRM firmware, and drivers. For example, + * if RSS hash is supported and enabled for TCP traffic only, + * then the values of tuple_extract_op corresponding to 4-tuples + * are only valid for TCP traffic. + */ + uint16_t agg_id; + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + /* unused2 is 9 b */ + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + #define RX_TPA_START_CMPL_AGG_ID_MASK UINT32_C(0xfe00) + #define RX_TPA_START_CMPL_AGG_ID_SFT 9 + uint32_t rss_hash; + /* + * This value is the RSS hash value calculated for the packet + * based on the mode bits and key value in the VNIC. + */ +} __attribute__((packed)); + +/* last 16 bytes of RX L2 TPA Start Completion Record */ +struct rx_tpa_start_cmpl_hi { + uint32_t flags2; + /* + * This indicates that the ip checksum was calculated for the + * inner packet and that the sum passed for all segments + * included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_IP_CS_CALC UINT32_C(0x1) + /* + * This indicates that the TCP, UDP or ICMP checksum was + * calculated for the inner packet and that the sum passed for + * all segments included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC UINT32_C(0x2) + /* + * This indicates that the ip checksum was calculated for the + * tunnel header and that the sum passed for all segments + * included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_T_IP_CS_CALC UINT32_C(0x4) + /* + * This indicates that the UDP checksum was calculated for the + * tunnel packet and that the sum passed for all segments + * included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_T_L4_CS_CALC UINT32_C(0x8) + /* This value indicates what format the metadata field is. */ + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_MASK UINT32_C(0xf0) + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_SFT 4 + /* No metadata informtaion. Value is zero. */ + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_NONE (UINT32_C(0x0) << 4) + /* + * The metadata field contains the VLAN tag and + * TPID value. - metadata[11:0] contains the + * vlan VID value. - metadata[12] contains the + * vlan DE value. - metadata[15:13] contains the + * vlan PRI value. - metadata[31:16] contains + * the vlan TPID value. + */ + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN (UINT32_C(0x1) << 4) + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_LAST \ + RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN + /* + * This field indicates the IP type for the inner-most IP + * header. A value of '0' indicates IPv4. A value of '1' + * indicates IPv6. + */ + #define RX_TPA_START_CMPL_FLAGS2_IP_TYPE UINT32_C(0x100) + uint32_t metadata; + /* + * This is data from the CFA block as indicated by the + * meta_format field. + */ + /* When meta_format=1, this value is the VLAN VID. */ + #define RX_TPA_START_CMPL_METADATA_VID_MASK UINT32_C(0xfff) + #define RX_TPA_START_CMPL_METADATA_VID_SFT 0 + /* When meta_format=1, this value is the VLAN DE. */ + #define RX_TPA_START_CMPL_METADATA_DE UINT32_C(0x1000) + /* When meta_format=1, this value is the VLAN PRI. */ + #define RX_TPA_START_CMPL_METADATA_PRI_MASK UINT32_C(0xe000) + #define RX_TPA_START_CMPL_METADATA_PRI_SFT 13 + /* When meta_format=1, this value is the VLAN TPID. */ + #define RX_TPA_START_CMPL_METADATA_TPID_MASK UINT32_C(0xffff0000) + #define RX_TPA_START_CMPL_METADATA_TPID_SFT 16 + uint16_t v2; + /* unused4 is 15 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_START_CMPL_V2 UINT32_C(0x1) + /* unused4 is 15 b */ + uint16_t cfa_code; + /* + * This field identifies the CFA action rule that was used for + * this packet. + */ + uint32_t inner_l4_size_inner_l3_offset_inner_l2_offset_outer_l3_offset; + /* + * This is the size in bytes of the inner most L4 header. This + * can be subtracted from the payload_offset to determine the + * start of the inner most L4 header. + */ + /* + * This is the offset from the beginning of the packet in bytes + * for the outer L3 header. If there is no outer L3 header, then + * this value is zero. + */ + #define RX_TPA_START_CMPL_OUTER_L3_OFFSET_MASK UINT32_C(0x1ff) + #define RX_TPA_START_CMPL_OUTER_L3_OFFSET_SFT 0 + /* + * This is the offset from the beginning of the packet in bytes + * for the inner most L2 header. + */ + #define RX_TPA_START_CMPL_INNER_L2_OFFSET_MASK UINT32_C(0x3fe00) + #define RX_TPA_START_CMPL_INNER_L2_OFFSET_SFT 9 + /* + * This is the offset from the beginning of the packet in bytes + * for the inner most L3 header. + */ + #define RX_TPA_START_CMPL_INNER_L3_OFFSET_MASK UINT32_C(0x7fc0000) + #define RX_TPA_START_CMPL_INNER_L3_OFFSET_SFT 18 + /* + * This is the size in bytes of the inner most L4 header. This + * can be subtracted from the payload_offset to determine the + * start of the inner most L4 header. + */ + #define RX_TPA_START_CMPL_INNER_L4_SIZE_MASK UINT32_C(0xf8000000) + #define RX_TPA_START_CMPL_INNER_L4_SIZE_SFT 27 +} __attribute__((packed)); + +/* RX TPA End Completion Record (32 bytes split to 2 16-byte struct) */ +struct rx_tpa_end_cmpl { + uint16_t flags_type; + /* + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. + */ + #define RX_TPA_END_CMPL_TYPE_MASK UINT32_C(0x3f) + #define RX_TPA_END_CMPL_TYPE_SFT 0 + /* + * RX L2 TPA End Completion: Completion at the + * end of a TPA operation. Length = 32B + */ + #define RX_TPA_END_CMPL_TYPE_RX_TPA_END UINT32_C(0x15) + /* + * When this bit is '1', it indicates a packet that has an error + * of some type. Type of error is indicated in error_flags. + */ + #define RX_TPA_END_CMPL_FLAGS_ERROR UINT32_C(0x40) + /* This field indicates how the packet was placed in the buffer. */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_MASK UINT32_C(0x380) + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_SFT 7 + /* + * Jumbo: TPA Packet was placed using jumbo + * algorithm. This means that the first buffer + * will be filled with data before moving to + * aggregation buffers. Each aggregation buffer + * will be filled before moving to the next + * aggregation buffer. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_JUMBO (UINT32_C(0x1) << 7) + /* + * Header/Data Separation: Packet was placed + * using Header/Data separation algorithm. The + * separation location is indicated by the itype + * field. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_HDS (UINT32_C(0x2) << 7) + /* + * GRO/Jumbo: Packet will be placed using + * GRO/Jumbo where the first packet is filled + * with data. Subsequent packets will be placed + * such that any one packet does not span two + * aggregation buffers unless it starts at the + * beginning of an aggregation buffer. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_JUMBO (UINT32_C(0x5) << 7) + /* + * GRO/Header-Data Separation: Packet will be + * placed using GRO/HDS where the header is in + * the first packet. Payload of each packet will + * be placed such that any one packet does not + * span two aggregation buffers unless it starts + * at the beginning of an aggregation buffer. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_HDS (UINT32_C(0x6) << 7) + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_LAST \ + RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_HDS + /* unused is 2 b */ + #define RX_TPA_END_CMPL_FLAGS_UNUSED_MASK UINT32_C(0xc00) + #define RX_TPA_END_CMPL_FLAGS_UNUSED_SFT 10 + /* + * This value indicates what the inner packet determined for the + * packet was. - 2 TCP Packet Indicates that the packet was IP + * and TCP. This indicates that the ip_cs field is valid and + * that the tcp_udp_cs field is valid and contains the TCP + * checksum. This also indicates that the payload_offset field + * is valid. + */ + #define RX_TPA_END_CMPL_FLAGS_ITYPE_MASK UINT32_C(0xf000) + #define RX_TPA_END_CMPL_FLAGS_ITYPE_SFT 12 + #define RX_TPA_END_CMPL_FLAGS_MASK UINT32_C(0xffc0) + #define RX_TPA_END_CMPL_FLAGS_SFT 6 + uint16_t len; + /* + * This value is zero for TPA End completions. There is no data + * in the buffer that corresponds to the opaque value in this + * completion. + */ + uint32_t opaque; + /* + * This is a copy of the opaque field from the RX BD this + * completion corresponds to. + */ + uint8_t agg_bufs_v1; + /* unused1 is 1 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_END_CMPL_V1 UINT32_C(0x1) + /* + * This value is the number of aggregation buffers that follow + * this entry in the completion ring that are a part of this + * aggregation packet. If the value is zero, then the packet is + * completely contained in the buffer space provided in the + * aggregation start completion. + */ + #define RX_TPA_END_CMPL_AGG_BUFS_MASK UINT32_C(0x7e) + #define RX_TPA_END_CMPL_AGG_BUFS_SFT 1 + /* unused1 is 1 b */ + uint8_t tpa_segs; + /* This value is the number of segments in the TPA operation. */ + uint8_t payload_offset; + /* + * This value indicates the offset in bytes from the beginning + * of the packet where the inner payload starts. This value is + * valid for TCP, UDP, FCoE, and RoCE packets. A value of zero + * indicates an offset of 256 bytes. + */ + uint8_t agg_id; + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + /* unused2 is 1 b */ + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + #define RX_TPA_END_CMPL_AGG_ID_MASK UINT32_C(0xfe) + #define RX_TPA_END_CMPL_AGG_ID_SFT 1 + uint32_t tsdelta; + /* + * For non-GRO packets, this value is the timestamp delta + * between earliest and latest timestamp values for TPA packet. + * If packets were not time stamped, then delta will be zero. + * For GRO packets, this field is zero except for the following + * sub-fields. - tsdelta[31] Timestamp present indication. When + * '0', no Timestamp option is in the packet. When '1', then a + * Timestamp option is present in the packet. + */ +} __attribute__((packed)); + +/* last 16 bytes of RX TPA End Completion Record */ +struct rx_tpa_end_cmpl_hi { + uint32_t tpa_dup_acks; + /* unused3 is 28 b */ + /* + * This value is the number of duplicate ACKs that have been + * received as part of the TPA operation. + */ + #define RX_TPA_END_CMPL_TPA_DUP_ACKS_MASK UINT32_C(0xf) + #define RX_TPA_END_CMPL_TPA_DUP_ACKS_SFT 0 + /* unused3 is 28 b */ + uint16_t tpa_seg_len; + /* + * This value is the valid when TPA completion is active. It + * indicates the length of the longest segment of the TPA + * operation for LRO mode and the length of the first segment in + * GRO mode. This value may be used by GRO software to re- + * construct the original packet stream from the TPA packet. + * This is the length of all but the last segment for GRO. In + * LRO mode this value may be used to indicate MSS size to the + * stack. + */ + uint16_t unused_3; + /* unused4 is 16 b */ + uint16_t errors_v2; + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_END_CMPL_V2 UINT32_C(0x1) + /* + * This error indicates that there was some sort of problem with + * the BDs for the packet that was found after part of the + * packet was already placed. The packet should be treated as + * invalid. + */ + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_MASK UINT32_C(0xe) + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_SFT 1 + /* + * This error occurs when there is a fatal HW + * problem in the chip only. It indicates that + * there were not BDs on chip but that there was + * adequate reservation. provided by the TPA + * block. + */ + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_NOT_ON_CHIP \ + (UINT32_C(0x2) << 1) + /* + * This error occurs when TPA block was not + * configured to reserve adequate BDs for TPA + * operations on this RX ring. All data for the + * TPA operation was not placed. This error can + * also be generated when the number of segments + * is not programmed correctly in TPA and the 33 + * total aggregation buffers allowed for the TPA + * operation has been exceeded. + */ + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_RSV_ERROR \ + (UINT32_C(0x4) << 1) + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_LAST \ + RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_RSV_ERROR + #define RX_TPA_END_CMPL_ERRORS_MASK UINT32_C(0xfffe) + #define RX_TPA_END_CMPL_ERRORS_SFT 1 + uint16_t unused_4; + /* unused5 is 16 b */ + uint32_t start_opaque; + /* + * This is the opaque value that was completed for the TPA start + * completion that corresponds to this TPA end completion. + */ +} __attribute__((packed)); + +/* HWRM Forwarded Request (16 bytes) */ struct hwrm_fwd_req_cmpl { uint16_t req_len_type; /* Length of forwarded request in bytes. */ @@ -1188,7 +1747,7 @@ struct hwrm_fwd_req_cmpl { #define HWRM_FWD_REQ_CMPL_REQ_BUF_ADDR_SFT 1 } __attribute__((packed)); -/* HWRM Asynchronous Event Completion Record (16 bytes) */ +/* HWRM Asynchronous Event Completion Record (16 bytes) */ struct hwrm_async_event_cmpl { uint16_t type; /* unused1 is 10 b */ @@ -1210,19 +1769,20 @@ struct hwrm_async_event_cmpl { /* Link MTU changed */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE UINT32_C(0x1) /* Link speed changed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE UINT32_C(0x2) + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE UINT32_C(0x2) /* DCB Configuration changed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE UINT32_C(0x3) + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE UINT32_C(0x3) /* Port connection not allowed */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED UINT32_C(0x4) /* Link speed configuration was not allowed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED UINT32_C(0x5) + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED \ + UINT32_C(0x5) /* Link speed configuration change */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE UINT32_C(0x6) /* Port PHY configuration change */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE UINT32_C(0x7) /* Function driver unloaded */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD UINT32_C(0x10) + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD UINT32_C(0x10) /* Function driver loaded */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD UINT32_C(0x11) /* Function FLR related processing has completed */ @@ -1231,12 +1791,13 @@ struct hwrm_async_event_cmpl { #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD UINT32_C(0x20) /* PF driver loaded */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD UINT32_C(0x21) - /* VF Function Level Reset (FLR) */ + /* VF Function Level Reset (FLR) */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR UINT32_C(0x30) /* VF MAC Address Change */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE UINT32_C(0x31) /* PF-VF communication channel status change. */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE UINT32_C(0x32) + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE \ + UINT32_C(0x32) /* VF Configuration Change */ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE UINT32_C(0x33) /* HWRM Error */ @@ -1255,69 +1816,13 @@ struct hwrm_async_event_cmpl { #define HWRM_ASYNC_EVENT_CMPL_OPAQUE_MASK UINT32_C(0xfe) #define HWRM_ASYNC_EVENT_CMPL_OPAQUE_SFT 1 uint8_t timestamp_lo; - /* 8-lsb timestamp from POR (100-msec resolution) */ + /* 8-lsb timestamp from POR (100-msec resolution) */ uint16_t timestamp_hi; - /* 16-lsb timestamp from POR (100-msec resolution) */ + /* 16-lsb timestamp from POR (100-msec resolution) */ uint32_t event_data1; /* Event specific data */ } __attribute__((packed)); -/* - * Note: The Hardware Resource Manager (HWRM) manages various hardware resources - * inside the chip. The HWRM is implemented in firmware, and runs on embedded - * processors inside the chip. This firmware service is vital part of the chip. - * The chip can not be used by a driver or HWRM client without the HWRM. - */ - -/* Input (16 bytes) */ -struct input { - uint16_t req_type; - /* - * This value indicates what type of request this is. The format - * for the rest of the command is determined by this field. - */ - uint16_t cmpl_ring; - /* - * This value indicates the what completion ring the request - * will be optionally completed on. If the value is -1, then no - * CR completion will be generated. Any other value must be a - * valid CR ring_id value for this function. - */ - uint16_t seq_id; - /* This value indicates the command sequence number. */ - uint16_t target_id; - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function - * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - * - HWRM - */ - uint64_t resp_addr; - /* - * This is the host address where the response will be written - * when the request is complete. This area must be 16B aligned - * and must be cleared to zero before the request is made. - */ -} __attribute__((packed)); - -/* Output (8 bytes) */ -struct output { - uint16_t error_code; - /* - * Pass/Fail or error type Note: receiver to verify the in - * parameters, and fail the call with an error when appropriate - */ - uint16_t req_type; - /* This field returns the type of original request. */ - uint16_t seq_id; - /* This field provides original sequence number of the command. */ - uint16_t resp_len; - /* - * This field is the length of the response in bytes. The last - * byte of the response is a valid flag that will read as '1' - * when the command has been completely written to memory. - */ -} __attribute__((packed)); - /* hwrm_ver_get */ /* * Description: This function is called by a driver to determine the HWRM @@ -1327,7 +1832,7 @@ struct output { * interface or firmware version with major = 0, minor = 0, and update = 0 shall * be considered an invalid version. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_ver_get_input { uint16_t req_type; /* @@ -1384,7 +1889,7 @@ struct hwrm_ver_get_input { uint8_t unused_0[5]; } __attribute__((packed)); -/* Output (128 bytes) */ +/* Output (128 bytes) */ struct hwrm_ver_get_output { uint16_t error_code; /* @@ -1454,7 +1959,7 @@ struct hwrm_ver_get_output { /* * This field is a reserved field. This field can be used to * represent firmware branches or customer specific releases - * tied to a specific (major,minor,update) version of the HWRM + * tied to a specific (major,minor,update) version of the HWRM * firmware. */ uint8_t mgmt_fw_maj; @@ -1477,7 +1982,7 @@ struct hwrm_ver_get_output { /* * This field is a reserved field. This field can be used to * represent firmware branches or customer specific releases - * tied to a specific (major,minor,update) version + * tied to a specific (major,minor,update) version */ uint8_t netctrl_fw_maj; /* @@ -1500,7 +2005,7 @@ struct hwrm_ver_get_output { /* * This field is a reserved field. This field can be used to * represent firmware branches or customer specific releases - * tied to a specific (major,minor,update) version + * tied to a specific (major,minor,update) version */ uint32_t dev_caps_cfg; /* @@ -1512,13 +2017,27 @@ struct hwrm_ver_get_output { * supported. If set to 0, then secure firmware update behavior * is not supported. */ - #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED UINT32_C(0x1) + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED \ + UINT32_C(0x1) /* * If set to 1, then firmware based DCBX agent is supported. If * set to 0, then firmware based DCBX agent capability is not * supported on this device. */ - #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED UINT32_C(0x2) + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, then HWRM short command format is supported. If + * set to 0, then HWRM short command format is not supported. + */ + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, then HWRM short command format is required. If + * set to 0, then HWRM short command format is not required. + */ + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_INPUTUIRED \ + UINT32_C(0x8) uint8_t roce_fw_maj; /* * This field represents the major version of RoCE firmware. A @@ -1539,22 +2058,22 @@ struct hwrm_ver_get_output { /* * This field is a reserved field. This field can be used to * represent firmware branches or customer specific releases - * tied to a specific (major,minor,update) version + * tied to a specific (major,minor,update) version */ char hwrm_fw_name[16]; /* - * This field represents the name of HWRM FW (ASCII chars with + * This field represents the name of HWRM FW (ASCII chars with * NULL at the end). */ char mgmt_fw_name[16]; /* - * This field represents the name of mgmt FW (ASCII chars with + * This field represents the name of mgmt FW (ASCII chars with * NULL at the end). */ char netctrl_fw_name[16]; /* * This field represents the name of network control firmware - * (ASCII chars with NULL at the end). + * (ASCII chars with NULL at the end). */ uint32_t reserved2[4]; /* @@ -1563,7 +2082,7 @@ struct hwrm_ver_get_output { */ char roce_fw_name[16]; /* - * This field represents the name of RoCE FW (ASCII chars with + * This field represents the name of RoCE FW (ASCII chars with * NULL at the end). */ uint16_t chip_num; @@ -1584,7 +2103,7 @@ struct hwrm_ver_get_output { /* FPGA platform of the chip. */ #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_FPGA UINT32_C(0x1) /* Palladium platform of the chip. */ - #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_PALLADIUM UINT32_C(0x2) + #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_PALLADIUM UINT32_C(0x2) uint16_t max_req_win_len; /* * This field returns the maximum value of request window that @@ -1614,7 +2133,7 @@ struct hwrm_ver_get_output { /* hwrm_func_reset */ /* - * Description: This command resets a hardware function (PCIe function) and + * Description: This command resets a hardware function (PCIe function) and * frees any resources used by the function. This command shall be initiated by * the driver after an FLR has occurred to prepare the function for re-use. This * command may also be initiated by a driver prior to doing it's own @@ -1626,7 +2145,7 @@ struct hwrm_ver_get_output { * idled. The command returns all the resources owned by the function so a new * driver may allocate and configure resources normally. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_func_reset_input { uint16_t req_type; /* @@ -1668,7 +2187,7 @@ struct hwrm_func_reset_input { /* This value indicates the level of a function reset. */ /* * Reset the caller function and its children - * VFs (if any). If no children functions exist, + * VFs (if any). If no children functions exist, * then reset the caller function only. */ #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETALL UINT32_C(0x0) @@ -1681,7 +2200,8 @@ struct hwrm_func_reset_input { * It is an error to specify this level by a PF * driver with no children VFs. */ - #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETCHILDREN UINT32_C(0x2) + #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETCHILDREN \ + UINT32_C(0x2) /* * Reset a specific VF of the caller function * driver if the caller is the parent PF driver. @@ -1694,7 +2214,7 @@ struct hwrm_func_reset_input { uint8_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_func_reset_output { uint16_t error_code; /* @@ -1734,7 +2254,7 @@ struct hwrm_func_reset_output { * physical function. The output FID value is needed to configure Rings and * MSI-X vectors so their DMA operations appear correctly on the PCI bus. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_func_qcaps_input { uint16_t req_type; /* @@ -1765,12 +2285,12 @@ struct hwrm_func_qcaps_input { uint16_t fid; /* * Function ID of the function that is being queried. 0xFF... - * (All Fs) if the query is for the requesting function. + * (All Fs) if the query is for the requesting function. */ uint16_t unused_0[3]; } __attribute__((packed)); -/* Output (80 bytes) */ +/* Output (80 bytes) */ struct hwrm_func_qcaps_output { uint16_t error_code; /* @@ -1795,54 +2315,56 @@ struct hwrm_func_qcaps_output { uint16_t port_id; /* * Port ID of port that this function is associated with. Valid - * only for the PF. 0xFF... (All Fs) if this function is not - * associated with any port. 0xFF... (All Fs) if this function + * only for the PF. 0xFF... (All Fs) if this function is not + * associated with any port. 0xFF... (All Fs) if this function * is called from a VF. */ uint32_t flags; /* If 1, then Push mode is supported on this function. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PUSH_MODE_SUPPORTED UINT32_C(0x1) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PUSH_MODE_SUPPORTED UINT32_C(0x1) /* * If 1, then the global MSI-X auto-masking is enabled for the * device. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_GLOBAL_MSIX_AUTOMASKING UINT32_C(0x2) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_GLOBAL_MSIX_AUTOMASKING \ + UINT32_C(0x2) /* - * If 1, then the Precision Time Protocol (PTP) processing is + * If 1, then the Precision Time Protocol (PTP) processing is * supported on this function. The HWRM should enable PTP on - * only a single Physical Function (PF) per port. + * only a single Physical Function (PF) per port. */ #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PTP_SUPPORTED UINT32_C(0x4) /* - * If 1, then RDMA over Converged Ethernet (RoCE) v1 is + * If 1, then RDMA over Converged Ethernet (RoCE) v1 is * supported on this function. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V1_SUPPORTED UINT32_C(0x8) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V1_SUPPORTED UINT32_C(0x8) /* - * If 1, then RDMA over Converged Ethernet (RoCE) v2 is + * If 1, then RDMA over Converged Ethernet (RoCE) v2 is * supported on this function. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V2_SUPPORTED UINT32_C(0x10) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V2_SUPPORTED UINT32_C(0x10) /* * If 1, then control and configuration of WoL magic packet are * supported on this function. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_MAGICPKT_SUPPORTED UINT32_C(0x20) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_MAGICPKT_SUPPORTED \ + UINT32_C(0x20) /* * If 1, then control and configuration of bitmap pattern packet * are supported on this function. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_BMP_SUPPORTED UINT32_C(0x40) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_BMP_SUPPORTED UINT32_C(0x40) /* * If set to 1, then the control and configuration of rate limit * of an allocated TX ring on the queried function is supported. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_RING_RL_SUPPORTED UINT32_C(0x80) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_RING_RL_SUPPORTED UINT32_C(0x80) /* * If 1, then control and configuration of minimum and maximum * bandwidths are supported on the queried function. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_BW_CFG_SUPPORTED UINT32_C(0x100) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_BW_CFG_SUPPORTED UINT32_C(0x100) /* * If the query is for a VF, then this flag shall be ignored. If * this query is for a PF and this flag is set to 1, then the PF @@ -1851,7 +2373,8 @@ struct hwrm_func_qcaps_output { * set to 0, then the PF does not have the capability to set the * rate limits on the TX rings of its children VFs. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_TX_RING_RL_SUPPORTED UINT32_C(0x200) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_TX_RING_RL_SUPPORTED \ + UINT32_C(0x200) /* * If the query is for a VF, then this flag shall be ignored. If * this query is for a PF and this flag is set to 1, then the PF @@ -1861,7 +2384,17 @@ struct hwrm_func_qcaps_output { * capability to set the minimum or maximum bandwidths for its * children VFs. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_BW_CFG_SUPPORTED UINT32_C(0x400) + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_BW_CFG_SUPPORTED UINT32_C(0x400) + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then standard TX ring mode is supported on the + * queried function. If set to 0, then standard TX ring mode is + * not available on the queried function. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_STD_TX_RING_MODE_SUPPORTED \ + UINT32_C(0x800) uint8_t mac_address[6]; /* * This value is current MAC address configured for this @@ -1901,7 +2434,7 @@ struct hwrm_func_qcaps_output { uint16_t first_vf_id; /* * The identifier for the first VF enabled on a PF. This is - * valid only on the PF with SR-IOV enabled. 0xFF... (All Fs) if + * valid only on the PF with SR-IOV enabled. 0xFF... (All Fs) if * this command is called on a PF with SR-IOV disabled or on a * VF. */ @@ -1909,7 +2442,7 @@ struct hwrm_func_qcaps_output { /* * The maximum number of VFs that can be allocated to the * function. This is valid only on the PF with SR-IOV enabled. - * 0xFF... (All Fs) if this command is called on a PF with SR- + * 0xFF... (All Fs) if this command is called on a PF with SR- * IOV disabled or on a VF. */ uint16_t max_stat_ctx; @@ -1929,22 +2462,22 @@ struct hwrm_func_qcaps_output { */ uint32_t max_tx_em_flows; /* - * The maximum number of Exact Match (EM) flows that can be + * The maximum number of Exact Match (EM) flows that can be * offloaded by this function on the TX side. */ uint32_t max_tx_wm_flows; /* - * The maximum number of Wildcard Match (WM) flows that can be + * The maximum number of Wildcard Match (WM) flows that can be * offloaded by this function on the TX side. */ uint32_t max_rx_em_flows; /* - * The maximum number of Exact Match (EM) flows that can be + * The maximum number of Exact Match (EM) flows that can be * offloaded by this function on the RX side. */ uint32_t max_rx_wm_flows; /* - * The maximum number of Wildcard Match (WM) flows that can be + * The maximum number of Wildcard Match (WM) flows that can be * offloaded by this function on the RX side. */ uint32_t max_mcast_filters; @@ -1968,7 +2501,7 @@ struct hwrm_func_qcaps_output { * be allocated to the function. This number indicates the * maximum number of TX rings that can be assigned strict * priorities out of the maximum number of TX rings that can be - * allocated (max_tx_rings) to the function. + * allocated (max_tx_rings) to the function. */ uint8_t unused_0; uint8_t valid; @@ -1989,9 +2522,13 @@ struct hwrm_func_qcaps_output { * allows a physical function driver to query virtual functions that are * children of the physical function. The output FID value is needed to * configure Rings and MSI-X vectors so their DMA operations appear correctly on - * the PCI bus. + * the PCI bus. This command should be called by every driver after + * 'hwrm_func_cfg' to get the actual number of resources allocated by the HWRM. + * The values returned by hwrm_func_qcfg are the values the driver shall use. + * These values may be different than what was originally requested in the + * 'hwrm_func_cfg' command. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_func_qcfg_input { uint16_t req_type; /* @@ -2022,12 +2559,12 @@ struct hwrm_func_qcfg_input { uint16_t fid; /* * Function ID of the function that is being queried. 0xFF... - * (All Fs) if the query is for the requesting function. + * (All Fs) if the query is for the requesting function. */ uint16_t unused_0[3]; } __attribute__((packed)); -/* Output (72 bytes) */ +/* Output (72 bytes) */ struct hwrm_func_qcfg_output { uint16_t error_code; /* @@ -2052,7 +2589,7 @@ struct hwrm_func_qcfg_output { uint16_t port_id; /* * Port ID of port that this function is associated with. - * 0xFF... (All Fs) if this function is not associated with any + * 0xFF... (All Fs) if this function is not associated with any * port. */ uint16_t vlan; @@ -2060,15 +2597,15 @@ struct hwrm_func_qcfg_output { * This value is the current VLAN setting for this function. The * value of 0 for this field indicates no priority tagging or * VLAN is used. This field's format is same as 802.1Q Tag's Tag - * Control Information (TCI) format that includes both Priority - * Code Point (PCP) and VLAN Identifier (VID). + * Control Information (TCI) format that includes both Priority + * Code Point (PCP) and VLAN Identifier (VID). */ uint16_t flags; /* * If 1, then magic packet based Out-Of-Box WoL is enabled on * the port associated with this function. */ - #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_OOB_WOL_MAGICPKT_ENABLED \ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_OOB_WOL_MAGICPKT_ENABLED \ UINT32_C(0x1) /* * If 1, then bitmap pattern based Out-Of-Box WoL packet is @@ -2080,8 +2617,33 @@ struct hwrm_func_qcfg_output { * on the port associated with this function. If set to 0, then * DCBX agent is not running in the firmware. */ - #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_DCBX_AGENT_ENABLED \ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_DCBX_AGENT_ENABLED \ UINT32_C(0x4) + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then standard TX ring mode is enabled on the + * queried function. If set to 0, then the standard TX ring mode + * is disabled on the queried function. In this extended TX ring + * resource mode, the minimum and maximum bandwidth settings are + * not supported to allow the allocation of TX rings to span + * multiple scheduler nodes. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_STD_TX_RING_MODE_ENABLED \ + UINT32_C(0x8) + /* + * If set to 1 then FW based LLDP agent is enabled and running + * on the port associated with this function. If set to 0 then + * the LLDP agent is not running in the firmware. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_LLDP_AGENT_ENABLED UINT32_C(0x10) + /* + * If set to 1, then multi-host mode is active for this + * function. If set to 0, then multi-host mode is inactive for + * this function or not applicable for this device. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_MULTI_HOST UINT32_C(0x20) uint8_t mac_address[6]; /* * This value is current MAC address configured for this @@ -2091,8 +2653,8 @@ struct hwrm_func_qcfg_output { uint16_t pci_id; /* * This value is current PCI ID of this function. If ARI is - * enabled, then it is Bus Number (8b):Function Number(8b). - * Otherwise, it is Bus Number (8b):Device Number (4b):Function + * enabled, then it is Bus Number (8b):Function Number(8b). + * Otherwise, it is Bus Number (8b):Device Number (4b):Function * Number(4b). */ uint16_t alloc_rsscos_ctx; @@ -2146,22 +2708,27 @@ struct hwrm_func_qcfg_output { /* Multiple physical functions */ #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_MPFS UINT32_C(0x1) /* Network Partitioning 1.0 */ - #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0 \ - UINT32_C(0x2) + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0 UINT32_C(0x2) /* Network Partitioning 1.5 */ - #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_5 \ - UINT32_C(0x3) + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_5 UINT32_C(0x3) /* Network Partitioning 2.0 */ - #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR2_0 \ - UINT32_C(0x4) + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR2_0 UINT32_C(0x4) /* Unknown */ - #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_UNKNOWN \ - UINT32_C(0xff) - uint8_t unused_0; + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_UNKNOWN UINT32_C(0xff) + uint8_t port_pf_cnt; + /* + * This field will indicate number of physical functions on this + * port_partition. HWRM shall return unavail (i.e. value of 0) + * for this field when this command is used to query VF's + * configuration or from older firmware that doesn't support + * this field. + */ + /* number of PFs is not available */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PF_CNT_UNAVAIL UINT32_C(0x0) uint16_t dflt_vnic_id; /* The default VNIC ID assigned to a function that is being queried. */ + uint8_t unused_0; uint8_t unused_1; - uint8_t unused_2; uint32_t min_bw; /* * Minimum BW allocated for this function. The HWRM will @@ -2169,26 +2736,41 @@ struct hwrm_func_qcfg_output { * for the scheduler inside the device. A value of 0 indicates * the minimum bandwidth is not configured. */ - /* Bandwidth value */ - #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_MASK \ - UINT32_C(0xfffffff) + /* The bandwidth value. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff) #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_SFT 0 - /* Reserved */ - #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_RSVD UINT32_C(0x10000000) + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BYTES \ + (UINT32_C(0x1) << 28) + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_LAST \ + FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BYTES /* bw_value_unit is 3 b */ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_MASK \ UINT32_C(0xe0000000) #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_SFT 29 - /* Value is in Mbps */ - #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_MBPS \ + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_MEGA \ (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) /* Value is in 1/100th of a percentage of total bandwidth. */ - #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 \ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 \ (UINT32_C(0x1) << 29) /* Invalid unit */ - #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_INVALID \ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_INVALID \ (UINT32_C(0x7) << 29) - #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_LAST \ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_LAST \ FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_INVALID uint32_t max_bw; /* @@ -2197,43 +2779,58 @@ struct hwrm_func_qcfg_output { * for the scheduler inside the device. A value of 0 indicates * that the maximum bandwidth is not configured. */ - /* Bandwidth value */ - #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_MASK \ - UINT32_C(0xfffffff) + /* The bandwidth value. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff) #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_SFT 0 - /* Reserved */ - #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_RSVD UINT32_C(0x10000000) + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BYTES \ + (UINT32_C(0x1) << 28) + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_LAST \ + FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BYTES /* bw_value_unit is 3 b */ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_MASK \ UINT32_C(0xe0000000) #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_SFT 29 - /* Value is in Mbps */ - #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_MBPS \ + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_MEGA \ (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) /* Value is in 1/100th of a percentage of total bandwidth. */ - #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ (UINT32_C(0x1) << 29) /* Invalid unit */ - #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ (UINT32_C(0x7) << 29) - #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_LAST \ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_LAST \ FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_INVALID uint8_t evb_mode; /* * This value indicates the Edge virtual bridge mode for the * domain that this function belongs to. */ - /* No Edge Virtual Bridging (EVB) */ + /* No Edge Virtual Bridging (EVB) */ #define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_NO_EVB UINT32_C(0x0) - /* Virtual Ethernet Bridge (VEB) */ + /* Virtual Ethernet Bridge (VEB) */ #define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_VEB UINT32_C(0x1) - /* Virtual Ethernet Port Aggregator (VEPA) */ + /* Virtual Ethernet Port Aggregator (VEPA) */ #define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_VEPA UINT32_C(0x2) - uint8_t unused_3; + uint8_t unused_2; uint16_t alloc_vfs; /* * The number of VFs that are allocated to the function. This is - * valid only on the PF with SR-IOV enabled. 0xFF... (All Fs) if + * valid only on the PF with SR-IOV enabled. 0xFF... (All Fs) if * this command is called on a PF with SR-IOV disabled or on a * VF. */ @@ -2247,9 +2844,957 @@ struct hwrm_func_qcfg_output { uint16_t alloc_sp_tx_rings; /* * The number of strict priority transmit rings out of currently - * allocated TX rings to the function (alloc_tx_rings). + * allocated TX rings to the function (alloc_tx_rings). + */ + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_vlan_qcfg */ +/* + * Description: This command should be called by PF driver to get the current + * C-TAG, S-TAG and correcponsing PCP and TPID values configured for the + * function. + */ +/* Input (24 bytes) */ +struct hwrm_func_vlan_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being configured. If set + * to 0xFF... (All Fs), then the configuration is for the + * requesting function. */ + uint16_t unused_0[3]; +}; + +/* Output (40 bytes) */ +struct hwrm_func_vlan_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ + uint16_t stag_vid; + /* S-TAG VLAN identifier configured for the function. */ + uint8_t stag_pcp; + /* S-TAG PCP value configured for the function. */ uint8_t unused_4; + uint16_t stag_tpid; + /* + * S-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint16_t ctag_vid; + /* C-TAG VLAN identifier configured for the function. */ + uint8_t ctag_pcp; + /* C-TAG PCP value configured for the function. */ + uint8_t unused_5; + uint16_t ctag_tpid; + /* + * C-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint32_t rsvd2; + /* Future use. */ + uint32_t rsvd3; + /* Future use. */ + uint32_t unused_6; +}; + +/* hwrm_func_vlan_cfg */ +/* + * Description: This command allows PF driver to configure C-TAG, S-TAG and + * corresponding PCP and TPID values for a function. + */ +/* Input (48 bytes) */ +struct hwrm_func_vlan_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being configured. If set + * to 0xFF... (All Fs), then the configuration is for the + * requesting function. + */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t enables; + /* This bit must be '1' for the stag_vid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_VID UINT32_C(0x1) + /* This bit must be '1' for the ctag_vid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_VID UINT32_C(0x2) + /* This bit must be '1' for the stag_pcp field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_PCP UINT32_C(0x4) + /* This bit must be '1' for the ctag_pcp field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_PCP UINT32_C(0x8) + /* This bit must be '1' for the stag_tpid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_TPID UINT32_C(0x10) + /* This bit must be '1' for the ctag_tpid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_TPID UINT32_C(0x20) + uint16_t stag_vid; + /* S-TAG VLAN identifier configured for the function. */ + uint8_t stag_pcp; + /* S-TAG PCP value configured for the function. */ + uint8_t unused_2; + uint16_t stag_tpid; + /* + * S-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint16_t ctag_vid; + /* C-TAG VLAN identifier configured for the function. */ + uint8_t ctag_pcp; + /* C-TAG PCP value configured for the function. */ + uint8_t unused_3; + uint16_t ctag_tpid; + /* + * C-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint32_t rsvd1; + /* Future use. */ + uint32_t rsvd2; + /* Future use. */ + uint32_t unused_4; +}; + +/* Output (16 bytes) */ +struct hwrm_func_vlan_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +}; + +/* hwrm_func_cfg */ +/* + * Description: This command allows configuration of a PF by the corresponding + * PF driver. This command also allows configuration of a child VF by its parent + * PF driver. The input FID value is used to indicate what function is being + * configured. This allows a PF driver to configure the PF owned by itself or a + * virtual function that is a child of the PF. This command allows to reserve + * resources for a VF by its parent PF. To reverse the process, the command + * should be called with all enables flags cleared for resources. This will free + * allocated resources for the VF and return them to the resource pool. If this + * command is requested by a VF driver to configure or reserve resources, then + * the HWRM shall fail this command. If default MAC address and/or VLAN are + * provided in this command, then the HWRM shall set up appropriate MAC/VLAN + * filters for the function that is being configured. If source properties + * checks are enabled and default MAC address and/or IP address are provided in + * this command, then the HWRM shall set appropriate source property checks + * based on provided MAC and/or IP addresses. The parent PF driver should not + * set MTU/MRU for a VF using this command. This is to allow MTU/MRU setting by + * the VF driver. If the MTU or MRU for a VF is set by the PF driver, then the + * HWRM should ignore it. A function's MTU/MRU should be set prior to allocating + * RX VNICs or TX rings. A PF driver calls hwrm_func_cfg to allocate resources + * for itself or its children VFs. All function drivers shall call hwrm_func_cfg + * to reserve resources. A request to hwrm_func_cfg may not be fully granted; + * that is, a request for resources may be larger than what can be supported by + * the device and the HWRM will allocate the best set of resources available, + * but that may be less than requested. If all the amounts requested could not + * be fulfilled, the HWRM shall allocate what it could and return a status code + * of success. A function driver should call hwrm_func_qcfg immediately after + * hwrm_func_cfg to determine what resources were assigned to the configured + * function. A call by a PF driver to hwrm_func_cfg to allocate resources for + * itself shall only allocate resources for the PF driver to use, not for its + * children VFs. Likewise, a call to hwrm_func_qcfg shall return the resources + * available for the PF driver to use, not what is available to its children + * VFs. + */ +/* Input (88 bytes) */ +struct hwrm_func_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being configured. If set + * to 0xFF... (All Fs), then the the configuration is for the + * requesting function. + */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t flags; + /* + * When this bit is '1', the function is disabled with source + * MAC address check. This is an anti-spoofing check. If this + * flag is set, then the function shall be configured to + * disallow transmission of frames with the source MAC address + * that is configured for this function. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE \ + UINT32_C(0x1) + /* + * When this bit is '1', the function is enabled with source MAC + * address check. This is an anti-spoofing check. If this flag + * is set, then the function shall be configured to allow + * transmission of frames with the source MAC address that is + * configured for this function. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE \ + UINT32_C(0x2) + /* reserved */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_RSVD_MASK UINT32_C(0x1fc) + #define HWRM_FUNC_CFG_INPUT_FLAGS_RSVD_SFT 2 + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then standard TX ring mode is requested to be + * enabled on the function being configured. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE \ + UINT32_C(0x200) + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then the standard TX ring mode is requested to be + * disabled on the function being configured. In this extended + * TX ring resource mode, the minimum and maximum bandwidth + * settings are not supported to allow the allocation of TX + * rings to span multiple scheduler nodes. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE \ + UINT32_C(0x400) + /* + * If this bit is set, virtual mac address configured in this + * command will be persistent over warm boot. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_VIRT_MAC_PERSIST UINT32_C(0x800) + /* + * This bit only applies to the VF. If this bit is set, the + * statistic context counters will not be cleared when the + * statistic context is freed or a function reset is called on + * VF. This bit will be cleared when the PF is unloaded or a + * function reset is called on the PF. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_NO_AUTOCLEAR_STATISTIC \ + UINT32_C(0x1000) + uint32_t enables; + /* This bit must be '1' for the mtu field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MTU UINT32_C(0x1) + /* This bit must be '1' for the mru field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MRU UINT32_C(0x2) + /* + * This bit must be '1' for the num_rsscos_ctxs field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS UINT32_C(0x4) + /* + * This bit must be '1' for the num_cmpl_rings field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_CMPL_RINGS UINT32_C(0x8) + /* This bit must be '1' for the num_tx_rings field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_TX_RINGS UINT32_C(0x10) + /* This bit must be '1' for the num_rx_rings field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RX_RINGS UINT32_C(0x20) + /* This bit must be '1' for the num_l2_ctxs field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_L2_CTXS UINT32_C(0x40) + /* This bit must be '1' for the num_vnics field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_VNICS UINT32_C(0x80) + /* + * This bit must be '1' for the num_stat_ctxs field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_STAT_CTXS UINT32_C(0x100) + /* + * This bit must be '1' for the dflt_mac_addr field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR UINT32_C(0x200) + /* This bit must be '1' for the dflt_vlan field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN UINT32_C(0x400) + /* This bit must be '1' for the dflt_ip_addr field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_IP_ADDR UINT32_C(0x800) + /* This bit must be '1' for the min_bw field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MIN_BW UINT32_C(0x1000) + /* This bit must be '1' for the max_bw field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MAX_BW UINT32_C(0x2000) + /* + * This bit must be '1' for the async_event_cr field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR UINT32_C(0x4000) + /* + * This bit must be '1' for the vlan_antispoof_mode field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_VLAN_ANTISPOOF_MODE UINT32_C(0x8000) + /* + * This bit must be '1' for the allowed_vlan_pris field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_ALLOWED_VLAN_PRIS UINT32_C(0x10000) + /* This bit must be '1' for the evb_mode field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_EVB_MODE UINT32_C(0x20000) + /* + * This bit must be '1' for the num_mcast_filters field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_MCAST_FILTERS UINT32_C(0x40000) + /* + * This bit must be '1' for the num_hw_ring_grps field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS UINT32_C(0x80000) + uint16_t mtu; + /* + * The maximum transmission unit of the function. The HWRM + * should make sure that the mtu of the function does not exceed + * the mtu of the physical port that this function is associated + * with. In addition to configuring mtu per function, it is + * possible to configure mtu per transmit ring. By default, the + * mtu of each transmit ring associated with a function is equal + * to the mtu of the function. The HWRM should make sure that + * the mtu of each transmit ring that is assigned to a function + * has a valid mtu. + */ + uint16_t mru; + /* + * The maximum receive unit of the function. The HWRM should + * make sure that the mru of the function does not exceed the + * mru of the physical port that this function is associated + * with. In addition to configuring mru per function, it is + * possible to configure mru per vnic. By default, the mru of + * each vnic associated with a function is equal to the mru of + * the function. The HWRM should make sure that the mru of each + * vnic that is assigned to a function has a valid mru. + */ + uint16_t num_rsscos_ctxs; + /* The number of RSS/COS contexts requested for the function. */ + uint16_t num_cmpl_rings; + /* + * The number of completion rings requested for the function. + * This does not include the rings allocated to any children + * functions if any. + */ + uint16_t num_tx_rings; + /* + * The number of transmit rings requested for the function. This + * does not include the rings allocated to any children + * functions if any. + */ + uint16_t num_rx_rings; + /* + * The number of receive rings requested for the function. This + * does not include the rings allocated to any children + * functions if any. + */ + uint16_t num_l2_ctxs; + /* The requested number of L2 contexts for the function. */ + uint16_t num_vnics; + /* The requested number of vnics for the function. */ + uint16_t num_stat_ctxs; + /* The requested number of statistic contexts for the function. */ + uint16_t num_hw_ring_grps; + /* + * The number of HW ring groups that should be reserved for this + * function. + */ + uint8_t dflt_mac_addr[6]; + /* The default MAC address for the function being configured. */ + uint16_t dflt_vlan; + /* + * The default VLAN for the function being configured. This + * field's format is same as 802.1Q Tag's Tag Control + * Information (TCI) format that includes both Priority Code + * Point (PCP) and VLAN Identifier (VID). + */ + uint32_t dflt_ip_addr[4]; + /* + * The default IP address for the function being configured. + * This address is only used in enabling source property check. + */ + uint32_t min_bw; + /* + * Minimum BW allocated for this function. The HWRM will + * translate this value into byte counter and time interval used + * for the scheduler inside the device. + */ + /* The bandwidth value. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_SFT 0 + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_LAST \ + FUNC_CFG_INPUT_MIN_BW_SCALE_BYTES + /* bw_value_unit is 3 b */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_MASK \ + UINT32_C(0xe0000000) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_SFT 29 + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_MEGA \ + (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) + /* Value is in 1/100th of a percentage of total bandwidth. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 \ + (UINT32_C(0x1) << 29) + /* Invalid unit */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_INVALID \ + (UINT32_C(0x7) << 29) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_LAST \ + FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_INVALID + uint32_t max_bw; + /* + * Maximum BW allocated for this function. The HWRM will + * translate this value into byte counter and time interval used + * for the scheduler inside the device. + */ + /* The bandwidth value. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_MASK \ + UINT32_C(0xfffffff) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_SFT 0 + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_LAST \ + FUNC_CFG_INPUT_MAX_BW_SCALE_BYTES + /* bw_value_unit is 3 b */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_MASK \ + UINT32_C(0xe0000000) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_SFT 29 + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_MEGA \ + (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) + /* Value is in 1/100th of a percentage of total bandwidth. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ + (UINT32_C(0x1) << 29) + /* Invalid unit */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ + (UINT32_C(0x7) << 29) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_LAST \ + FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID + uint16_t async_event_cr; + /* + * ID of the target completion ring for receiving asynchronous + * event completions. If this field is not valid, then the HWRM + * shall use the default completion ring of the function that is + * being configured as the target completion ring for providing + * any asynchronous event completions for that function. If this + * field is valid, then the HWRM shall use the completion ring + * identified by this ID as the target completion ring for + * providing any asynchronous event completions for the function + * that is being configured. + */ + uint8_t vlan_antispoof_mode; + /* VLAN Anti-spoofing mode. */ + /* No VLAN anti-spoofing checks are enabled */ + #define HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_NOCHECK UINT32_C(0x0) + /* Validate VLAN against the configured VLAN(s) */ + #define HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN \ + UINT32_C(0x1) + /* Insert VLAN if it does not exist, otherwise discard */ + #define HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_INSERT_IF_VLANDNE \ + UINT32_C(0x2) + /* + * Insert VLAN if it does not exist, override + * VLAN if it exists + */ + #define \ + HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN \ + UINT32_C(0x3) + uint8_t allowed_vlan_pris; + /* + * This bit field defines VLAN PRIs that are allowed on this + * function. If nth bit is set, then VLAN PRI n is allowed on + * this function. + */ + uint8_t evb_mode; + /* + * The HWRM shall allow a PF driver to change EVB mode for the + * partition it belongs to. The HWRM shall not allow a VF driver + * to change the EVB mode. The HWRM shall take into account the + * switching of EVB mode from one to another and reconfigure + * hardware resources as appropriately. The switching from VEB + * to VEPA mode requires the disabling of the loopback traffic. + * Additionally, source knock outs are handled differently in + * VEB and VEPA modes. + */ + /* No Edge Virtual Bridging (EVB) */ + #define HWRM_FUNC_CFG_INPUT_EVB_MODE_NO_EVB UINT32_C(0x0) + /* Virtual Ethernet Bridge (VEB) */ + #define HWRM_FUNC_CFG_INPUT_EVB_MODE_VEB UINT32_C(0x1) + /* Virtual Ethernet Port Aggregator (VEPA) */ + #define HWRM_FUNC_CFG_INPUT_EVB_MODE_VEPA UINT32_C(0x2) + uint8_t unused_2; + uint16_t num_mcast_filters; + /* + * The number of multicast filters that should be reserved for + * this function on the RX side. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_qstats */ +/* + * Description: This command returns statistics of a function. The input FID + * value is used to indicate what function is being queried. This allows a + * physical function driver to query virtual functions that are children of the + * physical function. The HWRM shall return any unsupported counter with a value + * of 0xFFFFFFFF for 32-bit counters and 0xFFFFFFFFFFFFFFFF for 64-bit counters. + */ +/* Input (24 bytes) */ +struct hwrm_func_qstats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being queried. 0xFF... + * (All Fs) if the query is for the requesting function. + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (176 bytes) */ +struct hwrm_func_qstats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint64_t tx_ucast_pkts; + /* Number of transmitted unicast packets on the function. */ + uint64_t tx_mcast_pkts; + /* Number of transmitted multicast packets on the function. */ + uint64_t tx_bcast_pkts; + /* Number of transmitted broadcast packets on the function. */ + uint64_t tx_err_pkts; + /* + * Number of transmitted packets that were discarded due to + * internal NIC resource problems. For transmit, this can only + * happen if TMP is configured to allow dropping in HOL blocking + * conditions, which is not a normal configuration. + */ + uint64_t tx_drop_pkts; + /* + * Number of dropped packets on transmit path on the function. + * These are packets that have been marked for drop by the TE + * CFA block or are packets that exceeded the transmit MTU limit + * for the function. + */ + uint64_t tx_ucast_bytes; + /* Number of transmitted bytes for unicast traffic on the function. */ + uint64_t tx_mcast_bytes; + /* + * Number of transmitted bytes for multicast traffic on the + * function. + */ + uint64_t tx_bcast_bytes; + /* + * Number of transmitted bytes for broadcast traffic on the + * function. + */ + uint64_t rx_ucast_pkts; + /* Number of received unicast packets on the function. */ + uint64_t rx_mcast_pkts; + /* Number of received multicast packets on the function. */ + uint64_t rx_bcast_pkts; + /* Number of received broadcast packets on the function. */ + uint64_t rx_err_pkts; + /* + * Number of received packets that were discarded on the + * function due to resource limitations. This can happen for 3 + * reasons. # The BD used for the packet has a bad format. # + * There were no BDs available in the ring for the packet. # + * There were no BDs available on-chip for the packet. + */ + uint64_t rx_drop_pkts; + /* + * Number of dropped packets on received path on the function. + * These are packets that have been marked for drop by the RE + * CFA. + */ + uint64_t rx_ucast_bytes; + /* Number of received bytes for unicast traffic on the function. */ + uint64_t rx_mcast_bytes; + /* Number of received bytes for multicast traffic on the function. */ + uint64_t rx_bcast_bytes; + /* Number of received bytes for broadcast traffic on the function. */ + uint64_t rx_agg_pkts; + /* Number of aggregated unicast packets on the function. */ + uint64_t rx_agg_bytes; + /* Number of aggregated unicast bytes on the function. */ + uint64_t rx_agg_events; + /* Number of aggregation events on the function. */ + uint64_t rx_agg_aborts; + /* Number of aborted aggregations on the function. */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_clr_stats */ +/* + * Description: This command clears statistics of a function. The input FID + * value is used to indicate what function's statistics is being cleared. This + * allows a physical function driver to clear statistics of virtual functions + * that are children of the physical function. + */ +/* Input (24 bytes) */ +struct hwrm_func_clr_stats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function. 0xFF... (All Fs) if the query is + * for the requesting function. + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_clr_stats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_vf_vnic_ids_query */ +/* Description: This command is used to query vf vnic ids. */ +/* Input (32 bytes) */ +struct hwrm_func_vf_vnic_ids_query_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t vf_id; + /* + * This value is used to identify a Virtual Function (VF). The + * scope of VF ID is local within a PF. + */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t max_vnic_id_cnt; + /* Max number of vnic ids in vnic id table */ + uint64_t vnic_id_tbl_addr; + /* This is the address for VF VNIC ID table */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_vf_vnic_ids_query_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t vnic_id_cnt; + /* + * Actual number of vnic ids Each VNIC ID is written as a 32-bit + * number. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; uint8_t valid; /* * This field is used in Output records to indicate that the @@ -2268,7 +3813,7 @@ struct hwrm_func_qcfg_output { * function driver shall use this command during the driver initialization right * after the HWRM version discovery and default ring resources allocation. */ -/* Input (80 bytes) */ +/* Input (80 bytes) */ struct hwrm_func_drv_rgtr_input { uint16_t req_type; /* @@ -2324,9 +3869,12 @@ struct hwrm_func_drv_rgtr_input { * This bit must be '1' for the async_event_fwd field to be * configured. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD UINT32_C(0x10) + #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD UINT32_C(0x10) uint16_t os_type; - /* This value indicates the type of OS. */ + /* + * This value indicates the type of OS. The values are based on + * CIM_OperatingSystem.mof file as published by the DMTF. + */ /* Unknown */ #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_UNKNOWN UINT32_C(0x0) /* Other OS not listed below. */ @@ -2374,17 +3922,17 @@ struct hwrm_func_drv_rgtr_input { uint32_t async_event_fwd[8]; /* * This is a 256-bit bit mask provided by the function driver - * (PF or VF driver) to indicate the list of asynchronous event + * (PF or VF driver) to indicate the list of asynchronous event * completions to be forwarded. Nth bit refers to the Nth * event_id. Setting Nth bit to 1 by the function driver shall * result in the HWRM forwarding asynchronous event completion - * with event_id equal to N. If all bits are set to 0 (value of + * with event_id equal to N. If all bits are set to 0 (value of * 0), then the HWRM shall not forward any asynchronous event * completion to this function driver. */ } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_func_drv_rgtr_output { uint16_t error_code; /* @@ -2422,7 +3970,7 @@ struct hwrm_func_drv_rgtr_output { * the HWRM. A function driver shall implement this command. A function driver * shall use this command during the driver unloading. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_func_drv_unrgtr_input { uint16_t req_type; /* @@ -2455,11 +4003,12 @@ struct hwrm_func_drv_unrgtr_input { * When this bit is '1', the function driver is notifying the * HWRM to prepare for the shutdown. */ - #define HWRM_FUNC_DRV_UNRGTR_INPUT_FLAGS_PREPARE_FOR_SHUTDOWN UINT32_C(0x1) + #define HWRM_FUNC_DRV_UNRGTR_INPUT_FLAGS_PREPARE_FOR_SHUTDOWN \ + UINT32_C(0x1) uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_func_drv_unrgtr_output { uint16_t error_code; /* @@ -2491,6 +4040,319 @@ struct hwrm_func_drv_unrgtr_output { */ } __attribute__((packed)); +/* hwrm_func_buf_rgtr */ +/* + * Description: This command is used by the PF driver to register buffers used + * in the PF-VF communication with the HWRM. The PF driver uses this command to + * register buffers for each PF-VF channel. A parent PF may issue this command + * per child VF. If VF ID is not valid, then this command is used to register + * buffers for all children VFs of the PF. + */ +/* Input (128 bytes) */ +struct hwrm_func_buf_rgtr_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the vf_id field to be configured. */ + #define HWRM_FUNC_BUF_RGTR_INPUT_ENABLES_VF_ID UINT32_C(0x1) + /* This bit must be '1' for the err_buf_addr field to be configured. */ + #define HWRM_FUNC_BUF_RGTR_INPUT_ENABLES_ERR_BUF_ADDR UINT32_C(0x2) + uint16_t vf_id; + /* + * This value is used to identify a Virtual Function (VF). The + * scope of VF ID is local within a PF. + */ + uint16_t req_buf_num_pages; + /* + * This field represents the number of pages used for request + * buffer(s). + */ + uint16_t req_buf_page_size; + /* This field represents the page size used for request buffer(s). */ + /* 16 bytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_16B UINT32_C(0x4) + /* 4 Kbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_4K UINT32_C(0xc) + /* 8 Kbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_8K UINT32_C(0xd) + /* 64 Kbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_64K UINT32_C(0x10) + /* 2 Mbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_2M UINT32_C(0x15) + /* 4 Mbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_4M UINT32_C(0x16) + /* 1 Gbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_1G UINT32_C(0x1e) + uint16_t req_buf_len; + /* The length of the request buffer per VF in bytes. */ + uint16_t resp_buf_len; + /* The length of the response buffer in bytes. */ + uint8_t unused_0; + uint8_t unused_1; + uint64_t req_buf_page_addr[10]; + /* This field represents the page address of req buffer. */ + uint64_t error_buf_addr; + /* + * This field is used to receive the error reporting from the + * chipset. Only applicable for PFs. + */ + uint64_t resp_buf_addr; + /* This field is used to receive the response forwarded by the HWRM. */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_buf_rgtr_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_buf_unrgtr */ +/* + * Description: This command is used by the PF driver to unregister buffers used + * in the PF-VF communication with the HWRM. The PF driver uses this command to + * unregister buffers for PF-VF communication. A parent PF may issue this + * command to unregister buffers for communication between the PF and a specific + * VF. If the VF ID is not valid, then this command is used to unregister + * buffers used for communications with all children VFs of the PF. + */ +/* Input (24 bytes) */ +struct hwrm_func_buf_unrgtr_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the vf_id field to be configured. */ + #define HWRM_FUNC_BUF_UNRGTR_INPUT_ENABLES_VF_ID UINT32_C(0x1) + uint16_t vf_id; + /* + * This value is used to identify a Virtual Function (VF). The + * scope of VF ID is local within a PF. + */ + uint16_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_buf_unrgtr_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_vf_cfg */ +/* + * Description: This command allows configuration of a VF by its driver. If this + * function is called by a PF driver, then the HWRM shall fail this command. If + * guest VLAN and/or MAC address are provided in this command, then the HWRM + * shall set up appropriate MAC/VLAN filters for the VF that is being + * configured. A VF driver should set VF MTU/MRU using this command prior to + * allocating RX VNICs or TX rings for the corresponding VF. + */ +/* Input (32 bytes) */ + +struct hwrm_func_vf_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format for the + * rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request will be + * optionally completed on. If the value is -1, then no CR completion + * will be generated. Any other value must be a valid CR ring_id value + * for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written when the + * request is complete. This area must be 16B aligned and must be + * cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the mtu field to be configured. */ + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_MTU UINT32_C(0x1) + /* This bit must be '1' for the guest_vlan field to be configured. */ + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_GUEST_VLAN UINT32_C(0x2) + /* + * This bit must be '1' for the async_event_cr field to be configured. + */ + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_ASYNC_EVENT_CR UINT32_C(0x4) + /* This bit must be '1' for the dflt_mac_addr field to be configured. */ + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_DFLT_MAC_ADDR UINT32_C(0x8) + uint16_t mtu; + /* + * The maximum transmission unit requested on the function. The HWRM + * should make sure that the mtu of the function does not exceed the mtu + * of the physical port that this function is associated with. In + * addition to requesting mtu per function, it is possible to configure + * mtu per transmit ring. By default, the mtu of each transmit ring + * associated with a function is equal to the mtu of the function. The + * HWRM should make sure that the mtu of each transmit ring that is + * assigned to a function has a valid mtu. + */ + uint16_t guest_vlan; + /* + * The guest VLAN for the function being configured. This field's format + * is same as 802.1Q Tag's Tag Control Information (TCI) format that + * includes both Priority Code Point (PCP) and VLAN Identifier (VID). + */ + uint16_t async_event_cr; + /* + * ID of the target completion ring for receiving asynchronous event + * completions. If this field is not valid, then the HWRM shall use the + * default completion ring of the function that is being configured as + * the target completion ring for providing any asynchronous event + * completions for that function. If this field is valid, then the HWRM + * shall use the completion ring identified by this ID as the target + * completion ring for providing any asynchronous event completions for + * the function that is being configured. + */ + uint8_t dflt_mac_addr[6]; + /* + * This value is the current MAC address requested by the VF driver to + * be configured on this VF. A value of 00-00-00-00-00-00 indicates no + * MAC address configuration is requested by the VF driver. The parent + * PF driver may reject or overwrite this MAC address. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ + +struct hwrm_func_vf_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in parameters, + * and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the output is + * completely written to RAM. This field should be read as '1' to + * indicate that the output has been completely written. When writing a + * command completion or response to an internal processor, the order of + * writes has to be such that this field is written last. + */ +} __attribute__((packed)); + /* hwrm_port_phy_cfg */ /* * Description: This command configures the PHY device for the port. It allows @@ -2500,7 +4362,7 @@ struct hwrm_func_drv_unrgtr_output { * configure PHY using this command. In a network partition mode, a PF driver * shall not be allowed to configure PHY using this command. */ -/* Input (56 bytes) */ +/* Input (56 bytes) */ struct hwrm_port_phy_cfg_input { uint16_t req_type; /* @@ -2540,19 +4402,8 @@ struct hwrm_port_phy_cfg_input { * PHY configuration and settings specified in this command. */ #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY UINT32_C(0x1) - /* - * When this bit is set to '1', the link shall be forced to be - * taken down. # When this bit is set to '1", all other command - * input settings related to the link speed shall be ignored. - * Once the link state is forced down, it can be explicitly - * cleared from that state by setting this flag to '0'. # If - * this flag is set to '0', then the link shall be cleared from - * forced down state if the link is in forced down state. There - * may be conditions (e.g. out-of-band or sideband configuration - * changes for the link) outside the scope of the HWRM - * implementation that may clear forced down link state. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DOWN UINT32_C(0x2) + /* deprecated bit. Do not use!!! */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_DEPRECATED UINT32_C(0x2) /* * When this bit is set to '1', the link shall be forced to the * force_link_speed value. When this bit is set to '1', the HWRM @@ -2570,14 +4421,14 @@ struct hwrm_port_phy_cfg_input { */ #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESTART_AUTONEG UINT32_C(0x8) /* - * When this bit is set to '1', Energy Efficient Ethernet (EEE) + * When this bit is set to '1', Energy Efficient Ethernet (EEE) * is requested to be enabled on this link. If EEE is not * supported on this port, then this flag shall be ignored by * the HWRM. */ #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_ENABLE UINT32_C(0x10) /* - * When this bit is set to '1', Energy Efficient Ethernet (EEE) + * When this bit is set to '1', Energy Efficient Ethernet (EEE) * is requested to be disabled on this link. If EEE is not * supported on this port, then this flag shall be ignored by * the HWRM. @@ -2598,49 +4449,67 @@ struct hwrm_port_phy_cfg_input { * ignored by the HWRM. If EEE is disabled on this port, then * this flag shall be ignored by the HWRM. */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_TX_LPI_DISABLE UINT32_C(0x80) + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_TX_LPI_DISABLE UINT32_C(0x80) /* * When set to 1, then the HWRM shall enable FEC * autonegotitation on this port if supported. When set to 0, * then this flag shall be ignored. If FEC autonegotiation is * not supported, then the HWRM shall ignore this flag. */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_AUTONEG_ENABLE UINT32_C(0x100) + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_AUTONEG_ENABLE UINT32_C(0x100) /* * When set to 1, then the HWRM shall disable FEC * autonegotiation on this port if supported. When set to 0, * then this flag shall be ignored. If FEC autonegotiation is * not supported, then the HWRM shall ignore this flag. */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_AUTONEG_DISABLE UINT32_C(0x200) + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_AUTONEG_DISABLE \ + UINT32_C(0x200) /* - * When set to 1, then the HWRM shall enable FEC CLAUSE 74 (Fire + * When set to 1, then the HWRM shall enable FEC CLAUSE 74 (Fire * Code) on this port if supported. When set to 0, then this * flag shall be ignored. If FEC CLAUSE 74 is not supported, * then the HWRM shall ignore this flag. */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE74_ENABLE UINT32_C(0x400) + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE74_ENABLE \ + UINT32_C(0x400) /* * When set to 1, then the HWRM shall disable FEC CLAUSE 74 - * (Fire Code) on this port if supported. When set to 0, then + * (Fire Code) on this port if supported. When set to 0, then * this flag shall be ignored. If FEC CLAUSE 74 is not * supported, then the HWRM shall ignore this flag. */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE74_DISABLE UINT32_C(0x800) + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE74_DISABLE \ + UINT32_C(0x800) /* - * When set to 1, then the HWRM shall enable FEC CLAUSE 91 (Reed + * When set to 1, then the HWRM shall enable FEC CLAUSE 91 (Reed * Solomon) on this port if supported. When set to 0, then this * flag shall be ignored. If FEC CLAUSE 91 is not supported, * then the HWRM shall ignore this flag. */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE91_ENABLE UINT32_C(0x1000) + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE91_ENABLE \ + UINT32_C(0x1000) /* * When set to 1, then the HWRM shall disable FEC CLAUSE 91 - * (Reed Solomon) on this port if supported. When set to 0, then + * (Reed Solomon) on this port if supported. When set to 0, then * this flag shall be ignored. If FEC CLAUSE 91 is not * supported, then the HWRM shall ignore this flag. */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE91_DISABLE UINT32_C(0x2000) + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE91_DISABLE \ + UINT32_C(0x2000) + /* + * When this bit is set to '1', the link shall be forced to be + * taken down. # When this bit is set to '1", all other command + * input settings related to the link speed shall be ignored. + * Once the link state is forced down, it can be explicitly + * cleared from that state by setting this flag to '0'. # If + * this flag is set to '0', then the link shall be cleared from + * forced down state if the link is in forced down state. There + * may be conditions (e.g. out-of-band or sideband configuration + * changes for the link) outside the scope of the HWRM + * implementation that may clear forced down link state. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DWN UINT32_C(0x4000) uint32_t enables; /* This bit must be '1' for the auto_mode field to be configured. */ #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE UINT32_C(0x1) @@ -2657,7 +4526,8 @@ struct hwrm_port_phy_cfg_input { * This bit must be '1' for the auto_link_speed_mask field to be * configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK UINT32_C(0x10) + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK \ + UINT32_C(0x10) /* This bit must be '1' for the wirespeed field to be configured. */ #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_WIOUTPUTEED UINT32_C(0x20) /* This bit must be '1' for the lpbk field to be configured. */ @@ -2670,7 +4540,8 @@ struct hwrm_port_phy_cfg_input { * This bit must be '1' for the eee_link_speed_mask field to be * configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_EEE_LINK_SPEED_MASK UINT32_C(0x200) + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_EEE_LINK_SPEED_MASK \ + UINT32_C(0x200) /* This bit must be '1' for the tx_lpi_timer field to be configured. */ #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_TX_LPI_TIMER UINT32_C(0x400) uint16_t port_id; @@ -2772,7 +4643,7 @@ struct hwrm_port_phy_cfg_input { * set to 1, auto_pause bits should be ignored and should be set * to 0. */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_AUTONEG_PAUSE UINT32_C(0x4) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_AUTONEG_PAUSE UINT32_C(0x4) uint8_t unused_0; uint16_t auto_link_speed; /* @@ -2808,34 +4679,46 @@ struct hwrm_port_phy_cfg_input { * autoneg_mode is "mask". If unsupported speed is enabled an * error will be generated. */ - /* 100Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MBHD UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MB UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GBHD UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GB UINT32_C(0x8) + /* 100Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MBHD \ + UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MB \ + UINT32_C(0x2) + /* 1Gb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GBHD \ + UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GB \ + UINT32_C(0x8) /* 2Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2GB UINT32_C(0x10) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2GB \ + UINT32_C(0x10) /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2_5GB UINT32_C(0x20) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2_5GB \ + UINT32_C(0x20) /* 10Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10GB UINT32_C(0x40) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10GB UINT32_C(0x40) /* 20Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_20GB UINT32_C(0x80) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_20GB UINT32_C(0x80) /* 25Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_25GB UINT32_C(0x100) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_25GB \ + UINT32_C(0x100) /* 40Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_40GB UINT32_C(0x200) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_40GB \ + UINT32_C(0x200) /* 50Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_50GB UINT32_C(0x400) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_50GB \ + UINT32_C(0x400) /* 100Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100GB UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MBHD UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MB UINT32_C(0x2000) + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100GB \ + UINT32_C(0x800) + /* 10Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MBHD \ + UINT32_C(0x1000) + /* 10Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MB \ + UINT32_C(0x2000) uint8_t wirespeed; /* This value controls the wirespeed feature. */ /* Wirespeed feature is disabled. */ @@ -2878,7 +4761,7 @@ struct hwrm_port_phy_cfg_input { uint32_t preemphasis; /* * This value controls the pre-emphasis to be used for the link. - * Driver should not set this value (use enable.preemphasis = 0) + * Driver should not set this value (use enable.preemphasis = 0) * unless driver is sure of setting. Normally HWRM FW will * determine proper pre-emphasis. */ @@ -2892,19 +4775,19 @@ struct hwrm_port_phy_cfg_input { * speed shall be provided in this mask. */ /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD1 UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_100MB UINT32_C(0x2) + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD1 UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_100MB UINT32_C(0x2) /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD2 UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD2 UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_1GB UINT32_C(0x8) /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD3 UINT32_C(0x10) + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD3 UINT32_C(0x10) /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD4 UINT32_C(0x20) + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD4 UINT32_C(0x20) /* 10Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_10GB UINT32_C(0x40) + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_10GB UINT32_C(0x40) uint8_t unused_2; uint8_t unused_3; uint32_t tx_lpi_timer; @@ -2913,11 +4796,11 @@ struct hwrm_port_phy_cfg_input { * Reuested setting of TX LPI timer in microseconds. This field * is valid only when EEE is enabled and TX LPI is enabled. */ - #define HWRM_PORT_PHY_CFG_INPUT_TX_LPI_TIMER_MASK UINT32_C(0xffffff) + #define HWRM_PORT_PHY_CFG_INPUT_TX_LPI_TIMER_MASK UINT32_C(0xffffff) #define HWRM_PORT_PHY_CFG_INPUT_TX_LPI_TIMER_SFT 0 } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_port_phy_cfg_output { uint16_t error_code; /* @@ -2951,7 +4834,7 @@ struct hwrm_port_phy_cfg_output { /* hwrm_port_phy_qcfg */ /* Description: This command queries the PHY configuration for the port. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_port_phy_qcfg_input { uint16_t req_type; /* @@ -2984,7 +4867,7 @@ struct hwrm_port_phy_qcfg_input { uint16_t unused_0[3]; } __attribute__((packed)); -/* Output (96 bytes) */ +/* Output (96 bytes) */ struct hwrm_port_phy_qcfg_output { uint16_t error_code; /* @@ -3062,50 +4945,47 @@ struct hwrm_port_phy_qcfg_output { * each speed that is supported, the corrresponding bit will be * set to '1'. */ - /* 100Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MBHD \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MB UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GBHD UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ + /* 100Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MBHD UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MB UINT32_C(0x2) + /* 1Gb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GBHD UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB UINT32_C(0x8) /* 2Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2GB UINT32_C(0x10) /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB UINT32_C(0x20) + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB UINT32_C(0x20) /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB UINT32_C(0x40) + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB UINT32_C(0x40) /* 20Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB UINT32_C(0x80) + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB UINT32_C(0x80) /* 25Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB UINT32_C(0x100) + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB UINT32_C(0x100) /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB UINT32_C(0x200) + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB UINT32_C(0x200) /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB UINT32_C(0x400) + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB UINT32_C(0x400) /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB UINT32_C(0x800) + /* 10Mb link speed (Half-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MBHD UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MB UINT32_C(0x2000) + /* 10Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MB UINT32_C(0x2000) uint16_t force_link_speed; /* * Current setting of forced link speed. When the link speed is * not being forced, this value shall be set to 0. */ /* 100Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100MB \ - UINT32_C(0x1) + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100MB UINT32_C(0x1) /* 1Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_1GB UINT32_C(0xa) /* 2Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_2GB UINT32_C(0x14) /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_2_5GB \ - UINT32_C(0x19) + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_2_5GB UINT32_C(0x19) /* 10Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_10GB UINT32_C(0x64) /* 20Mb link speed */ @@ -3113,14 +4993,13 @@ struct hwrm_port_phy_qcfg_output { /* 25Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_25GB UINT32_C(0xfa) /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_40GB UINT32_C(0x190) + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_40GB UINT32_C(0x190) /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_50GB UINT32_C(0x1f4) + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_50GB UINT32_C(0x1f4) /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100GB \ - UINT32_C(0x3e8) + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100GB UINT32_C(0x3e8) /* 10Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_10MB UINT32_C(0xffff) + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_10MB UINT32_C(0xffff) uint8_t auto_mode; /* Current setting of auto negotiation mode. */ /* @@ -3142,8 +5021,7 @@ struct hwrm_port_phy_qcfg_output { * DEPRECATED. An HWRM client should not use * this mode. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ONE_OR_BELOW \ - UINT32_C(0x3) + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ONE_OR_BELOW UINT32_C(0x3) /* * Select the speeds based on the corresponding * link speed mask value that is provided. @@ -3177,8 +5055,7 @@ struct hwrm_port_phy_qcfg_output { * set to 1, auto_pause bits should be ignored and should be set * to 0. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_AUTONEG_PAUSE \ - UINT32_C(0x4) + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_AUTONEG_PAUSE UINT32_C(0x4) uint16_t auto_link_speed; /* * Current setting for auto_link_speed. This field is only valid @@ -3203,9 +5080,9 @@ struct hwrm_port_phy_qcfg_output { /* 50Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_50GB UINT32_C(0x1f4) /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_100GB UINT32_C(0x3e8) + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_100GB UINT32_C(0x3e8) /* 10Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_10MB UINT32_C(0xffff) + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_10MB UINT32_C(0xffff) uint16_t auto_link_speed_mask; /* * Current setting for auto_link_speed_mask that is used to @@ -3214,23 +5091,22 @@ struct hwrm_port_phy_qcfg_output { * in this field shall be a subset of supported speeds on this * port. */ - /* 100Mb link speed (Half-duplex) */ + /* 100Mb link speed (Half-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100MBHD \ UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100MB \ + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100MB \ UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GBHD \ + /* 1Gb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GBHD \ UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GB \ - UINT32_C(0x8) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GB UINT32_C(0x8) /* 2Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_2GB \ UINT32_C(0x10) /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_2_5GB \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_2_5GB \ UINT32_C(0x20) /* 10Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10GB \ @@ -3248,12 +5124,12 @@ struct hwrm_port_phy_qcfg_output { #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_50GB \ UINT32_C(0x400) /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100GB \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100GB \ UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ + /* 10Mb link speed (Half-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10MBHD \ UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ + /* 10Mb link speed (Full-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10MB \ UINT32_C(0x2000) uint8_t wirespeed; @@ -3302,18 +5178,16 @@ struct hwrm_port_phy_qcfg_output { /* Module is inserted and accepted */ #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NONE UINT32_C(0x0) /* Module is rejected and transmit side Laser is disabled. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_DISABLETX \ - UINT32_C(0x1) + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_DISABLETX UINT32_C(0x1) /* Module mismatch warning. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG \ - UINT32_C(0x2) + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG UINT32_C(0x2) /* Module is rejected and powered down. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_PWRDOWN UINT32_C(0x3) /* Module is not inserted. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTINSERTED \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTINSERTED \ UINT32_C(0x4) /* Module status is not applicable. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTAPPLICABLE \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTAPPLICABLE \ UINT32_C(0xff) uint32_t preemphasis; /* Current setting for preemphasis. */ @@ -3329,13 +5203,13 @@ struct hwrm_port_phy_qcfg_output { #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN UINT32_C(0x0) /* BASE-CR */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR UINT32_C(0x1) - /* BASE-KR4 (Deprecated) */ + /* BASE-KR4 (Deprecated) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4 UINT32_C(0x2) /* BASE-LR */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR UINT32_C(0x3) /* BASE-SR */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR UINT32_C(0x4) - /* BASE-KR2 (Deprecated) */ + /* BASE-KR2 (Deprecated) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2 UINT32_C(0x5) /* BASE-KX */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX UINT32_C(0x6) @@ -3347,6 +5221,35 @@ struct hwrm_port_phy_qcfg_output { #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE UINT32_C(0x9) /* SGMII connected external PHY */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY UINT32_C(0xa) + /* 25G_BASECR_CA_L */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_L UINT32_C(0xb) + /* 25G_BASECR_CA_S */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_S UINT32_C(0xc) + /* 25G_BASECR_CA_N */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_N UINT32_C(0xd) + /* 25G_BASESR */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASESR UINT32_C(0xe) + /* 100G_BASECR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASECR4 UINT32_C(0xf) + /* 100G_BASESR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR4 UINT32_C(0x10) + /* 100G_BASELR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASELR4 UINT32_C(0x11) + /* 100G_BASEER4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASEER4 UINT32_C(0x12) + /* 100G_BASESR10 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR10 UINT32_C(0x13) + /* 40G_BASECR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASECR4 UINT32_C(0x14) + /* 40G_BASESR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASESR4 UINT32_C(0x15) + /* 40G_BASELR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASELR4 UINT32_C(0x16) + /* 40G_BASEER4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASEER4 UINT32_C(0x17) + /* 40G_ACTIVE_CABLE */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_ACTIVE_CABLE \ + UINT32_C(0x18) uint8_t media_type; /* This value represents a media type. */ /* Unknown */ @@ -3360,35 +5263,34 @@ struct hwrm_port_phy_qcfg_output { uint8_t xcvr_pkg_type; /* This value represents a transceiver type. */ /* PHY and MAC are in the same package */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_INTERNAL \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_INTERNAL \ UINT32_C(0x1) /* PHY and MAC are in different packages */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_EXTERNAL \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_EXTERNAL \ UINT32_C(0x2) uint8_t eee_config_phy_addr; /* * This field represents flags related to EEE configuration. * These EEE configuration flags are valid only when the - * auto_mode is not set to none (in other words autonegotiation + * auto_mode is not set to none (in other words autonegotiation * is enabled). */ /* This field represents PHY address. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_MASK UINT32_C(0x1f) #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_SFT 0 /* - * When set to 1, Energy Efficient Ethernet (EEE) mode is + * When set to 1, Energy Efficient Ethernet (EEE) mode is * enabled. Speeds for autoneg with EEE mode enabled are based * on eee_link_speed_mask. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_ENABLED \ - UINT32_C(0x20) + #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_ENABLED UINT32_C(0x20) /* * This flag is valid only when eee_enabled is set to 1. # If * eee_enabled is set to 0, then EEE mode is disabled and this * flag shall be ignored. # If eee_enabled is set to 1 and this - * flag is set to 1, then Energy Efficient Ethernet (EEE) mode + * flag is set to 1, then Energy Efficient Ethernet (EEE) mode * is enabled and in use. # If eee_enabled is set to 1 and this - * flag is set to 0, then Energy Efficient Ethernet (EEE) mode + * flag is set to 0, then Energy Efficient Ethernet (EEE) mode * is enabled but is currently not in use. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_ACTIVE UINT32_C(0x40) @@ -3396,16 +5298,16 @@ struct hwrm_port_phy_qcfg_output { * This flag is valid only when eee_enabled is set to 1. # If * eee_enabled is set to 0, then EEE mode is disabled and this * flag shall be ignored. # If eee_enabled is set to 1 and this - * flag is set to 1, then Energy Efficient Ethernet (EEE) mode + * flag is set to 1, then Energy Efficient Ethernet (EEE) mode * is enabled and TX LPI is enabled. # If eee_enabled is set to * 1 and this flag is set to 0, then Energy Efficient Ethernet - * (EEE) mode is enabled but TX LPI is disabled. + * (EEE) mode is enabled but TX LPI is disabled. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_TX_LPI UINT32_C(0x80) /* * This field represents flags related to EEE configuration. * These EEE configuration flags are valid only when the - * auto_mode is not set to none (in other words autonegotiation + * auto_mode is not set to none (in other words autonegotiation * is enabled). */ #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_MASK UINT32_C(0xe0) @@ -3427,16 +5329,16 @@ struct hwrm_port_phy_qcfg_output { * The advertised speeds for the port by the link partner. Each * advertised speed will be set to '1'. */ - /* 100Mb link speed (Half-duplex) */ + /* 100Mb link speed (Half-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100MBHD \ UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ + /* 100Mb link speed (Full-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100MB \ UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ + /* 1Gb link speed (Half-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_1GBHD \ UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ + /* 1Gb link speed (Full-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_1GB \ UINT32_C(0x8) /* 2Gb link speed */ @@ -3463,10 +5365,10 @@ struct hwrm_port_phy_qcfg_output { /* 100Gb link speed */ #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100GB \ UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ + /* 10Mb link speed (Half-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10MBHD \ UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ + /* 10Mb link speed (Full-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10MB \ UINT32_C(0x2000) uint8_t link_partner_adv_auto_mode; @@ -3483,13 +5385,14 @@ struct hwrm_port_phy_qcfg_output { /* Select all possible speeds for autoneg mode. */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS \ - UINT32_C(0x1) + UINT32_C(0x1) /* * Select only the auto_link_speed speed for * autoneg mode. This mode has been DEPRECATED. * An HWRM client should not use this mode. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED \ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED \ UINT32_C(0x2) /* * Select the auto_link_speed or any speed below @@ -3499,14 +5402,14 @@ struct hwrm_port_phy_qcfg_output { */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ONE_OR_BELOW \ - UINT32_C(0x3) + UINT32_C(0x3) /* * Select the speeds based on the corresponding * link speed mask value that is provided. */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK \ - UINT32_C(0x4) + UINT32_C(0x4) uint8_t link_partner_adv_pause; /* The advertised pause settings on the port by the link partner. */ /* @@ -3532,13 +5435,13 @@ struct hwrm_port_phy_qcfg_output { /* Reserved */ #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD1 \ UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ + /* 100Mb link speed (Full-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_100MB \ UINT32_C(0x2) /* Reserved */ #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD2 \ UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ + /* 1Gb link speed (Full-duplex) */ #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_1GB \ UINT32_C(0x8) /* Reserved */ @@ -3559,31 +5462,31 @@ struct hwrm_port_phy_qcfg_output { /* Reserved */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD1 \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ + UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_100MB \ - UINT32_C(0x2) + UINT32_C(0x2) /* Reserved */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD2 \ - UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ + UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_1GB \ - UINT32_C(0x8) + UINT32_C(0x8) /* Reserved */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD3 \ - UINT32_C(0x10) + UINT32_C(0x10) /* Reserved */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD4 \ - UINT32_C(0x20) + UINT32_C(0x20) /* 10Gb link speed */ #define \ HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_10GB \ - UINT32_C(0x40) + UINT32_C(0x40) uint32_t xcvr_identifier_type_tx_lpi_timer; /* This value represents transceiver identifier type. */ /* @@ -3591,32 +5494,31 @@ struct hwrm_port_phy_qcfg_output { * is valid only when_eee_enabled flag is set to 1 and * tx_lpi_enabled is set to 1. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_TX_LPI_TIMER_MASK \ - UINT32_C(0xffffff) + #define HWRM_PORT_PHY_QCFG_OUTPUT_TX_LPI_TIMER_MASK UINT32_C(0xffffff) #define HWRM_PORT_PHY_QCFG_OUTPUT_TX_LPI_TIMER_SFT 0 /* This value represents transceiver identifier type. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_MASK \ UINT32_C(0xff000000) #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_SFT 24 /* Unknown */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_UNKNOWN \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_UNKNOWN \ (UINT32_C(0x0) << 24) /* SFP/SFP+/SFP28 */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_SFP \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_SFP \ (UINT32_C(0x3) << 24) /* QSFP */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP \ (UINT32_C(0xc) << 24) /* QSFP+ */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFPPLUS \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFPPLUS \ (UINT32_C(0xd) << 24) /* QSFP28 */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP28 \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP28 \ (UINT32_C(0x11) << 24) uint16_t fec_cfg; /* * This value represents the current configuration of Forward - * Error Correction (FEC) on the port. + * Error Correction (FEC) on the port. */ /* * When set to 1, then FEC is not supported on this port. If @@ -3627,7 +5529,7 @@ struct hwrm_port_phy_qcfg_output { * then the HWRM shall set this flag to 1 when reporting FEC * capability. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_NONE_SUPPORTED \ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_NONE_SUPPORTED \ UINT32_C(0x1) /* * When set to 1, then FEC autonegotiation is supported on this @@ -3645,30 +5547,30 @@ struct hwrm_port_phy_qcfg_output { #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_AUTONEG_ENABLED \ UINT32_C(0x4) /* - * When set to 1, then FEC CLAUSE 74 (Fire Code) is supported on - * this port. When set to 0, then FEC CLAUSE 74 (Fire Code) is + * When set to 1, then FEC CLAUSE 74 (Fire Code) is supported on + * this port. When set to 0, then FEC CLAUSE 74 (Fire Code) is * not supported on this port. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE74_SUPPORTED \ UINT32_C(0x8) /* - * When set to 1, then FEC CLAUSE 74 (Fire Code) is enabled on - * this port. When set to 0, then FEC CLAUSE 74 (Fire Code) is + * When set to 1, then FEC CLAUSE 74 (Fire Code) is enabled on + * this port. When set to 0, then FEC CLAUSE 74 (Fire Code) is * disabled if supported. This flag should be ignored if FEC * CLAUSE 74 is not supported on this port. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE74_ENABLED \ UINT32_C(0x10) /* - * When set to 1, then FEC CLAUSE 91 (Reed Solomon) is supported - * on this port. When set to 0, then FEC CLAUSE 91 (Reed + * When set to 1, then FEC CLAUSE 91 (Reed Solomon) is supported + * on this port. When set to 0, then FEC CLAUSE 91 (Reed * Solomon) is not supported on this port. */ #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE91_SUPPORTED \ UINT32_C(0x20) /* - * When set to 1, then FEC CLAUSE 91 (Reed Solomon) is enabled - * on this port. When set to 0, then FEC CLAUSE 91 (Reed + * When set to 1, then FEC CLAUSE 91 (Reed Solomon) is enabled + * on this port. When set to 0, then FEC CLAUSE 91 (Reed * Solomon) is disabled if supported. This flag should be * ignored if FEC CLAUSE 91 is not supported on this port. */ @@ -3704,6 +5606,1132 @@ struct hwrm_port_phy_qcfg_output { */ } __attribute__((packed)); +/* hwrm_port_qstats */ +/* Description: This function returns per port Ethernet statistics. */ +/* Input (40 bytes) */ +struct hwrm_port_qstats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port that is being queried. */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2[3]; + uint8_t unused_3; + uint64_t tx_stat_host_addr; + /* This is the host address where Tx port statistics will be stored */ + uint64_t rx_stat_host_addr; + /* This is the host address where Rx port statistics will be stored */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_port_qstats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t tx_stat_size; + /* The size of TX port statistics block in bytes. */ + uint16_t rx_stat_size; + /* The size of RX port statistics block in bytes. */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_clr_stats */ +/* + * Description: This function clears per port statistics. The HWRM shall not + * allow a VF driver to clear port statistics. The HWRM shall not allow a PF + * driver to clear port statistics in a partitioning mode. The HWRM may allow a + * PF driver to clear port statistics in the non-partitioning mode. + */ +/* Input (24 bytes) */ +struct hwrm_port_clr_stats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port that is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_port_clr_stats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_led_cfg */ +/* + * Description: This function is used to configure LEDs on a given port. Each + * port has individual set of LEDs associated with it. These LEDs are used for + * speed/link configuration as well as activity indicator configuration. Up to + * three LEDs can be configured, one for activity and two for speeds. + */ +/* Input (64 bytes) */ +struct hwrm_port_led_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the led0_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_ID UINT32_C(0x1) + /* This bit must be '1' for the led0_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_STATE UINT32_C(0x2) + /* This bit must be '1' for the led0_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_COLOR UINT32_C(0x4) + /* + * This bit must be '1' for the led0_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_ON UINT32_C(0x8) + /* + * This bit must be '1' for the led0_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_OFF UINT32_C(0x10) + /* + * This bit must be '1' for the led0_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_GROUP_ID UINT32_C(0x20) + /* This bit must be '1' for the led1_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_ID UINT32_C(0x40) + /* This bit must be '1' for the led1_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_STATE UINT32_C(0x80) + /* This bit must be '1' for the led1_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_COLOR UINT32_C(0x100) + /* + * This bit must be '1' for the led1_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_BLINK_ON UINT32_C(0x200) + /* + * This bit must be '1' for the led1_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_BLINK_OFF UINT32_C(0x400) + /* + * This bit must be '1' for the led1_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_GROUP_ID UINT32_C(0x800) + /* This bit must be '1' for the led2_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_ID UINT32_C(0x1000) + /* This bit must be '1' for the led2_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_STATE UINT32_C(0x2000) + /* This bit must be '1' for the led2_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_COLOR UINT32_C(0x4000) + /* + * This bit must be '1' for the led2_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_BLINK_ON UINT32_C(0x8000) + /* + * This bit must be '1' for the led2_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_BLINK_OFF UINT32_C(0x10000) + /* + * This bit must be '1' for the led2_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_GROUP_ID UINT32_C(0x20000) + /* This bit must be '1' for the led3_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_ID UINT32_C(0x40000) + /* This bit must be '1' for the led3_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_STATE UINT32_C(0x80000) + /* This bit must be '1' for the led3_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_COLOR UINT32_C(0x100000) + /* + * This bit must be '1' for the led3_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_BLINK_ON UINT32_C(0x200000) + /* + * This bit must be '1' for the led3_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_BLINK_OFF \ + UINT32_C(0x400000) + /* + * This bit must be '1' for the led3_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_GROUP_ID UINT32_C(0x800000) + uint16_t port_id; + /* Port ID of port whose LEDs are configured. */ + uint8_t num_leds; + /* + * The number of LEDs that are being configured. Up to 4 LEDs + * can be configured with this command. + */ + uint8_t rsvd; + /* Reserved field. */ + uint8_t led0_id; + /* An identifier for the LED #0. */ + uint8_t led0_state; + /* The requested state of the LED #0. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINKALT UINT32_C(0x4) + uint8_t led0_color; + /* The requested color of LED #0. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_0; + uint16_t led0_blink_on; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led0_blink_off; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led0_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #0 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd0; + /* Reserved field. */ + uint8_t led1_id; + /* An identifier for the LED #1. */ + uint8_t led1_state; + /* The requested state of the LED #1. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_BLINKALT UINT32_C(0x4) + uint8_t led1_color; + /* The requested color of LED #1. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_1; + uint16_t led1_blink_on; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led1_blink_off; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led1_group_id; + /* + * An identifier for the group of LEDs that LED #1 belongs to. + * If set to 0, then the LED #1 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #1 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd1; + /* Reserved field. */ + uint8_t led2_id; + /* An identifier for the LED #2. */ + uint8_t led2_state; + /* The requested state of the LED #2. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_BLINKALT UINT32_C(0x4) + uint8_t led2_color; + /* The requested color of LED #2. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_2; + uint16_t led2_blink_on; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led2_blink_off; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led2_group_id; + /* + * An identifier for the group of LEDs that LED #2 belongs to. + * If set to 0, then the LED #2 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #2 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd2; + /* Reserved field. */ + uint8_t led3_id; + /* An identifier for the LED #3. */ + uint8_t led3_state; + /* The requested state of the LED #3. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_BLINKALT UINT32_C(0x4) + uint8_t led3_color; + /* The requested color of LED #3. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_3; + uint16_t led3_blink_on; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led3_blink_off; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led3_group_id; + /* + * An identifier for the group of LEDs that LED #3 belongs to. + * If set to 0, then the LED #3 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #3 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd3; + /* Reserved field. */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_port_led_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_led_qcfg */ +/* + * Description: This function is used to query configuration of LEDs on a given + * port. Each port has individual set of LEDs associated with it. These LEDs are + * used for speed/link configuration as well as activity indicator + * configuration. Up to three LEDs can be configured, one for activity and two + * for speeds. + */ +/* Input (24 bytes) */ +struct hwrm_port_led_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port whose LED configuration is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (56 bytes) */ +struct hwrm_port_led_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint8_t num_leds; + /* + * The number of LEDs that are configured on this port. Up to 4 + * LEDs can be returned in the response. + */ + uint8_t led0_id; + /* An identifier for the LED #0. */ + uint8_t led0_type; + /* The type of LED #0. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_INVALID UINT32_C(0xff) + uint8_t led0_state; + /* The current state of the LED #0. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINKALT UINT32_C(0x4) + uint8_t led0_color; + /* The color of LED #0. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_0; + uint16_t led0_blink_on; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led0_blink_off; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led0_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 is not grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t led1_id; + /* An identifier for the LED #1. */ + uint8_t led1_type; + /* The type of LED #1. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_INVALID UINT32_C(0xff) + uint8_t led1_state; + /* The current state of the LED #1. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_BLINKALT UINT32_C(0x4) + uint8_t led1_color; + /* The color of LED #1. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_1; + uint16_t led1_blink_on; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led1_blink_off; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led1_group_id; + /* + * An identifier for the group of LEDs that LED #1 belongs to. + * If set to 0, then the LED #1 is not grouped. For all other + * non-zero values of this field, LED #1 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t led2_id; + /* An identifier for the LED #2. */ + uint8_t led2_type; + /* The type of LED #2. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_INVALID UINT32_C(0xff) + uint8_t led2_state; + /* The current state of the LED #2. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_BLINKALT UINT32_C(0x4) + uint8_t led2_color; + /* The color of LED #2. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_2; + uint16_t led2_blink_on; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led2_blink_off; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led2_group_id; + /* + * An identifier for the group of LEDs that LED #2 belongs to. + * If set to 0, then the LED #2 is not grouped. For all other + * non-zero values of this field, LED #2 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t led3_id; + /* An identifier for the LED #3. */ + uint8_t led3_type; + /* The type of LED #3. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_INVALID UINT32_C(0xff) + uint8_t led3_state; + /* The current state of the LED #3. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_BLINKALT UINT32_C(0x4) + uint8_t led3_color; + /* The color of LED #3. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_3; + uint16_t led3_blink_on; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led3_blink_off; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led3_group_id; + /* + * An identifier for the group of LEDs that LED #3 belongs to. + * If set to 0, then the LED #3 is not grouped. For all other + * non-zero values of this field, LED #3 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_4; + uint16_t unused_5; + uint8_t unused_6; + uint8_t unused_7; + uint8_t unused_8; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_led_qcaps */ +/* + * Description: This function is used to query capabilities of LEDs on a given + * port. Each port has individual set of LEDs associated with it. These LEDs are + * used for speed/link configuration as well as activity indicator + * configuration. + */ +/* Input (24 bytes) */ +struct hwrm_port_led_qcaps_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port whose LED configuration is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (48 bytes) */ +struct hwrm_port_led_qcaps_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint8_t num_leds; + /* + * The number of LEDs that are configured on this port. Up to 4 + * LEDs can be returned in the response. + */ + uint8_t unused_0[3]; + /* Reserved for future use. */ + uint8_t led0_id; + /* An identifier for the LED #0. */ + uint8_t led0_type; + /* The type of LED #0. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_INVALID UINT32_C(0xff) + uint8_t led0_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_1; + uint16_t led0_state_caps; + /* The states supported by LED #0. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led0_color_caps; + /* The colors supported by LED #0. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t led1_id; + /* An identifier for the LED #1. */ + uint8_t led1_type; + /* The type of LED #1. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_INVALID UINT32_C(0xff) + uint8_t led1_group_id; + /* + * An identifier for the group of LEDs that LED #1 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_2; + uint16_t led1_state_caps; + /* The states supported by LED #1. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led1_color_caps; + /* The colors supported by LED #1. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t led2_id; + /* An identifier for the LED #2. */ + uint8_t led2_type; + /* The type of LED #2. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_INVALID UINT32_C(0xff) + uint8_t led2_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_3; + uint16_t led2_state_caps; + /* The states supported by LED #2. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led2_color_caps; + /* The colors supported by LED #2. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t led3_id; + /* An identifier for the LED #3. */ + uint8_t led3_type; + /* The type of LED #3. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_INVALID UINT32_C(0xff) + uint8_t led3_group_id; + /* + * An identifier for the group of LEDs that LED #3 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_4; + uint16_t led3_state_caps; + /* The states supported by LED #3. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led3_color_caps; + /* The colors supported by LED #3. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t unused_5; + uint8_t unused_6; + uint8_t unused_7; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + /* hwrm_queue_qportcfg */ /* * Description: This function is called by a driver to query queue configuration @@ -3715,7 +6743,7 @@ struct hwrm_port_phy_qcfg_output { * then the driver shall only use queues for which service profiles are pre- * configured. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_queue_qportcfg_input { uint16_t req_type; /* @@ -3754,7 +6782,7 @@ struct hwrm_queue_qportcfg_input { #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_TX UINT32_C(0x0) /* rx path */ #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX UINT32_C(0x1) - #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_LAST \ + #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_LAST \ QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX uint16_t port_id; /* @@ -3764,7 +6792,7 @@ struct hwrm_queue_qportcfg_input { uint16_t unused_0; } __attribute__((packed)); -/* Output (32 bytes) */ +/* Output (32 bytes) */ struct hwrm_queue_qportcfg_output { uint16_t error_code; /* @@ -3813,18 +6841,18 @@ struct hwrm_queue_qportcfg_output { * TX side is the same as the corresponding queue configuration * on the RX side. */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_ASYM_CFG \ - UINT32_C(0x1) + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_ASYM_CFG UINT32_C(0x1) uint8_t queue_pfcenable_cfg_allowed; /* * Bitmask indicating which queues can be configured by the * hwrm_queue_pfcenable_cfg command. Each bit represents a - * specific queue where bit 0 represents queue 0 and bit 7 - * represents queue 7. # A value of 0 indicates that the queue - * is not configurable by the hwrm_queue_pfcenable_cfg command. - * # A value of 1 indicates that the queue is configurable. # A - * hwrm_queue_pfcenable_cfg command shall return error when - * trying to configure a queue that is not configurable. + * specific priority where bit 0 represents priority 0 and bit 7 + * represents priority 7. # A value of 0 indicates that the + * priority is not configurable by the hwrm_queue_pfcenable_cfg + * command. # A value of 1 indicates that the priority is + * configurable. # A hwrm_queue_pfcenable_cfg command shall + * return error when trying to configure a priority that is not + * configurable. */ uint8_t queue_pri2cos_cfg_allowed; /* @@ -3860,14 +6888,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id0_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN \ @@ -3884,14 +6912,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id1_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN \ @@ -3908,14 +6936,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id2_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN \ @@ -3932,14 +6960,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id3_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN \ @@ -3956,14 +6984,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id4_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN \ @@ -3980,14 +7008,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id5_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN \ @@ -4004,14 +7032,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id6_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN \ @@ -4028,14 +7056,14 @@ struct hwrm_queue_qportcfg_output { */ uint8_t queue_id7_service_profile; /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ + /* Lossy (best-effort) */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_LOSSY \ UINT32_C(0x0) /* Lossless */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS \ UINT32_C(0x1) /* - * Set to 0xFF... (All Fs) if there is no + * Set to 0xFF... (All Fs) if there is no * service profile specified */ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN \ @@ -4068,7 +7096,7 @@ struct hwrm_queue_qportcfg_output { * enabled, then the internal VNIC to SVIF mapping data structures shall be * programmed at the time of VNIC allocation. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_vnic_alloc_input { uint16_t req_type; /* @@ -4105,7 +7133,7 @@ struct hwrm_vnic_alloc_input { uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_alloc_output { uint16_t error_code; /* @@ -4144,7 +7172,7 @@ struct hwrm_vnic_alloc_output { * VNIC as well as the VNIC. Reset and release all resources associated with the * VNIC. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_vnic_free_input { uint16_t req_type; /* @@ -4177,7 +7205,7 @@ struct hwrm_vnic_free_input { uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_free_output { uint16_t error_code; /* @@ -4211,7 +7239,7 @@ struct hwrm_vnic_free_output { /* hwrm_vnic_cfg */ /* Description: Configure the RX VNIC structure. */ -/* Input (40 bytes) */ +/* Input (40 bytes) */ struct hwrm_vnic_cfg_input { uint16_t req_type; /* @@ -4271,10 +7299,10 @@ struct hwrm_vnic_cfg_input { * roce_dual_vnic_mode flag is set to '1', then the HWRM client * shall not set this flag to '1'. */ - #define HWRM_VNIC_CFG_INPUT_FLAGS_ROCE_ONLY_VNIC_MODE UINT32_C(0x10) + #define HWRM_VNIC_CFG_INPUT_FLAGS_ROCE_ONLY_VNIC_MODE UINT32_C(0x10) /* * When a VNIC uses one destination ring group for certain - * application (e.g. Receive Flow Steering) where exact match is + * application (e.g. Receive Flow Steering) where exact match is * used to direct packets to a VNIC with one destination ring * group only, there is no need to configure RSS indirection * table for that VNIC as only one destination ring group is @@ -4310,17 +7338,17 @@ struct hwrm_vnic_cfg_input { */ uint16_t rss_rule; /* - * RSS ID for RSS rule/table structure. 0xFF... (All Fs) if + * RSS ID for RSS rule/table structure. 0xFF... (All Fs) if * there is no RSS rule. */ uint16_t cos_rule; /* - * RSS ID for COS rule/table structure. 0xFF... (All Fs) if + * RSS ID for COS rule/table structure. 0xFF... (All Fs) if * there is no COS rule. */ uint16_t lb_rule; /* - * RSS ID for load balancing rule/table structure. 0xFF... (All + * RSS ID for load balancing rule/table structure. 0xFF... (All * Fs) if there is no LB rule. */ uint16_t mru; @@ -4334,7 +7362,7 @@ struct hwrm_vnic_cfg_input { uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_cfg_output { uint16_t error_code; /* @@ -4366,9 +7394,153 @@ struct hwrm_vnic_cfg_output { */ } __attribute__((packed)); +/* hwrm_vnic_qcfg */ +/* + * Description: Query the RX VNIC structure. This function can be used by a PF + * driver to query its own VNIC resource or VNIC resource of its child VF. This + * function can also be used by a VF driver to query its own VNIC resource. + */ +/* Input (32 bytes) */ +struct hwrm_vnic_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the vf_id_valid field to be configured. */ + #define HWRM_VNIC_QCFG_INPUT_ENABLES_VF_ID_VALID UINT32_C(0x1) + uint32_t vnic_id; + /* Logical vnic ID */ + uint16_t vf_id; + /* ID of Virtual Function whose VNIC resource is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (32 bytes) */ +struct hwrm_vnic_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t dflt_ring_grp; + /* Default Completion ring for the VNIC. */ + uint16_t rss_rule; + /* + * RSS ID for RSS rule/table structure. 0xFF... (All Fs) if + * there is no RSS rule. + */ + uint16_t cos_rule; + /* + * RSS ID for COS rule/table structure. 0xFF... (All Fs) if + * there is no COS rule. + */ + uint16_t lb_rule; + /* + * RSS ID for load balancing rule/table structure. 0xFF... (All + * Fs) if there is no LB rule. + */ + uint16_t mru; + /* The maximum receive unit of the vnic. */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t flags; + /* + * When this bit is '1', the VNIC is the default VNIC for the + * function. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_DEFAULT UINT32_C(0x1) + /* + * When this bit is '1', the VNIC is configured to strip VLAN in + * the RX path. If set to '0', then VLAN stripping is disabled + * on this VNIC. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_VLAN_STRIP_MODE UINT32_C(0x2) + /* + * When this bit is '1', the VNIC is configured to buffer + * receive packets in the hardware until the host posts new + * receive buffers. If set to '0', then bd_stall is disabled on + * this VNIC. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_BD_STALL_MODE UINT32_C(0x4) + /* + * When this bit is '1', the VNIC is configured to receive both + * RoCE and non-RoCE traffic. If set to '0', then this VNIC is + * not configured to operate in dual VNIC mode. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_DUAL_VNIC_MODE UINT32_C(0x8) + /* + * When this flag is set to '1', the VNIC is configured to + * receive only RoCE traffic. When this flag is set to '0', the + * VNIC is not configured to receive only RoCE traffic. If + * roce_dual_vnic_mode flag and this flag both are set to '1', + * then it is an invalid configuration of the VNIC. The HWRM + * should not allow that type of mis-configuration by HWRM + * clients. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_ONLY_VNIC_MODE UINT32_C(0x10) + /* + * When a VNIC uses one destination ring group for certain + * application (e.g. Receive Flow Steering) where exact match is + * used to direct packets to a VNIC with one destination ring + * group only, there is no need to configure RSS indirection + * table for that VNIC as only one destination ring group is + * used. When this bit is set to '1', then the VNIC is enabled + * in a mode where RSS is enabled in the VNIC using a RSS + * context for computing RSS hash but the RSS indirection table + * is not configured. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE UINT32_C(0x20) + uint32_t unused_2; + uint8_t unused_3; + uint8_t unused_4; + uint8_t unused_5; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + /* hwrm_vnic_rss_cfg */ /* Description: This function is used to enable RSS configuration. */ -/* Input (48 bytes) */ +/* Input (48 bytes) */ struct hwrm_vnic_rss_cfg_input { uint16_t req_type; /* @@ -4441,7 +7613,7 @@ struct hwrm_vnic_rss_cfg_input { uint16_t unused_1[3]; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_rss_cfg_output { uint16_t error_code; /* @@ -4473,9 +7645,295 @@ struct hwrm_vnic_rss_cfg_output { */ } __attribute__((packed)); +/* hwrm_vnic_plcmodes_cfg */ +/* + * Description: This function can be used to set placement mode configuration of + * the VNIC. + */ +/* Input (40 bytes) */ + +struct hwrm_vnic_plcmodes_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format for the + * rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request will be + * optionally completed on. If the value is -1, then no CR completion + * will be generated. Any other value must be a valid CR ring_id value + * for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written when the + * request is complete. This area must be 16B aligned and must be + * cleared to zero before the request is made. + */ + uint32_t flags; + /* + * When this bit is '1', the VNIC shall be configured to use regular + * placement algorithm. By default, the regular placement algorithm + * shall be enabled on the VNIC. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_REGULAR_PLACEMENT \ + UINT32_C(0x1) + /* + * When this bit is '1', the VNIC shall be configured use the jumbo + * placement algorithm. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_JUMBO_PLACEMENT \ + UINT32_C(0x2) + /* + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for IPv4 packets according to the following rules: # If + * the packet is identified as TCP/IPv4, then the packet is split at the + * beginning of the TCP payload. # If the packet is identified as + * UDP/IPv4, then the packet is split at the beginning of UDP payload. # + * If the packet is identified as non-TCP and non-UDP IPv4 packet, then + * the packet is split at the beginning of the upper layer protocol + * header carried in the IPv4 packet. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_IPV4 UINT32_C(0x4) + /* + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for IPv6 packets according to the following rules: # If + * the packet is identified as TCP/IPv6, then the packet is split at the + * beginning of the TCP payload. # If the packet is identified as + * UDP/IPv6, then the packet is split at the beginning of UDP payload. # + * If the packet is identified as non-TCP and non-UDP IPv6 packet, then + * the packet is split at the beginning of the upper layer protocol + * header carried in the IPv6 packet. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_IPV6 UINT32_C(0x8) + /* + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for FCoE packets at the beginning of FC payload. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_FCOE UINT32_C(0x10) + /* + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for RoCE packets at the beginning of RoCE payload (after + * BTH/GRH headers). + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_ROCE UINT32_C(0x20) + uint32_t enables; + /* + * This bit must be '1' for the jumbo_thresh_valid field to be + * configured. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_JUMBO_THRESH_VALID \ + UINT32_C(0x1) + /* + * This bit must be '1' for the hds_offset_valid field to be configured. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_OFFSET_VALID \ + UINT32_C(0x2) + /* + * This bit must be '1' for the hds_threshold_valid field to be + * configured. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_THRESHOLD_VALID \ + UINT32_C(0x4) + uint32_t vnic_id; + /* Logical vnic ID */ + uint16_t jumbo_thresh; + /* + * When jumbo placement algorithm is enabled, this value is used to + * determine the threshold for jumbo placement. Packets with length + * larger than this value will be placed according to the jumbo + * placement algorithm. + */ + uint16_t hds_offset; + /* + * This value is used to determine the offset into packet buffer where + * the split data (payload) will be placed according to one of of HDS + * placement algorithm. The lengths of packet buffers provided for split + * data shall be larger than this value. + */ + uint16_t hds_threshold; + /* + * When one of the HDS placement algorithm is enabled, this value is + * used to determine the threshold for HDS placement. Packets with + * length larger than this value will be placed according to the HDS + * placement algorithm. This value shall be in multiple of 4 bytes. + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ + +struct hwrm_vnic_plcmodes_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in parameters, + * and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last byte of + * the response is a valid flag that will read as '1' when the command + * has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the output is + * completely written to RAM. This field should be read as '1' to + * indicate that the output has been completely written. When writing a + * command completion or response to an internal processor, the order of + * writes has to be such that this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_vnic_plcmodes_qcfg */ +/* + * Description: This function can be used to query placement mode configuration + * of the VNIC. + */ +/* Input (24 bytes) */ + +struct hwrm_vnic_plcmodes_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format for the + * rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request will be + * optionally completed on. If the value is -1, then no CR completion + * will be generated. Any other value must be a valid CR ring_id value + * for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written when the + * request is complete. This area must be 16B aligned and must be + * cleared to zero before the request is made. + */ + uint32_t vnic_id; + /* Logical vnic ID */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (24 bytes) */ + +struct hwrm_vnic_plcmodes_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in parameters, + * and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last byte of + * the response is a valid flag that will read as '1' when the command + * has been completely written to memory. + */ + uint32_t flags; + /* + * When this bit is '1', the VNIC is configured to use regular placement + * algorithm. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_REGULAR_PLACEMENT \ + UINT32_C(0x1) + /* + * When this bit is '1', the VNIC is configured to use the jumbo + * placement algorithm. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_JUMBO_PLACEMENT \ + UINT32_C(0x2) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for IPv4 packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_IPV4 UINT32_C(0x4) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for IPv6 packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_IPV6 UINT32_C(0x8) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for FCoE packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_FCOE UINT32_C(0x10) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for RoCE packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_ROCE UINT32_C(0x20) + /* + * When this bit is '1', the VNIC is configured to be the default VNIC + * of the requesting function. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_DFLT_VNIC UINT32_C(0x40) + uint16_t jumbo_thresh; + /* + * When jumbo placement algorithm is enabled, this value is used to + * determine the threshold for jumbo placement. Packets with length + * larger than this value will be placed according to the jumbo + * placement algorithm. + */ + uint16_t hds_offset; + /* + * This value is used to determine the offset into packet buffer where + * the split data (payload) will be placed according to one of of HDS + * placement algorithm. The lengths of packet buffers provided for split + * data shall be larger than this value. + */ + uint16_t hds_threshold; + /* + * When one of the HDS placement algorithm is enabled, this value is + * used to determine the threshold for HDS placement. Packets with + * length larger than this value will be placed according to the HDS + * placement algorithm. This value shall be in multiple of 4 bytes. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t unused_4; + uint8_t valid; + /* + * This field is used in Output records to indicate that the output is + * completely written to RAM. This field should be read as '1' to + * indicate that the output has been completely written. When writing a + * command completion or response to an internal processor, the order of + * writes has to be such that this field is written last. + */ +} __attribute__((packed)); + /* hwrm_vnic_rss_cos_lb_ctx_alloc */ /* Description: This function is used to allocate COS/Load Balance context. */ -/* Input (16 bytes) */ +/* Input (16 bytes) */ struct hwrm_vnic_rss_cos_lb_ctx_alloc_input { uint16_t req_type; /* @@ -4505,7 +7963,7 @@ struct hwrm_vnic_rss_cos_lb_ctx_alloc_input { */ } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_rss_cos_lb_ctx_alloc_output { uint16_t error_code; /* @@ -4542,7 +8000,7 @@ struct hwrm_vnic_rss_cos_lb_ctx_alloc_output { /* hwrm_vnic_rss_cos_lb_ctx_free */ /* Description: This function can be used to free COS/Load Balance context. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_vnic_rss_cos_lb_ctx_free_input { uint16_t req_type; /* @@ -4575,7 +8033,7 @@ struct hwrm_vnic_rss_cos_lb_ctx_free_input { uint16_t unused_0[3]; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_rss_cos_lb_ctx_free_output { uint16_t error_code; /* @@ -4607,11 +8065,187 @@ struct hwrm_vnic_rss_cos_lb_ctx_free_output { */ } __attribute__((packed)); +/* hwrm_vnic_tpa_cfg */ +/* Description: This function is used to enable/configure TPA on the VNIC. */ +/* Input (40 bytes) */ +struct hwrm_vnic_tpa_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) of non-tunneled TCP + * packets. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA UINT32_C(0x1) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) of tunneled TCP packets. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA UINT32_C(0x2) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) according to Windows + * Receive Segment Coalescing (RSC) rules. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE UINT32_C(0x4) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) according to Linux + * Generic Receive Offload (GRO) rules. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO UINT32_C(0x8) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) for TCP packets with IP + * ECN set to non-zero. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN UINT32_C(0x10) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) for GRE tunneled TCP + * packets only if all packets have the same GRE sequence. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ \ + UINT32_C(0x20) + /* + * When this bit is '1' and the GRO mode is enabled, the VNIC + * shall be configured to perform transparent packet aggregation + * (TPA) for TCP/IPv4 packets with consecutively increasing + * IPIDs. In other words, the last packet that is being + * aggregated to an already existing aggregation context shall + * have IPID 1 more than the IPID of the last packet that was + * aggregated in that aggregation context. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO_IPID_CHECK UINT32_C(0x40) + /* + * When this bit is '1' and the GRO mode is enabled, the VNIC + * shall be configured to perform transparent packet aggregation + * (TPA) for TCP packets with the same TTL (IPv4) or Hop limit + * (IPv6) value. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO_TTL_CHECK UINT32_C(0x80) + uint32_t enables; + /* This bit must be '1' for the max_agg_segs field to be configured. */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS UINT32_C(0x1) + /* This bit must be '1' for the max_aggs field to be configured. */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS UINT32_C(0x2) + /* + * This bit must be '1' for the max_agg_timer field to be + * configured. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_TIMER UINT32_C(0x4) + /* This bit must be '1' for the min_agg_len field to be configured. */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN UINT32_C(0x8) + uint16_t vnic_id; + /* Logical vnic ID */ + uint16_t max_agg_segs; + /* + * This is the maximum number of TCP segments that can be + * aggregated (unit is Log2). Max value is 31. + */ + /* 1 segment */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_1 UINT32_C(0x0) + /* 2 segments */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_2 UINT32_C(0x1) + /* 4 segments */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_4 UINT32_C(0x2) + /* 8 segments */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_8 UINT32_C(0x3) + /* Any segment size larger than this is not valid */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_MAX UINT32_C(0x1f) + uint16_t max_aggs; + /* + * This is the maximum number of aggregations this VNIC is + * allowed (unit is Log2). Max value is 7 + */ + /* 1 aggregation */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_1 UINT32_C(0x0) + /* 2 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_2 UINT32_C(0x1) + /* 4 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_4 UINT32_C(0x2) + /* 8 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_8 UINT32_C(0x3) + /* 16 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_16 UINT32_C(0x4) + /* Any aggregation size larger than this is not valid */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX UINT32_C(0x7) + uint8_t unused_0; + uint8_t unused_1; + uint32_t max_agg_timer; + /* + * This is the maximum amount of time allowed for an aggregation + * context to complete after it was initiated. + */ + uint32_t min_agg_len; + /* + * This is the minimum amount of payload length required to + * start an aggregation context. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_vnic_tpa_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + /* hwrm_ring_alloc */ /* * Description: This command allocates and does basic preparation for a ring. */ -/* Input (80 bytes) */ +/* Input (80 bytes) */ struct hwrm_ring_alloc_input { uint16_t req_type; /* @@ -4657,12 +8291,14 @@ struct hwrm_ring_alloc_input { #define HWRM_RING_ALLOC_INPUT_ENABLES_MAX_BW_VALID UINT32_C(0x20) uint8_t ring_type; /* Ring Type. */ - /* Completion Ring (CR) */ - #define HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL UINT32_C(0x0) - /* TX Ring (TR) */ + /* L2 Completion Ring (CR) */ + #define HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL UINT32_C(0x0) + /* TX Ring (TR) */ #define HWRM_RING_ALLOC_INPUT_RING_TYPE_TX UINT32_C(0x1) - /* RX Ring (RR) */ + /* RX Ring (RR) */ #define HWRM_RING_ALLOC_INPUT_RING_TYPE_RX UINT32_C(0x2) + /* RoCE Notification Completion Ring (ROCE_CR) */ + #define HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3) uint8_t unused_0; uint16_t unused_1; uint64_t page_tbl_addr; @@ -4725,23 +8361,22 @@ struct hwrm_ring_alloc_input { * a TX ring. */ /* Arbitration policy used for the ring. */ - #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_MASK \ - UINT32_C(0xf) + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_MASK UINT32_C(0xf) #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_SFT 0 /* * Use strict priority for the TX ring. Priority * value is specified in arb_policy_param */ - #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_SP \ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_SP \ (UINT32_C(0x1) << 0) /* * Use weighted fair queue arbitration for the * TX ring. Weight is specified in * arb_policy_param */ - #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_WFQ \ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_WFQ \ (UINT32_C(0x2) << 0) - #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_LAST \ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_LAST \ RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_WFQ /* Reserved field. */ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_RSVD_MASK UINT32_C(0xf0) @@ -4758,7 +8393,7 @@ struct hwrm_ring_alloc_input { */ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_PARAM_MASK \ UINT32_C(0xff00) - #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_PARAM_SFT 8 + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_PARAM_SFT 8 uint8_t unused_6; uint8_t unused_7; uint32_t reserved3; @@ -4778,26 +8413,40 @@ struct hwrm_ring_alloc_input { * translate this value into byte counter and time interval used * for this ring inside the device. */ - /* Bandwidth value */ - #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_MASK \ - UINT32_C(0xfffffff) + /* The bandwidth value. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff) #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_SFT 0 - /* Reserved */ - #define HWRM_RING_ALLOC_INPUT_MAX_BW_RSVD UINT32_C(0x10000000) + /* The granularity of the value (bits or bytes). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28) + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_LAST \ + RING_ALLOC_INPUT_MAX_BW_SCALE_BYTES /* bw_value_unit is 3 b */ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_MASK \ UINT32_C(0xe0000000) #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_SFT 29 - /* Value is in Mbps */ - #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_MBPS \ + /* Value is in Mb or MB (base 10). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_MEGA \ (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) /* Value is in 1/100th of a percentage of total bandwidth. */ - #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ (UINT32_C(0x1) << 29) /* Invalid unit */ - #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ (UINT32_C(0x7) << 29) - #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_LAST \ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_LAST \ RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID uint8_t int_mode; /* @@ -4817,7 +8466,7 @@ struct hwrm_ring_alloc_input { uint8_t unused_8[3]; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_ring_alloc_output { uint16_t error_code; /* @@ -4858,8 +8507,16 @@ struct hwrm_ring_alloc_output { /* hwrm_ring_free */ /* * Description: This command is used to free a ring and associated resources. + * With QoS and DCBx agents, it is possible the traffic classes will be moved + * from one CoS queue to another. When this occurs, the driver shall call + * 'hwrm_ring_free' to free the allocated rings and then call 'hwrm_ring_alloc' + * to re-allocate each ring and assign it to a new CoS queue. hwrm_ring_free + * shall be called on a ring only after it has been idle for 500ms or more and + * no frames have been posted to the ring during this time. All frames queued + * for transmission shall be completed and at least 500ms time elapsed from the + * last completion before calling this command. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_ring_free_input { uint16_t req_type; /* @@ -4889,19 +8546,21 @@ struct hwrm_ring_free_input { */ uint8_t ring_type; /* Ring Type. */ - /* Completion Ring (CR) */ - #define HWRM_RING_FREE_INPUT_RING_TYPE_CMPL UINT32_C(0x0) - /* TX Ring (TR) */ + /* L2 Completion Ring (CR) */ + #define HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL UINT32_C(0x0) + /* TX Ring (TR) */ #define HWRM_RING_FREE_INPUT_RING_TYPE_TX UINT32_C(0x1) - /* RX Ring (RR) */ + /* RX Ring (RR) */ #define HWRM_RING_FREE_INPUT_RING_TYPE_RX UINT32_C(0x2) + /* RoCE Notification Completion Ring (ROCE_CR) */ + #define HWRM_RING_FREE_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3) uint8_t unused_0; uint16_t ring_id; /* Physical number of ring allocated. */ uint32_t unused_1; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_ring_free_output { uint16_t error_code; /* @@ -4937,7 +8596,7 @@ struct hwrm_ring_free_output { /* * Description: This API allocates and does basic preparation for a ring group. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_ring_grp_alloc_input { uint16_t req_type; /* @@ -4972,7 +8631,7 @@ struct hwrm_ring_grp_alloc_input { uint16_t ar; /* * This value identifies the aggregation RR associated with the - * ring group. If this value is 0xFF... (All Fs), then no + * ring group. If this value is 0xFF... (All Fs), then no * Aggregation ring will be set. */ uint16_t sc; @@ -4982,7 +8641,7 @@ struct hwrm_ring_grp_alloc_input { */ } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_ring_grp_alloc_output { uint16_t error_code; /* @@ -5028,7 +8687,7 @@ struct hwrm_ring_grp_alloc_output { * a part of executing this command, the HWRM shall reset all associated ring * group resources. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_ring_grp_free_input { uint16_t req_type; /* @@ -5061,7 +8720,7 @@ struct hwrm_ring_grp_free_input { uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_ring_grp_free_output { uint16_t error_code; /* @@ -5095,12 +8754,61 @@ struct hwrm_ring_grp_free_output { /* hwrm_cfa_l2_filter_alloc */ /* - * A filter is used to identify traffic that contains a matching set of - * parameters like unicast or broadcast MAC address or a VLAN tag amongst - * other things which then allows the ASIC to direct the incoming traffic - * to an appropriate VNIC or Rx ring. + * Description: An L2 filter is a filter resource that is used to identify a + * vnic or ring for a packet based on layer 2 fields. Layer 2 fields for + * encapsulated packets include both outer L2 header and/or inner l2 header of + * encapsulated packet. The L2 filter resource covers the following OS specific + * L2 filters. Linux/FreeBSD (per function): # Broadcast enable/disable # List + * of individual multicast filters # All multicast enable/disable filter # + * Unicast filters # Promiscuous mode VMware: # Broadcast enable/disable (per + * physical function) # All multicast enable/disable (per function) # Unicast + * filters per ring or vnic # Promiscuous mode per PF Windows: # Broadcast + * enable/disable (per physical function) # List of individual multicast filters + * (Driver needs to advertise the maximum number of filters supported) # All + * multicast enable/disable per physical function # Unicast filters per vnic # + * Promiscuous mode per PF Implementation notes on the use of VNIC in this + * command: # By default, these filters belong to default vnic for the function. + * # Once these filters are set up, only destination VNIC can be modified. # If + * the destination VNIC is not specified in this command, then the HWRM shall + * only create an l2 context id. HWRM Implementation notes for multicast + * filters: # The hwrm_filter_alloc command can be used to set up multicast + * filters (perfect match or partial match). Each individual function driver can + * set up multicast filters independently. # The HWRM needs to keep track of + * multicast filters set up by function drivers and maintain multicast group + * replication records to enable a subset of functions to receive traffic for a + * specific multicast address. # When a specific multicast filter cannot be set, + * the HWRM shall return an error. In this error case, the driver should fall + * back to using one general filter (rather than specific) for all multicast + * traffic. # When the SR-IOV is enabled, the HWRM needs to additionally track + * source knockout per multicast group record. Examples of setting unicast + * filters: For a unicast MAC based filter, one can use a combination of the + * fields and masks provided in this command to set up the filter. Below are + * some examples: # MAC + no VLAN filter: This filter is used to identify + * traffic that does not contain any VLAN tags and matches destination (or + * source) MAC address. This filter can be set up by setting only l2_addr field + * to be a valid field. All other fields are not valid. The following value is + * set for l2_addr. l2_addr = MAC # MAC + Any VLAN filter: This filter is used + * to identify traffic that carries single VLAN tag and matches (destination or + * source) MAC address. This filter can be set up by setting only l2_addr and + * l2_ovlan_mask fields to be valid fields. All other fields are not valid. The + * following values are set for those two valid fields. l2_addr = MAC, + * l2_ovlan_mask = 0xFFFF # MAC + no VLAN or VLAN ID=0: This filter is used to + * identify untagged traffic that does not contain any VLAN tags or a VLAN tag + * with VLAN ID = 0 and matches destination (or source) MAC address. This filter + * can be set up by setting only l2_addr and l2_ovlan fields to be valid fields. + * All other fields are not valid. The following value are set for l2_addr and + * l2_ovlan. l2_addr = MAC, l2_ovlan = 0x0 # MAC + no VLAN or any VLAN: This + * filter is used to identify traffic that contains zero or 1 VLAN tag and + * matches destination (or source) MAC address. This filter can be set up by + * setting only l2_addr, l2_ovlan, and l2_mask fields to be valid fields. All + * other fields are not valid. The following value are set for l2_addr, + * l2_ovlan, and l2_mask fields. l2_addr = MAC, l2_ovlan = 0x0, l2_ovlan_mask = + * 0xFFFF # MAC + VLAN ID filter: This filter can be set up by setting only + * l2_addr, l2_ovlan, and l2_ovlan_mask fields to be valid fields. All other + * fields are not valid. The following values are set for those three valid + * fields. l2_addr = MAC, l2_ovlan = VLAN ID, l2_ovlan_mask = 0xF000 */ -/* Input (96 bytes) */ +/* Input (96 bytes) */ struct hwrm_cfa_l2_filter_alloc_input { uint16_t req_type; /* @@ -5136,10 +8844,12 @@ struct hwrm_cfa_l2_filter_alloc_input { */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH UINT32_C(0x1) /* tx path */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_TX (UINT32_C(0x0) << 0) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_TX \ + (UINT32_C(0x0) << 0) /* rx path */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX (UINT32_C(0x1) << 0) - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_LAST \ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX \ + (UINT32_C(0x1) << 0) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_LAST \ CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX /* * Setting of this flag indicates the applicability to the @@ -5156,60 +8866,70 @@ struct hwrm_cfa_l2_filter_alloc_input { * should not be specified. If this flag is set, then l2_* * fields refer to fields of outermost L2 header. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST UINT32_C(0x8) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST UINT32_C(0x8) uint32_t enables; /* This bit must be '1' for the l2_addr field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR UINT32_C(0x1) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR UINT32_C(0x1) /* This bit must be '1' for the l2_addr_mask field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK UINT32_C(0x2) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK \ + UINT32_C(0x2) /* This bit must be '1' for the l2_ovlan field to be configured. */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN UINT32_C(0x4) /* * This bit must be '1' for the l2_ovlan_mask field to be * configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK UINT32_C(0x8) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK \ + UINT32_C(0x8) /* This bit must be '1' for the l2_ivlan field to be configured. */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN UINT32_C(0x10) /* * This bit must be '1' for the l2_ivlan_mask field to be * configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK UINT32_C(0x20) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK \ + UINT32_C(0x20) /* This bit must be '1' for the t_l2_addr field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR UINT32_C(0x40) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR UINT32_C(0x40) /* * This bit must be '1' for the t_l2_addr_mask field to be * configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR_MASK UINT32_C(0x80) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR_MASK \ + UINT32_C(0x80) /* This bit must be '1' for the t_l2_ovlan field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN UINT32_C(0x100) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN \ + UINT32_C(0x100) /* * This bit must be '1' for the t_l2_ovlan_mask field to be * configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN_MASK UINT32_C(0x200) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN_MASK \ + UINT32_C(0x200) /* This bit must be '1' for the t_l2_ivlan field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN UINT32_C(0x400) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN \ + UINT32_C(0x400) /* * This bit must be '1' for the t_l2_ivlan_mask field to be * configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN_MASK UINT32_C(0x800) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN_MASK \ + UINT32_C(0x800) /* This bit must be '1' for the src_type field to be configured. */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_TYPE UINT32_C(0x1000) /* This bit must be '1' for the src_id field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID UINT32_C(0x2000) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID UINT32_C(0x2000) /* This bit must be '1' for the tunnel_type field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE UINT32_C(0x4000) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE \ + UINT32_C(0x4000) /* This bit must be '1' for the dst_id field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID UINT32_C(0x8000) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID UINT32_C(0x8000) /* * This bit must be '1' for the mirror_vnic_id field to be * configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID UINT32_C(0x10000) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID \ + UINT32_C(0x10000) uint8_t l2_addr[6]; /* * This value sets the match value for the L2 MAC address. @@ -5297,34 +9017,38 @@ struct hwrm_cfa_l2_filter_alloc_input { uint8_t tunnel_type; /* Tunnel Type. */ /* Non-tunnel */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL UINT32_C(0x0) - /* Virtual eXtensible Local Area Network (VXLAN) */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN UINT32_C(0x1) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL \ + UINT32_C(0x0) + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN \ + UINT32_C(0x1) /* * Network Virtualization Generic Routing - * Encapsulation (NVGRE) + * Encapsulation (NVGRE) */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE UINT32_C(0x2) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE \ + UINT32_C(0x2) /* - * Generic Routing Encapsulation (GRE) inside + * Generic Routing Encapsulation (GRE) inside * Ethernet payload */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_L2GRE UINT32_C(0x3) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_L2GRE UINT32_C(0x3) /* IP in IP */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPIP UINT32_C(0x4) - /* Generic Network Virtualization Encapsulation (Geneve) */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_GENEVE UINT32_C(0x5) - /* Multi-Protocol Lable Switching (MPLS) */ + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_GENEVE UINT32_C(0x5) + /* Multi-Protocol Lable Switching (MPLS) */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_MPLS UINT32_C(0x6) - /* Stateless Transport Tunnel (STT) */ + /* Stateless Transport Tunnel (STT) */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_STT UINT32_C(0x7) /* - * Generic Routing Encapsulation (GRE) inside IP + * Generic Routing Encapsulation (GRE) inside IP * datagram payload */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPGRE UINT32_C(0x8) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPGRE UINT32_C(0x8) /* Any tunneled traffic */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL UINT32_C(0xff) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL \ + UINT32_C(0xff) uint8_t unused_7; uint16_t dst_id; /* @@ -5340,11 +9064,14 @@ struct hwrm_cfa_l2_filter_alloc_input { * filter table. */ /* No preference */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_NO_PREFER UINT32_C(0x0) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_NO_PREFER \ + UINT32_C(0x0) /* Above the given filter */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_ABOVE_FILTER UINT32_C(0x1) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_ABOVE_FILTER \ + UINT32_C(0x1) /* Below the given filter */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_BELOW_FILTER UINT32_C(0x2) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_BELOW_FILTER \ + UINT32_C(0x2) /* As high as possible */ #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_MAX UINT32_C(0x3) /* As low as possible */ @@ -5359,7 +9086,7 @@ struct hwrm_cfa_l2_filter_alloc_input { */ } __attribute__((packed)); -/* Output (24 bytes) */ +/* Output (24 bytes) */ struct hwrm_cfa_l2_filter_alloc_output { uint16_t error_code; /* @@ -5407,7 +9134,7 @@ struct hwrm_cfa_l2_filter_alloc_output { * Description: Free a L2 filter. The HWRM shall free all associated filter * resources with the L2 filter. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_cfa_l2_filter_free_input { uint16_t req_type; /* @@ -5442,7 +9169,7 @@ struct hwrm_cfa_l2_filter_free_input { */ } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_cfa_l2_filter_free_output { uint16_t error_code; /* @@ -5476,7 +9203,7 @@ struct hwrm_cfa_l2_filter_free_output { /* hwrm_cfa_l2_filter_cfg */ /* Description: Change the configuration of an existing L2 filter */ -/* Input (40 bytes) */ +/* Input (40 bytes) */ struct hwrm_cfa_l2_filter_cfg_input { uint16_t req_type; /* @@ -5512,10 +9239,12 @@ struct hwrm_cfa_l2_filter_cfg_input { */ #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH UINT32_C(0x1) /* tx path */ - #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_TX (UINT32_C(0x0) << 0) + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_TX \ + (UINT32_C(0x0) << 0) /* rx path */ - #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX (UINT32_C(0x1) << 0) - #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_LAST \ + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX \ + (UINT32_C(0x1) << 0) + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_LAST \ CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX /* * Setting of this flag indicates drop action. If this flag is @@ -5529,7 +9258,8 @@ struct hwrm_cfa_l2_filter_cfg_input { * This bit must be '1' for the new_mirror_vnic_id field to be * configured. */ - #define HWRM_CFA_L2_FILTER_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID UINT32_C(0x2) + #define HWRM_CFA_L2_FILTER_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID \ + UINT32_C(0x2) uint64_t l2_filter_id; /* * This value identifies a set of CFA data structures used for @@ -5545,7 +9275,7 @@ struct hwrm_cfa_l2_filter_cfg_input { /* New Logical VNIC ID of the VNIC where traffic is mirrored. */ } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_cfa_l2_filter_cfg_output { uint16_t error_code; /* @@ -5579,7 +9309,7 @@ struct hwrm_cfa_l2_filter_cfg_output { /* hwrm_cfa_l2_set_rx_mask */ /* Description: This command will set rx mask of the function. */ -/* Input (56 bytes) */ +/* Input (56 bytes) */ struct hwrm_cfa_l2_set_rx_mask_input { uint16_t req_type; /* @@ -5632,14 +9362,14 @@ struct hwrm_cfa_l2_set_rx_mask_input { * the promiscuous mode. The HWRM should accept any function to * set up promiscuous mode. The HWRM shall follow the semantics * below for the promiscuous mode support. # When partitioning - * is not enabled on a port (i.e. single PF on the port), then + * is not enabled on a port (i.e. single PF on the port), then * the PF shall be allowed to be in the promiscuous mode. When * the PF is in the promiscuous mode, then it shall receive all * host bound traffic on that port. # When partitioning is - * enabled on a port (i.e. multiple PFs per port) and a PF on + * enabled on a port (i.e. multiple PFs per port) and a PF on * that port is in the promiscuous mode, then the PF receives * all traffic within that partition as identified by a unique - * identifier for the PF (e.g. S-Tag). If a unique outer VLAN + * identifier for the PF (e.g. S-Tag). If a unique outer VLAN * for the PF is specified, then the setting of promiscuous mode * on that PF shall result in the PF receiving all host bound * traffic with matching outer VLAN. # A VF shall can be set in @@ -5652,7 +9382,7 @@ struct hwrm_cfa_l2_set_rx_mask_input { * mode on a function independently from the promiscuous mode * settings on other functions. */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS UINT32_C(0x10) + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS UINT32_C(0x10) /* * If this flag is set, the corresponding RX filters shall be * set up to cover multicast/broadcast filters for the outermost @@ -5685,7 +9415,8 @@ struct hwrm_cfa_l2_set_rx_mask_input { * set at most one flag out of vlanonly, vlan_nonvlan, and * anyvlan_nonvlan. */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN UINT32_C(0x100) + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN \ + UINT32_C(0x100) uint64_t mc_tbl_addr; /* This is the address for mcast address tbl. */ uint32_t num_mc_entries; @@ -5708,7 +9439,7 @@ struct hwrm_cfa_l2_set_rx_mask_input { uint32_t unused_1; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_cfa_l2_set_rx_mask_output { uint16_t error_code; /* @@ -5740,12 +9471,364 @@ struct hwrm_cfa_l2_set_rx_mask_output { */ } __attribute__((packed)); +/* hwrm_cfa_vlan_antispoof_cfg */ +/* Description: Configures vlan anti-spoof filters for VF. */ +/* Input (32 bytes) */ +struct hwrm_cfa_vlan_antispoof_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format for the + * rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request will be + * optionally completed on. If the value is -1, then no CR completion + * will be generated. Any other value must be a valid CR ring_id value + * for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written when the + * request is complete. This area must be 16B aligned and must be + * cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being configured. Only valid for + * a VF FID configured by the PF. + */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t num_vlan_entries; + /* Number of VLAN entries in the vlan_tag_mask_tbl. */ + uint64_t vlan_tag_mask_tbl_addr; + /* + * The vlan_tag_mask_tbl_addr is the DMA address of the VLAN antispoof + * table. Each table entry contains the 16-bit TPID (0x8100 or 0x88a8 + * only), 16-bit VLAN ID, and a 16-bit mask, all in network order to + * match hwrm_cfa_l2_set_rx_mask. For an individual VLAN entry, the mask + * value should be 0xfff for the 12-bit VLAN ID. + */ +}; + +/* Output (16 bytes) */ +struct hwrm_cfa_vlan_antispoof_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in parameters, + * and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last byte of + * the response is a valid flag that will read as '1' when the command + * has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the output is + * completely written to RAM. This field should be read as '1' to + * indicate that the output has been completely written. When writing a + * command completion or response to an internal processor, the order of + * writes has to be such that this field is written last. + */ +}; + +/* hwrm_tunnel_dst_port_query */ +/* + * Description: This function is called by a driver to query tunnel type + * specific destination port configuration. + */ +/* Input (24 bytes) */ +struct hwrm_tunnel_dst_port_query_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_TUNNEL_DST_PORT_QUERY_INPUT_TUNNEL_TYPE_VXLAN \ + UINT32_C(0x1) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_TUNNEL_DST_PORT_QUERY_INPUT_TUNNEL_TYPE_GENEVE \ + UINT32_C(0x5) + uint8_t unused_0[7]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_tunnel_dst_port_query_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t tunnel_dst_port_id; + /* + * This field represents the identifier of L4 destination port + * used for the given tunnel type. This field is valid for + * specific tunnel types that use layer 4 (e.g. UDP) transports + * for tunneling. + */ + uint16_t tunnel_dst_port_val; + /* + * This field represents the value of L4 destination port + * identified by tunnel_dst_port_id. This field is valid for + * specific tunnel types that use layer 4 (e.g. UDP) transports + * for tunneling. This field is in network byte order. A value + * of 0 means that the destination port is not configured. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_tunnel_dst_port_alloc */ +/* + * Description: This function is called by a driver to allocate l4 destination + * port for a specific tunnel type. The destination port value is provided in + * the input. If the HWRM supports only one global destination port for a tunnel + * type, then the HWRM shall keep track of its usage as described below. # The + * first caller that allocates a destination port shall always succeed and the + * HWRM shall save the destination port configuration for that tunnel type and + * increment the usage count to 1. # Subsequent callers allocating the same + * destination port for that tunnel type shall succeed and the HWRM shall + * increment the usage count for that port for each subsequent caller that + * succeeds. # Any subsequent caller trying to allocate a different destination + * port for that tunnel type shall fail until the usage count for the original + * destination port goes to zero. # A caller that frees a port will cause the + * usage count for that port to decrement. + */ +/* Input (24 bytes) */ +struct hwrm_tunnel_dst_port_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN UINT32_C(0x1) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE \ + UINT32_C(0x5) + uint8_t unused_0; + uint16_t tunnel_dst_port_val; + /* + * This field represents the value of L4 destination port used + * for the given tunnel type. This field is valid for specific + * tunnel types that use layer 4 (e.g. UDP) transports for + * tunneling. This field is in network byte order. A value of 0 + * shall fail the command. + */ + uint32_t unused_1; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_tunnel_dst_port_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t tunnel_dst_port_id; + /* + * Identifier of a tunnel L4 destination port value. Only + * applies to tunnel types that has l4 destination port + * parameters. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t unused_4; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_tunnel_dst_port_free */ +/* + * Description: This function is called by a driver to free l4 destination port + * for a specific tunnel type. + */ +/* Input (24 bytes) */ +struct hwrm_tunnel_dst_port_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN UINT32_C(0x1) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE UINT32_C(0x5) + uint8_t unused_0; + uint16_t tunnel_dst_port_id; + /* + * Identifier of a tunnel L4 destination port value. Only + * applies to tunnel types that has l4 destination port + * parameters. + */ + uint32_t unused_1; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_tunnel_dst_port_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + /* hwrm_stat_ctx_alloc */ /* * Description: This command allocates and does basic preparation for a stat * context. */ -/* Input (32 bytes) */ +/* Input (32 bytes) */ struct hwrm_stat_ctx_alloc_input { uint16_t req_type; /* @@ -5783,10 +9866,24 @@ struct hwrm_stat_ctx_alloc_input { * used. In this case, the stat block can only be read by * hwrm_stat_ctx_query command. */ - uint32_t unused_0; + uint8_t stat_ctx_flags; + /* + * This field is used to specify statistics context specific + * configuration flags. + */ + /* + * When this bit is set to '1', the statistics context shall be + * allocated for RoCE traffic only. In this case, traffic other + * than offloaded RoCE traffic shall not be included in this + * statistic context. When this bit is set to '0', the + * statistics context shall be used for the network traffic + * other than offloaded RoCE traffic. + */ + #define HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE UINT32_C(0x1) + uint8_t unused_0[3]; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_stat_ctx_alloc_output { uint16_t error_code; /* @@ -5821,7 +9918,7 @@ struct hwrm_stat_ctx_alloc_output { /* hwrm_stat_ctx_free */ /* Description: This command is used to free a stat context. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_stat_ctx_free_input { uint16_t req_type; /* @@ -5854,7 +9951,7 @@ struct hwrm_stat_ctx_free_input { uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_stat_ctx_free_output { uint16_t error_code; /* @@ -5889,7 +9986,7 @@ struct hwrm_stat_ctx_free_output { /* hwrm_stat_ctx_clr_stats */ /* Description: This command clears statistics of a context. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_stat_ctx_clr_stats_input { uint16_t req_type; /* @@ -5922,7 +10019,7 @@ struct hwrm_stat_ctx_clr_stats_input { uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_stat_ctx_clr_stats_output { uint16_t error_code; /* @@ -5954,6 +10051,113 @@ struct hwrm_stat_ctx_clr_stats_output { */ } __attribute__((packed)); +/* hwrm_stat_ctx_query */ +/* Description: This command returns statistics of a context. */ +/* Input (24 bytes) */ + +struct hwrm_stat_ctx_query_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format for the + * rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request will be + * optionally completed on. If the value is -1, then no CR completion + * will be generated. Any other value must be a valid CR ring_id value + * for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written when the + * request is complete. This area must be 16B aligned and must be + * cleared to zero before the request is made. + */ + uint32_t stat_ctx_id; + /* ID of the statistics context that is being queried. */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (176 bytes) */ + +struct hwrm_stat_ctx_query_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in parameters, + * and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last byte of + * the response is a valid flag that will read as '1' when the command + * has been completely written to memory. + */ + uint64_t tx_ucast_pkts; + /* Number of transmitted unicast packets */ + uint64_t tx_mcast_pkts; + /* Number of transmitted multicast packets */ + uint64_t tx_bcast_pkts; + /* Number of transmitted broadcast packets */ + uint64_t tx_err_pkts; + /* Number of transmitted packets with error */ + uint64_t tx_drop_pkts; + /* Number of dropped packets on transmit path */ + uint64_t tx_ucast_bytes; + /* Number of transmitted bytes for unicast traffic */ + uint64_t tx_mcast_bytes; + /* Number of transmitted bytes for multicast traffic */ + uint64_t tx_bcast_bytes; + /* Number of transmitted bytes for broadcast traffic */ + uint64_t rx_ucast_pkts; + /* Number of received unicast packets */ + uint64_t rx_mcast_pkts; + /* Number of received multicast packets */ + uint64_t rx_bcast_pkts; + /* Number of received broadcast packets */ + uint64_t rx_err_pkts; + /* Number of received packets with error */ + uint64_t rx_drop_pkts; + /* Number of dropped packets on received path */ + uint64_t rx_ucast_bytes; + /* Number of received bytes for unicast traffic */ + uint64_t rx_mcast_bytes; + /* Number of received bytes for multicast traffic */ + uint64_t rx_bcast_bytes; + /* Number of received bytes for broadcast traffic */ + uint64_t rx_agg_pkts; + /* Number of aggregated unicast packets */ + uint64_t rx_agg_bytes; + /* Number of aggregated unicast bytes */ + uint64_t rx_agg_events; + /* Number of aggregation events */ + uint64_t rx_agg_aborts; + /* Number of aborted aggregations */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the output is + * completely written to RAM. This field should be read as '1' to + * indicate that the output has been completely written. When writing a + * command completion or response to an internal processor, the order of + * writes has to be such that this field is written last. + */ +} __attribute__((packed)); + /* hwrm_exec_fwd_resp */ /* * Description: This command is used to send an encapsulated request to the @@ -5964,7 +10168,7 @@ struct hwrm_stat_ctx_clr_stats_output { * acknowledge the receipt of the encapsulated request and forwarding of the * response. */ -/* Input (128 bytes) */ +/* Input (128 bytes) */ struct hwrm_exec_fwd_resp_input { uint16_t req_type; /* @@ -6008,7 +10212,7 @@ struct hwrm_exec_fwd_resp_input { uint16_t unused_0[3]; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_exec_fwd_resp_output { uint16_t error_code; /* @@ -6040,4 +10244,631 @@ struct hwrm_exec_fwd_resp_output { */ } __attribute__((packed)); -#endif +/* hwrm_reject_fwd_resp */ +/* + * Description: This command is used to send an encapsulated request to the + * HWRM. This command instructs the HWRM to reject the request and forward the + * error response of the encapsulated request to the location specified in the + * original request that is encapsulated. The target id of this command shall be + * set to 0xFFFF (HWRM). The response location in this command shall be used to + * acknowledge the receipt of the encapsulated request and forwarding of the + * response. + */ +/* Input (128 bytes) */ +struct hwrm_reject_fwd_resp_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t encap_request[26]; + /* + * This is an encapsulated request. This request should be + * rejected by the HWRM and the error response should be + * provided in the response buffer inside the encapsulated + * request. + */ + uint16_t encap_resp_target_id; + /* + * This value indicates the target id of the response to the + * encapsulated request. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - + * HWRM + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_reject_fwd_resp_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* Hardware Resource Manager Specification */ +/* Description: This structure is used to specify port description. */ +/* + * Note: The Hardware Resource Manager (HWRM) manages various hardware resources + * inside the chip. The HWRM is implemented in firmware, and runs on embedded + * processors inside the chip. This firmware service is vital part of the chip. + * The chip can not be used by a driver or HWRM client without the HWRM. + */ +/* Input (16 bytes) */ +struct input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ +} __attribute__((packed)); + +/* Output (8 bytes) */ +struct output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ +} __attribute__((packed)); + +/* Short Command Structure (16 bytes) */ +struct hwrm_short_input { + uint16_t req_type; + uint16_t signature; + #define HWRM_SHORT_REQ_SIGNATURE_SHORT_CMD (UINT32_C(0x4321)) + uint16_t unused_0; + uint16_t size; + uint64_t req_addr; +} __attribute__((packed)); + +#define HWRM_GET_HWRM_ERROR_CODE(arg) \ + { \ + typeof(arg) x = (arg); \ + ((x) == 0xf ? "HWRM_ERROR" : \ + ((x) == 0xffff ? "CMD_NOT_SUPPORTED" : \ + ((x) == 0xfffe ? "UNKNOWN_ERR" : \ + ((x) == 0x4 ? "RESOURCE_ALLOC_ERROR" : \ + ((x) == 0x5 ? "INVALID_FLAGS" : \ + ((x) == 0x6 ? "INVALID_ENABLES" : \ + ((x) == 0x0 ? "SUCCESS" : \ + ((x) == 0x1 ? "FAIL" : \ + ((x) == 0x2 ? "INVALID_PARAMS" : \ + ((x) == 0x3 ? "RESOURCE_ACCESS_DENIED" : \ + "Unknown error_code")))))))))) \ + } + +/* Return Codes (8 bytes) */ +struct ret_codes { + uint16_t error_code; + /* These are numbers assigned to return/error codes. */ + /* Request was successfully executed by the HWRM. */ + #define HWRM_ERR_CODE_SUCCESS (UINT32_C(0x0)) + /* THe HWRM failed to execute the request. */ + #define HWRM_ERR_CODE_FAIL (UINT32_C(0x1)) + /* + * The request contains invalid argument(s) or + * input parameters. + */ + #define HWRM_ERR_CODE_INVALID_PARAMS (UINT32_C(0x2)) + /* + * The requester is not allowed to access the + * requested resource. This error code shall be + * provided in a response to a request to query + * or modify an existing resource that is not + * accessible by the requester. + */ + #define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED (UINT32_C(0x3)) + /* + * The HWRM is unable to allocate the requested + * resource. This code only applies to requests + * for HWRM resource allocations. + */ + #define HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR (UINT32_C(0x4)) + /* Invalid combination of flags is specified in the request. */ + #define HWRM_ERR_CODE_INVALID_FLAGS (UINT32_C(0x5)) + /* + * Invalid combination of enables fields is + * specified in the request. + */ + #define HWRM_ERR_CODE_INVALID_ENABLES (UINT32_C(0x6)) + /* + * Generic HWRM execution error that represents + * an internal error. + */ + #define HWRM_ERR_CODE_HWRM_ERROR (UINT32_C(0xf)) + /* Unknown error */ + #define HWRM_ERR_CODE_UNKNOWN_ERR (UINT32_C(0xfffe)) + /* Unsupported or invalid command */ + #define HWRM_ERR_CODE_CMD_NOT_SUPPORTED (UINT32_C(0xffff)) + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_err_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t opaque_0; + /* debug info for this error response. */ + uint16_t opaque_1; + /* debug info for this error response. */ + uint8_t cmd_err; + /* + * In the case of an error response, command specific error code + * is returned in this field. + */ + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* Port Tx Statistics Formats (408 bytes) */ +struct tx_port_stats { + uint64_t tx_64b_frames; + /* Total Number of 64 Bytes frames transmitted */ + uint64_t tx_65b_127b_frames; + /* Total Number of 65-127 Bytes frames transmitted */ + uint64_t tx_128b_255b_frames; + /* Total Number of 128-255 Bytes frames transmitted */ + uint64_t tx_256b_511b_frames; + /* Total Number of 256-511 Bytes frames transmitted */ + uint64_t tx_512b_1023b_frames; + /* Total Number of 512-1023 Bytes frames transmitted */ + uint64_t tx_1024b_1518_frames; + /* Total Number of 1024-1518 Bytes frames transmitted */ + uint64_t tx_good_vlan_frames; + /* + * Total Number of each good VLAN (exludes FCS errors) frame + * transmitted which is 1519 to 1522 bytes in length inclusive + * (excluding framing bits but including FCS bytes). + */ + uint64_t tx_1519b_2047_frames; + /* Total Number of 1519-2047 Bytes frames transmitted */ + uint64_t tx_2048b_4095b_frames; + /* Total Number of 2048-4095 Bytes frames transmitted */ + uint64_t tx_4096b_9216b_frames; + /* Total Number of 4096-9216 Bytes frames transmitted */ + uint64_t tx_9217b_16383b_frames; + /* Total Number of 9217-16383 Bytes frames transmitted */ + uint64_t tx_good_frames; + /* Total Number of good frames transmitted */ + uint64_t tx_total_frames; + /* Total Number of frames transmitted */ + uint64_t tx_ucast_frames; + /* Total number of unicast frames transmitted */ + uint64_t tx_mcast_frames; + /* Total number of multicast frames transmitted */ + uint64_t tx_bcast_frames; + /* Total number of broadcast frames transmitted */ + uint64_t tx_pause_frames; + /* Total number of PAUSE control frames transmitted */ + uint64_t tx_pfc_frames; + /* Total number of PFC/per-priority PAUSE control frames transmitted */ + uint64_t tx_jabber_frames; + /* Total number of jabber frames transmitted */ + uint64_t tx_fcs_err_frames; + /* Total number of frames transmitted with FCS error */ + uint64_t tx_control_frames; + /* Total number of control frames transmitted */ + uint64_t tx_oversz_frames; + /* Total number of over-sized frames transmitted */ + uint64_t tx_single_dfrl_frames; + /* Total number of frames with single deferral */ + uint64_t tx_multi_dfrl_frames; + /* Total number of frames with multiple deferrals */ + uint64_t tx_single_coll_frames; + /* Total number of frames with single collision */ + uint64_t tx_multi_coll_frames; + /* Total number of frames with multiple collisions */ + uint64_t tx_late_coll_frames; + /* Total number of frames with late collisions */ + uint64_t tx_excessive_coll_frames; + /* Total number of frames with excessive collisions */ + uint64_t tx_frag_frames; + /* Total number of fragmented frames transmitted */ + uint64_t tx_err; + /* Total number of transmit errors */ + uint64_t tx_tagged_frames; + /* Total number of single VLAN tagged frames transmitted */ + uint64_t tx_dbl_tagged_frames; + /* Total number of double VLAN tagged frames transmitted */ + uint64_t tx_runt_frames; + /* Total number of runt frames transmitted */ + uint64_t tx_fifo_underruns; + /* Total number of TX FIFO under runs */ + uint64_t tx_pfc_ena_frames_pri0; + /* + * Total number of PFC frames with PFC enabled bit for Pri 0 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri1; + /* + * Total number of PFC frames with PFC enabled bit for Pri 1 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri2; + /* + * Total number of PFC frames with PFC enabled bit for Pri 2 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri3; + /* + * Total number of PFC frames with PFC enabled bit for Pri 3 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri4; + /* + * Total number of PFC frames with PFC enabled bit for Pri 4 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri5; + /* + * Total number of PFC frames with PFC enabled bit for Pri 5 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri6; + /* + * Total number of PFC frames with PFC enabled bit for Pri 6 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri7; + /* + * Total number of PFC frames with PFC enabled bit for Pri 7 + * transmitted + */ + uint64_t tx_eee_lpi_events; + /* Total number of EEE LPI Events on TX */ + uint64_t tx_eee_lpi_duration; + /* EEE LPI Duration Counter on TX */ + uint64_t tx_llfc_logical_msgs; + /* + * Total number of Link Level Flow Control (LLFC) messages + * transmitted + */ + uint64_t tx_hcfc_msgs; + /* Total number of HCFC messages transmitted */ + uint64_t tx_total_collisions; + /* Total number of TX collisions */ + uint64_t tx_bytes; + /* Total number of transmitted bytes */ + uint64_t tx_xthol_frames; + /* Total number of end-to-end HOL frames */ + uint64_t tx_stat_discard; + /* Total Tx Drops per Port reported by STATS block */ + uint64_t tx_stat_error; + /* Total Tx Error Drops per Port reported by STATS block */ +} __attribute__((packed)); + +/* Port Rx Statistics Formats (528 bytes) */ +struct rx_port_stats { + uint64_t rx_64b_frames; + /* Total Number of 64 Bytes frames received */ + uint64_t rx_65b_127b_frames; + /* Total Number of 65-127 Bytes frames received */ + uint64_t rx_128b_255b_frames; + /* Total Number of 128-255 Bytes frames received */ + uint64_t rx_256b_511b_frames; + /* Total Number of 256-511 Bytes frames received */ + uint64_t rx_512b_1023b_frames; + /* Total Number of 512-1023 Bytes frames received */ + uint64_t rx_1024b_1518_frames; + /* Total Number of 1024-1518 Bytes frames received */ + uint64_t rx_good_vlan_frames; + /* + * Total Number of each good VLAN (exludes FCS errors) frame + * received which is 1519 to 1522 bytes in length inclusive + * (excluding framing bits but including FCS bytes). + */ + uint64_t rx_1519b_2047b_frames; + /* Total Number of 1519-2047 Bytes frames received */ + uint64_t rx_2048b_4095b_frames; + /* Total Number of 2048-4095 Bytes frames received */ + uint64_t rx_4096b_9216b_frames; + /* Total Number of 4096-9216 Bytes frames received */ + uint64_t rx_9217b_16383b_frames; + /* Total Number of 9217-16383 Bytes frames received */ + uint64_t rx_total_frames; + /* Total number of frames received */ + uint64_t rx_ucast_frames; + /* Total number of unicast frames received */ + uint64_t rx_mcast_frames; + /* Total number of multicast frames received */ + uint64_t rx_bcast_frames; + /* Total number of broadcast frames received */ + uint64_t rx_fcs_err_frames; + /* Total number of received frames with FCS error */ + uint64_t rx_ctrl_frames; + /* Total number of control frames received */ + uint64_t rx_pause_frames; + /* Total number of PAUSE frames received */ + uint64_t rx_pfc_frames; + /* Total number of PFC frames received */ + uint64_t rx_unsupported_opcode_frames; + /* Total number of frames received with an unsupported opcode */ + uint64_t rx_unsupported_da_pausepfc_frames; + /* + * Total number of frames received with an unsupported DA for + * pause and PFC + */ + uint64_t rx_wrong_sa_frames; + /* Total number of frames received with an unsupported SA */ + uint64_t rx_align_err_frames; + /* Total number of received packets with alignment error */ + uint64_t rx_oor_len_frames; + /* Total number of received frames with out-of-range length */ + uint64_t rx_code_err_frames; + /* Total number of received frames with error termination */ + uint64_t rx_false_carrier_frames; + /* + * Total number of received frames with a false carrier is + * detected during idle, as defined by RX_ER samples active and + * RXD is 0xE. The event is reported along with the statistics + * generated on the next received frame. Only one false carrier + * condition can be detected and logged between frames. Carrier + * event, valid for 10M/100M speed modes only. + */ + uint64_t rx_ovrsz_frames; + /* Total number of over-sized frames received */ + uint64_t rx_jbr_frames; + /* Total number of jabber packets received */ + uint64_t rx_mtu_err_frames; + /* Total number of received frames with MTU error */ + uint64_t rx_match_crc_frames; + /* Total number of received frames with CRC match */ + uint64_t rx_promiscuous_frames; + /* Total number of frames received promiscuously */ + uint64_t rx_tagged_frames; + /* Total number of received frames with one or two VLAN tags */ + uint64_t rx_double_tagged_frames; + /* Total number of received frames with two VLAN tags */ + uint64_t rx_trunc_frames; + /* Total number of truncated frames received */ + uint64_t rx_good_frames; + /* Total number of good frames (without errors) received */ + uint64_t rx_pfc_xon2xoff_frames_pri0; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 0 + */ + uint64_t rx_pfc_xon2xoff_frames_pri1; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 1 + */ + uint64_t rx_pfc_xon2xoff_frames_pri2; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 2 + */ + uint64_t rx_pfc_xon2xoff_frames_pri3; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 3 + */ + uint64_t rx_pfc_xon2xoff_frames_pri4; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 4 + */ + uint64_t rx_pfc_xon2xoff_frames_pri5; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 5 + */ + uint64_t rx_pfc_xon2xoff_frames_pri6; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 6 + */ + uint64_t rx_pfc_xon2xoff_frames_pri7; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 7 + */ + uint64_t rx_pfc_ena_frames_pri0; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 0 + */ + uint64_t rx_pfc_ena_frames_pri1; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 1 + */ + uint64_t rx_pfc_ena_frames_pri2; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 2 + */ + uint64_t rx_pfc_ena_frames_pri3; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 3 + */ + uint64_t rx_pfc_ena_frames_pri4; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 4 + */ + uint64_t rx_pfc_ena_frames_pri5; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 5 + */ + uint64_t rx_pfc_ena_frames_pri6; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 6 + */ + uint64_t rx_pfc_ena_frames_pri7; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 7 + */ + uint64_t rx_sch_crc_err_frames; + /* Total Number of frames received with SCH CRC error */ + uint64_t rx_undrsz_frames; + /* Total Number of under-sized frames received */ + uint64_t rx_frag_frames; + /* Total Number of fragmented frames received */ + uint64_t rx_eee_lpi_events; + /* Total number of RX EEE LPI Events */ + uint64_t rx_eee_lpi_duration; + /* EEE LPI Duration Counter on RX */ + uint64_t rx_llfc_physical_msgs; + /* + * Total number of physical type Link Level Flow Control (LLFC) + * messages received + */ + uint64_t rx_llfc_logical_msgs; + /* + * Total number of logical type Link Level Flow Control (LLFC) + * messages received + */ + uint64_t rx_llfc_msgs_with_crc_err; + /* + * Total number of logical type Link Level Flow Control (LLFC) + * messages received with CRC error + */ + uint64_t rx_hcfc_msgs; + /* Total number of HCFC messages received */ + uint64_t rx_hcfc_msgs_with_crc_err; + /* Total number of HCFC messages received with CRC error */ + uint64_t rx_bytes; + /* Total number of received bytes */ + uint64_t rx_runt_bytes; + /* Total number of bytes received in runt frames */ + uint64_t rx_runt_frames; + /* Total number of runt frames received */ + uint64_t rx_stat_discard; + /* Total Rx Discards per Port reported by STATS block */ + uint64_t rx_stat_err; + /* Total Rx Error Drops per Port reported by STATS block */ +} __attribute__((packed)); + +/* Periodic Statistics Context DMA to host (160 bytes) */ +/* + * per-context HW statistics -- chip view + */ + +struct ctx_hw_stats64 { + uint64_t rx_ucast_pkts; + uint64_t rx_mcast_pkts; + uint64_t rx_bcast_pkts; + uint64_t rx_drop_pkts; + uint64_t rx_discard_pkts; + uint64_t rx_ucast_bytes; + uint64_t rx_mcast_bytes; + uint64_t rx_bcast_bytes; + + uint64_t tx_ucast_pkts; + uint64_t tx_mcast_pkts; + uint64_t tx_bcast_pkts; + uint64_t tx_drop_pkts; + uint64_t tx_discard_pkts; + uint64_t tx_ucast_bytes; + uint64_t tx_mcast_bytes; + uint64_t tx_bcast_bytes; + + uint64_t tpa_pkts; + uint64_t tpa_bytes; + uint64_t tpa_events; + uint64_t tpa_aborts; +} __attribute__((packed)); + +#endif /* _HSI_STRUCT_DEF_DPDK_ */ diff --git a/drivers/net/bnxt/rte_pmd_bnxt.c b/drivers/net/bnxt/rte_pmd_bnxt.c new file mode 100644 index 00000000..c343d903 --- /dev/null +++ b/drivers/net/bnxt/rte_pmd_bnxt.c @@ -0,0 +1,843 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Broadcom Limited. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <inttypes.h> +#include <stdbool.h> +#include <unistd.h> + +#include <rte_dev.h> +#include <rte_ethdev.h> +#include <rte_malloc.h> +#include <rte_cycles.h> +#include <rte_byteorder.h> + +#include "bnxt.h" +#include "bnxt_filter.h" +#include "bnxt_hwrm.h" +#include "bnxt_vnic.h" +#include "rte_pmd_bnxt.h" +#include "hsi_struct_def_dpdk.h" + +int bnxt_rcv_msg_from_vf(struct bnxt *bp, uint16_t vf_id, void *msg) +{ + struct rte_pmd_bnxt_mb_event_param ret_param; + + ret_param.retval = RTE_PMD_BNXT_MB_EVENT_PROCEED; + ret_param.vf_id = vf_id; + ret_param.msg = msg; + + _rte_eth_dev_callback_process(bp->eth_dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); + + /* Default to approve */ + if (ret_param.retval == RTE_PMD_BNXT_MB_EVENT_PROCEED) + ret_param.retval = RTE_PMD_BNXT_MB_EVENT_NOOP_ACK; + + return ret_param.retval == RTE_PMD_BNXT_MB_EVENT_NOOP_ACK ? + true : false; +} + +int rte_pmd_bnxt_set_tx_loopback(uint8_t port, uint8_t on) +{ + struct rte_eth_dev *eth_dev; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + eth_dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(eth_dev)) + return -ENOTSUP; + + bp = (struct bnxt *)eth_dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set Tx loopback on non-PF port %d!\n", + port); + return -ENOTSUP; + } + + if (on) + bp->pf.evb_mode = BNXT_EVB_MODE_VEB; + else + bp->pf.evb_mode = BNXT_EVB_MODE_VEPA; + + rc = bnxt_hwrm_pf_evb_mode(bp); + + return rc; +} + +static void +rte_pmd_bnxt_set_all_queues_drop_en_cb(struct bnxt_vnic_info *vnic, void *onptr) +{ + uint8_t *on = onptr; + vnic->bd_stall = !(*on); +} + +int rte_pmd_bnxt_set_all_queues_drop_en(uint8_t port, uint8_t on) +{ + struct rte_eth_dev *eth_dev; + struct bnxt *bp; + uint32_t i; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + eth_dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(eth_dev)) + return -ENOTSUP; + + bp = (struct bnxt *)eth_dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set all queues drop on non-PF port!\n"); + return -ENOTSUP; + } + + if (bp->vnic_info == NULL) + return -ENODEV; + + /* Stall PF */ + for (i = 0; i < bp->nr_vnics; i++) { + bp->vnic_info[i].bd_stall = !on; + rc = bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info[i]); + if (rc) { + RTE_LOG(ERR, PMD, "Failed to update PF VNIC %d.\n", i); + return rc; + } + } + + /* Stall all active VFs */ + for (i = 0; i < bp->pf.active_vfs; i++) { + rc = bnxt_hwrm_func_vf_vnic_query_and_config(bp, i, + rte_pmd_bnxt_set_all_queues_drop_en_cb, &on, + bnxt_hwrm_vnic_cfg); + if (rc) { + RTE_LOG(ERR, PMD, "Failed to update VF VNIC %d.\n", i); + break; + } + } + + return rc; +} + +int rte_pmd_bnxt_set_vf_mac_addr(uint8_t port, uint16_t vf, + struct ether_addr *mac_addr) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf >= dev_info.max_vfs || mac_addr == NULL) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VF %d mac address on non-PF port %d!\n", + vf, port); + return -ENOTSUP; + } + + rc = bnxt_hwrm_func_vf_mac(bp, vf, (uint8_t *)mac_addr); + + return rc; +} + +int rte_pmd_bnxt_set_vf_rate_limit(uint8_t port, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk) +{ + struct rte_eth_dev *eth_dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + uint16_t tot_rate = 0; + uint64_t idx; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + eth_dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(eth_dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)eth_dev->data->dev_private; + + if (!bp->pf.active_vfs) + return -EINVAL; + + if (vf >= bp->pf.max_vfs) + return -EINVAL; + + /* Add up the per queue BW and configure MAX BW of the VF */ + for (idx = 0; idx < 64; idx++) { + if ((1ULL << idx) & q_msk) + tot_rate += tx_rate; + } + + /* Requested BW can't be greater than link speed */ + if (tot_rate > eth_dev->data->dev_link.link_speed) { + RTE_LOG(ERR, PMD, "Rate > Link speed. Set to %d\n", tot_rate); + return -EINVAL; + } + + /* Requested BW already configured */ + if (tot_rate == bp->pf.vf_info[vf].max_tx_rate) + return 0; + + rc = bnxt_hwrm_func_bw_cfg(bp, vf, tot_rate, + HWRM_FUNC_CFG_INPUT_ENABLES_MAX_BW); + + if (!rc) + bp->pf.vf_info[vf].max_tx_rate = tot_rate; + + return rc; +} + +int rte_pmd_bnxt_set_vf_mac_anti_spoof(uint8_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev_info dev_info; + struct rte_eth_dev *dev; + uint32_t func_flags; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set mac spoof on non-PF port %d!\n", port); + return -EINVAL; + } + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + /* Prev setting same as new setting. */ + if (on == bp->pf.vf_info[vf].mac_spoof_en) + return 0; + + func_flags = bp->pf.vf_info[vf].func_cfg_flags; + func_flags &= ~(HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE | + HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE); + + if (on) + func_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE; + else + func_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE; + + rc = bnxt_hwrm_func_cfg_vf_set_flags(bp, vf, func_flags); + if (!rc) { + bp->pf.vf_info[vf].mac_spoof_en = on; + bp->pf.vf_info[vf].func_cfg_flags = func_flags; + } + + return rc; +} + +int rte_pmd_bnxt_set_vf_vlan_anti_spoof(uint8_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev_info dev_info; + struct rte_eth_dev *dev; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VLAN spoof on non-PF port %d!\n", port); + return -EINVAL; + } + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + if (on == bp->pf.vf_info[vf].vlan_spoof_en) + return 0; + + rc = bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(bp, vf, on); + if (!rc) { + bp->pf.vf_info[vf].vlan_spoof_en = on; + if (on) { + if (bnxt_hwrm_cfa_vlan_antispoof_cfg(bp, + bp->pf.first_vf_id + vf, + bp->pf.vf_info[vf].vlan_count, + bp->pf.vf_info[vf].vlan_as_table)) + rc = -1; + } + } else { + RTE_LOG(ERR, PMD, "Failed to update VF VNIC %d.\n", vf); + } + + return rc; +} + +static void +rte_pmd_bnxt_set_vf_vlan_stripq_cb(struct bnxt_vnic_info *vnic, void *onptr) +{ + uint8_t *on = onptr; + vnic->vlan_strip = *on; +} + +int +rte_pmd_bnxt_set_vf_vlan_stripq(uint8_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VF %d stripq on non-PF port %d!\n", + vf, port); + return -ENOTSUP; + } + + rc = bnxt_hwrm_func_vf_vnic_query_and_config(bp, vf, + rte_pmd_bnxt_set_vf_vlan_stripq_cb, &on, + bnxt_hwrm_vnic_cfg); + if (rc) + RTE_LOG(ERR, PMD, "Failed to update VF VNIC %d.\n", vf); + + return rc; +} + +int rte_pmd_bnxt_set_vf_rxmode(uint8_t port, uint16_t vf, + uint16_t rx_mask, uint8_t on) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + uint16_t flag = 0; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!bp->pf.vf_info) + return -EINVAL; + + if (vf >= bp->pdev->max_vfs) + return -EINVAL; + + if (rx_mask & (ETH_VMDQ_ACCEPT_UNTAG | ETH_VMDQ_ACCEPT_HASH_MC)) { + RTE_LOG(ERR, PMD, "Currently cannot toggle this setting\n"); + return -ENOTSUP; + } + + if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC && !on) { + RTE_LOG(ERR, PMD, "Currently cannot disable UC Rx\n"); + return -ENOTSUP; + } + + if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST) + flag |= BNXT_VNIC_INFO_BCAST; + if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST) + flag |= BNXT_VNIC_INFO_ALLMULTI; + + if (on) + bp->pf.vf_info[vf].l2_rx_mask |= flag; + else + bp->pf.vf_info[vf].l2_rx_mask &= ~flag; + + rc = bnxt_hwrm_func_vf_vnic_query_and_config(bp, vf, + vf_vnic_set_rxmask_cb, + &bp->pf.vf_info[vf].l2_rx_mask, + bnxt_set_rx_mask_no_vlan); + if (rc) + RTE_LOG(ERR, PMD, "bnxt_hwrm_func_vf_vnic_set_rxmask failed\n"); + + return rc; +} + +static int bnxt_set_vf_table(struct bnxt *bp, uint16_t vf) +{ + int rc = 0; + int dflt_vnic; + struct bnxt_vnic_info vnic; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VLAN table on non-PF port!\n"); + return -EINVAL; + } + + if (vf >= bp->pdev->max_vfs) + return -EINVAL; + + dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf); + if (dflt_vnic < 0) { + /* This simply indicates there's no driver loaded. + * This is not an error. + */ + RTE_LOG(ERR, PMD, "Unable to get default VNIC for VF %d\n", vf); + } else { + memset(&vnic, 0, sizeof(vnic)); + vnic.fw_vnic_id = dflt_vnic; + if (bnxt_hwrm_vnic_qcfg(bp, &vnic, + bp->pf.first_vf_id + vf) == 0) { + if (bnxt_hwrm_cfa_l2_set_rx_mask(bp, &vnic, + bp->pf.vf_info[vf].vlan_count, + bp->pf.vf_info[vf].vlan_table)) + rc = -1; + } else { + rc = -1; + } + } + + return rc; +} + +int rte_pmd_bnxt_set_vf_vlan_filter(uint8_t port, uint16_t vlan, + uint64_t vf_mask, uint8_t vlan_on) +{ + struct bnxt_vlan_table_entry *ve; + struct bnxt_vlan_antispoof_table_entry *vase; + struct rte_eth_dev *dev; + struct bnxt *bp; + uint16_t cnt; + int rc = 0; + int i, j; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + bp = (struct bnxt *)dev->data->dev_private; + if (!bp->pf.vf_info) + return -EINVAL; + + for (i = 0; vf_mask; i++, vf_mask >>= 1) { + cnt = bp->pf.vf_info[i].vlan_count; + if ((vf_mask & 1) == 0) + continue; + + if (bp->pf.vf_info[i].vlan_table == NULL) { + rc = -1; + continue; + } + if (bp->pf.vf_info[i].vlan_as_table == NULL) { + rc = -1; + continue; + } + if (vlan_on) { + /* First, search for a duplicate... */ + for (j = 0; j < cnt; j++) { + if (rte_be_to_cpu_16( + bp->pf.vf_info[i].vlan_table[j].vid) == vlan) + break; + } + if (j == cnt) { + /* Now check that there's space */ + if (cnt == getpagesize() / sizeof(struct + bnxt_vlan_antispoof_table_entry)) { + RTE_LOG(ERR, PMD, + "VLAN anti-spoof table is full\n"); + RTE_LOG(ERR, PMD, + "VF %d cannot add VLAN %u\n", + i, vlan); + rc = -1; + continue; + } + + /* cnt is one less than vlan_count */ + cnt = bp->pf.vf_info[i].vlan_count++; + /* + * And finally, add to the + * end of the table + */ + vase = &bp->pf.vf_info[i].vlan_as_table[cnt]; + // TODO: Hardcoded TPID + vase->tpid = rte_cpu_to_be_16(0x8100); + vase->vid = rte_cpu_to_be_16(vlan); + vase->mask = rte_cpu_to_be_16(0xfff); + ve = &bp->pf.vf_info[i].vlan_table[cnt]; + /* TODO: Hardcoded TPID */ + ve->tpid = rte_cpu_to_be_16(0x8100); + ve->vid = rte_cpu_to_be_16(vlan); + } + } else { + for (j = 0; j < cnt; j++) { + if (rte_be_to_cpu_16( + bp->pf.vf_info[i].vlan_table[j].vid) != vlan) + continue; + memmove(&bp->pf.vf_info[i].vlan_table[j], + &bp->pf.vf_info[i].vlan_table[j + 1], + getpagesize() - ((j + 1) * + sizeof(struct bnxt_vlan_table_entry))); + memmove(&bp->pf.vf_info[i].vlan_as_table[j], + &bp->pf.vf_info[i].vlan_as_table[j + 1], + getpagesize() - ((j + 1) * sizeof(struct + bnxt_vlan_antispoof_table_entry))); + j--; + cnt = --bp->pf.vf_info[i].vlan_count; + } + } + bnxt_set_vf_table(bp, i); + } + + return rc; +} + +int rte_pmd_bnxt_get_vf_stats(uint8_t port, + uint16_t vf_id, + struct rte_eth_stats *stats) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to get VF %d stats on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_hwrm_func_qstats(bp, bp->pf.first_vf_id + vf_id, stats); +} + +int rte_pmd_bnxt_reset_vf_stats(uint8_t port, + uint16_t vf_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to reset VF %d stats on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_hwrm_func_clr_stats(bp, bp->pf.first_vf_id + vf_id); +} + +int rte_pmd_bnxt_get_vf_rx_status(uint8_t port, uint16_t vf_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to query VF %d RX stats on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_vf_vnic_count(bp, vf_id); +} + +int rte_pmd_bnxt_get_vf_tx_drop_count(uint8_t port, uint16_t vf_id, + uint64_t *count) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to query VF %d TX drops on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_hwrm_func_qstats_tx_drop(bp, bp->pf.first_vf_id + vf_id, + count); +} + +int rte_pmd_bnxt_mac_addr_add(uint8_t port, struct ether_addr *addr, + uint32_t vf_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + struct bnxt_filter_info *filter; + struct bnxt_vnic_info vnic; + struct ether_addr dflt_mac; + int rc; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to config VF %d MAC on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + /* If the VF currently uses a random MAC, update default to this one */ + if (bp->pf.vf_info[vf_id].random_mac) { + if (rte_pmd_bnxt_get_vf_rx_status(port, vf_id) <= 0) + rc = bnxt_hwrm_func_vf_mac(bp, vf_id, (uint8_t *)addr); + } + + /* query the default VNIC id used by the function */ + rc = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf_id); + if (rc < 0) + goto exit; + + memset(&vnic, 0, sizeof(struct bnxt_vnic_info)); + vnic.fw_vnic_id = rte_le_to_cpu_16(rc); + rc = bnxt_hwrm_vnic_qcfg(bp, &vnic, bp->pf.first_vf_id + vf_id); + if (rc < 0) + goto exit; + + STAILQ_FOREACH(filter, &bp->pf.vf_info[vf_id].filter, next) { + if (filter->flags == + HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX && + filter->enables == + (HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK) && + memcmp(addr, filter->l2_addr, ETHER_ADDR_LEN) == 0) { + bnxt_hwrm_clear_filter(bp, filter); + break; + } + } + + if (filter == NULL) + filter = bnxt_alloc_vf_filter(bp, vf_id); + + filter->fw_l2_filter_id = UINT64_MAX; + filter->flags = HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX; + filter->enables = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK; + memcpy(filter->l2_addr, addr, ETHER_ADDR_LEN); + memset(filter->l2_addr_mask, 0xff, ETHER_ADDR_LEN); + + /* Do not add a filter for the default MAC */ + if (bnxt_hwrm_func_qcfg_vf_default_mac(bp, vf_id, &dflt_mac) || + memcmp(filter->l2_addr, dflt_mac.addr_bytes, ETHER_ADDR_LEN)) + rc = bnxt_hwrm_set_filter(bp, vnic.fw_vnic_id, filter); + +exit: + return rc; +} + +int +rte_pmd_bnxt_set_vf_vlan_insert(uint8_t port, uint16_t vf, + uint16_t vlan_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VF %d vlan insert on non-PF port %d!\n", + vf, port); + return -ENOTSUP; + } + + bp->pf.vf_info[vf].dflt_vlan = vlan_id; + if (bnxt_hwrm_func_qcfg_current_vf_vlan(bp, vf) == + bp->pf.vf_info[vf].dflt_vlan) + return 0; + + rc = bnxt_hwrm_set_vf_vlan(bp, vf); + + return rc; +} + +int rte_pmd_bnxt_set_vf_persist_stats(uint8_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev_info dev_info; + struct rte_eth_dev *dev; + uint32_t func_flags; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + dev = &rte_eth_devices[port]; + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set persist stats on non-PF port %d!\n", + port); + return -EINVAL; + } + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + /* Prev setting same as new setting. */ + if (on == bp->pf.vf_info[vf].persist_stats) + return 0; + + func_flags = bp->pf.vf_info[vf].func_cfg_flags; + + if (on) + func_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_NO_AUTOCLEAR_STATISTIC; + else + func_flags &= + ~HWRM_FUNC_CFG_INPUT_FLAGS_NO_AUTOCLEAR_STATISTIC; + + rc = bnxt_hwrm_func_cfg_vf_set_flags(bp, vf, func_flags); + if (!rc) { + bp->pf.vf_info[vf].persist_stats = on; + bp->pf.vf_info[vf].func_cfg_flags = func_flags; + } + + return rc; +} diff --git a/drivers/net/bnxt/rte_pmd_bnxt.h b/drivers/net/bnxt/rte_pmd_bnxt.h new file mode 100644 index 00000000..c4c4770e --- /dev/null +++ b/drivers/net/bnxt/rte_pmd_bnxt.h @@ -0,0 +1,354 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Broadcom Limited. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _PMD_BNXT_H_ +#define _PMD_BNXT_H_ + +#include <rte_ethdev.h> + +/* + * Response sent back to the caller after callback + */ +enum rte_pmd_bnxt_mb_event_rsp { + RTE_PMD_BNXT_MB_EVENT_NOOP_ACK, /**< skip mbox request and ACK */ + RTE_PMD_BNXT_MB_EVENT_NOOP_NACK, /**< skip mbox request and NACK */ + RTE_PMD_BNXT_MB_EVENT_PROCEED, /**< proceed with mbox request */ + RTE_PMD_BNXT_MB_EVENT_MAX /**< max value of this enum */ +}; + +/* mailbox message types */ +#define BNXT_VF_RESET 0x01 /* VF requests reset */ +#define BNXT_VF_SET_MAC_ADDR 0x02 /* VF requests PF to set MAC addr */ +#define BNXT_VF_SET_VLAN 0x03 /* VF requests PF to set VLAN */ +#define BNXT_VF_SET_MTU 0x04 /* VF requests PF to set MTU */ +#define BNXT_VF_SET_MRU 0x05 /* VF requests PF to set MRU */ + +/* + * Data sent to the caller when the callback is executed. + */ +struct rte_pmd_bnxt_mb_event_param { + uint16_t vf_id; /* Virtual Function number */ + int retval; /* return value */ + void *msg; /* pointer to message */ +}; + +/** + * Enable/Disable VF MAC anti spoof + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param on + * 1 - Enable VF MAC anti spoof. + * 0 - Disable VF MAC anti spoof. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_mac_anti_spoof(uint8_t port, uint16_t vf, uint8_t on); + +/** + * Set the VF MAC address. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param mac_addr + * VF MAC address. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* or *mac_addr* is invalid. + */ +int rte_pmd_bnxt_set_vf_mac_addr(uint8_t port, uint16_t vf, + struct ether_addr *mac_addr); + +/** + * Enable/Disable vf vlan strip for all queues in a pool + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param on + * 1 - Enable VF's vlan strip on RX queues. + * 0 - Disable VF's vlan strip on RX queues. + * + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support this feature. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int +rte_pmd_bnxt_set_vf_vlan_stripq(uint8_t port, uint16_t vf, uint8_t on); + +/** + * Enable/Disable vf vlan insert + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param vlan_id + * 0 - Disable VF's vlan insert. + * n - Enable; n is inserted as the vlan id. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int +rte_pmd_bnxt_set_vf_vlan_insert(uint8_t port, uint16_t vf, + uint16_t vlan_id); + +/** + * Enable/Disable hardware VF VLAN filtering by an Ethernet device of + * received VLAN packets tagged with a given VLAN Tag Identifier. + * + * @param port + * The port identifier of the Ethernet device. + * @param vlan + * The VLAN Tag Identifier whose filtering must be enabled or disabled. + * @param vf_mask + * Bitmap listing which VFs participate in the VLAN filtering. + * @param vlan_on + * 1 - Enable VFs VLAN filtering. + * 0 - Disable VFs VLAN filtering. + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-ENODEV) if *port_id* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_vlan_filter(uint8_t port, uint16_t vlan, + uint64_t vf_mask, uint8_t vlan_on); + +/** + * Enable/Disable tx loopback + * + * @param port + * The port identifier of the Ethernet device. + * @param on + * 1 - Enable tx loopback. + * 0 - Disable tx loopback. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_tx_loopback(uint8_t port, uint8_t on); + +/** + * set all queues drop enable bit + * + * @param port + * The port identifier of the Ethernet device. + * @param on + * 1 - set the queue drop enable bit for all pools. + * 0 - reset the queue drop enable bit for all pools. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_all_queues_drop_en(uint8_t port, uint8_t on); + +/** + * Set the VF rate limit. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param tx_rate + * Tx rate for the VF + * @param q_msk + * Mask of the Tx queue + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* or *mac_addr* is invalid. + */ +int rte_pmd_bnxt_set_vf_rate_limit(uint8_t port, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk); + +/** + * Get VF's statistics + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @param stats + * A pointer to a structure of type *rte_eth_stats* to be filled with + * the values of device counters supported statistics: + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ + +int rte_pmd_bnxt_get_vf_stats(uint8_t port, + uint16_t vf_id, + struct rte_eth_stats *stats); + +/** + * Clear VF's statistics + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_reset_vf_stats(uint8_t port, + uint16_t vf_id); + +/** + * Enable/Disable VF VLAN anti spoof + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param on + * 1 - Enable VF VLAN anti spoof. + * 0 - Disable VF VLAN anti spoof. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_vlan_anti_spoof(uint8_t port, uint16_t vf, uint8_t on); + +/** + * Set RX L2 Filtering mode of a VF of an Ethernet device. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param rx_mask + * The RX mode mask + * @param on + * 1 - Enable a VF RX mode. + * 0 - Disable a VF RX mode. + * @return + * - (0) if successful. + * - (-ENODEV) if *port_id* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_rxmode(uint8_t port, uint16_t vf, + uint16_t rx_mask, uint8_t on); + +/** + * Returns the number of default RX queues on a VF + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @return + * - Non-negative value - Number of default RX queues + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) if on a function without VFs + * - (-ENOMEM) on an allocation failure + * - (-1) firmware interface error + */ +int rte_pmd_bnxt_get_vf_rx_status(uint8_t port, uint16_t vf_id); + +/** + * Queries the TX drop counter for the function + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @param count + * Pointer to a uint64_t that will be populated with the counter value. + * @return + * - Positive Non-zero value - Error code from HWRM + * - (-EINVAL) invalid vf_id specified. + * - (-ENOTSUP) Ethernet device is not a PF + */ +int rte_pmd_bnxt_get_vf_tx_drop_count(uint8_t port, uint16_t vf_id, + uint64_t *count); + +/** + * Programs the MAC address for the function specified + * + * @param port + * The port identifier of the Ethernet device. + * @param mac_addr + * The MAC address to be programmed in the filter. + * @param vf_id + * VF on which to get. + * @return + * - Positive Non-zero value - Error code from HWRM + * - (-EINVAL) invalid vf_id specified. + * - (-ENOTSUP) Ethernet device is not a PF + * - (-ENOMEM) on an allocation failure + */ +int rte_pmd_bnxt_mac_addr_add(uint8_t port, struct ether_addr *mac_addr, + uint32_t vf_id); + +/** + * Enable/Disable VF statistics retention + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param on + * 1 - Prevent VF statistics from automatically resetting + * 0 - Allow VF statistics to automatically reset + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_persist_stats(uint8_t port, uint16_t vf, uint8_t on); +#endif /* _PMD_BNXT_H_ */ diff --git a/drivers/net/bnxt/rte_pmd_bnxt_version.map b/drivers/net/bnxt/rte_pmd_bnxt_version.map index 349c6e1c..4750d40a 100644 --- a/drivers/net/bnxt/rte_pmd_bnxt_version.map +++ b/drivers/net/bnxt/rte_pmd_bnxt_version.map @@ -1,4 +1,22 @@ -DPDK_16.04 { +DPDK_17.08 { + global: + + rte_pmd_bnxt_get_vf_rx_status; + rte_pmd_bnxt_get_vf_stats; + rte_pmd_bnxt_get_vf_tx_drop_count; + rte_pmd_bnxt_mac_addr_add; + rte_pmd_bnxt_reset_vf_stats; + rte_pmd_bnxt_set_all_queues_drop_en; + rte_pmd_bnxt_set_tx_loopback; + rte_pmd_bnxt_set_vf_mac_addr; + rte_pmd_bnxt_set_vf_mac_anti_spoof; + rte_pmd_bnxt_set_vf_rate_limit; + rte_pmd_bnxt_set_vf_rxmode; + rte_pmd_bnxt_set_vf_vlan_anti_spoof; + rte_pmd_bnxt_set_vf_vlan_filter; + rte_pmd_bnxt_set_vf_vlan_insert; + rte_pmd_bnxt_set_vf_vlan_stripq; + rte_pmd_bnxt_set_vf_persist_stats; local: *; }; diff --git a/drivers/net/bonding/rte_eth_bond_8023ad.c b/drivers/net/bonding/rte_eth_bond_8023ad.c index 7b863d6e..20b5a896 100644 --- a/drivers/net/bonding/rte_eth_bond_8023ad.c +++ b/drivers/net/bonding/rte_eth_bond_8023ad.c @@ -44,7 +44,6 @@ #include "rte_eth_bond_private.h" static void bond_mode_8023ad_ext_periodic_cb(void *arg); - #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD #define MODE4_DEBUG(fmt, ...) RTE_LOG(DEBUG, PMD, "%6u [Port %u: %s] " fmt, \ bond_dbg_get_time_diff_ms(), slave_id, \ @@ -435,7 +434,7 @@ periodic_machine(struct bond_dev_private *internals, uint8_t slave_id) * In other case (was fast and now it is slow) just switch * timeout to slow without forcing send of LACP (because standard * say so)*/ - if (!is_partner_fast) + if (is_partner_fast) SM_FLAG_SET(port, NTT); } else return; /* Nothing changed */ @@ -632,21 +631,53 @@ tx_machine(struct bond_dev_private *internals, uint8_t slave_id) lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION; lacpdu->terminator_length = 0; - if (rte_ring_enqueue(port->tx_ring, lacp_pkt) == -ENOBUFS) { - /* If TX ring full, drop packet and free message. Retransmission - * will happen in next function call. */ - rte_pktmbuf_free(lacp_pkt); - set_warning_flags(port, WRN_TX_QUEUE_FULL); - return; + MODE4_DEBUG("Sending LACP frame\n"); + BOND_PRINT_LACP(lacpdu); + + if (internals->mode4.dedicated_queues.enabled == 0) { + int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt); + if (retval != 0) { + /* If TX ring full, drop packet and free message. + Retransmission will happen in next function call. */ + rte_pktmbuf_free(lacp_pkt); + set_warning_flags(port, WRN_TX_QUEUE_FULL); + return; + } + } else { + uint16_t pkts_sent = rte_eth_tx_burst(slave_id, + internals->mode4.dedicated_queues.tx_qid, + &lacp_pkt, 1); + if (pkts_sent != 1) { + rte_pktmbuf_free(lacp_pkt); + set_warning_flags(port, WRN_TX_QUEUE_FULL); + return; + } } - MODE4_DEBUG("sending LACP frame\n"); - BOND_PRINT_LACP(lacpdu); timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout); SM_FLAG_CLR(port, NTT); } +static uint8_t +max_index(uint64_t *a, int n) +{ + if (n <= 0) + return -1; + + int i, max_i = 0; + uint64_t max = a[0]; + + for (i = 1; i < n; ++i) { + if (a[i] > max) { + max = a[i]; + max_i = i; + } + } + + return max_i; +} + /** * Function assigns port to aggregator. * @@ -657,8 +688,13 @@ static void selection_logic(struct bond_dev_private *internals, uint8_t slave_id) { struct port *agg, *port; - uint8_t slaves_count, new_agg_id, i; + uint8_t slaves_count, new_agg_id, i, j = 0; uint8_t *slaves; + uint64_t agg_bandwidth[8] = {0}; + uint64_t agg_count[8] = {0}; + uint8_t default_slave = 0; + uint8_t mode_count_id, mode_band_id; + struct rte_eth_link link_info; slaves = internals->active_slaves; slaves_count = internals->active_slave_count; @@ -671,6 +707,10 @@ selection_logic(struct bond_dev_private *internals, uint8_t slave_id) if (agg->aggregator_port_id != slaves[i]) continue; + agg_count[agg->aggregator_port_id] += 1; + rte_eth_link_get_nowait(slaves[i], &link_info); + agg_bandwidth[agg->aggregator_port_id] += link_info.link_speed; + /* Actors system ID is not checked since all slave device have the same * ID (MAC address). */ if ((agg->actor.key == port->actor.key && @@ -681,15 +721,36 @@ selection_logic(struct bond_dev_private *internals, uint8_t slave_id) (agg->actor.key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) { - break; + if (j == 0) + default_slave = i; + j++; } } - /* By default, port uses it self as agregator */ - if (i == slaves_count) - new_agg_id = slave_id; - else - new_agg_id = slaves[i]; + switch (internals->mode4.agg_selection) { + case AGG_COUNT: + mode_count_id = max_index( + (uint64_t *)agg_count, slaves_count); + new_agg_id = mode_count_id; + break; + case AGG_BANDWIDTH: + mode_band_id = max_index( + (uint64_t *)agg_bandwidth, slaves_count); + new_agg_id = mode_band_id; + break; + case AGG_STABLE: + if (default_slave == slaves_count) + new_agg_id = slave_id; + else + new_agg_id = slaves[default_slave]; + break; + default: + if (default_slave == slaves_count) + new_agg_id = slave_id; + else + new_agg_id = slaves[default_slave]; + break; + } if (new_agg_id != port->aggregator_port_id) { port->aggregator_port_id = new_agg_id; @@ -741,6 +802,22 @@ link_speed_key(uint16_t speed) { } static void +rx_machine_update(struct bond_dev_private *internals, uint8_t slave_id, + struct rte_mbuf *lacp_pkt) { + struct lacpdu_header *lacp; + + if (lacp_pkt != NULL) { + lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *); + RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP); + + /* This is LACP frame so pass it to rx_machine */ + rx_machine(internals, slave_id, &lacp->lacpdu); + rte_pktmbuf_free(lacp_pkt); + } else + rx_machine(internals, slave_id, NULL); +} + +static void bond_mode_8023ad_periodic_cb(void *arg) { struct rte_eth_dev *bond_dev = arg; @@ -748,8 +825,8 @@ bond_mode_8023ad_periodic_cb(void *arg) struct port *port; struct rte_eth_link link_info; struct ether_addr slave_addr; + struct rte_mbuf *lacp_pkt = NULL; - void *pkt = NULL; uint8_t i, slave_id; @@ -758,7 +835,7 @@ bond_mode_8023ad_periodic_cb(void *arg) uint16_t key; slave_id = internals->active_slaves[i]; - rte_eth_link_get(slave_id, &link_info); + rte_eth_link_get_nowait(slave_id, &link_info); rte_eth_macaddr_get(slave_id, &slave_addr); if (link_info.link_status != 0) { @@ -809,20 +886,28 @@ bond_mode_8023ad_periodic_cb(void *arg) SM_FLAG_SET(port, LACP_ENABLED); - /* Find LACP packet to this port. Do not check subtype, it is done in - * function that queued packet */ - if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) { - struct rte_mbuf *lacp_pkt = pkt; - struct lacpdu_header *lacp; + if (internals->mode4.dedicated_queues.enabled == 0) { + /* Find LACP packet to this port. Do not check subtype, + * it is done in function that queued packet + */ + int retval = rte_ring_dequeue(port->rx_ring, + (void **)&lacp_pkt); - lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *); - RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP); + if (retval != 0) + lacp_pkt = NULL; - /* This is LACP frame so pass it to rx_machine */ - rx_machine(internals, slave_id, &lacp->lacpdu); - rte_pktmbuf_free(lacp_pkt); - } else - rx_machine(internals, slave_id, NULL); + rx_machine_update(internals, slave_id, lacp_pkt); + } else { + uint16_t rx_count = rte_eth_rx_burst(slave_id, + internals->mode4.dedicated_queues.rx_qid, + &lacp_pkt, 1); + + if (rx_count == 1) + bond_mode_8023ad_handle_slow_pkt(internals, + slave_id, lacp_pkt); + else + rx_machine_update(internals, slave_id, NULL); + } periodic_machine(internals, slave_id); mux_machine(internals, slave_id); @@ -872,7 +957,7 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) /* default states */ port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED; - port->partner_state = STATE_LACP_ACTIVE; + port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION; port->sm_flags = SM_FLAGS_BEGIN; /* use this port as agregator */ @@ -886,7 +971,10 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) RTE_ASSERT(port->rx_ring == NULL); RTE_ASSERT(port->tx_ring == NULL); - socket_id = rte_eth_devices[slave_id].data->numa_node; + + socket_id = rte_eth_dev_socket_id(slave_id); + if (socket_id == (int)LCORE_ID_ANY) + socket_id = rte_socket_id(); element_size = sizeof(struct slow_protocol_frame) + RTE_PKTMBUF_HEADROOM; @@ -905,7 +993,7 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) 32 : RTE_MEMPOOL_CACHE_MAX_SIZE, 0, element_size, socket_id); - /* Any memory allocation failure in initalization is critical because + /* Any memory allocation failure in initialization is critical because * resources can't be free, so reinitialization is impossible. */ if (port->mbuf_pool == NULL) { rte_panic("Slave %u: Failed to create memory pool '%s': %s\n", @@ -1037,6 +1125,18 @@ bond_mode_8023ad_conf_get_v1607(struct rte_eth_dev *dev, } static void +bond_mode_8023ad_conf_get_v1708(struct rte_eth_dev *dev, + struct rte_eth_bond_8023ad_conf *conf) +{ + struct bond_dev_private *internals = dev->data->dev_private; + struct mode8023ad_private *mode4 = &internals->mode4; + + bond_mode_8023ad_conf_get(dev, conf); + conf->slowrx_cb = mode4->slowrx_cb; + conf->agg_selection = mode4->agg_selection; +} + +static void bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf) { conf->fast_periodic_ms = BOND_8023AD_FAST_PERIODIC_MS; @@ -1048,6 +1148,7 @@ bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf) conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS; conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS; conf->slowrx_cb = NULL; + conf->agg_selection = AGG_STABLE; } static void @@ -1064,6 +1165,10 @@ bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4, mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks; mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks; mode4->update_timeout_us = conf->update_timeout_ms * 1000; + + mode4->dedicated_queues.enabled = 0; + mode4->dedicated_queues.rx_qid = UINT16_MAX; + mode4->dedicated_queues.tx_qid = UINT16_MAX; } static void @@ -1102,7 +1207,29 @@ bond_mode_8023ad_setup(struct rte_eth_dev *dev, bond_mode_8023ad_stop(dev); bond_mode_8023ad_conf_assign(mode4, conf); + + + if (dev->data->dev_started) + bond_mode_8023ad_start(dev); +} + +static void +bond_mode_8023ad_setup_v1708(struct rte_eth_dev *dev, + struct rte_eth_bond_8023ad_conf *conf) +{ + struct rte_eth_bond_8023ad_conf def_conf; + struct bond_dev_private *internals = dev->data->dev_private; + struct mode8023ad_private *mode4 = &internals->mode4; + + if (conf == NULL) { + conf = &def_conf; + bond_mode_8023ad_conf_get_default(conf); + } + + bond_mode_8023ad_stop(dev); + bond_mode_8023ad_conf_assign(mode4, conf); mode4->slowrx_cb = conf->slowrx_cb; + mode4->agg_selection = AGG_STABLE; if (dev->data->dev_started) bond_mode_8023ad_start(dev); @@ -1188,18 +1315,36 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals, m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP; rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.s_addr); - if (unlikely(rte_ring_enqueue(port->tx_ring, pkt) == -ENOBUFS)) { - /* reset timer */ - port->rx_marker_timer = 0; - wrn = WRN_TX_QUEUE_FULL; - goto free_out; + if (internals->mode4.dedicated_queues.enabled == 0) { + int retval = rte_ring_enqueue(port->tx_ring, pkt); + if (retval != 0) { + /* reset timer */ + port->rx_marker_timer = 0; + wrn = WRN_TX_QUEUE_FULL; + goto free_out; + } + } else { + /* Send packet directly to the slow queue */ + uint16_t tx_count = rte_eth_tx_burst(slave_id, + internals->mode4.dedicated_queues.tx_qid, + &pkt, 1); + if (tx_count != 1) { + /* reset timer */ + port->rx_marker_timer = 0; + wrn = WRN_TX_QUEUE_FULL; + goto free_out; + } } } else if (likely(subtype == SLOW_SUBTYPE_LACP)) { - if (unlikely(rte_ring_enqueue(port->rx_ring, pkt) == -ENOBUFS)) { - /* If RX fing full free lacpdu message and drop packet */ - wrn = WRN_RX_QUEUE_FULL; - goto free_out; - } + if (internals->mode4.dedicated_queues.enabled == 0) { + int retval = rte_ring_enqueue(port->rx_ring, pkt); + if (retval != 0) { + /* If RX fing full free lacpdu message and drop packet */ + wrn = WRN_RX_QUEUE_FULL; + goto free_out; + } + } else + rx_machine_update(internals, slave_id, pkt); } else { wrn = WRN_UNKNOWN_SLOW_TYPE; goto free_out; @@ -1246,10 +1391,71 @@ rte_eth_bond_8023ad_conf_get_v1607(uint8_t port_id, bond_mode_8023ad_conf_get_v1607(bond_dev, conf); return 0; } -BIND_DEFAULT_SYMBOL(rte_eth_bond_8023ad_conf_get, _v1607, 16.07); +VERSION_SYMBOL(rte_eth_bond_8023ad_conf_get, _v1607, 16.07); + +int +rte_eth_bond_8023ad_conf_get_v1708(uint8_t port_id, + struct rte_eth_bond_8023ad_conf *conf) +{ + struct rte_eth_dev *bond_dev; + + if (valid_bonded_port_id(port_id) != 0) + return -EINVAL; + + if (conf == NULL) + return -EINVAL; + + bond_dev = &rte_eth_devices[port_id]; + bond_mode_8023ad_conf_get_v1708(bond_dev, conf); + return 0; +} MAP_STATIC_SYMBOL(int rte_eth_bond_8023ad_conf_get(uint8_t port_id, struct rte_eth_bond_8023ad_conf *conf), - rte_eth_bond_8023ad_conf_get_v1607); + rte_eth_bond_8023ad_conf_get_v1708); +BIND_DEFAULT_SYMBOL(rte_eth_bond_8023ad_conf_get, _v1708, 17.08); + +int +rte_eth_bond_8023ad_agg_selection_set(uint8_t port_id, + enum rte_bond_8023ad_agg_selection agg_selection) +{ + struct rte_eth_dev *bond_dev; + struct bond_dev_private *internals; + struct mode8023ad_private *mode4; + + bond_dev = &rte_eth_devices[port_id]; + internals = bond_dev->data->dev_private; + + if (valid_bonded_port_id(port_id) != 0) + return -EINVAL; + if (internals->mode != 4) + return -EINVAL; + + mode4 = &internals->mode4; + if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH + || agg_selection == AGG_STABLE) + mode4->agg_selection = agg_selection; + return 0; +} + +int rte_eth_bond_8023ad_agg_selection_get(uint8_t port_id) +{ + struct rte_eth_dev *bond_dev; + struct bond_dev_private *internals; + struct mode8023ad_private *mode4; + + bond_dev = &rte_eth_devices[port_id]; + internals = bond_dev->data->dev_private; + + if (valid_bonded_port_id(port_id) != 0) + return -EINVAL; + if (internals->mode != 4) + return -EINVAL; + mode4 = &internals->mode4; + + return mode4->agg_selection; +} + + static int bond_8023ad_setup_validate(uint8_t port_id, @@ -1310,10 +1516,34 @@ rte_eth_bond_8023ad_setup_v1607(uint8_t port_id, return 0; } -BIND_DEFAULT_SYMBOL(rte_eth_bond_8023ad_setup, _v1607, 16.07); +VERSION_SYMBOL(rte_eth_bond_8023ad_setup, _v1607, 16.07); + + +int +rte_eth_bond_8023ad_setup_v1708(uint8_t port_id, + struct rte_eth_bond_8023ad_conf *conf) +{ + struct rte_eth_dev *bond_dev; + int err; + + err = bond_8023ad_setup_validate(port_id, conf); + if (err != 0) + return err; + + bond_dev = &rte_eth_devices[port_id]; + bond_mode_8023ad_setup_v1708(bond_dev, conf); + + return 0; +} +BIND_DEFAULT_SYMBOL(rte_eth_bond_8023ad_setup, _v1708, 17.08); MAP_STATIC_SYMBOL(int rte_eth_bond_8023ad_setup(uint8_t port_id, struct rte_eth_bond_8023ad_conf *conf), - rte_eth_bond_8023ad_setup_v1607); + rte_eth_bond_8023ad_setup_v1708); + + + + + int rte_eth_bond_8023ad_slave_info(uint8_t port_id, uint8_t slave_id, @@ -1504,3 +1734,49 @@ bond_mode_8023ad_ext_periodic_cb(void *arg) rte_eal_alarm_set(internals->mode4.update_timeout_us, bond_mode_8023ad_ext_periodic_cb, arg); } + +int +rte_eth_bond_8023ad_dedicated_queues_enable(uint8_t port) +{ + int retval = 0; + struct rte_eth_dev *dev = &rte_eth_devices[port]; + struct bond_dev_private *internals = (struct bond_dev_private *) + dev->data->dev_private; + + if (check_for_bonded_ethdev(dev) != 0) + return -1; + + if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0) + return -1; + + /* Device must be stopped to set up slow queue */ + if (dev->data->dev_started) + return -1; + + internals->mode4.dedicated_queues.enabled = 1; + + bond_ethdev_mode_set(dev, internals->mode); + return retval; +} + +int +rte_eth_bond_8023ad_dedicated_queues_disable(uint8_t port) +{ + int retval = 0; + struct rte_eth_dev *dev = &rte_eth_devices[port]; + struct bond_dev_private *internals = (struct bond_dev_private *) + dev->data->dev_private; + + if (check_for_bonded_ethdev(dev) != 0) + return -1; + + /* Device must be stopped to set up slow queue */ + if (dev->data->dev_started) + return -1; + + internals->mode4.dedicated_queues.enabled = 0; + + bond_ethdev_mode_set(dev, internals->mode); + + return retval; +} diff --git a/drivers/net/bonding/rte_eth_bond_8023ad.h b/drivers/net/bonding/rte_eth_bond_8023ad.h index 6b8ff575..1d353c73 100644 --- a/drivers/net/bonding/rte_eth_bond_8023ad.h +++ b/drivers/net/bonding/rte_eth_bond_8023ad.h @@ -73,6 +73,12 @@ enum rte_bond_8023ad_selection { SELECTED }; +enum rte_bond_8023ad_agg_selection { + AGG_BANDWIDTH, + AGG_COUNT, + AGG_STABLE +}; + /** Generic slow protocol structure */ struct slow_protocol { uint8_t subtype; @@ -161,6 +167,7 @@ struct rte_eth_bond_8023ad_conf { uint32_t rx_marker_period_ms; uint32_t update_timeout_ms; rte_eth_bond_8023ad_ext_slowrx_fn slowrx_cb; + enum rte_bond_8023ad_agg_selection agg_selection; }; struct rte_eth_bond_8023ad_slave_info { @@ -193,6 +200,9 @@ rte_eth_bond_8023ad_conf_get_v20(uint8_t port_id, int rte_eth_bond_8023ad_conf_get_v1607(uint8_t port_id, struct rte_eth_bond_8023ad_conf *conf); +int +rte_eth_bond_8023ad_conf_get_v1708(uint8_t port_id, + struct rte_eth_bond_8023ad_conf *conf); /** * @internal @@ -214,6 +224,9 @@ rte_eth_bond_8023ad_setup_v20(uint8_t port_id, int rte_eth_bond_8023ad_setup_v1607(uint8_t port_id, struct rte_eth_bond_8023ad_conf *conf); +int +rte_eth_bond_8023ad_setup_v1708(uint8_t port_id, + struct rte_eth_bond_8023ad_conf *conf); /** * @internal @@ -302,4 +315,65 @@ int rte_eth_bond_8023ad_ext_slowtx(uint8_t port_id, uint8_t slave_id, struct rte_mbuf *lacp_pkt); +/** + * Enable dedicated hw queues for 802.3ad control plane traffic on on slaves + * + * This function creates an additional tx and rx queue on each slave for + * dedicated 802.3ad control plane traffic . A flow filtering rule is + * programmed on each slave to redirect all LACP slow packets to that rx queue + * for processing in the LACP state machine, this removes the need to filter + * these packets in the bonded devices data path. The additional tx queue is + * used to enable the LACP state machine to enqueue LACP packets directly to + * slave hw independently of the bonded devices data path. + * + * To use this feature all slaves must support the programming of the flow + * filter rule required for rx and have enough queues that one rx and tx queue + * can be reserved for the LACP state machines control packets. + * + * Bonding port must be stopped to change this configuration. + * + * @param port_id Bonding device id + * + * @return + * 0 on success, negative value otherwise. + */ +int +rte_eth_bond_8023ad_dedicated_queues_enable(uint8_t port_id); + +/** + * Disable slow queue on slaves + * + * This function disables hardware slow packet filter. + * + * Bonding port must be stopped to change this configuration. + * + * @see rte_eth_bond_8023ad_slow_pkt_hw_filter_enable + * + * @param port_id Bonding device id + * @return + * 0 on success, negative value otherwise. + * + */ +int +rte_eth_bond_8023ad_dedicated_queues_disable(uint8_t port_id); + +/* + * Get aggregator mode for 8023ad + * @param port_id Bonding device id + * + * @return + * agregator mode on success, negative value otherwise + */ +int +rte_eth_bond_8023ad_agg_selection_get(uint8_t port_id); + +/** + * Set aggregator mode for 8023ad + * @param port_id Bonding device id + * @return + * 0 on success, negative value otherwise + */ +int +rte_eth_bond_8023ad_agg_selection_set(uint8_t port_id, + enum rte_bond_8023ad_agg_selection agg_selection); #endif /* RTE_ETH_BOND_8023AD_H_ */ diff --git a/drivers/net/bonding/rte_eth_bond_8023ad_private.h b/drivers/net/bonding/rte_eth_bond_8023ad_private.h index ca8858be..d46e44a8 100644 --- a/drivers/net/bonding/rte_eth_bond_8023ad_private.h +++ b/drivers/net/bonding/rte_eth_bond_8023ad_private.h @@ -39,6 +39,7 @@ #include <rte_ether.h> #include <rte_byteorder.h> #include <rte_atomic.h> +#include <rte_flow.h> #include "rte_eth_bond_8023ad.h" @@ -162,6 +163,9 @@ struct port { uint64_t warning_timer; volatile uint16_t warnings_to_show; + + /** Memory pool used to allocate slow queues */ + struct rte_mempool *slow_pool; }; struct mode8023ad_private { @@ -175,6 +179,23 @@ struct mode8023ad_private { uint64_t update_timeout_us; rte_eth_bond_8023ad_ext_slowrx_fn slowrx_cb; uint8_t external_sm; + + struct rte_eth_link slave_link; + /***< slave link properties */ + + /** + * Configuration of dedicated hardware queues for control plane + * traffic + */ + struct { + uint8_t enabled; + + struct rte_flow *flow[RTE_MAX_ETHPORTS]; + + uint16_t rx_qid; + uint16_t tx_qid; + } dedicated_queues; + enum rte_bond_8023ad_agg_selection agg_selection; }; /** @@ -295,4 +316,14 @@ bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *dev, uint8_t slave_pos); void bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev); +int +bond_ethdev_8023ad_flow_verify(struct rte_eth_dev *bond_dev, + uint8_t slave_port); + +int +bond_ethdev_8023ad_flow_set(struct rte_eth_dev *bond_dev, uint8_t slave_port); + +int +bond_8023ad_slow_pkt_hw_filter_supported(uint8_t port_id); + #endif /* RTE_ETH_BOND_8023AD_H_ */ diff --git a/drivers/net/bonding/rte_eth_bond_alb.c b/drivers/net/bonding/rte_eth_bond_alb.c index 38f5c4d4..d9d37495 100644 --- a/drivers/net/bonding/rte_eth_bond_alb.c +++ b/drivers/net/bonding/rte_eth_bond_alb.c @@ -80,7 +80,8 @@ bond_mode_alb_enable(struct rte_eth_dev *bond_dev) * The value is chosen to be cache aligned. */ data_size = 256 + RTE_PKTMBUF_HEADROOM; - snprintf(mem_name, sizeof(mem_name), "%s_MODE6", bond_dev->data->name); + snprintf(mem_name, sizeof(mem_name), "%s_ALB", + bond_dev->device->name); internals->mode6.mempool = rte_pktmbuf_pool_create(mem_name, 512 * RTE_MAX_ETHPORTS, RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ? @@ -89,7 +90,7 @@ bond_mode_alb_enable(struct rte_eth_dev *bond_dev) if (internals->mode6.mempool == NULL) { RTE_LOG(ERR, PMD, "%s: Failed to initialize ALB mempool.\n", - bond_dev->data->name); + bond_dev->device->name); goto mempool_alloc_error; } } diff --git a/drivers/net/bonding/rte_eth_bond_api.c b/drivers/net/bonding/rte_eth_bond_api.c index 36ec65d6..de1d9e0d 100644 --- a/drivers/net/bonding/rte_eth_bond_api.c +++ b/drivers/net/bonding/rte_eth_bond_api.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -48,11 +48,11 @@ int check_for_bonded_ethdev(const struct rte_eth_dev *eth_dev) { /* Check valid pointer */ - if (eth_dev->data->drv_name == NULL) + if (eth_dev->device->driver->name == NULL) return -1; /* return 0 if driver name matches */ - return eth_dev->data->drv_name != pmd_bond_drv.driver.name; + return eth_dev->device->driver->name != pmd_bond_drv.driver.name; } int @@ -63,13 +63,18 @@ valid_bonded_port_id(uint8_t port_id) } int -valid_slave_port_id(uint8_t port_id) +valid_slave_port_id(uint8_t port_id, uint8_t mode) { RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1); /* Verify that port_id refers to a non bonded port */ - if (check_for_bonded_ethdev(&rte_eth_devices[port_id]) == 0) + if (check_for_bonded_ethdev(&rte_eth_devices[port_id]) == 0 && + mode == BONDING_MODE_8023AD) { + RTE_BOND_LOG(ERR, "Cannot add slave to bonded device in 802.3ad" + " mode as slave is also a bonded device, only " + "physical devices can be support in this mode."); return -1; + } return 0; } @@ -143,22 +148,6 @@ deactivate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id) } } -uint8_t -number_of_sockets(void) -{ - int sockets = 0; - int i; - const struct rte_memseg *ms = rte_eal_get_physmem_layout(); - - for (i = 0; ((i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL)); i++) { - if (sockets < ms[i].socket_id) - sockets = ms[i].socket_id; - } - - /* Number of sockets = maximum socket_id + 1 */ - return ++sockets; -} - int rte_eth_bond_create(const char *name, uint8_t mode, uint8_t socket_id) { @@ -251,12 +240,12 @@ __eth_bond_slave_add_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) struct rte_eth_link link_props; struct rte_eth_dev_info dev_info; - if (valid_slave_port_id(slave_port_id) != 0) - return -1; - bonded_eth_dev = &rte_eth_devices[bonded_port_id]; internals = bonded_eth_dev->data->dev_private; + if (valid_slave_port_id(slave_port_id, internals->mode) != 0) + return -1; + slave_eth_dev = &rte_eth_devices[slave_port_id]; if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_BONDED_SLAVE) { RTE_BOND_LOG(ERR, "Slave device is already a slave of a bonded device"); @@ -313,6 +302,9 @@ __eth_bond_slave_add_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) internals->tx_offload_capa &= dev_info.tx_offload_capa; internals->flow_type_rss_offloads &= dev_info.flow_type_rss_offloads; + link_properties_valid(bonded_eth_dev, + &slave_eth_dev->data->dev_link); + /* RETA size is GCD of all slaves RETA sizes, so, if all sizes will be * the power of 2, the lower one is GCD */ @@ -402,12 +394,12 @@ __eth_bond_slave_remove_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) struct rte_eth_dev *slave_eth_dev; int i, slave_idx; - if (valid_slave_port_id(slave_port_id) != 0) - return -1; - bonded_eth_dev = &rte_eth_devices[bonded_port_id]; internals = bonded_eth_dev->data->dev_private; + if (valid_slave_port_id(slave_port_id, internals->mode) < 0) + return -1; + /* first remove from active slave list */ slave_idx = find_slave_by_id(internals->active_slaves, internals->active_slave_count, slave_port_id); @@ -455,9 +447,6 @@ __eth_bond_slave_remove_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) } if (internals->active_slave_count < 1) { - /* reset device link properties as no slaves are active */ - link_properties_reset(&rte_eth_devices[bonded_port_id]); - /* if no slaves are any longer attached to bonded device and MAC is not * user defined then clear MAC of bonded device as it will be reset * when a new slave is added */ @@ -528,10 +517,10 @@ rte_eth_bond_primary_set(uint8_t bonded_port_id, uint8_t slave_port_id) if (valid_bonded_port_id(bonded_port_id) != 0) return -1; - if (valid_slave_port_id(slave_port_id) != 0) - return -1; + internals = rte_eth_devices[bonded_port_id].data->dev_private; - internals = rte_eth_devices[bonded_port_id].data->dev_private; + if (valid_slave_port_id(slave_port_id, internals->mode) != 0) + return -1; internals->user_defined_primary_port = 1; internals->primary_port = slave_port_id; diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c index e3bdad9d..bb634c62 100644 --- a/drivers/net/bonding/rte_eth_bond_args.c +++ b/drivers/net/bonding/rte_eth_bond_args.c @@ -32,6 +32,7 @@ */ #include <rte_devargs.h> +#include <rte_pci.h> #include <rte_kvargs.h> #include <cmdline_parse.h> @@ -47,6 +48,7 @@ const char *pmd_bond_init_valid_arguments[] = { PMD_BOND_XMIT_POLICY_KVARG, PMD_BOND_SOCKET_ID_KVARG, PMD_BOND_MAC_ADDR_KVARG, + PMD_BOND_AGG_MODE_KVARG, "driver", NULL }; @@ -69,7 +71,7 @@ find_port_id_by_pci_addr(const struct rte_pci_addr *pci_addr) rte_eth_devices[i].data->kdrv == RTE_KDRV_NONE) continue; - pci_dev = RTE_DEV_TO_PCI(rte_eth_devices[i].device); + pci_dev = RTE_ETH_DEV_TO_PCI(&rte_eth_devices[i]); eth_pci_addr = &pci_dev->addr; if (pci_addr->bus == eth_pci_addr->bus && @@ -90,7 +92,7 @@ find_port_id_by_dev_name(const char *name) if (rte_eth_devices[i].data == NULL) continue; - if (strcmp(rte_eth_devices[i].data->name, name) == 0) + if (strcmp(rte_eth_devices[i].device->name, name) == 0) return i; } return -1; @@ -134,7 +136,7 @@ parse_port_id(const char *port_str) } int -bond_ethdev_parse_slave_port_kvarg(const char *key __rte_unused, +bond_ethdev_parse_slave_port_kvarg(const char *key, const char *value, void *extra_args) { struct bond_ethdev_slave_ports *slave_ports; @@ -190,6 +192,38 @@ bond_ethdev_parse_slave_mode_kvarg(const char *key __rte_unused, } int +bond_ethdev_parse_slave_agg_mode_kvarg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + uint8_t *agg_mode; + + if (value == NULL || extra_args == NULL) + return -1; + + agg_mode = extra_args; + + errno = 0; + if (strncmp(value, "stable", 6) == 0) + *agg_mode = AGG_STABLE; + + if (strncmp(value, "bandwidth", 9) == 0) + *agg_mode = AGG_BANDWIDTH; + + if (strncmp(value, "count", 5) == 0) + *agg_mode = AGG_COUNT; + + switch (*agg_mode) { + case AGG_STABLE: + case AGG_BANDWIDTH: + case AGG_COUNT: + return 0; + default: + RTE_BOND_LOG(ERR, "Invalid agg mode value stable/bandwidth/count"); + return -1; + } +} + +int bond_ethdev_parse_socket_id_kvarg(const char *key __rte_unused, const char *value, void *extra_args) { @@ -204,8 +238,8 @@ bond_ethdev_parse_socket_id_kvarg(const char *key __rte_unused, if (*endptr != 0 || errno != 0) return -1; - /* validate mode value */ - if (socket_id >= 0 && socket_id < number_of_sockets()) { + /* validate socket id value */ + if (socket_id >= 0) { *(uint8_t *)extra_args = (uint8_t)socket_id; return 0; } diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c index 82959abc..3ee70baa 100644 --- a/drivers/net/bonding/rte_eth_bond_pmd.c +++ b/drivers/net/bonding/rte_eth_bond_pmd.c @@ -133,6 +133,254 @@ is_lacp_packets(uint16_t ethertype, uint8_t subtype, uint16_t vlan_tci) (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP)); } +/***************************************************************************** + * Flow director's setup for mode 4 optimization + */ + +static struct rte_flow_item_eth flow_item_eth_type_8023ad = { + .dst.addr_bytes = { 0 }, + .src.addr_bytes = { 0 }, + .type = RTE_BE16(ETHER_TYPE_SLOW), +}; + +static struct rte_flow_item_eth flow_item_eth_mask_type_8023ad = { + .dst.addr_bytes = { 0 }, + .src.addr_bytes = { 0 }, + .type = 0xFFFF, +}; + +static struct rte_flow_item flow_item_8023ad[] = { + { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .spec = &flow_item_eth_type_8023ad, + .last = NULL, + .mask = &flow_item_eth_mask_type_8023ad, + }, + { + .type = RTE_FLOW_ITEM_TYPE_END, + .spec = NULL, + .last = NULL, + .mask = NULL, + } +}; + +const struct rte_flow_attr flow_attr_8023ad = { + .group = 0, + .priority = 0, + .ingress = 1, + .egress = 0, + .reserved = 0, +}; + +int +bond_ethdev_8023ad_flow_verify(struct rte_eth_dev *bond_dev, + uint8_t slave_port) { + struct rte_flow_error error; + struct bond_dev_private *internals = (struct bond_dev_private *) + (bond_dev->data->dev_private); + + struct rte_flow_action_queue lacp_queue_conf = { + .index = internals->mode4.dedicated_queues.rx_qid, + }; + + const struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_QUEUE, + .conf = &lacp_queue_conf + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + } + }; + + int ret = rte_flow_validate(slave_port, &flow_attr_8023ad, + flow_item_8023ad, actions, &error); + if (ret < 0) + return -1; + + return 0; +} + +int +bond_8023ad_slow_pkt_hw_filter_supported(uint8_t port_id) { + struct rte_eth_dev *bond_dev = &rte_eth_devices[port_id]; + struct bond_dev_private *internals = (struct bond_dev_private *) + (bond_dev->data->dev_private); + struct rte_eth_dev_info bond_info, slave_info; + uint8_t idx; + + /* Verify if all slaves in bonding supports flow director and */ + if (internals->slave_count > 0) { + rte_eth_dev_info_get(bond_dev->data->port_id, &bond_info); + + internals->mode4.dedicated_queues.rx_qid = bond_info.nb_rx_queues; + internals->mode4.dedicated_queues.tx_qid = bond_info.nb_tx_queues; + + for (idx = 0; idx < internals->slave_count; idx++) { + rte_eth_dev_info_get(internals->slaves[idx].port_id, + &slave_info); + + if (bond_ethdev_8023ad_flow_verify(bond_dev, + internals->slaves[idx].port_id) != 0) + return -1; + } + } + + return 0; +} + +int +bond_ethdev_8023ad_flow_set(struct rte_eth_dev *bond_dev, uint8_t slave_port) { + + struct rte_flow_error error; + struct bond_dev_private *internals = (struct bond_dev_private *) + (bond_dev->data->dev_private); + + struct rte_flow_action_queue lacp_queue_conf = { + .index = internals->mode4.dedicated_queues.rx_qid, + }; + + const struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_QUEUE, + .conf = &lacp_queue_conf + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + } + }; + + internals->mode4.dedicated_queues.flow[slave_port] = rte_flow_create(slave_port, + &flow_attr_8023ad, flow_item_8023ad, actions, &error); + if (internals->mode4.dedicated_queues.flow[slave_port] == NULL) { + RTE_BOND_LOG(ERR, "bond_ethdev_8023ad_flow_set: %s " + "(slave_port=%d queue_id=%d)", + error.message, slave_port, + internals->mode4.dedicated_queues.rx_qid); + return -1; + } + + return 0; +} + +static uint16_t +bond_ethdev_rx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs, + uint16_t nb_pkts) +{ + struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue; + struct bond_dev_private *internals = bd_rx_q->dev_private; + uint16_t num_rx_total = 0; /* Total number of received packets */ + uint8_t slaves[RTE_MAX_ETHPORTS]; + uint8_t slave_count; + + uint8_t i, idx; + + /* Copy slave list to protect against slave up/down changes during tx + * bursting */ + slave_count = internals->active_slave_count; + memcpy(slaves, internals->active_slaves, + sizeof(internals->active_slaves[0]) * slave_count); + + for (i = 0, idx = internals->active_slave; + i < slave_count && num_rx_total < nb_pkts; i++, idx++) { + idx = idx % slave_count; + + /* Read packets from this slave */ + num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id, + &bufs[num_rx_total], nb_pkts - num_rx_total); + } + + internals->active_slave = idx; + + return num_rx_total; +} + +static uint16_t +bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs, + uint16_t nb_pkts) +{ + struct bond_dev_private *internals; + struct bond_tx_queue *bd_tx_q; + + uint8_t num_of_slaves; + uint8_t slaves[RTE_MAX_ETHPORTS]; + /* positions in slaves, not ID */ + uint8_t distributing_offsets[RTE_MAX_ETHPORTS]; + uint8_t distributing_count; + + uint16_t num_tx_slave, num_tx_total = 0, num_tx_fail_total = 0; + uint16_t i, op_slave_idx; + + struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts]; + + /* Total amount of packets in slave_bufs */ + uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; + /* Slow packets placed in each slave */ + + if (unlikely(nb_pkts == 0)) + return 0; + + bd_tx_q = (struct bond_tx_queue *)queue; + internals = bd_tx_q->dev_private; + + /* Copy slave list to protect against slave up/down changes during tx + * bursting */ + num_of_slaves = internals->active_slave_count; + if (num_of_slaves < 1) + return num_tx_total; + + memcpy(slaves, internals->active_slaves, sizeof(slaves[0]) * + num_of_slaves); + + distributing_count = 0; + for (i = 0; i < num_of_slaves; i++) { + struct port *port = &mode_8023ad_ports[slaves[i]]; + if (ACTOR_STATE(port, DISTRIBUTING)) + distributing_offsets[distributing_count++] = i; + } + + if (likely(distributing_count > 0)) { + /* Populate slaves mbuf with the packets which are to be sent */ + for (i = 0; i < nb_pkts; i++) { + /* Select output slave using hash based on xmit policy */ + op_slave_idx = internals->xmit_hash(bufs[i], + distributing_count); + + /* Populate slave mbuf arrays with mbufs for that slave. + * Use only slaves that are currently distributing. + */ + uint8_t slave_offset = + distributing_offsets[op_slave_idx]; + slave_bufs[slave_offset][slave_nb_pkts[slave_offset]] = + bufs[i]; + slave_nb_pkts[slave_offset]++; + } + } + + /* Send packet burst on each slave device */ + for (i = 0; i < num_of_slaves; i++) { + if (slave_nb_pkts[i] == 0) + continue; + + num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, + slave_bufs[i], slave_nb_pkts[i]); + + num_tx_total += num_tx_slave; + num_tx_fail_total += slave_nb_pkts[i] - num_tx_slave; + + /* If tx burst fails move packets to end of bufs */ + if (unlikely(num_tx_slave < slave_nb_pkts[i])) { + uint16_t j = nb_pkts - num_tx_fail_total; + for ( ; num_tx_slave < slave_nb_pkts[i]; j++, + num_tx_slave++) + bufs[j] = slave_bufs[i][num_tx_slave]; + } + } + + return num_tx_total; +} + + static uint16_t bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) @@ -180,6 +428,13 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, /* Handle slow protocol packets. */ while (j < num_rx_total) { + + /* If packet is not pure L2 and is known, skip it */ + if ((bufs[j]->packet_type & ~RTE_PTYPE_L2_ETHER) != 0) { + j++; + continue; + } + if (j + 3 < num_rx_total) rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *)); @@ -187,7 +442,7 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype; /* Remove packet from array if it is slow packet or slave is not - * in collecting state or bondign interface is not in promiscus + * in collecting state or bonding interface is not in promiscuous * mode and packet address does not match. */ if (unlikely(is_lacp_packets(hdr->ether_type, subtype, bufs[j]->vlan_tci) || !collecting || (!promisc && @@ -654,7 +909,7 @@ bandwidth_left(uint8_t port_id, uint64_t load, uint8_t update_idx, { struct rte_eth_link link_status; - rte_eth_link_get(port_id, &link_status); + rte_eth_link_get_nowait(port_id, &link_status); uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8; if (link_bwg == 0) return; @@ -793,8 +1048,8 @@ bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 }; /* - * We create separate transmit buffers for update packets as they wont be - * counted in num_tx_total. + * We create separate transmit buffers for update packets as they won't + * be counted in num_tx_total. */ struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE]; uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 }; @@ -1131,39 +1386,44 @@ bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs, } void -link_properties_set(struct rte_eth_dev *bonded_eth_dev, - struct rte_eth_link *slave_dev_link) +link_properties_set(struct rte_eth_dev *ethdev, struct rte_eth_link *slave_link) { - struct rte_eth_link *bonded_dev_link = &bonded_eth_dev->data->dev_link; - struct bond_dev_private *internals = bonded_eth_dev->data->dev_private; + struct bond_dev_private *bond_ctx = ethdev->data->dev_private; - if (slave_dev_link->link_status && - bonded_eth_dev->data->dev_started) { - bonded_dev_link->link_duplex = slave_dev_link->link_duplex; - bonded_dev_link->link_speed = slave_dev_link->link_speed; + if (bond_ctx->mode == BONDING_MODE_8023AD) { + /** + * If in mode 4 then save the link properties of the first + * slave, all subsequent slaves must match these properties + */ + struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link; - internals->link_props_set = 1; + bond_link->link_autoneg = slave_link->link_autoneg; + bond_link->link_duplex = slave_link->link_duplex; + bond_link->link_speed = slave_link->link_speed; + } else { + /** + * In any other mode the link properties are set to default + * values of AUTONEG/DUPLEX + */ + ethdev->data->dev_link.link_autoneg = ETH_LINK_AUTONEG; + ethdev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; } } -void -link_properties_reset(struct rte_eth_dev *bonded_eth_dev) +int +link_properties_valid(struct rte_eth_dev *ethdev, + struct rte_eth_link *slave_link) { - struct bond_dev_private *internals = bonded_eth_dev->data->dev_private; + struct bond_dev_private *bond_ctx = ethdev->data->dev_private; - memset(&(bonded_eth_dev->data->dev_link), 0, - sizeof(bonded_eth_dev->data->dev_link)); + if (bond_ctx->mode == BONDING_MODE_8023AD) { + struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link; - internals->link_props_set = 0; -} - -int -link_properties_valid(struct rte_eth_link *bonded_dev_link, - struct rte_eth_link *slave_dev_link) -{ - if (bonded_dev_link->link_duplex != slave_dev_link->link_duplex || - bonded_dev_link->link_speed != slave_dev_link->link_speed) - return -1; + if (bond_link->link_duplex != slave_link->link_duplex || + bond_link->link_autoneg != slave_link->link_autoneg || + bond_link->link_speed != slave_link->link_speed) + return -1; + } return 0; } @@ -1295,11 +1555,19 @@ bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode) if (bond_mode_8023ad_enable(eth_dev) != 0) return -1; - eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad; - eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad; - RTE_LOG(WARNING, PMD, - "Using mode 4, it is necessary to do TX burst and RX burst " - "at least every 100ms.\n"); + if (internals->mode4.dedicated_queues.enabled == 0) { + eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad; + eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad; + RTE_LOG(WARNING, PMD, + "Using mode 4, it is necessary to do TX burst " + "and RX burst at least every 100ms.\n"); + } else { + /* Use flow director's optimization */ + eth_dev->rx_pkt_burst = + bond_ethdev_rx_burst_8023ad_fast_queue; + eth_dev->tx_pkt_burst = + bond_ethdev_tx_burst_8023ad_fast_queue; + } break; case BONDING_MODE_TLB: eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb; @@ -1321,15 +1589,81 @@ bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode) return 0; } + +static int +slave_configure_slow_queue(struct rte_eth_dev *bonded_eth_dev, + struct rte_eth_dev *slave_eth_dev) +{ + int errval = 0; + struct bond_dev_private *internals = (struct bond_dev_private *) + bonded_eth_dev->data->dev_private; + struct port *port = &mode_8023ad_ports[slave_eth_dev->data->port_id]; + + if (port->slow_pool == NULL) { + char mem_name[256]; + int slave_id = slave_eth_dev->data->port_id; + + snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_slow_pool", + slave_id); + port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191, + 250, 0, RTE_MBUF_DEFAULT_BUF_SIZE, + slave_eth_dev->data->numa_node); + + /* Any memory allocation failure in initialization is critical because + * resources can't be free, so reinitialization is impossible. */ + if (port->slow_pool == NULL) { + rte_panic("Slave %u: Failed to create memory pool '%s': %s\n", + slave_id, mem_name, rte_strerror(rte_errno)); + } + } + + if (internals->mode4.dedicated_queues.enabled == 1) { + /* Configure slow Rx queue */ + + errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.rx_qid, 128, + rte_eth_dev_socket_id(slave_eth_dev->data->port_id), + NULL, port->slow_pool); + if (errval != 0) { + RTE_BOND_LOG(ERR, + "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.rx_qid, + errval); + return errval; + } + + errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.tx_qid, 512, + rte_eth_dev_socket_id(slave_eth_dev->data->port_id), + NULL); + if (errval != 0) { + RTE_BOND_LOG(ERR, + "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.tx_qid, + errval); + return errval; + } + } + return 0; +} + int slave_configure(struct rte_eth_dev *bonded_eth_dev, struct rte_eth_dev *slave_eth_dev) { struct bond_rx_queue *bd_rx_q; struct bond_tx_queue *bd_tx_q; + uint16_t nb_rx_queues; + uint16_t nb_tx_queues; int errval; uint16_t q_id; + struct rte_flow_error flow_error; + + struct bond_dev_private *internals = (struct bond_dev_private *) + bonded_eth_dev->data->dev_private; /* Stop slave */ rte_eth_dev_stop(slave_eth_dev->data->port_id); @@ -1359,10 +1693,19 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, slave_eth_dev->data->dev_conf.rxmode.hw_vlan_filter = bonded_eth_dev->data->dev_conf.rxmode.hw_vlan_filter; + nb_rx_queues = bonded_eth_dev->data->nb_rx_queues; + nb_tx_queues = bonded_eth_dev->data->nb_tx_queues; + + if (internals->mode == BONDING_MODE_8023AD) { + if (internals->mode4.dedicated_queues.enabled == 1) { + nb_rx_queues++; + nb_tx_queues++; + } + } + /* Configure device */ errval = rte_eth_dev_configure(slave_eth_dev->data->port_id, - bonded_eth_dev->data->nb_rx_queues, - bonded_eth_dev->data->nb_tx_queues, + nb_rx_queues, nb_tx_queues, &(slave_eth_dev->data->dev_conf)); if (errval != 0) { RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u , err (%d)", @@ -1396,12 +1739,35 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, &bd_tx_q->tx_conf); if (errval != 0) { RTE_BOND_LOG(ERR, - "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", - slave_eth_dev->data->port_id, q_id, errval); + "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, q_id, errval); return errval; } } + if (internals->mode == BONDING_MODE_8023AD && + internals->mode4.dedicated_queues.enabled == 1) { + if (slave_configure_slow_queue(bonded_eth_dev, slave_eth_dev) + != 0) + return errval; + + if (bond_ethdev_8023ad_flow_verify(bonded_eth_dev, + slave_eth_dev->data->port_id) != 0) { + RTE_BOND_LOG(ERR, + "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, q_id, errval); + return -1; + } + + if (internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id] != NULL) + rte_flow_destroy(slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id], + &flow_error); + + bond_ethdev_8023ad_flow_set(bonded_eth_dev, + slave_eth_dev->data->port_id); + } + /* Start device */ errval = rte_eth_dev_start(slave_eth_dev->data->port_id); if (errval != 0) { @@ -1438,7 +1804,8 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0); bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id, - RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id); + RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id, + NULL); } return 0; @@ -1559,13 +1926,25 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) if (internals->promiscuous_en) bond_ethdev_promiscuous_enable(eth_dev); + if (internals->mode == BONDING_MODE_8023AD) { + if (internals->mode4.dedicated_queues.enabled == 1) { + internals->mode4.dedicated_queues.rx_qid = + eth_dev->data->nb_rx_queues; + internals->mode4.dedicated_queues.tx_qid = + eth_dev->data->nb_tx_queues; + } + } + + /* Reconfigure each slave device if starting bonded device */ for (i = 0; i < internals->slave_count; i++) { - if (slave_configure(eth_dev, - &(rte_eth_devices[internals->slaves[i].port_id])) != 0) { + struct rte_eth_dev *slave_ethdev = + &(rte_eth_devices[internals->slaves[i].port_id]); + if (slave_configure(eth_dev, slave_ethdev) != 0) { RTE_BOND_LOG(ERR, - "bonded port (%d) failed to reconfigure slave device (%d)", - eth_dev->data->port_id, internals->slaves[i].port_id); + "bonded port (%d) failed to reconfigure slave device (%d)", + eth_dev->data->port_id, + internals->slaves[i].port_id); return -1; } /* We will need to poll for link status if any slave doesn't @@ -1666,7 +2045,7 @@ bond_ethdev_close(struct rte_eth_dev *dev) uint8_t bond_port_id = internals->port_id; int skipped = 0; - RTE_LOG(INFO, EAL, "Closing bonded device %s\n", dev->data->name); + RTE_LOG(INFO, EAL, "Closing bonded device %s\n", dev->device->name); while (internals->slave_count != skipped) { uint8_t port_id = internals->slaves[skipped].port_id; @@ -1675,7 +2054,7 @@ bond_ethdev_close(struct rte_eth_dev *dev) if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) { RTE_LOG(ERR, EAL, "Failed to remove port %d from bonded device " - "%s\n", port_id, dev->data->name); + "%s\n", port_id, dev->device->name); skipped++; } } @@ -1691,14 +2070,47 @@ bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct bond_dev_private *internals = dev->data->dev_private; + uint16_t max_nb_rx_queues = UINT16_MAX; + uint16_t max_nb_tx_queues = UINT16_MAX; + dev_info->max_mac_addrs = 1; - dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen - ? internals->candidate_max_rx_pktlen - : ETHER_MAX_JUMBO_FRAME_LEN; + dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen ? + internals->candidate_max_rx_pktlen : + ETHER_MAX_JUMBO_FRAME_LEN; + + /* Max number of tx/rx queues that the bonded device can support is the + * minimum values of the bonded slaves, as all slaves must be capable + * of supporting the same number of tx/rx queues. + */ + if (internals->slave_count > 0) { + struct rte_eth_dev_info slave_info; + uint8_t idx; - dev_info->max_rx_queues = (uint16_t)128; - dev_info->max_tx_queues = (uint16_t)512; + for (idx = 0; idx < internals->slave_count; idx++) { + rte_eth_dev_info_get(internals->slaves[idx].port_id, + &slave_info); + + if (slave_info.max_rx_queues < max_nb_rx_queues) + max_nb_rx_queues = slave_info.max_rx_queues; + + if (slave_info.max_tx_queues < max_nb_tx_queues) + max_nb_tx_queues = slave_info.max_tx_queues; + } + } + + dev_info->max_rx_queues = max_nb_rx_queues; + dev_info->max_tx_queues = max_nb_tx_queues; + + /** + * If dedicated hw queues enabled for link bonding device in LACP mode + * then we need to reduce the maximum number of data path queues by 1. + */ + if (internals->mode == BONDING_MODE_8023AD && + internals->mode4.dedicated_queues.enabled == 1) { + dev_info->max_rx_queues--; + dev_info->max_tx_queues--; + } dev_info->min_rx_bufsize = 0; @@ -1849,7 +2261,8 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg) bond_ethdev_lsc_event_callback(internals->slaves[i].port_id, RTE_ETH_EVENT_INTR_LSC, - &bonded_ethdev->data->port_id); + &bonded_ethdev->data->port_id, + NULL); } } rte_spinlock_unlock(&internals->lock); @@ -1862,36 +2275,90 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg) } static int -bond_ethdev_link_update(struct rte_eth_dev *bonded_eth_dev, - int wait_to_complete) +bond_ethdev_link_update(struct rte_eth_dev *ethdev, int wait_to_complete) { - struct bond_dev_private *internals = bonded_eth_dev->data->dev_private; + void (*link_update)(uint8_t port_id, struct rte_eth_link *eth_link); - if (!bonded_eth_dev->data->dev_started || - internals->active_slave_count == 0) { - bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; + struct bond_dev_private *bond_ctx; + struct rte_eth_link slave_link; + + uint32_t idx; + + bond_ctx = ethdev->data->dev_private; + + ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; + + if (ethdev->data->dev_started == 0 || + bond_ctx->active_slave_count == 0) { + ethdev->data->dev_link.link_status = ETH_LINK_DOWN; return 0; - } else { - struct rte_eth_dev *slave_eth_dev; - int i, link_up = 0; + } - for (i = 0; i < internals->active_slave_count; i++) { - slave_eth_dev = &rte_eth_devices[internals->active_slaves[i]]; + ethdev->data->dev_link.link_status = ETH_LINK_UP; - (*slave_eth_dev->dev_ops->link_update)(slave_eth_dev, - wait_to_complete); - if (slave_eth_dev->data->dev_link.link_status == ETH_LINK_UP) { - link_up = 1; - break; - } + if (wait_to_complete) + link_update = rte_eth_link_get; + else + link_update = rte_eth_link_get_nowait; + + switch (bond_ctx->mode) { + case BONDING_MODE_BROADCAST: + /** + * Setting link speed to UINT32_MAX to ensure we pick up the + * value of the first active slave + */ + ethdev->data->dev_link.link_speed = UINT32_MAX; + + /** + * link speed is minimum value of all the slaves link speed as + * packet loss will occur on this slave if transmission at rates + * greater than this are attempted + */ + for (idx = 1; idx < bond_ctx->active_slave_count; idx++) { + link_update(bond_ctx->active_slaves[0], &slave_link); + + if (slave_link.link_speed < + ethdev->data->dev_link.link_speed) + ethdev->data->dev_link.link_speed = + slave_link.link_speed; } + break; + case BONDING_MODE_ACTIVE_BACKUP: + /* Current primary slave */ + link_update(bond_ctx->current_primary_port, &slave_link); - bonded_eth_dev->data->dev_link.link_status = link_up; + ethdev->data->dev_link.link_speed = slave_link.link_speed; + break; + case BONDING_MODE_8023AD: + ethdev->data->dev_link.link_autoneg = + bond_ctx->mode4.slave_link.link_autoneg; + ethdev->data->dev_link.link_duplex = + bond_ctx->mode4.slave_link.link_duplex; + /* fall through to update link speed */ + case BONDING_MODE_ROUND_ROBIN: + case BONDING_MODE_BALANCE: + case BONDING_MODE_TLB: + case BONDING_MODE_ALB: + default: + /** + * In theses mode the maximum theoretical link speed is the sum + * of all the slaves + */ + ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; + + for (idx = 0; idx < bond_ctx->active_slave_count; idx++) { + link_update(bond_ctx->active_slaves[idx], &slave_link); + + ethdev->data->dev_link.link_speed += + slave_link.link_speed; + } } + return 0; } + static void bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { @@ -1995,35 +2462,35 @@ bond_ethdev_delayed_lsc_propagation(void *arg) return; _rte_eth_dev_callback_process((struct rte_eth_dev *)arg, - RTE_ETH_EVENT_INTR_LSC, NULL); + RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } -void +int bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, - void *param) + void *param, void *ret_param __rte_unused) { - struct rte_eth_dev *bonded_eth_dev, *slave_eth_dev; + struct rte_eth_dev *bonded_eth_dev; struct bond_dev_private *internals; struct rte_eth_link link; + int rc = -1; int i, valid_slave = 0; uint8_t active_pos; uint8_t lsc_flag = 0; if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL) - return; + return rc; bonded_eth_dev = &rte_eth_devices[*(uint8_t *)param]; - slave_eth_dev = &rte_eth_devices[port_id]; if (check_for_bonded_ethdev(bonded_eth_dev)) - return; + return rc; internals = bonded_eth_dev->data->dev_private; /* If the device isn't started don't handle interrupts */ if (!bonded_eth_dev->data->dev_started) - return; + return rc; /* verify that port_id is a valid slave of bonded port */ for (i = 0; i < internals->slave_count; i++) { @@ -2034,7 +2501,7 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, } if (!valid_slave) - return; + return rc; /* Search for port in active port list */ active_pos = find_slave_by_id(internals->active_slaves, @@ -2043,7 +2510,7 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, rte_eth_link_get_nowait(port_id, &link); if (link.link_status) { if (active_pos < internals->active_slave_count) - return; + return rc; /* if no active slave ports then set this port to be primary port */ if (internals->active_slave_count < 1) { @@ -2053,20 +2520,6 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, lsc_flag = 1; mac_address_slaves_update(bonded_eth_dev); - - /* Inherit eth dev link properties from first active slave */ - link_properties_set(bonded_eth_dev, - &(slave_eth_dev->data->dev_link)); - } else { - if (link_properties_valid( - &bonded_eth_dev->data->dev_link, &link) != 0) { - slave_eth_dev->data->dev_flags &= - (~RTE_ETH_DEV_BONDED_SLAVE); - RTE_LOG(ERR, PMD, - "port %u invalid speed/duplex\n", - port_id); - return; - } } activate_slave(bonded_eth_dev, port_id); @@ -2077,19 +2530,13 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, bond_ethdev_primary_set(internals, port_id); } else { if (active_pos == internals->active_slave_count) - return; + return rc; /* Remove from active slave list */ deactivate_slave(bonded_eth_dev, port_id); - /* No active slaves, change link status to down and reset other - * link properties */ - if (internals->active_slave_count < 1) { + if (internals->active_slave_count < 1) lsc_flag = 1; - bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; - - link_properties_reset(bonded_eth_dev); - } /* Update primary id, take first active slave from list or if none * available set to -1 */ @@ -2102,6 +2549,12 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, } } + /** + * Update bonded device link properties after any change to active + * slaves + */ + bond_ethdev_link_update(bonded_eth_dev, 0); + if (lsc_flag) { /* Cancel any possible outstanding interrupts if delays are enabled */ if (internals->link_up_delay_ms > 0 || @@ -2116,7 +2569,8 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, (void *)bonded_eth_dev); else _rte_eth_dev_callback_process(bonded_eth_dev, - RTE_ETH_EVENT_INTR_LSC, NULL); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } else { if (internals->link_down_delay_ms > 0) @@ -2125,9 +2579,11 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, (void *)bonded_eth_dev); else _rte_eth_dev_callback_process(bonded_eth_dev, - RTE_ETH_EVENT_INTR_LSC, NULL); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } } + return 0; } static int @@ -2272,12 +2728,6 @@ bond_alloc(struct rte_vdev_device *dev, uint8_t mode) * and internal (private) data */ - if (socket_id >= number_of_sockets()) { - RTE_BOND_LOG(ERR, - "Invalid socket id specified to create bonded device on."); - goto err; - } - /* reserve an ethdev entry */ eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals)); if (eth_dev == NULL) { @@ -2308,7 +2758,6 @@ bond_alloc(struct rte_vdev_device *dev, uint8_t mode) internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2; internals->xmit_hash = xmit_l2_hash; internals->user_defined_mac = 0; - internals->link_props_set = 0; internals->link_status_polling_enabled = 0; @@ -2376,7 +2825,7 @@ bond_probe(struct rte_vdev_device *dev) const char *name; struct bond_dev_private *internals; struct rte_kvargs *kvlist; - uint8_t bonding_mode, socket_id; + uint8_t bonding_mode, socket_id/*, agg_mode*/; int arg_count, port_id; if (!dev) @@ -2498,11 +2947,12 @@ bond_remove(struct rte_vdev_device *dev) static int bond_ethdev_configure(struct rte_eth_dev *dev) { - char *name = dev->data->name; + const char *name = dev->device->name; struct bond_dev_private *internals = dev->data->dev_private; struct rte_kvargs *kvlist = internals->kvlist; int arg_count; uint8_t port_id = dev - rte_eth_devices; + uint8_t agg_mode; static const uint8_t default_rss_key[40] = { 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D, @@ -2590,6 +3040,21 @@ bond_ethdev_configure(struct rte_eth_dev *dev) return -1; } + if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) { + if (rte_kvargs_process(kvlist, + PMD_BOND_AGG_MODE_KVARG, + &bond_ethdev_parse_slave_agg_mode_kvarg, + &agg_mode) != 0) { + RTE_LOG(ERR, EAL, + "Failed to parse agg selection mode for bonded device %s\n", + name); + } + if (internals->mode == BONDING_MODE_8023AD) + if (agg_mode != 0) + rte_eth_bond_8023ad_agg_selection_set(port_id, + agg_mode); + } + /* Parse/add slave ports to bonded device */ if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) { struct bond_ethdev_slave_ports slave_ports; @@ -2753,6 +3218,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_bonding, "primary=<ifc> " "mode=[0-6] " "xmit_policy=[l2 | l23 | l34] " + "agg_mode=[count | stable | bandwidth] " "socket_id=<int> " "mac=<mac addr> " "lsc_poll_period_ms=<int> " diff --git a/drivers/net/bonding/rte_eth_bond_private.h b/drivers/net/bonding/rte_eth_bond_private.h index c8db0900..1fe6ff88 100644 --- a/drivers/net/bonding/rte_eth_bond_private.h +++ b/drivers/net/bonding/rte_eth_bond_private.h @@ -45,6 +45,7 @@ #define PMD_BOND_SLAVE_PORT_KVARG ("slave") #define PMD_BOND_PRIMARY_SLAVE_KVARG ("primary") #define PMD_BOND_MODE_KVARG ("mode") +#define PMD_BOND_AGG_MODE_KVARG ("agg_mode") #define PMD_BOND_XMIT_POLICY_KVARG ("xmit_policy") #define PMD_BOND_SOCKET_ID_KVARG ("socket_id") #define PMD_BOND_MAC_ADDR_KVARG ("mac") @@ -132,8 +133,7 @@ struct bond_dev_private { /**< Flag for whether MAC address is user defined or not */ uint8_t promiscuous_en; /**< Enabled/disable promiscuous mode on bonding device */ - uint8_t link_props_set; - /**< flag to denote if the link properties are set */ + uint8_t link_status_polling_enabled; uint32_t link_status_polling_interval_ms; @@ -184,7 +184,7 @@ extern const struct eth_dev_ops default_dev_ops; int check_for_bonded_ethdev(const struct rte_eth_dev *eth_dev); -/* Search given slave array to find possition of given id. +/* Search given slave array to find position of given id. * Return slave pos or slaves_count if not found. */ static inline uint8_t find_slave_by_id(uint8_t *slaves, uint8_t slaves_count, uint8_t slave_id) { @@ -205,7 +205,7 @@ int valid_bonded_port_id(uint8_t port_id); int -valid_slave_port_id(uint8_t port_id); +valid_slave_port_id(uint8_t port_id, uint8_t mode); void deactivate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id); @@ -216,11 +216,8 @@ activate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id); void link_properties_set(struct rte_eth_dev *bonded_eth_dev, struct rte_eth_link *slave_dev_link); -void -link_properties_reset(struct rte_eth_dev *bonded_eth_dev); - int -link_properties_valid(struct rte_eth_link *bonded_dev_link, +link_properties_valid(struct rte_eth_dev *bonded_eth_dev, struct rte_eth_link *slave_dev_link); int @@ -232,9 +229,6 @@ mac_address_get(struct rte_eth_dev *eth_dev, struct ether_addr *dst_mac_addr); int mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev); -uint8_t -number_of_sockets(void); - int bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode); @@ -263,36 +257,40 @@ void bond_ethdev_primary_set(struct bond_dev_private *internals, uint8_t slave_port_id); -void +int bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, - void *param); + void *param, void *ret_param); + +int +bond_ethdev_parse_slave_port_kvarg(const char *key, + const char *value, void *extra_args); int -bond_ethdev_parse_slave_port_kvarg(const char *key __rte_unused, +bond_ethdev_parse_slave_mode_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_slave_mode_kvarg(const char *key __rte_unused, +bond_ethdev_parse_slave_agg_mode_kvarg(const char *key __rte_unused, const char *value, void *extra_args); int -bond_ethdev_parse_socket_id_kvarg(const char *key __rte_unused, +bond_ethdev_parse_socket_id_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_primary_slave_port_id_kvarg(const char *key __rte_unused, +bond_ethdev_parse_primary_slave_port_id_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_balance_xmit_policy_kvarg(const char *key __rte_unused, +bond_ethdev_parse_balance_xmit_policy_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_bond_mac_addr_kvarg(const char *key __rte_unused, +bond_ethdev_parse_bond_mac_addr_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_time_ms_kvarg(const char *key __rte_unused, +bond_ethdev_parse_time_ms_kvarg(const char *key, const char *value, void *extra_args); void diff --git a/drivers/net/bonding/rte_eth_bond_version.map b/drivers/net/bonding/rte_eth_bond_version.map index 2de0a7d3..0f4e847d 100644 --- a/drivers/net/bonding/rte_eth_bond_version.map +++ b/drivers/net/bonding/rte_eth_bond_version.map @@ -43,3 +43,16 @@ DPDK_16.07 { rte_eth_bond_8023ad_setup; } DPDK_16.04; + +DPDK_17.08 { + global: + + rte_eth_bond_8023ad_dedicated_queues_enable; + rte_eth_bond_8023ad_dedicated_queues_disable; + rte_eth_bond_8023ad_agg_selection_get; + rte_eth_bond_8023ad_agg_selection_set; + rte_eth_bond_8023ad_conf_get; + rte_eth_bond_8023ad_setup; + + +} DPDK_16.07; diff --git a/drivers/net/cxgbe/base/adapter.h b/drivers/net/cxgbe/base/adapter.h index 26807900..5e5f221e 100644 --- a/drivers/net/cxgbe/base/adapter.h +++ b/drivers/net/cxgbe/base/adapter.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -148,6 +148,7 @@ struct sge_rspq { /* state for an SGE response queue */ void __iomem *bar2_addr; /* address of BAR2 Queue registers */ unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ + struct sge_qstat *stat; unsigned int cidx; /* consumer index */ unsigned int gts_idx; /* last gts write sent */ @@ -459,7 +460,10 @@ static inline void t4_write_reg64(struct adapter *adapter, u32 reg_addr, #define PCI_CAP_ID_EXP 0x10 /* PCI Express */ #define PCI_CAP_LIST_ID 0 /* Capability ID */ #define PCI_CAP_LIST_NEXT 1 /* Next capability in the list */ +#define PCI_EXP_DEVCTL 0x0008 /* Device control */ #define PCI_EXP_DEVCTL2 40 /* Device Control 2 */ +#define PCI_EXP_DEVCTL_EXT_TAG 0x0100 /* Extended Tag Field Enable */ +#define PCI_EXP_DEVCTL_PAYLOAD 0x00E0 /* Max payload */ #define PCI_CAP_ID_VPD 0x03 /* Vital Product Data */ #define PCI_VPD_ADDR 2 /* Address to access (15 bits!) */ #define PCI_VPD_ADDR_F 0x8000 /* Write 0, 1 indicates completion */ @@ -706,7 +710,8 @@ void reclaim_completed_tx(struct sge_txq *q); void t4_free_sge_resources(struct adapter *adap); void t4_sge_tx_monitor_start(struct adapter *adap); void t4_sge_tx_monitor_stop(struct adapter *adap); -int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf); +int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, + uint16_t nb_pkts); int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, const struct pkt_gl *gl); int t4_sge_init(struct adapter *adap); diff --git a/drivers/net/cxgbe/base/common.h b/drivers/net/cxgbe/base/common.h index 11f139c9..1eda57d0 100644 --- a/drivers/net/cxgbe/base/common.h +++ b/drivers/net/cxgbe/base/common.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -68,6 +68,12 @@ enum { PAUSE_AUTONEG = 1 << 2 }; +enum { + FEC_RS = 1 << 0, + FEC_BASER_RS = 1 << 1, + FEC_RESERVED = 1 << 2, +}; + struct port_stats { u64 tx_octets; /* total # of octets in good frames */ u64 tx_frames; /* all good frames */ @@ -151,6 +157,11 @@ struct tp_params { u32 vlan_pri_map; /* cached TP_VLAN_PRI_MAP */ u32 ingress_config; /* cached TP_INGRESS_CONFIG */ + /* cached TP_OUT_CONFIG compressed error vector + * and passing outer header info for encapsulated packets. + */ + int rx_pkt_encap; + /* * TP_VLAN_PRI_MAP Compressed Filter Tuple field offsets. This is a * subset of the set of fields which may be present in the Compressed @@ -210,7 +221,9 @@ struct adapter_params { unsigned int sf_nsec; /* # of flash sectors */ unsigned int fw_vers; + unsigned int bs_vers; unsigned int tp_vers; + unsigned int er_vers; unsigned short mtus[NMTUS]; unsigned short a_wnd[NCCTRL_WIN]; @@ -231,10 +244,12 @@ struct adapter_params { struct link_config { unsigned short supported; /* link capabilities */ unsigned short advertising; /* advertised capabilities */ - unsigned short requested_speed; /* speed user has requested */ - unsigned short speed; /* actual link speed */ + unsigned int requested_speed; /* speed user has requested */ + unsigned int speed; /* actual link speed */ unsigned char requested_fc; /* flow control user has requested */ unsigned char fc; /* actual link flow control */ + unsigned char requested_fec; /* Forward Error Correction user */ + unsigned char fec; /* has requested and actual FEC */ unsigned char autoneg; /* autonegotiating? */ unsigned char link_ok; /* link up? */ }; @@ -272,6 +287,7 @@ int t4_fw_bye(struct adapter *adap, unsigned int mbox); int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset); int t4_fw_halt(struct adapter *adap, unsigned int mbox, int reset); int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset); +int t4_fl_pkt_align(struct adapter *adap); int t4_fixup_host_params_compat(struct adapter *adap, unsigned int page_size, unsigned int cache_line_size, enum chip_type chip_compat); @@ -374,7 +390,8 @@ int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p); int t4_read_flash(struct adapter *adapter, unsigned int addr, unsigned int nwords, u32 *data, int byte_oriented); int t4_flash_cfg_addr(struct adapter *adapter); -unsigned int t4_get_mps_bg_map(struct adapter *adapter, int idx); +unsigned int t4_get_mps_bg_map(struct adapter *adapter, unsigned int pidx); +unsigned int t4_get_tp_ch_map(struct adapter *adapter, unsigned int pidx); const char *t4_get_port_type_description(enum fw_port_type port_type); void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p); void t4_get_port_stats_offset(struct adapter *adap, int idx, @@ -382,9 +399,10 @@ void t4_get_port_stats_offset(struct adapter *adap, int idx, struct port_stats *offset); void t4_clr_port_stats(struct adapter *adap, int idx); void t4_reset_link_config(struct adapter *adap, int idx); -int t4_get_fw_version(struct adapter *adapter, u32 *vers); -int t4_get_tp_version(struct adapter *adapter, u32 *vers); +int t4_get_version_info(struct adapter *adapter); +void t4_dump_version_info(struct adapter *adapter); int t4_get_flash_params(struct adapter *adapter); +int t4_get_chip_type(struct adapter *adap, int ver); int t4_prep_adapter(struct adapter *adapter); int t4_port_init(struct adapter *adap, int mbox, int pf, int vf); int t4_init_rss_mode(struct adapter *adap, int mbox); diff --git a/drivers/net/cxgbe/base/t4_chip_type.h b/drivers/net/cxgbe/base/t4_chip_type.h index 1ca68039..cd7a9282 100644 --- a/drivers/net/cxgbe/base/t4_chip_type.h +++ b/drivers/net/cxgbe/base/t4_chip_type.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -49,6 +49,7 @@ #define CHELSIO_T4 0x4 #define CHELSIO_T5 0x5 +#define CHELSIO_T6 0x6 #define CHELSIO_CHIP_CODE(version, revision) (((version) << 4) | (revision)) #define CHELSIO_CHIP_VERSION(code) (((code) >> 4) & 0xf) @@ -64,6 +65,10 @@ enum chip_type { T5_A1 = CHELSIO_CHIP_CODE(CHELSIO_T5, 1), T5_FIRST_REV = T5_A0, T5_LAST_REV = T5_A1, + + T6_A0 = CHELSIO_CHIP_CODE(CHELSIO_T6, 0), + T6_FIRST_REV = T6_A0, + T6_LAST_REV = T6_A0, }; static inline int is_t4(enum chip_type chip) @@ -76,4 +81,8 @@ static inline int is_t5(enum chip_type chip) return (CHELSIO_CHIP_VERSION(chip) == CHELSIO_T5); } +static inline int is_t6(enum chip_type chip) +{ + return (CHELSIO_CHIP_VERSION(chip) == CHELSIO_T6); +} #endif /* __T4_CHIP_TYPE_H__ */ diff --git a/drivers/net/cxgbe/base/t4_hw.c b/drivers/net/cxgbe/base/t4_hw.c index 19afdac9..a8ccea00 100644 --- a/drivers/net/cxgbe/base/t4_hw.c +++ b/drivers/net/cxgbe/base/t4_hw.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -46,7 +46,6 @@ #include <rte_alarm.h> #include <rte_ether.h> #include <rte_ethdev.h> -#include <rte_atomic.h> #include <rte_malloc.h> #include <rte_random.h> #include <rte_dev.h> @@ -57,7 +56,8 @@ #include "t4_regs_values.h" #include "t4fw_interface.h" -static void init_link_config(struct link_config *lc, unsigned int caps); +static void init_link_config(struct link_config *lc, unsigned int pcaps, + unsigned int acaps); /** * t4_read_mtu_tbl - returns the values in the HW path MTU table @@ -584,6 +584,7 @@ unsigned int t4_get_regs_len(struct adapter *adapter) switch (chip_version) { case CHELSIO_T5: + case CHELSIO_T6: return T5_REGMAP_SIZE; } @@ -1379,6 +1380,567 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size) 0x51300, 0x51308, }; + static const unsigned int t6_reg_ranges[] = { + 0x1008, 0x101c, + 0x1024, 0x10a8, + 0x10b4, 0x10f8, + 0x1100, 0x1114, + 0x111c, 0x112c, + 0x1138, 0x113c, + 0x1144, 0x114c, + 0x1180, 0x1184, + 0x1190, 0x1194, + 0x11a0, 0x11a4, + 0x11b0, 0x11b4, + 0x11fc, 0x1274, + 0x1280, 0x133c, + 0x1800, 0x18fc, + 0x3000, 0x302c, + 0x3060, 0x30b0, + 0x30b8, 0x30d8, + 0x30e0, 0x30fc, + 0x3140, 0x357c, + 0x35a8, 0x35cc, + 0x35ec, 0x35ec, + 0x3600, 0x5624, + 0x56cc, 0x56ec, + 0x56f4, 0x5720, + 0x5728, 0x575c, + 0x580c, 0x5814, + 0x5890, 0x589c, + 0x58a4, 0x58ac, + 0x58b8, 0x58bc, + 0x5940, 0x595c, + 0x5980, 0x598c, + 0x59b0, 0x59c8, + 0x59d0, 0x59dc, + 0x59fc, 0x5a18, + 0x5a60, 0x5a6c, + 0x5a80, 0x5a8c, + 0x5a94, 0x5a9c, + 0x5b94, 0x5bfc, + 0x5c10, 0x5e48, + 0x5e50, 0x5e94, + 0x5ea0, 0x5eb0, + 0x5ec0, 0x5ec0, + 0x5ec8, 0x5ed0, + 0x5ee0, 0x5ee0, + 0x5ef0, 0x5ef0, + 0x5f00, 0x5f00, + 0x6000, 0x6020, + 0x6028, 0x6040, + 0x6058, 0x609c, + 0x60a8, 0x619c, + 0x7700, 0x7798, + 0x77c0, 0x7880, + 0x78cc, 0x78fc, + 0x7b00, 0x7b58, + 0x7b60, 0x7b84, + 0x7b8c, 0x7c54, + 0x7d00, 0x7d38, + 0x7d40, 0x7d84, + 0x7d8c, 0x7ddc, + 0x7de4, 0x7e04, + 0x7e10, 0x7e1c, + 0x7e24, 0x7e38, + 0x7e40, 0x7e44, + 0x7e4c, 0x7e78, + 0x7e80, 0x7edc, + 0x7ee8, 0x7efc, + 0x8dc0, 0x8de4, + 0x8df8, 0x8e04, + 0x8e10, 0x8e84, + 0x8ea0, 0x8f88, + 0x8fb8, 0x9058, + 0x9060, 0x9060, + 0x9068, 0x90f8, + 0x9100, 0x9124, + 0x9400, 0x9470, + 0x9600, 0x9600, + 0x9608, 0x9638, + 0x9640, 0x9704, + 0x9710, 0x971c, + 0x9800, 0x9808, + 0x9820, 0x983c, + 0x9850, 0x9864, + 0x9c00, 0x9c6c, + 0x9c80, 0x9cec, + 0x9d00, 0x9d6c, + 0x9d80, 0x9dec, + 0x9e00, 0x9e6c, + 0x9e80, 0x9eec, + 0x9f00, 0x9f6c, + 0x9f80, 0xa020, + 0xd004, 0xd03c, + 0xd100, 0xd118, + 0xd200, 0xd214, + 0xd220, 0xd234, + 0xd240, 0xd254, + 0xd260, 0xd274, + 0xd280, 0xd294, + 0xd2a0, 0xd2b4, + 0xd2c0, 0xd2d4, + 0xd2e0, 0xd2f4, + 0xd300, 0xd31c, + 0xdfc0, 0xdfe0, + 0xe000, 0xf008, + 0xf010, 0xf018, + 0xf020, 0xf028, + 0x11000, 0x11014, + 0x11048, 0x1106c, + 0x11074, 0x11088, + 0x11098, 0x11120, + 0x1112c, 0x1117c, + 0x11190, 0x112e0, + 0x11300, 0x1130c, + 0x12000, 0x1206c, + 0x19040, 0x1906c, + 0x19078, 0x19080, + 0x1908c, 0x190e8, + 0x190f0, 0x190f8, + 0x19100, 0x19110, + 0x19120, 0x19124, + 0x19150, 0x19194, + 0x1919c, 0x191b0, + 0x191d0, 0x191e8, + 0x19238, 0x19290, + 0x192a4, 0x192b0, + 0x192bc, 0x192bc, + 0x19348, 0x1934c, + 0x193f8, 0x19418, + 0x19420, 0x19428, + 0x19430, 0x19444, + 0x1944c, 0x1946c, + 0x19474, 0x19474, + 0x19490, 0x194cc, + 0x194f0, 0x194f8, + 0x19c00, 0x19c48, + 0x19c50, 0x19c80, + 0x19c94, 0x19c98, + 0x19ca0, 0x19cbc, + 0x19ce4, 0x19ce4, + 0x19cf0, 0x19cf8, + 0x19d00, 0x19d28, + 0x19d50, 0x19d78, + 0x19d94, 0x19d98, + 0x19da0, 0x19dc8, + 0x19df0, 0x19e10, + 0x19e50, 0x19e6c, + 0x19ea0, 0x19ebc, + 0x19ec4, 0x19ef4, + 0x19f04, 0x19f2c, + 0x19f34, 0x19f34, + 0x19f40, 0x19f50, + 0x19f90, 0x19fac, + 0x19fc4, 0x19fc8, + 0x19fd0, 0x19fe4, + 0x1a000, 0x1a004, + 0x1a010, 0x1a06c, + 0x1a0b0, 0x1a0e4, + 0x1a0ec, 0x1a0f8, + 0x1a100, 0x1a108, + 0x1a114, 0x1a120, + 0x1a128, 0x1a130, + 0x1a138, 0x1a138, + 0x1a190, 0x1a1c4, + 0x1a1fc, 0x1a1fc, + 0x1e008, 0x1e00c, + 0x1e040, 0x1e044, + 0x1e04c, 0x1e04c, + 0x1e284, 0x1e290, + 0x1e2c0, 0x1e2c0, + 0x1e2e0, 0x1e2e0, + 0x1e300, 0x1e384, + 0x1e3c0, 0x1e3c8, + 0x1e408, 0x1e40c, + 0x1e440, 0x1e444, + 0x1e44c, 0x1e44c, + 0x1e684, 0x1e690, + 0x1e6c0, 0x1e6c0, + 0x1e6e0, 0x1e6e0, + 0x1e700, 0x1e784, + 0x1e7c0, 0x1e7c8, + 0x1e808, 0x1e80c, + 0x1e840, 0x1e844, + 0x1e84c, 0x1e84c, + 0x1ea84, 0x1ea90, + 0x1eac0, 0x1eac0, + 0x1eae0, 0x1eae0, + 0x1eb00, 0x1eb84, + 0x1ebc0, 0x1ebc8, + 0x1ec08, 0x1ec0c, + 0x1ec40, 0x1ec44, + 0x1ec4c, 0x1ec4c, + 0x1ee84, 0x1ee90, + 0x1eec0, 0x1eec0, + 0x1eee0, 0x1eee0, + 0x1ef00, 0x1ef84, + 0x1efc0, 0x1efc8, + 0x1f008, 0x1f00c, + 0x1f040, 0x1f044, + 0x1f04c, 0x1f04c, + 0x1f284, 0x1f290, + 0x1f2c0, 0x1f2c0, + 0x1f2e0, 0x1f2e0, + 0x1f300, 0x1f384, + 0x1f3c0, 0x1f3c8, + 0x1f408, 0x1f40c, + 0x1f440, 0x1f444, + 0x1f44c, 0x1f44c, + 0x1f684, 0x1f690, + 0x1f6c0, 0x1f6c0, + 0x1f6e0, 0x1f6e0, + 0x1f700, 0x1f784, + 0x1f7c0, 0x1f7c8, + 0x1f808, 0x1f80c, + 0x1f840, 0x1f844, + 0x1f84c, 0x1f84c, + 0x1fa84, 0x1fa90, + 0x1fac0, 0x1fac0, + 0x1fae0, 0x1fae0, + 0x1fb00, 0x1fb84, + 0x1fbc0, 0x1fbc8, + 0x1fc08, 0x1fc0c, + 0x1fc40, 0x1fc44, + 0x1fc4c, 0x1fc4c, + 0x1fe84, 0x1fe90, + 0x1fec0, 0x1fec0, + 0x1fee0, 0x1fee0, + 0x1ff00, 0x1ff84, + 0x1ffc0, 0x1ffc8, + 0x30000, 0x30030, + 0x30100, 0x30168, + 0x30190, 0x301a0, + 0x301a8, 0x301b8, + 0x301c4, 0x301c8, + 0x301d0, 0x301d0, + 0x30200, 0x30320, + 0x30400, 0x304b4, + 0x304c0, 0x3052c, + 0x30540, 0x3061c, + 0x30800, 0x308a0, + 0x308c0, 0x30908, + 0x30910, 0x309b8, + 0x30a00, 0x30a04, + 0x30a0c, 0x30a14, + 0x30a1c, 0x30a2c, + 0x30a44, 0x30a50, + 0x30a74, 0x30a74, + 0x30a7c, 0x30afc, + 0x30b08, 0x30c24, + 0x30d00, 0x30d14, + 0x30d1c, 0x30d3c, + 0x30d44, 0x30d4c, + 0x30d54, 0x30d74, + 0x30d7c, 0x30d7c, + 0x30de0, 0x30de0, + 0x30e00, 0x30ed4, + 0x30f00, 0x30fa4, + 0x30fc0, 0x30fc4, + 0x31000, 0x31004, + 0x31080, 0x310fc, + 0x31208, 0x31220, + 0x3123c, 0x31254, + 0x31300, 0x31300, + 0x31308, 0x3131c, + 0x31338, 0x3133c, + 0x31380, 0x31380, + 0x31388, 0x313a8, + 0x313b4, 0x313b4, + 0x31400, 0x31420, + 0x31438, 0x3143c, + 0x31480, 0x31480, + 0x314a8, 0x314a8, + 0x314b0, 0x314b4, + 0x314c8, 0x314d4, + 0x31a40, 0x31a4c, + 0x31af0, 0x31b20, + 0x31b38, 0x31b3c, + 0x31b80, 0x31b80, + 0x31ba8, 0x31ba8, + 0x31bb0, 0x31bb4, + 0x31bc8, 0x31bd4, + 0x32140, 0x3218c, + 0x321f0, 0x321f4, + 0x32200, 0x32200, + 0x32218, 0x32218, + 0x32400, 0x32400, + 0x32408, 0x3241c, + 0x32618, 0x32620, + 0x32664, 0x32664, + 0x326a8, 0x326a8, + 0x326ec, 0x326ec, + 0x32a00, 0x32abc, + 0x32b00, 0x32b38, + 0x32b20, 0x32b38, + 0x32b40, 0x32b58, + 0x32b60, 0x32b78, + 0x32c00, 0x32c00, + 0x32c08, 0x32c3c, + 0x33000, 0x3302c, + 0x33034, 0x33050, + 0x33058, 0x33058, + 0x33060, 0x3308c, + 0x3309c, 0x330ac, + 0x330c0, 0x330c0, + 0x330c8, 0x330d0, + 0x330d8, 0x330e0, + 0x330ec, 0x3312c, + 0x33134, 0x33150, + 0x33158, 0x33158, + 0x33160, 0x3318c, + 0x3319c, 0x331ac, + 0x331c0, 0x331c0, + 0x331c8, 0x331d0, + 0x331d8, 0x331e0, + 0x331ec, 0x33290, + 0x33298, 0x332c4, + 0x332e4, 0x33390, + 0x33398, 0x333c4, + 0x333e4, 0x3342c, + 0x33434, 0x33450, + 0x33458, 0x33458, + 0x33460, 0x3348c, + 0x3349c, 0x334ac, + 0x334c0, 0x334c0, + 0x334c8, 0x334d0, + 0x334d8, 0x334e0, + 0x334ec, 0x3352c, + 0x33534, 0x33550, + 0x33558, 0x33558, + 0x33560, 0x3358c, + 0x3359c, 0x335ac, + 0x335c0, 0x335c0, + 0x335c8, 0x335d0, + 0x335d8, 0x335e0, + 0x335ec, 0x33690, + 0x33698, 0x336c4, + 0x336e4, 0x33790, + 0x33798, 0x337c4, + 0x337e4, 0x337fc, + 0x33814, 0x33814, + 0x33854, 0x33868, + 0x33880, 0x3388c, + 0x338c0, 0x338d0, + 0x338e8, 0x338ec, + 0x33900, 0x3392c, + 0x33934, 0x33950, + 0x33958, 0x33958, + 0x33960, 0x3398c, + 0x3399c, 0x339ac, + 0x339c0, 0x339c0, + 0x339c8, 0x339d0, + 0x339d8, 0x339e0, + 0x339ec, 0x33a90, + 0x33a98, 0x33ac4, + 0x33ae4, 0x33b10, + 0x33b24, 0x33b28, + 0x33b38, 0x33b50, + 0x33bf0, 0x33c10, + 0x33c24, 0x33c28, + 0x33c38, 0x33c50, + 0x33cf0, 0x33cfc, + 0x34000, 0x34030, + 0x34100, 0x34168, + 0x34190, 0x341a0, + 0x341a8, 0x341b8, + 0x341c4, 0x341c8, + 0x341d0, 0x341d0, + 0x34200, 0x34320, + 0x34400, 0x344b4, + 0x344c0, 0x3452c, + 0x34540, 0x3461c, + 0x34800, 0x348a0, + 0x348c0, 0x34908, + 0x34910, 0x349b8, + 0x34a00, 0x34a04, + 0x34a0c, 0x34a14, + 0x34a1c, 0x34a2c, + 0x34a44, 0x34a50, + 0x34a74, 0x34a74, + 0x34a7c, 0x34afc, + 0x34b08, 0x34c24, + 0x34d00, 0x34d14, + 0x34d1c, 0x34d3c, + 0x34d44, 0x34d4c, + 0x34d54, 0x34d74, + 0x34d7c, 0x34d7c, + 0x34de0, 0x34de0, + 0x34e00, 0x34ed4, + 0x34f00, 0x34fa4, + 0x34fc0, 0x34fc4, + 0x35000, 0x35004, + 0x35080, 0x350fc, + 0x35208, 0x35220, + 0x3523c, 0x35254, + 0x35300, 0x35300, + 0x35308, 0x3531c, + 0x35338, 0x3533c, + 0x35380, 0x35380, + 0x35388, 0x353a8, + 0x353b4, 0x353b4, + 0x35400, 0x35420, + 0x35438, 0x3543c, + 0x35480, 0x35480, + 0x354a8, 0x354a8, + 0x354b0, 0x354b4, + 0x354c8, 0x354d4, + 0x35a40, 0x35a4c, + 0x35af0, 0x35b20, + 0x35b38, 0x35b3c, + 0x35b80, 0x35b80, + 0x35ba8, 0x35ba8, + 0x35bb0, 0x35bb4, + 0x35bc8, 0x35bd4, + 0x36140, 0x3618c, + 0x361f0, 0x361f4, + 0x36200, 0x36200, + 0x36218, 0x36218, + 0x36400, 0x36400, + 0x36408, 0x3641c, + 0x36618, 0x36620, + 0x36664, 0x36664, + 0x366a8, 0x366a8, + 0x366ec, 0x366ec, + 0x36a00, 0x36abc, + 0x36b00, 0x36b38, + 0x36b20, 0x36b38, + 0x36b40, 0x36b58, + 0x36b60, 0x36b78, + 0x36c00, 0x36c00, + 0x36c08, 0x36c3c, + 0x37000, 0x3702c, + 0x37034, 0x37050, + 0x37058, 0x37058, + 0x37060, 0x3708c, + 0x3709c, 0x370ac, + 0x370c0, 0x370c0, + 0x370c8, 0x370d0, + 0x370d8, 0x370e0, + 0x370ec, 0x3712c, + 0x37134, 0x37150, + 0x37158, 0x37158, + 0x37160, 0x3718c, + 0x3719c, 0x371ac, + 0x371c0, 0x371c0, + 0x371c8, 0x371d0, + 0x371d8, 0x371e0, + 0x371ec, 0x37290, + 0x37298, 0x372c4, + 0x372e4, 0x37390, + 0x37398, 0x373c4, + 0x373e4, 0x3742c, + 0x37434, 0x37450, + 0x37458, 0x37458, + 0x37460, 0x3748c, + 0x3749c, 0x374ac, + 0x374c0, 0x374c0, + 0x374c8, 0x374d0, + 0x374d8, 0x374e0, + 0x374ec, 0x3752c, + 0x37534, 0x37550, + 0x37558, 0x37558, + 0x37560, 0x3758c, + 0x3759c, 0x375ac, + 0x375c0, 0x375c0, + 0x375c8, 0x375d0, + 0x375d8, 0x375e0, + 0x375ec, 0x37690, + 0x37698, 0x376c4, + 0x376e4, 0x37790, + 0x37798, 0x377c4, + 0x377e4, 0x377fc, + 0x37814, 0x37814, + 0x37854, 0x37868, + 0x37880, 0x3788c, + 0x378c0, 0x378d0, + 0x378e8, 0x378ec, + 0x37900, 0x3792c, + 0x37934, 0x37950, + 0x37958, 0x37958, + 0x37960, 0x3798c, + 0x3799c, 0x379ac, + 0x379c0, 0x379c0, + 0x379c8, 0x379d0, + 0x379d8, 0x379e0, + 0x379ec, 0x37a90, + 0x37a98, 0x37ac4, + 0x37ae4, 0x37b10, + 0x37b24, 0x37b28, + 0x37b38, 0x37b50, + 0x37bf0, 0x37c10, + 0x37c24, 0x37c28, + 0x37c38, 0x37c50, + 0x37cf0, 0x37cfc, + 0x40040, 0x40040, + 0x40080, 0x40084, + 0x40100, 0x40100, + 0x40140, 0x401bc, + 0x40200, 0x40214, + 0x40228, 0x40228, + 0x40240, 0x40258, + 0x40280, 0x40280, + 0x40304, 0x40304, + 0x40330, 0x4033c, + 0x41304, 0x413c8, + 0x413d0, 0x413dc, + 0x413f0, 0x413f0, + 0x41400, 0x4140c, + 0x41414, 0x4141c, + 0x41480, 0x414d0, + 0x44000, 0x4407c, + 0x440c0, 0x441ac, + 0x441b4, 0x4427c, + 0x442c0, 0x443ac, + 0x443b4, 0x4447c, + 0x444c0, 0x445ac, + 0x445b4, 0x4467c, + 0x446c0, 0x447ac, + 0x447b4, 0x4487c, + 0x448c0, 0x449ac, + 0x449b4, 0x44a7c, + 0x44ac0, 0x44bac, + 0x44bb4, 0x44c7c, + 0x44cc0, 0x44dac, + 0x44db4, 0x44e7c, + 0x44ec0, 0x44fac, + 0x44fb4, 0x4507c, + 0x450c0, 0x451ac, + 0x451b4, 0x451fc, + 0x45800, 0x45804, + 0x45810, 0x45830, + 0x45840, 0x45860, + 0x45868, 0x45868, + 0x45880, 0x45884, + 0x458a0, 0x458b0, + 0x45a00, 0x45a04, + 0x45a10, 0x45a30, + 0x45a40, 0x45a60, + 0x45a68, 0x45a68, + 0x45a80, 0x45a84, + 0x45aa0, 0x45ab0, + 0x460c0, 0x460e4, + 0x47000, 0x4703c, + 0x47044, 0x4708c, + 0x47200, 0x47250, + 0x47400, 0x47408, + 0x47414, 0x47420, + 0x47600, 0x47618, + 0x47800, 0x47814, + 0x47820, 0x4782c, + 0x50000, 0x50084, + 0x50090, 0x500cc, + 0x50300, 0x50384, + 0x50400, 0x50400, + 0x50800, 0x50884, + 0x50890, 0x508cc, + 0x50b00, 0x50b84, + 0x50c00, 0x50c00, + 0x51000, 0x51020, + 0x51028, 0x510b0, + 0x51300, 0x51324, + }; + u32 *buf_end = (u32 *)((char *)buf + buf_size); const unsigned int *reg_ranges; int reg_ranges_size, range; @@ -1393,6 +1955,11 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size) reg_ranges_size = ARRAY_SIZE(t5_reg_ranges); break; + case CHELSIO_T6: + reg_ranges = t6_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t6_reg_ranges); + break; + default: dev_err(adap, "Unsupported chip version %d\n", chip_version); @@ -1929,35 +2496,180 @@ int t4_read_flash(struct adapter *adapter, unsigned int addr, } /** + * t4_get_exprom_version - return the Expansion ROM version (if any) + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the Expansion ROM header from FLASH and returns the version + * number (if present) through the @vers return value pointer. We return + * this in the Firmware Version Format since it's convenient. Return + * 0 on success, -ENOENT if no Expansion ROM is present. + */ +static int t4_get_exprom_version(struct adapter *adapter, u32 *vers) +{ + struct exprom_header { + unsigned char hdr_arr[16]; /* must start with 0x55aa */ + unsigned char hdr_ver[4]; /* Expansion ROM version */ + } *hdr; + u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header), + sizeof(u32))]; + int ret; + + ret = t4_read_flash(adapter, FLASH_EXP_ROM_START, + ARRAY_SIZE(exprom_header_buf), + exprom_header_buf, 0); + if (ret) + return ret; + + hdr = (struct exprom_header *)exprom_header_buf; + if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa) + return -ENOENT; + + *vers = (V_FW_HDR_FW_VER_MAJOR(hdr->hdr_ver[0]) | + V_FW_HDR_FW_VER_MINOR(hdr->hdr_ver[1]) | + V_FW_HDR_FW_VER_MICRO(hdr->hdr_ver[2]) | + V_FW_HDR_FW_VER_BUILD(hdr->hdr_ver[3])); + return 0; +} + +/** * t4_get_fw_version - read the firmware version * @adapter: the adapter * @vers: where to place the version * * Reads the FW version from flash. */ -int t4_get_fw_version(struct adapter *adapter, u32 *vers) +static int t4_get_fw_version(struct adapter *adapter, u32 *vers) { return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr, fw_ver), 1, vers, 0); } /** + * t4_get_bs_version - read the firmware bootstrap version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the FW Bootstrap version from flash. + */ +static int t4_get_bs_version(struct adapter *adapter, u32 *vers) +{ + return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START + + offsetof(struct fw_hdr, fw_ver), 1, + vers, 0); +} + +/** * t4_get_tp_version - read the TP microcode version * @adapter: the adapter * @vers: where to place the version * * Reads the TP microcode version from flash. */ -int t4_get_tp_version(struct adapter *adapter, u32 *vers) +static int t4_get_tp_version(struct adapter *adapter, u32 *vers) { return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr, tp_microcode_ver), 1, vers, 0); } -#define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\ - FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_40G | \ - FW_PORT_CAP_SPEED_100G | FW_PORT_CAP_ANEG) +/** + * t4_get_version_info - extract various chip/firmware version information + * @adapter: the adapter + * + * Reads various chip/firmware version numbers and stores them into the + * adapter Adapter Parameters structure. If any of the efforts fails + * the first failure will be returned, but all of the version numbers + * will be read. + */ +int t4_get_version_info(struct adapter *adapter) +{ + int ret = 0; + +#define FIRST_RET(__getvinfo) \ + do { \ + int __ret = __getvinfo; \ + if (__ret && !ret) \ + ret = __ret; \ + } while (0) + + FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers)); + FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers)); + FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers)); + FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers)); + +#undef FIRST_RET + + return ret; +} + +/** + * t4_dump_version_info - dump all of the adapter configuration IDs + * @adapter: the adapter + * + * Dumps all of the various bits of adapter configuration version/revision + * IDs information. This is typically called at some point after + * t4_get_version_info() has been called. + */ +void t4_dump_version_info(struct adapter *adapter) +{ + /** + * Device information. + */ + dev_info(adapter, "Chelsio rev %d\n", + CHELSIO_CHIP_RELEASE(adapter->params.chip)); + + /** + * Firmware Version. + */ + if (!adapter->params.fw_vers) + dev_warn(adapter, "No firmware loaded\n"); + else + dev_info(adapter, "Firmware version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.fw_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.fw_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.fw_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.fw_vers)); + + /** + * Bootstrap Firmware Version. + */ + if (!adapter->params.bs_vers) + dev_warn(adapter, "No bootstrap loaded\n"); + else + dev_info(adapter, "Bootstrap version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.bs_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.bs_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.bs_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.bs_vers)); + + /** + * TP Microcode Version. + */ + if (!adapter->params.tp_vers) + dev_warn(adapter, "No TP Microcode loaded\n"); + else + dev_info(adapter, "TP Microcode version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.tp_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.tp_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.tp_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.tp_vers)); + + /** + * Expansion ROM version. + */ + if (!adapter->params.er_vers) + dev_info(adapter, "No Expansion ROM loaded\n"); + else + dev_info(adapter, "Expansion ROM version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.er_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.er_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.er_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.er_vers)); +} + +#define ADVERT_MASK (V_FW_PORT_CAP_SPEED(M_FW_PORT_CAP_SPEED) | \ + FW_PORT_CAP_ANEG) /** * t4_link_l1cfg - apply link configuration to MAC/PHY @@ -1976,14 +2688,24 @@ int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port, struct link_config *lc) { struct fw_port_cmd c; - unsigned int fc = 0, mdi = V_FW_PORT_CAP_MDI(FW_PORT_CAP_MDI_AUTO); + unsigned int mdi = V_FW_PORT_CAP_MDI(FW_PORT_CAP_MDI_AUTO); + unsigned int fc, fec; lc->link_ok = 0; + fc = 0; if (lc->requested_fc & PAUSE_RX) fc |= FW_PORT_CAP_FC_RX; if (lc->requested_fc & PAUSE_TX) fc |= FW_PORT_CAP_FC_TX; + fec = 0; + if (lc->requested_fec & FEC_RS) + fec |= FW_PORT_CAP_FEC_RS; + if (lc->requested_fec & FEC_BASER_RS) + fec |= FW_PORT_CAP_FEC_BASER_RS; + if (lc->requested_fec & FEC_RESERVED) + fec |= FW_PORT_CAP_FEC_RESERVED; + memset(&c, 0, sizeof(c)); c.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | @@ -1994,13 +2716,16 @@ int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port, if (!(lc->supported & FW_PORT_CAP_ANEG)) { c.u.l1cfg.rcap = cpu_to_be32((lc->supported & ADVERT_MASK) | - fc); - lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + fc | fec); + lc->fc = lc->requested_fc & ~PAUSE_AUTONEG; + lc->fec = lc->requested_fec; } else if (lc->autoneg == AUTONEG_DISABLE) { - c.u.l1cfg.rcap = cpu_to_be32(lc->requested_speed | fc | mdi); - lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + c.u.l1cfg.rcap = cpu_to_be32(lc->requested_speed | fc | + fec | mdi); + lc->fc = lc->requested_fc & ~PAUSE_AUTONEG; + lc->fec = lc->requested_fec; } else { - c.u.l1cfg.rcap = cpu_to_be32(lc->advertising | fc | mdi); + c.u.l1cfg.rcap = cpu_to_be32(lc->advertising | fc | fec | mdi); } return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); @@ -2044,7 +2769,9 @@ int t4_flash_cfg_addr(struct adapter *adapter) void t4_intr_enable(struct adapter *adapter) { u32 val = 0; - u32 pf = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI)); + u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI); + u32 pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami); if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) val = F_ERR_DROPPED_DB | F_ERR_EGR_CTXT_PRIO | F_DBFIFO_HP_INT; @@ -2069,7 +2796,9 @@ void t4_intr_enable(struct adapter *adapter) */ void t4_intr_disable(struct adapter *adapter) { - u32 pf = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI)); + u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI); + u32 pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami); t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), 0); t4_set_reg_field(adapter, A_PL_INT_MAP0, 1 << pf, 0); @@ -2098,6 +2827,12 @@ const char *t4_get_port_type_description(enum fw_port_type port_type) "QSA", "QSFP", "BP40_BA", + "KR4_100G", + "CR4_QSFP", + "CR_QSFP", + "CR2_QSFP", + "SFP28", + "KR_SFP28", }; if (port_type < ARRAY_SIZE(port_type_description)) @@ -2108,21 +2843,90 @@ const char *t4_get_port_type_description(enum fw_port_type port_type) /** * t4_get_mps_bg_map - return the buffer groups associated with a port * @adap: the adapter - * @idx: the port index + * @pidx: the port index * * Returns a bitmap indicating which MPS buffer groups are associated * with the given port. Bit i is set if buffer group i is used by the * port. */ -unsigned int t4_get_mps_bg_map(struct adapter *adap, int idx) +unsigned int t4_get_mps_bg_map(struct adapter *adap, unsigned int pidx) { - u32 n = G_NUMPORTS(t4_read_reg(adap, A_MPS_CMN_CTL)); + unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip); + unsigned int nports = 1 << G_NUMPORTS(t4_read_reg(adap, + A_MPS_CMN_CTL)); + + if (pidx >= nports) { + dev_warn(adap, "MPS Port Index %d >= Nports %d\n", + pidx, nports); + return 0; + } + + switch (chip_version) { + case CHELSIO_T4: + case CHELSIO_T5: + switch (nports) { + case 1: return 0xf; + case 2: return 3 << (2 * pidx); + case 4: return 1 << pidx; + } + break; + + case CHELSIO_T6: + switch (nports) { + case 2: return 1 << (2 * pidx); + } + break; + } - if (n == 0) - return idx == 0 ? 0xf : 0; - if (n == 1) - return idx < 2 ? (3 << (2 * idx)) : 0; - return 1 << idx; + dev_err(adap, "Need MPS Buffer Group Map for Chip %0x, Nports %d\n", + chip_version, nports); + return 0; +} + +/** + * t4_get_tp_ch_map - return TP ingress channels associated with a port + * @adapter: the adapter + * @pidx: the port index + * + * Returns a bitmap indicating which TP Ingress Channels are associated with + * a given Port. Bit i is set if TP Ingress Channel i is used by the Port. + */ +unsigned int t4_get_tp_ch_map(struct adapter *adapter, unsigned int pidx) +{ + unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip); + unsigned int nports = 1 << G_NUMPORTS(t4_read_reg(adapter, + A_MPS_CMN_CTL)); + + if (pidx >= nports) { + dev_warn(adap, "TP Port Index %d >= Nports %d\n", + pidx, nports); + return 0; + } + + switch (chip_version) { + case CHELSIO_T4: + case CHELSIO_T5: + /* Note that this happens to be the same values as the MPS + * Buffer Group Map for these Chips. But we replicate the code + * here because they're really separate concepts. + */ + switch (nports) { + case 1: return 0xf; + case 2: return 3 << (2 * pidx); + case 4: return 1 << pidx; + } + break; + + case CHELSIO_T6: + switch (nports) { + case 2: return 1 << pidx; + } + break; + } + + dev_err(adapter, "Need TP Channel Map for Chip %0x, Nports %d\n", + chip_version, nports); + return 0; } /** @@ -2573,6 +3377,49 @@ int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset) } /** + * t4_fl_pkt_align - return the fl packet alignment + * @adap: the adapter + * + * T4 has a single field to specify the packing and padding boundary. + * T5 onwards has separate fields for this and hence the alignment for + * next packet offset is maximum of these two. + */ +int t4_fl_pkt_align(struct adapter *adap) +{ + u32 sge_control, sge_control2; + unsigned int ingpadboundary, ingpackboundary, fl_align, ingpad_shift; + + sge_control = t4_read_reg(adap, A_SGE_CONTROL); + + /* T4 uses a single control field to specify both the PCIe Padding and + * Packing Boundary. T5 introduced the ability to specify these + * separately. The actual Ingress Packet Data alignment boundary + * within Packed Buffer Mode is the maximum of these two + * specifications. + */ + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + ingpad_shift = X_INGPADBOUNDARY_SHIFT; + else + ingpad_shift = X_T6_INGPADBOUNDARY_SHIFT; + + ingpadboundary = 1 << (G_INGPADBOUNDARY(sge_control) + ingpad_shift); + + fl_align = ingpadboundary; + if (!is_t4(adap->params.chip)) { + sge_control2 = t4_read_reg(adap, A_SGE_CONTROL2); + ingpackboundary = G_INGPACKBOUNDARY(sge_control2); + if (ingpackboundary == X_INGPACKBOUNDARY_16B) + ingpackboundary = 16; + else + ingpackboundary = 1 << (ingpackboundary + + X_INGPACKBOUNDARY_SHIFT); + + fl_align = max(ingpadboundary, ingpackboundary); + } + return fl_align; +} + +/** * t4_fixup_host_params_compat - fix up host-dependent parameters * @adap: the adapter * @page_size: the host's Base Page Size @@ -2617,6 +3464,10 @@ int t4_fixup_host_params_compat(struct adapter *adap, X_INGPADBOUNDARY_SHIFT) | V_EGRSTATUSPAGESIZE(stat_len != 64)); else { + unsigned int pack_align; + unsigned int ingpad, ingpack; + unsigned int pcie_cap; + /* * T5 introduced the separation of the Free List Padding and * Packing Boundaries. Thus, we can select a smaller Padding @@ -2630,11 +3481,33 @@ int t4_fixup_host_params_compat(struct adapter *adap, * Size (the minimum unit of transfer to/from Memory). If we * have a Padding Boundary which is smaller than the Memory * Line Size, that'll involve a Read-Modify-Write cycle on the - * Memory Controller which is never good. For T5 the smallest - * Padding Boundary which we can select is 32 bytes which is - * larger than any known Memory Controller Line Size so we'll - * use that. + * Memory Controller which is never good. + */ + + /* We want the Packing Boundary to be based on the Cache Line + * Size in order to help avoid False Sharing performance + * issues between CPUs, etc. We also want the Packing + * Boundary to incorporate the PCI-E Maximum Payload Size. We + * get best performance when the Packing Boundary is a + * multiple of the Maximum Payload Size. */ + pack_align = fl_align; + pcie_cap = t4_os_find_pci_capability(adap, PCI_CAP_ID_EXP); + if (pcie_cap) { + unsigned int mps, mps_log; + u16 devctl; + + /* The PCIe Device Control Maximum Payload Size field + * [bits 7:5] encodes sizes as powers of 2 starting at + * 128 bytes. + */ + t4_os_pci_read_cfg2(adap, pcie_cap + PCI_EXP_DEVCTL, + &devctl); + mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7; + mps = 1 << mps_log; + if (mps > pack_align) + pack_align = mps; + } /* * N.B. T5 has a different interpretation of the "0" value for @@ -2644,19 +3517,36 @@ int t4_fixup_host_params_compat(struct adapter *adap, * on the other hand, if we wanted 32 bytes, the best we can * really do is 64 bytes ... */ - if (fl_align <= 32) { + if (pack_align <= 16) { + ingpack = X_INGPACKBOUNDARY_16B; + fl_align = 16; + } else if (pack_align == 32) { + ingpack = X_INGPACKBOUNDARY_64B; fl_align = 64; - fl_align_log = 6; + } else { + unsigned int pack_align_log = cxgbe_fls(pack_align) - 1; + + ingpack = pack_align_log - X_INGPACKBOUNDARY_SHIFT; + fl_align = pack_align; } + + /* Use the smallest Ingress Padding which isn't smaller than + * the Memory Controller Read/Write Size. We'll take that as + * being 8 bytes since we don't know of any system with a + * wider Memory Controller Bus Width. + */ + if (is_t5(adap->params.chip)) + ingpad = X_INGPADBOUNDARY_32B; + else + ingpad = X_T6_INGPADBOUNDARY_8B; t4_set_reg_field(adap, A_SGE_CONTROL, V_INGPADBOUNDARY(M_INGPADBOUNDARY) | F_EGRSTATUSPAGESIZE, - V_INGPADBOUNDARY(X_INGPCIEBOUNDARY_32B) | + V_INGPADBOUNDARY(ingpad) | V_EGRSTATUSPAGESIZE(stat_len != 64)); t4_set_reg_field(adap, A_SGE_CONTROL2, V_INGPACKBOUNDARY(M_INGPACKBOUNDARY), - V_INGPACKBOUNDARY(fl_align_log - - X_INGPACKBOUNDARY_SHIFT)); + V_INGPACKBOUNDARY(ingpack)); } /* @@ -3194,7 +4084,7 @@ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl) if (opcode == FW_PORT_CMD && action == FW_PORT_ACTION_GET_PORT_INFO) { /* link/module state change message */ - int speed = 0, fc = 0, i; + unsigned int speed = 0, fc = 0, i; int chan = G_FW_PORT_CMD_PORTID(be32_to_cpu(p->op_to_portid)); struct port_info *pi = NULL; struct link_config *lc; @@ -3212,8 +4102,12 @@ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl) speed = ETH_SPEED_NUM_1G; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_10G)) speed = ETH_SPEED_NUM_10G; + else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_25G)) + speed = ETH_SPEED_NUM_25G; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_40G)) speed = ETH_SPEED_NUM_40G; + else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_100G)) + speed = ETH_SPEED_NUM_100G; for_each_port(adap, i) { pi = adap2pinfo(adap, i); @@ -3271,19 +4165,37 @@ void t4_reset_link_config(struct adapter *adap, int idx) /** * init_link_config - initialize a link's SW state * @lc: structure holding the link state - * @caps: link capabilities + * @pcaps: link Port Capabilities + * @acaps: link current Advertised Port Capabilities * * Initializes the SW state maintained for each link, including the link's * capabilities and default speed/flow-control/autonegotiation settings. */ -static void init_link_config(struct link_config *lc, - unsigned int caps) +static void init_link_config(struct link_config *lc, unsigned int pcaps, + unsigned int acaps) { - lc->supported = caps; + unsigned int fec; + + lc->supported = pcaps; lc->requested_speed = 0; lc->speed = 0; lc->requested_fc = 0; lc->fc = 0; + + /** + * For Forward Error Control, we default to whatever the Firmware + * tells us the Link is currently advertising. + */ + fec = 0; + if (acaps & FW_PORT_CAP_FEC_RS) + fec |= FEC_RS; + if (acaps & FW_PORT_CAP_FEC_BASER_RS) + fec |= FEC_BASER_RS; + if (acaps & FW_PORT_CAP_FEC_RESERVED) + fec |= FEC_RESERVED; + lc->requested_fec = fec; + lc->fec = fec; + if (lc->supported & FW_PORT_CAP_ANEG) { lc->advertising = lc->supported & ADVERT_MASK; lc->autoneg = AUTONEG_ENABLE; @@ -3312,8 +4224,12 @@ static int t4_wait_dev_ready(struct adapter *adapter) msleep(500); whoami = t4_read_reg(adapter, A_PL_WHOAMI); - return (whoami != 0xffffffff && whoami != X_CIM_PF_NOACCESS - ? 0 : -EIO); + if (whoami != 0xffffffff && whoami != X_CIM_PF_NOACCESS) + return 0; + + dev_err(adapter, "Device didn't become ready for access, whoami = %#x\n", + whoami); + return -EIO; } struct flash_desc { @@ -3329,47 +4245,96 @@ int t4_get_flash_params(struct adapter *adapter) * sectors. */ static struct flash_desc supported_flash[] = { - { 0x150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ + { 0x00150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ }; int ret; - unsigned int i; - u32 info = 0; - + u32 flashid = 0; + unsigned int part, manufacturer; + unsigned int density, size; + + /** + * Issue a Read ID Command to the Flash part. We decode supported + * Flash parts and their sizes from this. There's a newer Query + * Command which can retrieve detailed geometry information but + * many Flash parts don't support it. + */ ret = sf1_write(adapter, 1, 1, 0, SF_RD_ID); if (!ret) - ret = sf1_read(adapter, 3, 0, 1, &info); + ret = sf1_read(adapter, 3, 0, 1, &flashid); t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */ if (ret < 0) return ret; - for (i = 0; i < ARRAY_SIZE(supported_flash); ++i) - if (supported_flash[i].vendor_and_model_id == info) { - adapter->params.sf_size = supported_flash[i].size_mb; + for (part = 0; part < ARRAY_SIZE(supported_flash); part++) { + if (supported_flash[part].vendor_and_model_id == flashid) { + adapter->params.sf_size = + supported_flash[part].size_mb; adapter->params.sf_nsec = adapter->params.sf_size / SF_SEC_SIZE; - return 0; + goto found; } + } - if ((info & 0xff) != 0x20) /* not a Numonix flash */ - return -EINVAL; - info >>= 16; /* log2 of size */ - if (info >= 0x14 && info < 0x18) - adapter->params.sf_nsec = 1 << (info - 16); - else if (info == 0x18) - adapter->params.sf_nsec = 64; - else + manufacturer = flashid & 0xff; + switch (manufacturer) { + case 0x20: { /* Micron/Numonix */ + /** + * This Density -> Size decoding table is taken from Micron + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x14: + size = 1 << 20; /* 1MB */ + break; + case 0x15: + size = 1 << 21; /* 2MB */ + break; + case 0x16: + size = 1 << 22; /* 4MB */ + break; + case 0x17: + size = 1 << 23; /* 8MB */ + break; + case 0x18: + size = 1 << 24; /* 16MB */ + break; + case 0x19: + size = 1 << 25; /* 32MB */ + break; + case 0x20: + size = 1 << 26; /* 64MB */ + break; + case 0x21: + size = 1 << 27; /* 128MB */ + break; + case 0x22: + size = 1 << 28; /* 256MB */ + break; + default: + dev_err(adapter, "Micron Flash Part has bad size, ID = %#x, Density code = %#x\n", + flashid, density); + return -EINVAL; + } + + adapter->params.sf_size = size; + adapter->params.sf_nsec = size / SF_SEC_SIZE; + break; + } + default: + dev_err(adapter, "Unsupported Flash Part, ID = %#x\n", flashid); return -EINVAL; - adapter->params.sf_size = 1 << info; + } +found: /* * We should reject adapters with FLASHes which are too small. So, emit * a warning. */ - if (adapter->params.sf_size < FLASH_MIN_SIZE) { - dev_warn(adapter, "WARNING!!! FLASH size %#x < %#x!!!\n", - adapter->params.sf_size, FLASH_MIN_SIZE); - } + if (adapter->params.sf_size < FLASH_MIN_SIZE) + dev_warn(adapter, "WARNING: Flash Part ID %#x, size %#x < %#x\n", + flashid, adapter->params.sf_size, FLASH_MIN_SIZE); return 0; } @@ -3390,6 +4355,33 @@ static void set_pcie_completion_timeout(struct adapter *adapter, } /** + * t4_get_chip_type - Determine chip type from device ID + * @adap: the adapter + * @ver: adapter version + */ +int t4_get_chip_type(struct adapter *adap, int ver) +{ + enum chip_type chip = 0; + u32 pl_rev = G_REV(t4_read_reg(adap, A_PL_REV)); + + /* Retrieve adapter's device ID */ + switch (ver) { + case CHELSIO_T5: + chip |= CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); + break; + case CHELSIO_T6: + chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); + break; + default: + dev_err(adap, "Device %d is not supported\n", + adap->params.pci.device_id); + return -EINVAL; + } + + return chip; +} + +/** * t4_prep_adapter - prepare SW and HW for operation * @adapter: the adapter * @@ -3426,6 +4418,15 @@ int t4_prep_adapter(struct adapter *adapter) adapter->params.arch.nchan = NCHAN; adapter->params.arch.vfcount = 128; break; + case CHELSIO_T6: + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); + adapter->params.arch.sge_fl_db = 0; + adapter->params.arch.mps_tcam_size = + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + adapter->params.arch.mps_rplc_size = 256; + adapter->params.arch.nchan = 2; + adapter->params.arch.vfcount = 256; + break; default: dev_err(adapter, "%s: Device %d is not supported\n", __func__, adapter->params.pci.device_id); @@ -3436,8 +4437,11 @@ int t4_prep_adapter(struct adapter *adapter) t4_os_find_pci_capability(adapter, PCI_CAP_ID_VPD); ret = t4_get_flash_params(adapter); - if (ret < 0) + if (ret < 0) { + dev_err(adapter, "Unable to retrieve Flash Parameters, ret = %d\n", + -ret); return ret; + } adapter->params.cim_la_size = CIMLA_SIZE; @@ -3609,6 +4613,14 @@ int t4_init_tp_params(struct adapter *adap) &adap->params.tp.ingress_config, 1, A_TP_INGRESS_CONFIG); + /* For T6, cache the adapter's compressed error vector + * and passing outer header info for encapsulated packets. + */ + if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { + v = t4_read_reg(adap, A_TP_OUT_CONFIG); + adap->params.tp.rx_pkt_encap = (v & F_CRXPKTENC) ? 1 : 0; + } + /* * Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field * shift positions of several elements of the Compressed Filter Tuple @@ -3747,7 +4759,8 @@ int t4_port_init(struct adapter *adap, int mbox, int pf, int vf) p->port_type = G_FW_PORT_CMD_PTYPE(ret); p->mod_type = FW_PORT_MOD_TYPE_NA; - init_link_config(&p->link_cfg, be16_to_cpu(c.u.info.pcap)); + init_link_config(&p->link_cfg, be16_to_cpu(c.u.info.pcap), + be16_to_cpu(c.u.info.acap)); j++; } return 0; diff --git a/drivers/net/cxgbe/base/t4_hw.h b/drivers/net/cxgbe/base/t4_hw.h index 5e62c417..07498841 100644 --- a/drivers/net/cxgbe/base/t4_hw.h +++ b/drivers/net/cxgbe/base/t4_hw.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -124,6 +124,14 @@ struct rsp_ctrl { enum { /* + * Various Expansion-ROM boot images, etc. + */ + FLASH_EXP_ROM_START_SEC = 0, + FLASH_EXP_ROM_NSECS = 6, + FLASH_EXP_ROM_START = FLASH_START(FLASH_EXP_ROM_START_SEC), + FLASH_EXP_ROM_MAX_SIZE = FLASH_MAX_SIZE(FLASH_EXP_ROM_NSECS), + + /* * Location of firmware image in FLASH. */ FLASH_FW_START_SEC = 8, @@ -132,6 +140,14 @@ enum { FLASH_FW_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FW_NSECS), /* + * Location of bootstrap firmware image in FLASH. + */ + FLASH_FWBOOTSTRAP_START_SEC = 27, + FLASH_FWBOOTSTRAP_NSECS = 1, + FLASH_FWBOOTSTRAP_START = FLASH_START(FLASH_FWBOOTSTRAP_START_SEC), + FLASH_FWBOOTSTRAP_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FWBOOTSTRAP_NSECS), + + /* * Location of Firmware Configuration File in FLASH. */ FLASH_CFG_START_SEC = 31, diff --git a/drivers/net/cxgbe/base/t4_msg.h b/drivers/net/cxgbe/base/t4_msg.h index 4b04cd0d..6acd749a 100644 --- a/drivers/net/cxgbe/base/t4_msg.h +++ b/drivers/net/cxgbe/base/t4_msg.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -262,6 +262,20 @@ struct cpl_rx_pkt { #define V_RXF_IP6(x) ((x) << S_RXF_IP6) #define F_RXF_IP6 V_RXF_IP6(1U) +/* rx_pkt.err_vec fields */ +/* In T6, rx_pkt.err_vec indicates + * RxError Error vector (16b) or + * Encapsulating header length (8b), + * Outer encapsulation type (2b) and + * compressed error vector (6b) if CRxPktEnc is + * enabled in TP_OUT_CONFIG + */ +#define S_T6_COMPR_RXERR_VEC 0 +#define M_T6_COMPR_RXERR_VEC 0x3F +#define V_T6_COMPR_RXERR_VEC(x) ((x) << S_T6_COMPR_RXERR_VEC) +#define G_T6_COMPR_RXERR_VEC(x) \ + (((x) >> S_T6_COMPR_RXERR_VEC) & M_T6_COMPR_RXERR_VEC) + /* cpl_fw*.type values */ enum { FW_TYPE_RSSCPL = 4, diff --git a/drivers/net/cxgbe/base/t4_pci_id_tbl.h b/drivers/net/cxgbe/base/t4_pci_id_tbl.h index 110fadb0..1230e738 100644 --- a/drivers/net/cxgbe/base/t4_pci_id_tbl.h +++ b/drivers/net/cxgbe/base/t4_pci_id_tbl.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -144,6 +144,19 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN CH_PCI_ID_TABLE_FENTRY(0x5090), /* Custom T540-CR */ CH_PCI_ID_TABLE_FENTRY(0x5091), /* Custom T522-CR */ CH_PCI_ID_TABLE_FENTRY(0x5092), /* Custom T520-CR */ + + /* T6 adapter */ + CH_PCI_ID_TABLE_FENTRY(0x6001), /* T6225-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6002), /* T6225-SO-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6003), /* T6425-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6005), /* T6225-OCP */ + CH_PCI_ID_TABLE_FENTRY(0x6007), /* T62100-LP-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6008), /* T62100-SO-CR */ + CH_PCI_ID_TABLE_FENTRY(0x600d), /* T62100-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6011), /* T6225-LL-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6014), /* T61100-OCP-SO */ + CH_PCI_ID_TABLE_FENTRY(0x6080), /* Custom T6225-CR SFP28 */ + CH_PCI_ID_TABLE_FENTRY(0x6081), /* Custom T62100-CR */ CH_PCI_DEVICE_ID_TABLE_DEFINE_END; #endif /* CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN */ diff --git a/drivers/net/cxgbe/base/t4_regs.h b/drivers/net/cxgbe/base/t4_regs.h index 9c132dcd..1100e16f 100644 --- a/drivers/net/cxgbe/base/t4_regs.h +++ b/drivers/net/cxgbe/base/t4_regs.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -420,6 +420,26 @@ #define A_PCIE_FW 0x30b8 #define A_PCIE_FW_PF 0x30bc +#define A_PCIE_CFG2 0x3018 + +#define S_TOTMAXTAG 0 +#define M_TOTMAXTAG 0x3U +#define V_TOTMAXTAG(x) ((x) << S_TOTMAXTAG) + +#define S_T6_TOTMAXTAG 0 +#define M_T6_TOTMAXTAG 0x7U +#define V_T6_TOTMAXTAG(x) ((x) << S_T6_TOTMAXTAG) + +#define A_PCIE_CMD_CFG 0x5980 + +#define S_MINTAG 0 +#define M_MINTAG 0xffU +#define V_MINTAG(x) ((x) << S_MINTAG) + +#define S_T6_MINTAG 0 +#define M_T6_MINTAG 0xffU +#define V_T6_MINTAG(x) ((x) << S_T6_MINTAG) + /* registers for module CIM */ #define CIM_BASE_ADDR 0x7b00 @@ -443,6 +463,12 @@ #define F_UPCRST V_UPCRST(1U) /* registers for module TP */ +#define A_TP_OUT_CONFIG 0x7d04 + +#define S_CRXPKTENC 3 +#define V_CRXPKTENC(x) ((x) << S_CRXPKTENC) +#define F_CRXPKTENC V_CRXPKTENC(1U) + #define TP_BASE_ADDR 0x7d00 #define A_TP_TIMER_RESOLUTION 0x7d90 @@ -781,6 +807,11 @@ #define V_SOURCEPF(x) ((x) << S_SOURCEPF) #define G_SOURCEPF(x) (((x) >> S_SOURCEPF) & M_SOURCEPF) +#define S_T6_SOURCEPF 9 +#define M_T6_SOURCEPF 0x7U +#define V_T6_SOURCEPF(x) ((x) << S_T6_SOURCEPF) +#define G_T6_SOURCEPF(x) (((x) >> S_T6_SOURCEPF) & M_T6_SOURCEPF) + #define A_PL_PF_INT_ENABLE 0x3c4 #define S_PFSW 3 diff --git a/drivers/net/cxgbe/base/t4_regs_values.h b/drivers/net/cxgbe/base/t4_regs_values.h index d7d3144c..9085ff6d 100644 --- a/drivers/net/cxgbe/base/t4_regs_values.h +++ b/drivers/net/cxgbe/base/t4_regs_values.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -55,10 +55,15 @@ #define X_RXPKTCPLMODE_SPLIT 1 #define X_INGPCIEBOUNDARY_32B 0 #define X_INGPADBOUNDARY_SHIFT 5 +#define X_INGPADBOUNDARY_32B 0 + +#define X_T6_INGPADBOUNDARY_SHIFT 3 +#define X_T6_INGPADBOUNDARY_8B 0 /* CONTROL2 register */ #define X_INGPACKBOUNDARY_SHIFT 5 #define X_INGPACKBOUNDARY_16B 0 +#define X_INGPACKBOUNDARY_64B 1 /* GTS register */ #define X_TIMERREG_RESTART_COUNTER 6 @@ -77,7 +82,7 @@ /* * Ingress Context field values */ -#define X_UPDATEDELIVERY_INTERRUPT 1 +#define X_UPDATEDELIVERY_STATUS_PAGE 2 #define X_RSPD_TYPE_FLBUF 0 #define X_RSPD_TYPE_CPL 1 diff --git a/drivers/net/cxgbe/base/t4fw_interface.h b/drivers/net/cxgbe/base/t4fw_interface.h index 74f19fe7..6ca4f318 100644 --- a/drivers/net/cxgbe/base/t4fw_interface.h +++ b/drivers/net/cxgbe/base/t4fw_interface.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -84,6 +84,7 @@ enum fw_memtype { enum fw_wr_opcodes { FW_ETH_TX_PKT_WR = 0x08, FW_ETH_TX_PKTS_WR = 0x09, + FW_ETH_TX_PKTS2_WR = 0x78, }; /* @@ -591,6 +592,13 @@ struct fw_iq_cmd { #define G_FW_IQ_CMD_IQESIZE(x) \ (((x) >> S_FW_IQ_CMD_IQESIZE) & M_FW_IQ_CMD_IQESIZE) +#define S_FW_IQ_CMD_IQRO 30 +#define M_FW_IQ_CMD_IQRO 0x1 +#define V_FW_IQ_CMD_IQRO(x) ((x) << S_FW_IQ_CMD_IQRO) +#define G_FW_IQ_CMD_IQRO(x) \ + (((x) >> S_FW_IQ_CMD_IQRO) & M_FW_IQ_CMD_IQRO) +#define F_FW_IQ_CMD_IQRO V_FW_IQ_CMD_IQRO(1U) + #define S_FW_IQ_CMD_IQFLINTCONGEN 27 #define M_FW_IQ_CMD_IQFLINTCONGEN 0x1 #define V_FW_IQ_CMD_IQFLINTCONGEN(x) ((x) << S_FW_IQ_CMD_IQFLINTCONGEN) @@ -1061,7 +1069,7 @@ struct fw_vi_stats_cmd { enum fw_port_cap { FW_PORT_CAP_SPEED_100M = 0x0001, FW_PORT_CAP_SPEED_1G = 0x0002, - FW_PORT_CAP_SPEED_2_5G = 0x0004, + FW_PORT_CAP_SPEED_25G = 0x0004, FW_PORT_CAP_SPEED_10G = 0x0008, FW_PORT_CAP_SPEED_40G = 0x0010, FW_PORT_CAP_SPEED_100G = 0x0020, @@ -1070,13 +1078,19 @@ enum fw_port_cap { FW_PORT_CAP_ANEG = 0x0100, FW_PORT_CAP_MDIX = 0x0200, FW_PORT_CAP_MDIAUTO = 0x0400, - FW_PORT_CAP_FEC = 0x0800, - FW_PORT_CAP_TECHKR = 0x1000, - FW_PORT_CAP_TECHKX4 = 0x2000, + FW_PORT_CAP_FEC_RS = 0x0800, + FW_PORT_CAP_FEC_BASER_RS = 0x1000, + FW_PORT_CAP_FEC_RESERVED = 0x2000, FW_PORT_CAP_802_3_PAUSE = 0x4000, FW_PORT_CAP_802_3_ASM_DIR = 0x8000, }; +#define S_FW_PORT_CAP_SPEED 0 +#define M_FW_PORT_CAP_SPEED 0x3f +#define V_FW_PORT_CAP_SPEED(x) ((x) << S_FW_PORT_CAP_SPEED) +#define G_FW_PORT_CAP_SPEED(x) \ + (((x) >> S_FW_PORT_CAP_SPEED) & M_FW_PORT_CAP_SPEED) + enum fw_port_mdi { FW_PORT_CAP_MDI_AUTO, }; @@ -1279,7 +1293,12 @@ enum fw_port_type { FW_PORT_TYPE_QSFP = 14, /* No, 4, Yes, No, No, No, 40G */ FW_PORT_TYPE_BP40_BA = 15, /* No, 4, No, No, Yes, Yes, 40G/10G/1G, BP ANGE */ - + FW_PORT_TYPE_KR4_100G = 16, /* No, 4, 100G/40G/25G, Backplane */ + FW_PORT_TYPE_CR4_QSFP = 17, /* No, 4, 100G/40G/25G */ + FW_PORT_TYPE_CR_QSFP = 18, /* No, 1, 25G Spider cable */ + FW_PORT_TYPE_CR2_QSFP = 19, /* No, 2, 50G */ + FW_PORT_TYPE_SFP28 = 20, /* No, 1, 25G/10G/1G */ + FW_PORT_TYPE_KR_SFP28 = 21, /* No, 1, 25G/10G/1G using Backplane */ FW_PORT_TYPE_NONE = M_FW_PORT_CMD_PTYPE }; diff --git a/drivers/net/cxgbe/cxgbe.h b/drivers/net/cxgbe/cxgbe.h index 9120c439..f9891545 100644 --- a/drivers/net/cxgbe/cxgbe.h +++ b/drivers/net/cxgbe/cxgbe.h @@ -47,6 +47,7 @@ #define CXGBE_MAX_RX_PKTLEN (9000 + ETHER_HDR_LEN + ETHER_CRC_LEN) /* max pkt */ int cxgbe_probe(struct adapter *adapter); +void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps); int cxgbe_up(struct adapter *adap); int cxgbe_down(struct port_info *pi); void cxgbe_close(struct adapter *adapter); diff --git a/drivers/net/cxgbe/cxgbe_compat.h b/drivers/net/cxgbe/cxgbe_compat.h index 1551cbf5..03bba9fe 100644 --- a/drivers/net/cxgbe/cxgbe_compat.h +++ b/drivers/net/cxgbe/cxgbe_compat.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -226,6 +226,15 @@ static inline int cxgbe_fls(int x) return x ? sizeof(x) * 8 - __builtin_clz(x) : 0; } +/** + * cxgbe_ffs - find first bit set + * @x: the word to search + */ +static inline int cxgbe_ffs(int x) +{ + return x ? __builtin_ffs(x) : 0; +} + static inline unsigned long ilog2(unsigned long n) { unsigned int e = 0; diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c index 598a7440..7bca4561 100644 --- a/drivers/net/cxgbe/cxgbe_ethdev.c +++ b/drivers/net/cxgbe/cxgbe_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -58,7 +58,6 @@ #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_ethdev_pci.h> -#include <rte_atomic.h> #include <rte_malloc.h> #include <rte_random.h> #include <rte_dev.h> @@ -104,7 +103,8 @@ static uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, pkts_remain = nb_pkts - total_sent; for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) { - ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent]); + ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent], + nb_pkts); if (ret < 0) break; } @@ -148,7 +148,7 @@ static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev, .nb_align = 1, }; - device_info->pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + device_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); device_info->min_rx_bufsize = CXGBE_MIN_RX_BUFSIZE; device_info->max_rx_pktlen = CXGBE_MAX_RX_PKTLEN; @@ -174,7 +174,7 @@ static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev, device_info->rx_desc_lim = cxgbe_desc_lim; device_info->tx_desc_lim = cxgbe_desc_lim; - device_info->speed_capa = ETH_LINK_SPEED_10G | ETH_LINK_SPEED_40G; + cxgbe_get_speed_caps(pi, &device_info->speed_capa); } static void cxgbe_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) @@ -619,7 +619,7 @@ static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx, &rxq->fl, t4_ethrx_handler, - t4_get_mps_bg_map(adapter, pi->tx_chan), mp, + t4_get_tp_ch_map(adapter, pi->tx_chan), mp, queue_idx, socket_id); dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n", @@ -1010,7 +1010,7 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev) if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); snprintf(name, sizeof(name), "cxgbeadapter%d", eth_dev->data->port_id); adapter = rte_zmalloc(name, sizeof(*adapter), 0); @@ -1056,7 +1056,7 @@ static int eth_cxgbe_pci_remove(struct rte_pci_device *pci_dev) static struct rte_pci_driver rte_cxgbe_pmd = { .id_table = cxgb4_pci_tbl, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, .probe = eth_cxgbe_pci_probe, .remove = eth_cxgbe_pci_remove, }; diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c index 71c3671d..b709fe2b 100644 --- a/drivers/net/cxgbe/cxgbe_main.c +++ b/drivers/net/cxgbe/cxgbe_main.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -58,7 +58,6 @@ #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_ethdev_pci.h> -#include <rte_atomic.h> #include <rte_malloc.h> #include <rte_random.h> #include <rte_dev.h> @@ -206,9 +205,12 @@ static inline bool is_x_1g_port(const struct link_config *lc) static inline bool is_x_10g_port(const struct link_config *lc) { - return ((lc->supported & FW_PORT_CAP_SPEED_10G) != 0 || - (lc->supported & FW_PORT_CAP_SPEED_40G) != 0 || - (lc->supported & FW_PORT_CAP_SPEED_100G) != 0); + unsigned int speeds, high_speeds; + + speeds = V_FW_PORT_CAP_SPEED(G_FW_PORT_CAP_SPEED(lc->supported)); + high_speeds = speeds & ~(FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G); + + return high_speeds != 0; } inline void init_rspq(struct adapter *adap, struct sge_rspq *q, @@ -298,7 +300,7 @@ void cfg_queues(struct rte_eth_dev *eth_dev) for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { struct sge_eth_rxq *r = &s->ethrxq[i]; - init_rspq(adap, &r->rspq, 0, 0, 1024, 64); + init_rspq(adap, &r->rspq, 5, 32, 1024, 64); r->usembufs = 1; r->fl.size = (r->usembufs ? 1024 : 72); } @@ -363,6 +365,17 @@ static int init_rss(struct adapter *adap) return 0; } +/** + * Dump basic information about the adapter. + */ +static void print_adapter_info(struct adapter *adap) +{ + /** + * Hardware/Firmware/etc. Version/Revision IDs. + */ + t4_dump_version_info(adap); +} + static void print_port_info(struct adapter *adap) { int i; @@ -374,13 +387,17 @@ static void print_port_info(struct adapter *adap) char *bufp = buf; if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M) - bufp += sprintf(bufp, "100/"); + bufp += sprintf(bufp, "100M/"); if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G) - bufp += sprintf(bufp, "1000/"); + bufp += sprintf(bufp, "1G/"); if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) bufp += sprintf(bufp, "10G/"); + if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_25G) + bufp += sprintf(bufp, "25G/"); if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) bufp += sprintf(bufp, "40G/"); + if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100G) + bufp += sprintf(bufp, "100G/"); if (bufp != buf) --bufp; sprintf(bufp, "BASE-%s", @@ -396,6 +413,36 @@ static void print_port_info(struct adapter *adap) } } +static void configure_pcie_ext_tag(struct adapter *adapter) +{ + u16 v; + int pos = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP); + + if (!pos) + return; + + if (pos > 0) { + t4_os_pci_read_cfg2(adapter, pos + PCI_EXP_DEVCTL, &v); + v |= PCI_EXP_DEVCTL_EXT_TAG; + t4_os_pci_write_cfg2(adapter, pos + PCI_EXP_DEVCTL, v); + if (is_t6(adapter->params.chip)) { + t4_set_reg_field(adapter, A_PCIE_CFG2, + V_T6_TOTMAXTAG(M_T6_TOTMAXTAG), + V_T6_TOTMAXTAG(7)); + t4_set_reg_field(adapter, A_PCIE_CMD_CFG, + V_T6_MINTAG(M_T6_MINTAG), + V_T6_MINTAG(8)); + } else { + t4_set_reg_field(adapter, A_PCIE_CFG2, + V_TOTMAXTAG(M_TOTMAXTAG), + V_TOTMAXTAG(3)); + t4_set_reg_field(adapter, A_PCIE_CMD_CFG, + V_MINTAG(M_MINTAG), + V_MINTAG(8)); + } + } +} + /* * Tweak configuration based on system architecture, etc. Most of these have * defaults assigned to them by Firmware Configuration Files (if we're using @@ -427,6 +474,9 @@ static int adap_init0_tweaks(struct adapter *adapter) V_CREDITCNT(M_CREDITCNT) | M_CREDITCNTPACKING, V_CREDITCNT(3) | V_CREDITCNTPACKING(1)); + t4_set_reg_field(adapter, A_SGE_INGRESS_RX_THRESHOLD, + V_THRESHOLD_3(M_THRESHOLD_3), V_THRESHOLD_3(32U)); + t4_set_reg_field(adapter, A_SGE_CONTROL2, V_IDMAARBROUNDROBIN(1U), V_IDMAARBROUNDROBIN(1U)); @@ -575,7 +625,7 @@ static int adap_init0_config(struct adapter *adapter, int reset) /* * Return successfully and note that we're operating with parameters * not supplied by the driver, rather than from hard-wired - * initialization constants burried in the driver. + * initialization constants buried in the driver. */ dev_info(adapter, "Successfully configured using Firmware Configuration File \"%s\", version %#x, computed checksum %#x\n", @@ -641,18 +691,7 @@ static int adap_init0(struct adapter *adap) state = (enum dev_state)((unsigned)state & ~DEV_STATE_INIT); } - t4_get_fw_version(adap, &adap->params.fw_vers); - t4_get_tp_version(adap, &adap->params.tp_vers); - - dev_info(adap, "fw: %u.%u.%u.%u, TP: %u.%u.%u.%u\n", - G_FW_HDR_FW_VER_MAJOR(adap->params.fw_vers), - G_FW_HDR_FW_VER_MINOR(adap->params.fw_vers), - G_FW_HDR_FW_VER_MICRO(adap->params.fw_vers), - G_FW_HDR_FW_VER_BUILD(adap->params.fw_vers), - G_FW_HDR_FW_VER_MAJOR(adap->params.tp_vers), - G_FW_HDR_FW_VER_MINOR(adap->params.tp_vers), - G_FW_HDR_FW_VER_MICRO(adap->params.tp_vers), - G_FW_HDR_FW_VER_BUILD(adap->params.tp_vers)); + t4_get_version_info(adap); ret = t4_get_core_clock(adap, &adap->params.vpd); if (ret < 0) { @@ -662,26 +701,6 @@ static int adap_init0(struct adapter *adap) } /* - * Find out what ports are available to us. Note that we need to do - * this before calling adap_init0_no_config() since it needs nports - * and portvec ... - */ - v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | - V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC); - ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); - if (ret < 0) { - dev_err(adap, "%s: failure in t4_queury_params; error = %d\n", - __func__, ret); - goto bye; - } - - adap->params.nports = hweight32(port_vec); - adap->params.portvec = port_vec; - - dev_debug(adap, "%s: adap->params.nports = %u\n", __func__, - adap->params.nports); - - /* * If the firmware is initialized already (and we're not forcing a * master initialization), note that we're living with existing * adapter parameters. Otherwise, it's time to try initializing the @@ -705,6 +724,22 @@ static int adap_init0(struct adapter *adap) goto bye; } + /* Find out what ports are available to us. */ + v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | + V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); + if (ret < 0) { + dev_err(adap, "%s: failure in t4_query_params; error = %d\n", + __func__, ret); + goto bye; + } + + adap->params.nports = hweight32(port_vec); + adap->params.portvec = port_vec; + + dev_debug(adap, "%s: adap->params.nports = %u\n", __func__, + adap->params.nports); + /* * Give the SGE code a chance to pull in anything that it needs ... * Note that this must be called after we retrieve our VPD parameters @@ -793,6 +828,7 @@ static int adap_init0(struct adapter *adap) } t4_init_sge_params(adap); t4_init_tp_params(adap); + configure_pcie_ext_tag(adap); adap->params.drv_memwin = MEMWIN_NIC; adap->flags |= FW_OK; @@ -833,10 +869,10 @@ void t4_os_portmod_changed(const struct adapter *adap, int port_id) dev_info(adap, "Port%d: %s port module inserted\n", pi->port_id, mod_str[pi->mod_type]); else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) - dev_info(adap, "Port%d: unsupported optical port module inserted\n", + dev_info(adap, "Port%d: unsupported port module inserted\n", pi->port_id); else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) - dev_info(adap, "Port%d: unknown port module inserted, forcing TWINAX\n", + dev_info(adap, "Port%d: unknown port module inserted\n", pi->port_id); else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) dev_info(adap, "Port%d: transceiver module error\n", @@ -997,6 +1033,110 @@ void cxgbe_enable_rx_queues(struct port_info *pi) } /** + * fw_caps_to_speed_caps - translate Firmware Port Caps to Speed Caps. + * @port_type: Firmware Port Type + * @fw_caps: Firmware Port Capabilities + * @speed_caps: Device Info Speed Capabilities + * + * Translate a Firmware Port Capabilities specification to Device Info + * Speed Capabilities. + */ +static void fw_caps_to_speed_caps(enum fw_port_type port_type, + unsigned int fw_caps, + u32 *speed_caps) +{ +#define SET_SPEED(__speed_name) \ + do { \ + *speed_caps |= ETH_LINK_ ## __speed_name; \ + } while (0) + +#define FW_CAPS_TO_SPEED(__fw_name) \ + do { \ + if (fw_caps & FW_PORT_CAP_ ## __fw_name) \ + SET_SPEED(__fw_name); \ + } while (0) + + switch (port_type) { + case FW_PORT_TYPE_BT_SGMII: + case FW_PORT_TYPE_BT_XFI: + case FW_PORT_TYPE_BT_XAUI: + FW_CAPS_TO_SPEED(SPEED_100M); + FW_CAPS_TO_SPEED(SPEED_1G); + FW_CAPS_TO_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_KX4: + case FW_PORT_TYPE_KX: + case FW_PORT_TYPE_FIBER_XFI: + case FW_PORT_TYPE_FIBER_XAUI: + case FW_PORT_TYPE_SFP: + case FW_PORT_TYPE_QSFP_10G: + case FW_PORT_TYPE_QSA: + FW_CAPS_TO_SPEED(SPEED_1G); + FW_CAPS_TO_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_KR: + SET_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_BP_AP: + case FW_PORT_TYPE_BP4_AP: + SET_SPEED(SPEED_1G); + SET_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_BP40_BA: + case FW_PORT_TYPE_QSFP: + SET_SPEED(SPEED_40G); + break; + + case FW_PORT_TYPE_CR_QSFP: + case FW_PORT_TYPE_SFP28: + case FW_PORT_TYPE_KR_SFP28: + FW_CAPS_TO_SPEED(SPEED_1G); + FW_CAPS_TO_SPEED(SPEED_10G); + FW_CAPS_TO_SPEED(SPEED_25G); + break; + + case FW_PORT_TYPE_CR2_QSFP: + SET_SPEED(SPEED_50G); + break; + + case FW_PORT_TYPE_KR4_100G: + case FW_PORT_TYPE_CR4_QSFP: + FW_CAPS_TO_SPEED(SPEED_25G); + FW_CAPS_TO_SPEED(SPEED_40G); + FW_CAPS_TO_SPEED(SPEED_100G); + break; + + default: + break; + } + +#undef FW_CAPS_TO_SPEED +#undef SET_SPEED +} + +/** + * cxgbe_get_speed_caps - Fetch supported speed capabilities + * @pi: Underlying port's info + * @speed_caps: Device Info speed capabilities + * + * Fetch supported speed capabilities of the underlying port. + */ +void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps) +{ + *speed_caps = 0; + + fw_caps_to_speed_caps(pi->port_type, pi->link_cfg.supported, + speed_caps); + + if (!(pi->link_cfg.supported & FW_PORT_CAP_ANEG)) + *speed_caps |= ETH_LINK_SPEED_FIXED; +} + +/** * cxgb_up - enable the adapter * @adap: adapter being enabled * @@ -1062,10 +1202,20 @@ void cxgbe_close(struct adapter *adapter) int cxgbe_probe(struct adapter *adapter) { struct port_info *pi; + int chip; int func, i; int err = 0; + u32 whoami; + + whoami = t4_read_reg(adapter, A_PL_WHOAMI); + chip = t4_get_chip_type(adapter, + CHELSIO_PCI_ID_VER(adapter->pdev->id.device_id)); + if (chip < 0) + return chip; + + func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ? + G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami); - func = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI)); adapter->mbox = func; adapter->pf = func; @@ -1182,6 +1332,7 @@ allocate_mac: cfg_queues(adapter->eth_dev); + print_adapter_info(adapter); print_port_info(adapter); err = init_rss(adapter); diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c index 2f9e12c9..5376fc50 100644 --- a/drivers/net/cxgbe/sge.c +++ b/drivers/net/cxgbe/sge.c @@ -57,7 +57,6 @@ #include <rte_alarm.h> #include <rte_ether.h> #include <rte_ethdev.h> -#include <rte_atomic.h> #include <rte_malloc.h> #include <rte_random.h> #include <rte_dev.h> @@ -388,7 +387,7 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, struct rte_pktmbuf_pool_private *mbp_priv; u8 jumbo_en = rxq->rspq.eth_dev->data->dev_conf.rxmode.jumbo_frame; - /* Use jumbo mtu buffers iff mbuf data room size can fit jumbo data. */ + /* Use jumbo mtu buffers if mbuf data room size can fit jumbo data. */ mbp_priv = rte_mempool_get_priv(rxq->rspq.mb_pool); if (jumbo_en && ((mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM) >= 9000)) @@ -415,12 +414,18 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, } rte_mbuf_refcnt_set(mbuf, 1); - mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->data_off = + (uint16_t)(RTE_PTR_ALIGN((char *)mbuf->buf_addr + + RTE_PKTMBUF_HEADROOM, + adap->sge.fl_align) - + (char *)mbuf->buf_addr); mbuf->next = NULL; mbuf->nb_segs = 1; mbuf->port = rxq->rspq.port_id; - mapping = (dma_addr_t)(mbuf->buf_physaddr + mbuf->data_off); + mapping = (dma_addr_t)RTE_ALIGN(mbuf->buf_physaddr + + mbuf->data_off, + adap->sge.fl_align); mapping |= buf_size_idx; *d++ = cpu_to_be64(mapping); set_rx_sw_desc(sd, mbuf, mapping); @@ -591,7 +596,7 @@ static inline unsigned int calc_tx_flits(const struct rte_mbuf *m) * Write Header (incorporated as part of the cpl_tx_pkt_lso and * cpl_tx_pkt structures), followed by either a TX Packet Write CPL * message or, if we're doing a Large Send Offload, an LSO CPL message - * with an embeded TX Packet Write CPL message. + * with an embedded TX Packet Write CPL message. */ flits = sgl_len(m->nb_segs); if (m->tso_segsz) @@ -681,6 +686,10 @@ static void write_sgl(struct rte_mbuf *mbuf, struct sge_txq *q, #define Q_IDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->size) #define R_IDXDIFF(q, idx) IDXDIFF((q)->cidx, (q)->idx, (q)->size) +#define PIDXDIFF(head, tail, wrap) \ + ((tail) >= (head) ? (tail) - (head) : (wrap) - (head) + (tail)) +#define P_IDXDIFF(q, idx) PIDXDIFF((q)->cidx, idx, (q)->size) + /** * ring_tx_db - ring a Tx queue's doorbell * @adap: the adapter @@ -771,7 +780,7 @@ static u64 hwcsum(enum chip_type chip, const struct rte_mbuf *m) } if (likely(csum_type >= TX_CSUM_TCPIP)) { - int hdr_len = V_TXPKT_IPHDR_LEN(m->l3_len); + u64 hdr_len = V_TXPKT_IPHDR_LEN(m->l3_len); int eth_hdr_len = m->l2_len; if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) @@ -846,7 +855,7 @@ static inline void ship_tx_pkt_coalesce_wr(struct adapter *adap, /* fill the pkts WR header */ wr = (void *)&q->desc[q->pidx]; - wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); + wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR)); wr_mid = V_FW_WR_LEN16(DIV_ROUND_UP(q->coalesce.flits, 2)); ndesc = flits_to_desc(q->coalesce.flits); @@ -969,7 +978,7 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, int flits, struct adapter *adap, const struct port_info *pi, - dma_addr_t *addr) + dma_addr_t *addr, uint16_t nb_pkts) { u64 cntrl, *end; struct sge_txq *q = &txq->q; @@ -979,6 +988,10 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, struct tx_sw_desc *sd; unsigned int idx = q->coalesce.idx, len = mbuf->pkt_len; +#ifdef RTE_LIBRTE_CXGBE_TPUT + RTE_SET_USED(nb_pkts); +#endif + if (q->coalesce.type == 0) { mc = (struct ulp_txpkt *)q->coalesce.ptr; mc->cmd_dest = htonl(V_ULPTX_CMD(4) | V_ULP_TXPKT_DEST(0) | @@ -1048,7 +1061,11 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, sd->coalesce.idx = (idx & 1) + 1; /* send the coaelsced work request if max reached */ - if (++q->coalesce.idx == ETH_COALESCE_PKT_NUM) + if (++q->coalesce.idx == ETH_COALESCE_PKT_NUM +#ifndef RTE_LIBRTE_CXGBE_TPUT + || q->coalesce.idx >= nb_pkts +#endif + ) ship_tx_pkt_coalesce_wr(adap, txq); return 0; } @@ -1060,7 +1077,8 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, * * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled. */ -int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf) +int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, + uint16_t nb_pkts) { const struct port_info *pi; struct cpl_tx_pkt_lso_core *lso; @@ -1114,7 +1132,7 @@ out_free: } rte_prefetch0((volatile void *)addr); return tx_do_packet_coalesce(txq, mbuf, cflits, adap, - pi, addr); + pi, addr, nb_pkts); } else { return -EBUSY; } @@ -1185,9 +1203,15 @@ out_free: else lso->len = htonl(V_LSO_T5_XFER_SIZE(m->pkt_len)); cpl = (void *)(lso + 1); - cntrl = V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | - V_TXPKT_IPHDR_LEN(l3hdr_len) | - V_TXPKT_ETHHDR_LEN(eth_xtra_len); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + cntrl = V_TXPKT_ETHHDR_LEN(eth_xtra_len); + else + cntrl = V_T6_TXPKT_ETHHDR_LEN(eth_xtra_len); + + cntrl |= V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : + TX_CSUM_TCPIP) | + V_TXPKT_IPHDR_LEN(l3hdr_len); txq->stats.tso++; txq->stats.tx_cso += m->tso_segsz; } @@ -1344,10 +1368,16 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, const struct rss_header *rss_hdr; bool csum_ok; struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + u16 err_vec; rss_hdr = (const void *)rsp; pkt = (const void *)&rsp[1]; - csum_ok = pkt->csum_calc && !pkt->err_vec; + /* Compressed error vector is enabled for T6 only */ + if (q->adapter->params.tp.rx_pkt_encap) + err_vec = G_T6_COMPR_RXERR_VEC(ntohs(pkt->err_vec)); + else + err_vec = ntohs(pkt->err_vec); + csum_ok = pkt->csum_calc && !err_vec; mbuf = t4_pktgl_to_mbuf(si); if (unlikely(!mbuf)) { @@ -1384,20 +1414,6 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, return 0; } -/** - * is_new_response - check if a response is newly written - * @r: the response descriptor - * @q: the response queue - * - * Returns true if a response descriptor contains a yet unprocessed - * response. - */ -static inline bool is_new_response(const struct rsp_ctrl *r, - const struct sge_rspq *q) -{ - return (r->u.type_gen >> S_RSPD_GEN) == q->gen; -} - #define CXGB4_MSG_AN ((void *)1) /** @@ -1439,12 +1455,12 @@ static int process_responses(struct sge_rspq *q, int budget, struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); while (likely(budget_left)) { + if (q->cidx == ntohs(q->stat->pidx)) + break; + rc = (const struct rsp_ctrl *) ((const char *)q->cur_desc + (q->iqe_len - sizeof(*rc))); - if (!is_new_response(rc, q)) - break; - /* * Ensure response has been read */ @@ -1452,63 +1468,101 @@ static int process_responses(struct sge_rspq *q, int budget, rsp_type = G_RSPD_TYPE(rc->u.type_gen); if (likely(rsp_type == X_RSPD_TYPE_FLBUF)) { - const struct rx_sw_desc *rsd = - &rxq->fl.sdesc[rxq->fl.cidx]; - const struct rss_header *rss_hdr = - (const void *)q->cur_desc; - const struct cpl_rx_pkt *cpl = - (const void *)&q->cur_desc[1]; - bool csum_ok = cpl->csum_calc && !cpl->err_vec; - struct rte_mbuf *pkt, *npkt; - u32 len, bufsz; - - len = ntohl(rc->pldbuflen_qid); - BUG_ON(!(len & F_RSPD_NEWBUF)); - pkt = rsd->buf; - npkt = pkt; - len = G_RSPD_LEN(len); - pkt->pkt_len = len; - - /* Chain mbufs into len if necessary */ - while (len) { - struct rte_mbuf *new_pkt = rsd->buf; - - bufsz = min(get_buf_size(q->adapter, rsd), len); - new_pkt->data_len = bufsz; - unmap_rx_buf(&rxq->fl); - len -= bufsz; - npkt->next = new_pkt; - npkt = new_pkt; - pkt->nb_segs++; - rsd = &rxq->fl.sdesc[rxq->fl.cidx]; - } - npkt->next = NULL; - pkt->nb_segs--; - - if (cpl->l2info & htonl(F_RXF_IP)) { - pkt->packet_type = RTE_PTYPE_L3_IPV4; - if (unlikely(!csum_ok)) - pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; - - if ((cpl->l2info & - htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok) - pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; - } else if (cpl->l2info & htonl(F_RXF_IP6)) { - pkt->packet_type = RTE_PTYPE_L3_IPV6; - } + unsigned int stat_pidx; + int stat_pidx_diff; + + stat_pidx = ntohs(q->stat->pidx); + stat_pidx_diff = P_IDXDIFF(q, stat_pidx); + while (stat_pidx_diff && budget_left) { + const struct rx_sw_desc *rsd = + &rxq->fl.sdesc[rxq->fl.cidx]; + const struct rss_header *rss_hdr = + (const void *)q->cur_desc; + const struct cpl_rx_pkt *cpl = + (const void *)&q->cur_desc[1]; + struct rte_mbuf *pkt, *npkt; + u32 len, bufsz; + bool csum_ok; + u16 err_vec; + + rc = (const struct rsp_ctrl *) + ((const char *)q->cur_desc + + (q->iqe_len - sizeof(*rc))); + + rsp_type = G_RSPD_TYPE(rc->u.type_gen); + if (unlikely(rsp_type != X_RSPD_TYPE_FLBUF)) + break; + + len = ntohl(rc->pldbuflen_qid); + BUG_ON(!(len & F_RSPD_NEWBUF)); + pkt = rsd->buf; + npkt = pkt; + len = G_RSPD_LEN(len); + pkt->pkt_len = len; + + /* Compressed error vector is enabled for + * T6 only + */ + if (q->adapter->params.tp.rx_pkt_encap) + err_vec = G_T6_COMPR_RXERR_VEC( + ntohs(cpl->err_vec)); + else + err_vec = ntohs(cpl->err_vec); + csum_ok = cpl->csum_calc && !err_vec; + + /* Chain mbufs into len if necessary */ + while (len) { + struct rte_mbuf *new_pkt = rsd->buf; + + bufsz = min(get_buf_size(q->adapter, + rsd), len); + new_pkt->data_len = bufsz; + unmap_rx_buf(&rxq->fl); + len -= bufsz; + npkt->next = new_pkt; + npkt = new_pkt; + pkt->nb_segs++; + rsd = &rxq->fl.sdesc[rxq->fl.cidx]; + } + npkt->next = NULL; + pkt->nb_segs--; + + if (cpl->l2info & htonl(F_RXF_IP)) { + pkt->packet_type = RTE_PTYPE_L3_IPV4; + if (unlikely(!csum_ok)) + pkt->ol_flags |= + PKT_RX_IP_CKSUM_BAD; + + if ((cpl->l2info & + htonl(F_RXF_UDP | F_RXF_TCP)) && + !csum_ok) + pkt->ol_flags |= + PKT_RX_L4_CKSUM_BAD; + } else if (cpl->l2info & htonl(F_RXF_IP6)) { + pkt->packet_type = RTE_PTYPE_L3_IPV6; + } - if (!rss_hdr->filter_tid && rss_hdr->hash_type) { - pkt->ol_flags |= PKT_RX_RSS_HASH; - pkt->hash.rss = ntohl(rss_hdr->hash_val); - } + if (!rss_hdr->filter_tid && + rss_hdr->hash_type) { + pkt->ol_flags |= PKT_RX_RSS_HASH; + pkt->hash.rss = + ntohl(rss_hdr->hash_val); + } + + if (cpl->vlan_ex) { + pkt->ol_flags |= PKT_RX_VLAN_PKT; + pkt->vlan_tci = ntohs(cpl->vlan); + } + + rxq->stats.pkts++; + rxq->stats.rx_bytes += pkt->pkt_len; + rx_pkts[budget - budget_left] = pkt; - if (cpl->vlan_ex) { - pkt->ol_flags |= PKT_RX_VLAN_PKT; - pkt->vlan_tci = ntohs(cpl->vlan); + rspq_next(q); + budget_left--; + stat_pidx_diff--; } - rxq->stats.pkts++; - rxq->stats.rx_bytes += pkt->pkt_len; - rx_pkts[budget - budget_left] = pkt; + continue; } else if (likely(rsp_type == X_RSPD_TYPE_CPL)) { ret = q->handler(q, q->cur_desc, NULL); } else { @@ -1523,35 +1577,6 @@ static int process_responses(struct sge_rspq *q, int budget, rspq_next(q); budget_left--; - - if (R_IDXDIFF(q, gts_idx) >= 64) { - unsigned int cidx_inc = R_IDXDIFF(q, gts_idx); - unsigned int params; - u32 val; - - if (fl_cap(&rxq->fl) - rxq->fl.avail >= 64) - __refill_fl(q->adapter, &rxq->fl); - params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX); - q->next_intr_params = params; - val = V_CIDXINC(cidx_inc) | V_SEINTARM(params); - - if (unlikely(!q->bar2_addr)) - t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), - val | - V_INGRESSQID((u32)q->cntxt_id)); - else { - writel(val | V_INGRESSQID(q->bar2_qid), - (void *)((uintptr_t)q->bar2_addr + - SGE_UDB_GTS)); - /* - * This Write memory Barrier will force the - * write to the User Doorbell area to be - * flushed. - */ - wmb(); - } - q->gts_idx = q->cidx; - } } /* @@ -1569,10 +1594,38 @@ static int process_responses(struct sge_rspq *q, int budget, int cxgbe_poll(struct sge_rspq *q, struct rte_mbuf **rx_pkts, unsigned int budget, unsigned int *work_done) { - int err = 0; + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + unsigned int cidx_inc; + unsigned int params; + u32 val; *work_done = process_responses(q, budget, rx_pkts); - return err; + + if (*work_done) { + cidx_inc = R_IDXDIFF(q, gts_idx); + + if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 64) + __refill_fl(q->adapter, &rxq->fl); + + params = q->intr_params; + q->next_intr_params = params; + val = V_CIDXINC(cidx_inc) | V_SEINTARM(params); + + if (unlikely(!q->bar2_addr)) { + t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), + val | V_INGRESSQID((u32)q->cntxt_id)); + } else { + writel(val | V_INGRESSQID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + SGE_UDB_GTS)); + /* This Write memory Barrier will force the + * write to the User Doorbell area to be + * flushed. + */ + wmb(); + } + q->gts_idx = q->cidx; + } + return 0; } /** @@ -1641,7 +1694,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, iq->size = cxgbe_roundup(iq->size, 16); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->data->drv_name, + eth_dev->device->driver->name, fwevtq ? "fwq_ring" : "rx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1662,24 +1715,25 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, V_FW_IQ_CMD_IQASYNCH(fwevtq) | V_FW_IQ_CMD_VIID(pi->viid) | V_FW_IQ_CMD_IQANDST(intr_idx < 0) | - V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT) | + V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_STATUS_PAGE) | V_FW_IQ_CMD_IQANDSTINDEX(intr_idx >= 0 ? intr_idx : -intr_idx - 1)); c.iqdroprss_to_iqesize = - htons(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | + htons(V_FW_IQ_CMD_IQPCIECH(cong > 0 ? cxgbe_ffs(cong) - 1 : + pi->tx_chan) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(iq->pktcnt_idx) | V_FW_IQ_CMD_IQESIZE(ilog2(iq->iqe_len) - 4)); c.iqsize = htons(iq->size); c.iqaddr = cpu_to_be64(iq->phys_addr); if (cong >= 0) - c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN); + c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN | + F_FW_IQ_CMD_IQRO); if (fl) { struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq, fl); - enum chip_type chip = (enum chip_type)CHELSIO_CHIP_VERSION( - adap->params.chip); + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); /* * Allocate the ring for the hardware free list (with space @@ -1694,7 +1748,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, fl->size = cxgbe_roundup(fl->size, 8); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->data->drv_name, + eth_dev->device->driver->name, fwevtq ? "fwq_ring" : "fl_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1725,9 +1779,12 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, * Hence maximum allowed burst size will be 448 bytes. */ c.fl0dcaen_to_fl0cidxfthresh = - htons(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_128B) | - V_FW_IQ_CMD_FL0FBMAX((chip <= CHELSIO_T5) ? - X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); + htons(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ? + X_FETCHBURSTMIN_128B : + X_FETCHBURSTMIN_64B) | + V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ? + X_FETCHBURSTMAX_512B : + X_FETCHBURSTMAX_256B)); c.fl0size = htons(flsz); c.fl0addr = cpu_to_be64(fl->addr); } @@ -1746,6 +1803,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, iq->bar2_addr = bar2_address(adap, iq->cntxt_id, T4_BAR2_QTYPE_INGRESS, &iq->bar2_qid); iq->size--; /* subtract status entry */ + iq->stat = (void *)&iq->desc[iq->size * 8]; iq->eth_dev = eth_dev; iq->handler = hnd; iq->port_id = pi->port_id; @@ -1890,7 +1948,7 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->data->drv_name, "tx_ring", + eth_dev->device->driver->name, "tx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -2183,8 +2241,7 @@ static int t4_sge_init_soft(struct adapter *adap) int t4_sge_init(struct adapter *adap) { struct sge *s = &adap->sge; - u32 sge_control, sge_control2, sge_conm_ctrl; - unsigned int ingpadboundary, ingpackboundary; + u32 sge_control, sge_conm_ctrl; int ret, egress_threshold; /* @@ -2194,34 +2251,7 @@ int t4_sge_init(struct adapter *adap) sge_control = t4_read_reg(adap, A_SGE_CONTROL); s->pktshift = G_PKTSHIFT(sge_control); s->stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; - - /* - * T4 uses a single control field to specify both the PCIe Padding and - * Packing Boundary. T5 introduced the ability to specify these - * separately. The actual Ingress Packet Data alignment boundary - * within Packed Buffer Mode is the maximum of these two - * specifications. - */ - ingpadboundary = 1 << (G_INGPADBOUNDARY(sge_control) + - X_INGPADBOUNDARY_SHIFT); - s->fl_align = ingpadboundary; - - if (!is_t4(adap->params.chip) && !adap->use_unpacked_mode) { - /* - * T5 has a weird interpretation of one of the PCIe Packing - * Boundary values. No idea why ... - */ - sge_control2 = t4_read_reg(adap, A_SGE_CONTROL2); - ingpackboundary = G_INGPACKBOUNDARY(sge_control2); - if (ingpackboundary == X_INGPACKBOUNDARY_16B) - ingpackboundary = 16; - else - ingpackboundary = 1 << (ingpackboundary + - X_INGPACKBOUNDARY_SHIFT); - - s->fl_align = max(ingpadboundary, ingpackboundary); - } - + s->fl_align = t4_fl_pkt_align(adap); ret = t4_sge_init_soft(adap); if (ret < 0) { dev_err(adap, "%s: t4_sge_init_soft failed, error %d\n", diff --git a/drivers/net/dpaa2/Makefile b/drivers/net/dpaa2/Makefile index ce65542a..32cd819b 100644 --- a/drivers/net/dpaa2/Makefile +++ b/drivers/net/dpaa2/Makefile @@ -1,7 +1,7 @@ # BSD LICENSE # # Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -# Copyright (c) 2016 NXP. All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/drivers/net/dpaa2/base/dpaa2_hw_dpni.c b/drivers/net/dpaa2/base/dpaa2_hw_dpni.c index 3dc60ccc..4c82aa87 100644 --- a/drivers/net/dpaa2/base/dpaa2_hw_dpni.c +++ b/drivers/net/dpaa2/base/dpaa2_hw_dpni.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -42,7 +42,6 @@ #include <rte_cycles.h> #include <rte_kvargs.h> #include <rte_dev.h> -#include <rte_ethdev.h> #include <fslmc_logs.h> #include <dpaa2_hw_pvt.h> @@ -91,8 +90,9 @@ dpaa2_setup_flow_dist(struct rte_eth_dev *eth_dev, &tc_cfg); rte_free(p_params); if (ret) { - RTE_LOG(ERR, PMD, "Setting distribution for Rx failed with" - " err code: %d\n", ret); + RTE_LOG(ERR, PMD, + "Setting distribution for Rx failed with err: %d\n", + ret); return ret; } @@ -134,8 +134,9 @@ int dpaa2_remove_flow_dist( &tc_cfg); rte_free(p_params); if (ret) { - RTE_LOG(ERR, PMD, "Setting distribution for Rx failed with" - " err code: %d\n", ret); + RTE_LOG(ERR, PMD, + "Setting distribution for Rx failed with err: %d\n", + ret); return ret; } return ret; @@ -306,15 +307,22 @@ dpaa2_attach_bp_list(struct dpaa2_dev_priv *priv, */ /* ... rx buffer layout ... */ - tot_size = DPAA2_HW_BUF_RESERVE + RTE_PKTMBUF_HEADROOM; - tot_size = RTE_ALIGN_CEIL(tot_size, - DPAA2_PACKET_LAYOUT_ALIGN); + tot_size = RTE_PKTMBUF_HEADROOM; + tot_size = RTE_ALIGN_CEIL(tot_size, DPAA2_PACKET_LAYOUT_ALIGN); memset(&layout, 0, sizeof(struct dpni_buffer_layout)); - layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; - - layout.data_head_room = - tot_size - DPAA2_FD_PTA_SIZE - DPAA2_MBUF_HW_ANNOTATION; + layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | + DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | + DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | + DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | + DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE; + + layout.pass_frame_status = 1; + layout.private_data_size = DPAA2_FD_PTA_SIZE; + layout.pass_parser_result = 1; + layout.data_align = DPAA2_PACKET_LAYOUT_ALIGN; + layout.data_head_room = tot_size - DPAA2_FD_PTA_SIZE - + DPAA2_MBUF_HW_ANNOTATION; retcode = dpni_set_buffer_layout(dpni, CMD_PRI_LOW, priv->token, DPNI_QUEUE_RX, &layout); if (retcode) { @@ -327,9 +335,8 @@ dpaa2_attach_bp_list(struct dpaa2_dev_priv *priv, bpool_cfg.num_dpbp = 1; bpool_cfg.pools[0].dpbp_id = bp_list->buf_pool.dpbp_node->dpbp_id; bpool_cfg.pools[0].backup_pool = 0; - bpool_cfg.pools[0].buffer_size = - RTE_ALIGN_CEIL(bp_list->buf_pool.size, - 256 /*DPAA2_PACKET_LAYOUT_ALIGN*/); + bpool_cfg.pools[0].buffer_size = RTE_ALIGN_CEIL(bp_list->buf_pool.size, + DPAA2_PACKET_LAYOUT_ALIGN); retcode = dpni_set_pools(dpni, CMD_PRI_LOW, priv->token, &bpool_cfg); if (retcode != 0) { diff --git a/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h b/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h index 9324c6a3..e68febff 100644 --- a/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h +++ b/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c b/drivers/net/dpaa2/dpaa2_ethdev.c index 45764420..429b3a08 100644 --- a/drivers/net/dpaa2/dpaa2_ethdev.c +++ b/drivers/net/dpaa2/dpaa2_ethdev.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -42,7 +42,6 @@ #include <rte_cycles.h> #include <rte_kvargs.h> #include <rte_dev.h> -#include <rte_ethdev.h> #include <rte_fslmc.h> #include <fslmc_logs.h> @@ -50,10 +49,14 @@ #include <dpaa2_hw_pvt.h> #include <dpaa2_hw_mempool.h> #include <dpaa2_hw_dpio.h> - +#include <mc/fsl_dpmng.h> #include "dpaa2_ethdev.h" static struct rte_dpaa2_driver rte_dpaa2_pmd; +static int dpaa2_dev_uninit(struct rte_eth_dev *eth_dev); +static int dpaa2_dev_set_link_up(struct rte_eth_dev *dev); +static int dpaa2_dev_set_link_down(struct rte_eth_dev *dev); +static int dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); /** * Atomically reads the link status information from global @@ -107,6 +110,89 @@ dpaa2_dev_atomic_write_link_status(struct rte_eth_dev *dev, return 0; } +static int +dpaa2_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return -1; + } + + if (on) + ret = dpni_add_vlan_id(dpni, CMD_PRI_LOW, + priv->token, vlan_id); + else + ret = dpni_remove_vlan_id(dpni, CMD_PRI_LOW, + priv->token, vlan_id); + + if (ret < 0) + PMD_DRV_LOG(ERR, "ret = %d Unable to add/rem vlan %d hwid =%d", + ret, vlan_id, priv->hw_id); + + return ret; +} + +static void +dpaa2_vlan_offload_set(struct rte_eth_dev *dev, int mask) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (mask & ETH_VLAN_FILTER_MASK) { + if (dev->data->dev_conf.rxmode.hw_vlan_filter) + ret = dpni_enable_vlan_filter(dpni, CMD_PRI_LOW, + priv->token, true); + else + ret = dpni_enable_vlan_filter(dpni, CMD_PRI_LOW, + priv->token, false); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to set vlan filter = %d\n", + ret); + } +} + +static int +dpaa2_fw_version_get(struct rte_eth_dev *dev, + char *fw_version, + size_t fw_size) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + struct mc_soc_version mc_plat_info = {0}; + struct mc_version mc_ver_info = {0}; + + PMD_INIT_FUNC_TRACE(); + + if (mc_get_soc_version(dpni, CMD_PRI_LOW, &mc_plat_info)) + RTE_LOG(WARNING, PMD, "\tmc_get_soc_version failed\n"); + + if (mc_get_version(dpni, CMD_PRI_LOW, &mc_ver_info)) + RTE_LOG(WARNING, PMD, "\tmc_get_version failed\n"); + + ret = snprintf(fw_version, fw_size, + "%x-%d.%d.%d", + mc_plat_info.svr, + mc_ver_info.major, + mc_ver_info.minor, + mc_ver_info.revision); + + ret += 1; /* add the size of '\0' */ + if (fw_size < (uint32_t)ret) + return ret; + else + return 0; +} + static void dpaa2_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { @@ -176,13 +262,17 @@ dpaa2_alloc_rx_tx_queues(struct rte_eth_dev *dev) for (i = 0; i < priv->nb_tx_queues; i++) { mc_q->dev = dev; - mc_q->flow_id = DPNI_NEW_FLOW_ID; + mc_q->flow_id = 0xffff; priv->tx_vq[i] = mc_q++; + dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i]; + dpaa2_q->cscn = rte_malloc(NULL, + sizeof(struct qbman_result), 16); + if (!dpaa2_q->cscn) + goto fail_tx; } vq_id = 0; - for (dist_idx = 0; dist_idx < priv->num_dist_per_tc[DPAA2_DEF_TC]; - dist_idx++) { + for (dist_idx = 0; dist_idx < priv->nb_rx_queues; dist_idx++) { mcq = (struct dpaa2_queue *)priv->rx_vq[vq_id]; mcq->tc_index = DPAA2_DEF_TC; mcq->flow_id = dist_idx; @@ -190,6 +280,14 @@ dpaa2_alloc_rx_tx_queues(struct rte_eth_dev *dev) } return 0; +fail_tx: + i -= 1; + while (i >= 0) { + dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i]; + rte_free(dpaa2_q->cscn); + priv->tx_vq[i--] = NULL; + } + i = priv->nb_rx_queues; fail: i -= 1; mc_q = priv->rx_vq[0]; @@ -212,6 +310,20 @@ dpaa2_eth_dev_configure(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); + if (eth_conf->rxmode.jumbo_frame == 1) { + if (eth_conf->rxmode.max_rx_pkt_len <= DPAA2_MAX_RX_PKT_LEN) { + ret = dpaa2_dev_mtu_set(dev, + eth_conf->rxmode.max_rx_pkt_len); + if (ret) { + PMD_INIT_LOG(ERR, + "unable to set mtu. check config\n"); + return ret; + } + } else { + return -1; + } + } + /* Check for correct configuration */ if (eth_conf->rxmode.mq_mode != ETH_MQ_RX_RSS && data->nb_rx_queues > 1) { @@ -248,6 +360,7 @@ dpaa2_dev_rx_queue_setup(struct rte_eth_dev *dev, { struct dpaa2_dev_priv *priv = dev->data->dev_private; struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + struct mc_soc_version mc_plat_info = {0}; struct dpaa2_queue *dpaa2_q; struct dpni_queue cfg; uint8_t options = 0; @@ -270,21 +383,25 @@ dpaa2_dev_rx_queue_setup(struct rte_eth_dev *dev, dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[rx_queue_id]; dpaa2_q->mb_pool = mb_pool; /**< mbuf pool to populate RX ring. */ - /*Get the tc id and flow id from given VQ id*/ - flow_id = rx_queue_id % priv->num_dist_per_tc[dpaa2_q->tc_index]; + /*Get the flow id from given VQ id*/ + flow_id = rx_queue_id % priv->nb_rx_queues; memset(&cfg, 0, sizeof(struct dpni_queue)); options = options | DPNI_QUEUE_OPT_USER_CTX; cfg.user_context = (uint64_t)(dpaa2_q); /*if ls2088 or rev2 device, enable the stashing */ - if ((qbman_get_version() & 0xFFFF0000) > QMAN_REV_4000) { + + if (mc_get_soc_version(dpni, CMD_PRI_LOW, &mc_plat_info)) + PMD_INIT_LOG(ERR, "\tmc_get_soc_version failed\n"); + + if ((mc_plat_info.svr & 0xffff0000) != SVR_LS2080A) { options |= DPNI_QUEUE_OPT_FLC; cfg.flc.stash_control = true; cfg.flc.value &= 0xFFFFFFFFFFFFFFC0; /* 00 00 00 - last 6 bit represent annotation, context stashing, * data stashing setting 01 01 00 (0x14) to enable - * 1 line annotation, 1 line context + * 1 line data, 1 line annotation */ cfg.flc.value |= 0x14; } @@ -295,6 +412,25 @@ dpaa2_dev_rx_queue_setup(struct rte_eth_dev *dev, return -1; } + if (!(priv->flags & DPAA2_RX_TAILDROP_OFF)) { + struct dpni_taildrop taildrop; + + taildrop.enable = 1; + /*enabling per rx queue congestion control */ + taildrop.threshold = CONG_THRESHOLD_RX_Q; + taildrop.units = DPNI_CONGESTION_UNIT_BYTES; + PMD_INIT_LOG(DEBUG, "Enabling Early Drop on queue = %d", + rx_queue_id); + ret = dpni_set_taildrop(dpni, CMD_PRI_LOW, priv->token, + DPNI_CP_QUEUE, DPNI_QUEUE_RX, + dpaa2_q->tc_index, flow_id, &taildrop); + if (ret) { + PMD_INIT_LOG(ERR, "Error in setting the rx flow" + " err : = %d\n", ret); + return -1; + } + } + dev->data->rx_queues[rx_queue_id] = dpaa2_q; return 0; } @@ -319,19 +455,14 @@ dpaa2_dev_tx_queue_setup(struct rte_eth_dev *dev, PMD_INIT_FUNC_TRACE(); /* Return if queue already configured */ - if (dpaa2_q->flow_id != DPNI_NEW_FLOW_ID) + if (dpaa2_q->flow_id != 0xffff) return 0; memset(&tx_conf_cfg, 0, sizeof(struct dpni_queue)); memset(&tx_flow_cfg, 0, sizeof(struct dpni_queue)); - if (priv->num_tc == 1) { - tc_id = 0; - flow_id = tx_queue_id % priv->num_dist_per_tc[tc_id]; - } else { - tc_id = tx_queue_id; - flow_id = 0; - } + tc_id = tx_queue_id; + flow_id = 0; ret = dpni_set_queue(dpni, CMD_PRI_LOW, priv->token, DPNI_QUEUE_TX, tc_id, flow_id, options, &tx_flow_cfg); @@ -357,6 +488,35 @@ dpaa2_dev_tx_queue_setup(struct rte_eth_dev *dev, } dpaa2_q->tc_index = tc_id; + if (!(priv->flags & DPAA2_TX_CGR_OFF)) { + struct dpni_congestion_notification_cfg cong_notif_cfg; + + cong_notif_cfg.units = DPNI_CONGESTION_UNIT_FRAMES; + cong_notif_cfg.threshold_entry = CONG_ENTER_TX_THRESHOLD; + /* Notify that the queue is not congested when the data in + * the queue is below this thershold. + */ + cong_notif_cfg.threshold_exit = CONG_EXIT_TX_THRESHOLD; + cong_notif_cfg.message_ctx = 0; + cong_notif_cfg.message_iova = (uint64_t)dpaa2_q->cscn; + cong_notif_cfg.dest_cfg.dest_type = DPNI_DEST_NONE; + cong_notif_cfg.notification_mode = + DPNI_CONG_OPT_WRITE_MEM_ON_ENTER | + DPNI_CONG_OPT_WRITE_MEM_ON_EXIT | + DPNI_CONG_OPT_COHERENT_WRITE; + + ret = dpni_set_congestion_notification(dpni, CMD_PRI_LOW, + priv->token, + DPNI_QUEUE_TX, + tc_id, + &cong_notif_cfg); + if (ret) { + PMD_INIT_LOG(ERR, + "Error in setting tx congestion notification: = %d", + -ret); + return -ret; + } + } dev->data->tx_queues[tx_queue_id] = dpaa2_q; return 0; } @@ -390,7 +550,7 @@ dpaa2_supported_ptypes_get(struct rte_eth_dev *dev) RTE_PTYPE_UNKNOWN }; - if (dev->rx_pkt_burst == dpaa2_dev_rx) + if (dev->rx_pkt_burst == dpaa2_dev_prefetch_rx) return ptypes; return NULL; } @@ -417,6 +577,9 @@ dpaa2_dev_start(struct rte_eth_dev *dev) return ret; } + /* Power up the phy. Needed to make the link go Up */ + dpaa2_dev_set_link_up(dev); + ret = dpni_get_qdid(dpni, CMD_PRI_LOW, priv->token, DPNI_QUEUE_TX, &qdid); if (ret) { @@ -479,6 +642,9 @@ dpaa2_dev_start(struct rte_eth_dev *dev) "code = %d\n", ret); return ret; } + /* VLAN Offload Settings */ + if (priv->max_vlan_filters) + dpaa2_vlan_offload_set(dev, ETH_VLAN_FILTER_MASK); return 0; } @@ -497,6 +663,8 @@ dpaa2_dev_stop(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); + dpaa2_dev_set_link_down(dev); + ret = dpni_disable(dpni, CMD_PRI_LOW, priv->token); if (ret) { PMD_INIT_LOG(ERR, "Failure (ret %d) in disabling dpni %d dev\n", @@ -512,12 +680,23 @@ dpaa2_dev_stop(struct rte_eth_dev *dev) static void dpaa2_dev_close(struct rte_eth_dev *dev) { + struct rte_eth_dev_data *data = dev->data; struct dpaa2_dev_priv *priv = dev->data->dev_private; struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; - int ret; + int i, ret; + struct rte_eth_link link; + struct dpaa2_queue *dpaa2_q; PMD_INIT_FUNC_TRACE(); + for (i = 0; i < data->nb_tx_queues; i++) { + dpaa2_q = (struct dpaa2_queue *)data->tx_queues[i]; + if (!dpaa2_q->cscn) { + rte_free(dpaa2_q->cscn); + dpaa2_q->cscn = NULL; + } + } + /* Clean the device first */ ret = dpni_reset(dpni, CMD_PRI_LOW, priv->token); if (ret) { @@ -525,6 +704,9 @@ dpaa2_dev_close(struct rte_eth_dev *dev) " error code %d\n", ret); return; } + + memset(&link, 0, sizeof(link)); + dpaa2_dev_atomic_write_link_status(dev, &link); } static void @@ -538,13 +720,17 @@ dpaa2_dev_promiscuous_enable( PMD_INIT_FUNC_TRACE(); if (dpni == NULL) { - RTE_LOG(ERR, PMD, "dpni is NULL"); + RTE_LOG(ERR, PMD, "dpni is NULL\n"); return; } ret = dpni_set_unicast_promisc(dpni, CMD_PRI_LOW, priv->token, true); if (ret < 0) - RTE_LOG(ERR, PMD, "Unable to enable promiscuous mode %d", ret); + RTE_LOG(ERR, PMD, "Unable to enable U promisc mode %d\n", ret); + + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, true); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to enable M promisc mode %d\n", ret); } static void @@ -558,13 +744,65 @@ dpaa2_dev_promiscuous_disable( PMD_INIT_FUNC_TRACE(); if (dpni == NULL) { - RTE_LOG(ERR, PMD, "dpni is NULL"); + RTE_LOG(ERR, PMD, "dpni is NULL\n"); return; } ret = dpni_set_unicast_promisc(dpni, CMD_PRI_LOW, priv->token, false); if (ret < 0) - RTE_LOG(ERR, PMD, "Unable to disable promiscuous mode %d", ret); + RTE_LOG(ERR, PMD, "Unable to disable U promisc mode %d\n", ret); + + if (dev->data->all_multicast == 0) { + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, + priv->token, false); + if (ret < 0) + RTE_LOG(ERR, PMD, + "Unable to disable M promisc mode %d\n", + ret); + } +} + +static void +dpaa2_dev_allmulticast_enable( + struct rte_eth_dev *dev) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, true); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to enable multicast mode %d\n", ret); +} + +static void +dpaa2_dev_allmulticast_disable(struct rte_eth_dev *dev) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + /* must remain on for all promiscuous */ + if (dev->data->promiscuous == 1) + return; + + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, false); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to disable multicast mode %d\n", ret); } static int @@ -578,7 +816,7 @@ dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) PMD_INIT_FUNC_TRACE(); if (dpni == NULL) { - RTE_LOG(ERR, PMD, "dpni is NULL"); + RTE_LOG(ERR, PMD, "dpni is NULL\n"); return -EINVAL; } @@ -586,6 +824,11 @@ dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) if ((mtu < ETHER_MIN_MTU) || (frame_size > DPAA2_MAX_RX_PKT_LEN)) return -EINVAL; + if (frame_size > ETHER_MAX_LEN) + dev->data->dev_conf.rxmode.jumbo_frame = 1; + else + dev->data->dev_conf.rxmode.jumbo_frame = 0; + /* Set the Max Rx frame length as 'mtu' + * Maximum Ethernet header length */ @@ -599,6 +842,79 @@ dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) return 0; } +static int +dpaa2_dev_add_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *addr, + __rte_unused uint32_t index, + __rte_unused uint32_t pool) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return -1; + } + + ret = dpni_add_mac_addr(dpni, CMD_PRI_LOW, + priv->token, addr->addr_bytes); + if (ret) + RTE_LOG(ERR, PMD, + "error: Adding the MAC ADDR failed: err = %d\n", ret); + return 0; +} + +static void +dpaa2_dev_remove_mac_addr(struct rte_eth_dev *dev, + uint32_t index) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + struct rte_eth_dev_data *data = dev->data; + struct ether_addr *macaddr; + + PMD_INIT_FUNC_TRACE(); + + macaddr = &data->mac_addrs[index]; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_remove_mac_addr(dpni, CMD_PRI_LOW, + priv->token, macaddr->addr_bytes); + if (ret) + RTE_LOG(ERR, PMD, + "error: Removing the MAC ADDR failed: err = %d\n", ret); +} + +static void +dpaa2_dev_set_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *addr) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_set_primary_mac_addr(dpni, CMD_PRI_LOW, + priv->token, addr->addr_bytes); + + if (ret) + RTE_LOG(ERR, PMD, + "error: Setting the MAC ADDR failed %d\n", ret); +} static void dpaa2_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) @@ -614,12 +930,12 @@ void dpaa2_dev_stats_get(struct rte_eth_dev *dev, PMD_INIT_FUNC_TRACE(); if (!dpni) { - RTE_LOG(ERR, PMD, "dpni is NULL"); + RTE_LOG(ERR, PMD, "dpni is NULL\n"); return; } if (!stats) { - RTE_LOG(ERR, PMD, "stats is NULL"); + RTE_LOG(ERR, PMD, "stats is NULL\n"); return; } @@ -647,7 +963,11 @@ void dpaa2_dev_stats_get(struct rte_eth_dev *dev, if (retcode) goto err; - stats->ierrors = value.page_2.ingress_discarded_frames; + /* Ingress drop frame count due to configured rules */ + stats->ierrors = value.page_2.ingress_filtered_frames; + /* Ingress drop frame count due to error */ + stats->ierrors += value.page_2.ingress_discarded_frames; + stats->oerrors = value.page_2.egress_discarded_frames; stats->imissed = value.page_2.ingress_nobuffer_discards; @@ -668,7 +988,7 @@ void dpaa2_dev_stats_reset(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); if (dpni == NULL) { - RTE_LOG(ERR, PMD, "dpni is NULL"); + RTE_LOG(ERR, PMD, "dpni is NULL\n"); return; } @@ -697,7 +1017,7 @@ dpaa2_dev_link_update(struct rte_eth_dev *dev, PMD_INIT_FUNC_TRACE(); if (dpni == NULL) { - RTE_LOG(ERR, PMD, "error : dpni is NULL"); + RTE_LOG(ERR, PMD, "dpni is NULL\n"); return 0; } memset(&old, 0, sizeof(old)); @@ -705,7 +1025,7 @@ dpaa2_dev_link_update(struct rte_eth_dev *dev, ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state); if (ret < 0) { - RTE_LOG(ERR, PMD, "error: dpni_get_link_state %d", ret); + RTE_LOG(ERR, PMD, "error: dpni_get_link_state %d\n", ret); return -1; } @@ -732,6 +1052,253 @@ dpaa2_dev_link_update(struct rte_eth_dev *dev, return 0; } +/** + * Toggle the DPNI to enable, if not already enabled. + * This is not strictly PHY up/down - it is more of logical toggling. + */ +static int +dpaa2_dev_set_link_up(struct rte_eth_dev *dev) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + int en = 0; + + PMD_INIT_FUNC_TRACE(); + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "DPNI is NULL\n"); + return ret; + } + + /* Check if DPNI is currently enabled */ + ret = dpni_is_enabled(dpni, CMD_PRI_LOW, priv->token, &en); + if (ret) { + /* Unable to obtain dpni status; Not continuing */ + PMD_DRV_LOG(ERR, "Interface Link UP failed (%d)", ret); + return -EINVAL; + } + + /* Enable link if not already enabled */ + if (!en) { + ret = dpni_enable(dpni, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_DRV_LOG(ERR, "Interface Link UP failed (%d)", ret); + return -EINVAL; + } + } + /* changing tx burst function to start enqueues */ + dev->tx_pkt_burst = dpaa2_dev_tx; + dev->data->dev_link.link_status = 1; + + PMD_DRV_LOG(INFO, "Port %d Link UP successful", dev->data->port_id); + return ret; +} + +/** + * Toggle the DPNI to disable, if not already disabled. + * This is not strictly PHY up/down - it is more of logical toggling. + */ +static int +dpaa2_dev_set_link_down(struct rte_eth_dev *dev) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + int dpni_enabled = 0; + int retries = 10; + + PMD_INIT_FUNC_TRACE(); + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "Device has not yet been configured\n"); + return ret; + } + + /*changing tx burst function to avoid any more enqueues */ + dev->tx_pkt_burst = dummy_dev_tx; + + /* Loop while dpni_disable() attempts to drain the egress FQs + * and confirm them back to us. + */ + do { + ret = dpni_disable(dpni, 0, priv->token); + if (ret) { + PMD_DRV_LOG(ERR, "dpni disable failed (%d)", ret); + return ret; + } + ret = dpni_is_enabled(dpni, 0, priv->token, &dpni_enabled); + if (ret) { + PMD_DRV_LOG(ERR, "dpni_is_enabled failed (%d)", ret); + return ret; + } + if (dpni_enabled) + /* Allow the MC some slack */ + rte_delay_us(100 * 1000); + } while (dpni_enabled && --retries); + + if (!retries) { + PMD_DRV_LOG(WARNING, "Retry count exceeded disabling DPNI\n"); + /* todo- we may have to manually cleanup queues. + */ + } else { + PMD_DRV_LOG(INFO, "Port %d Link DOWN successful", + dev->data->port_id); + } + + dev->data->dev_link.link_status = 0; + + return ret; +} + +static int +dpaa2_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + struct dpni_link_state state = {0}; + + PMD_INIT_FUNC_TRACE(); + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL || fc_conf == NULL) { + RTE_LOG(ERR, PMD, "device not configured\n"); + return ret; + } + + ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state); + if (ret) { + RTE_LOG(ERR, PMD, "error: dpni_get_link_state %d\n", ret); + return ret; + } + + memset(fc_conf, 0, sizeof(struct rte_eth_fc_conf)); + if (state.options & DPNI_LINK_OPT_PAUSE) { + /* DPNI_LINK_OPT_PAUSE set + * if ASYM_PAUSE not set, + * RX Side flow control (handle received Pause frame) + * TX side flow control (send Pause frame) + * if ASYM_PAUSE set, + * RX Side flow control (handle received Pause frame) + * No TX side flow control (send Pause frame disabled) + */ + if (!(state.options & DPNI_LINK_OPT_ASYM_PAUSE)) + fc_conf->mode = RTE_FC_FULL; + else + fc_conf->mode = RTE_FC_RX_PAUSE; + } else { + /* DPNI_LINK_OPT_PAUSE not set + * if ASYM_PAUSE set, + * TX side flow control (send Pause frame) + * No RX side flow control (No action on pause frame rx) + * if ASYM_PAUSE not set, + * Flow control disabled + */ + if (state.options & DPNI_LINK_OPT_ASYM_PAUSE) + fc_conf->mode = RTE_FC_TX_PAUSE; + else + fc_conf->mode = RTE_FC_NONE; + } + + return ret; +} + +static int +dpaa2_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + struct dpni_link_state state = {0}; + struct dpni_link_cfg cfg = {0}; + + PMD_INIT_FUNC_TRACE(); + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return ret; + } + + /* It is necessary to obtain the current state before setting fc_conf + * as MC would return error in case rate, autoneg or duplex values are + * different. + */ + ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state); + if (ret) { + RTE_LOG(ERR, PMD, "Unable to get link state (err=%d)\n", ret); + return -1; + } + + /* Disable link before setting configuration */ + dpaa2_dev_set_link_down(dev); + + /* Based on fc_conf, update cfg */ + cfg.rate = state.rate; + cfg.options = state.options; + + /* update cfg with fc_conf */ + switch (fc_conf->mode) { + case RTE_FC_FULL: + /* Full flow control; + * OPT_PAUSE set, ASYM_PAUSE not set + */ + cfg.options |= DPNI_LINK_OPT_PAUSE; + cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; + break; + case RTE_FC_TX_PAUSE: + /* Enable RX flow control + * OPT_PAUSE not set; + * ASYM_PAUSE set; + */ + cfg.options |= DPNI_LINK_OPT_ASYM_PAUSE; + cfg.options &= ~DPNI_LINK_OPT_PAUSE; + break; + case RTE_FC_RX_PAUSE: + /* Enable TX Flow control + * OPT_PAUSE set + * ASYM_PAUSE set + */ + cfg.options |= DPNI_LINK_OPT_PAUSE; + cfg.options |= DPNI_LINK_OPT_ASYM_PAUSE; + break; + case RTE_FC_NONE: + /* Disable Flow control + * OPT_PAUSE not set + * ASYM_PAUSE not set + */ + cfg.options &= ~DPNI_LINK_OPT_PAUSE; + cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; + break; + default: + RTE_LOG(ERR, PMD, "Incorrect Flow control flag (%d)\n", + fc_conf->mode); + return -1; + } + + ret = dpni_set_link_cfg(dpni, CMD_PRI_LOW, priv->token, &cfg); + if (ret) + RTE_LOG(ERR, PMD, + "Unable to set Link configuration (err=%d)\n", + ret); + + /* Enable link */ + dpaa2_dev_set_link_up(dev); + + return ret; +} + static struct eth_dev_ops dpaa2_ethdev_ops = { .dev_configure = dpaa2_eth_dev_configure, .dev_start = dpaa2_dev_start, @@ -739,16 +1306,28 @@ static struct eth_dev_ops dpaa2_ethdev_ops = { .dev_close = dpaa2_dev_close, .promiscuous_enable = dpaa2_dev_promiscuous_enable, .promiscuous_disable = dpaa2_dev_promiscuous_disable, + .allmulticast_enable = dpaa2_dev_allmulticast_enable, + .allmulticast_disable = dpaa2_dev_allmulticast_disable, + .dev_set_link_up = dpaa2_dev_set_link_up, + .dev_set_link_down = dpaa2_dev_set_link_down, .link_update = dpaa2_dev_link_update, .stats_get = dpaa2_dev_stats_get, .stats_reset = dpaa2_dev_stats_reset, + .fw_version_get = dpaa2_fw_version_get, .dev_infos_get = dpaa2_dev_info_get, .dev_supported_ptypes_get = dpaa2_supported_ptypes_get, .mtu_set = dpaa2_dev_mtu_set, + .vlan_filter_set = dpaa2_vlan_filter_set, + .vlan_offload_set = dpaa2_vlan_offload_set, .rx_queue_setup = dpaa2_dev_rx_queue_setup, .rx_queue_release = dpaa2_dev_rx_queue_release, .tx_queue_setup = dpaa2_dev_tx_queue_setup, .tx_queue_release = dpaa2_dev_tx_queue_release, + .flow_ctrl_get = dpaa2_flow_ctrl_get, + .flow_ctrl_set = dpaa2_flow_ctrl_set, + .mac_addr_add = dpaa2_dev_add_mac_addr, + .mac_addr_remove = dpaa2_dev_remove_mac_addr, + .mac_addr_set = dpaa2_dev_set_mac_addr, }; static int @@ -760,8 +1339,7 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev) struct dpni_attr attr; struct dpaa2_dev_priv *priv = eth_dev->data->dev_private; struct dpni_buffer_layout layout; - int i, ret, hw_id; - int tot_size; + int ret, hw_id; PMD_INIT_FUNC_TRACE(); @@ -773,7 +1351,7 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev) hw_id = dpaa2_dev->object_id; - dpni_dev = (struct fsl_mc_io *)malloc(sizeof(struct fsl_mc_io)); + dpni_dev = rte_malloc(NULL, sizeof(struct fsl_mc_io), 0); if (!dpni_dev) { PMD_INIT_LOG(ERR, "malloc failed for dpni device\n"); return -1; @@ -782,43 +1360,45 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev) dpni_dev->regs = rte_mcp_ptr_list[0]; ret = dpni_open(dpni_dev, CMD_PRI_LOW, hw_id, &priv->token); if (ret) { - PMD_INIT_LOG(ERR, "Failure in opening dpni@%d device with" - " error code %d\n", hw_id, ret); + PMD_INIT_LOG(ERR, + "Failure in opening dpni@%d with err code %d\n", + hw_id, ret); + rte_free(dpni_dev); return -1; } /* Clean the device first */ ret = dpni_reset(dpni_dev, CMD_PRI_LOW, priv->token); if (ret) { - PMD_INIT_LOG(ERR, "Failure cleaning dpni@%d device with" - " error code %d\n", hw_id, ret); - return -1; + PMD_INIT_LOG(ERR, + "Failure cleaning dpni@%d with err code %d\n", + hw_id, ret); + goto init_err; } ret = dpni_get_attributes(dpni_dev, CMD_PRI_LOW, priv->token, &attr); if (ret) { - PMD_INIT_LOG(ERR, "Failure in getting dpni@%d attribute, " - " error code %d\n", hw_id, ret); - return -1; + PMD_INIT_LOG(ERR, + "Failure in get dpni@%d attribute, err code %d\n", + hw_id, ret); + goto init_err; } priv->num_tc = attr.num_tcs; - for (i = 0; i < attr.num_tcs; i++) { - priv->num_dist_per_tc[i] = attr.num_queues; - break; - } - /* Distribution is per Tc only, - * so choosing RX queues from default TC only + /* Resetting the "num_rx_vqueues" to equal number of queues in first TC + * as only one TC is supported on Rx Side. Once Multiple TCs will be + * in use for Rx processing then this will be changed or removed. */ - priv->nb_rx_queues = priv->num_dist_per_tc[DPAA2_DEF_TC]; + priv->nb_rx_queues = attr.num_queues; - if (attr.num_tcs == 1) - priv->nb_tx_queues = attr.num_queues; - else - priv->nb_tx_queues = attr.num_tcs; + /* TODO:Using hard coded value for number of TX queues due to dependency + * in MC. + */ + priv->nb_tx_queues = 8; - PMD_INIT_LOG(DEBUG, "num_tc %d", priv->num_tc); + PMD_INIT_LOG(DEBUG, "num TC - RX %d", priv->num_tc); + PMD_INIT_LOG(DEBUG, "nb_tx_queues %d", priv->nb_tx_queues); PMD_INIT_LOG(DEBUG, "nb_rx_queues %d", priv->nb_rx_queues); priv->hw = dpni_dev; @@ -832,51 +1412,27 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev) ret = dpaa2_alloc_rx_tx_queues(eth_dev); if (ret) { PMD_INIT_LOG(ERR, "dpaa2_alloc_rx_tx_queuesFailed\n"); - return -ret; + goto init_err; } /* Allocate memory for storing MAC addresses */ eth_dev->data->mac_addrs = rte_zmalloc("dpni", ETHER_ADDR_LEN * attr.mac_filter_entries, 0); if (eth_dev->data->mac_addrs == NULL) { - PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " - "store MAC addresses", - ETHER_ADDR_LEN * attr.mac_filter_entries); - return -ENOMEM; + PMD_INIT_LOG(ERR, + "Failed to allocate %d bytes needed to store MAC addresses", + ETHER_ADDR_LEN * attr.mac_filter_entries); + ret = -ENOMEM; + goto init_err; } ret = dpni_get_primary_mac_addr(dpni_dev, CMD_PRI_LOW, priv->token, (uint8_t *)(eth_dev->data->mac_addrs[0].addr_bytes)); if (ret) { - PMD_INIT_LOG(ERR, "DPNI get mac address failed:" - " Error Code = %d\n", ret); - return -ret; - } - - /* ... rx buffer layout ... */ - tot_size = DPAA2_HW_BUF_RESERVE + RTE_PKTMBUF_HEADROOM; - tot_size = RTE_ALIGN_CEIL(tot_size, - DPAA2_PACKET_LAYOUT_ALIGN); - - memset(&layout, 0, sizeof(struct dpni_buffer_layout)); - layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | - DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | - DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | - DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE; - - layout.pass_frame_status = 1; - layout.data_head_room = tot_size - - DPAA2_FD_PTA_SIZE - DPAA2_MBUF_HW_ANNOTATION; - layout.private_data_size = DPAA2_FD_PTA_SIZE; - layout.pass_parser_result = 1; - PMD_INIT_LOG(DEBUG, "Tot_size = %d, head room = %d, private = %d", - tot_size, layout.data_head_room, layout.private_data_size); - ret = dpni_set_buffer_layout(dpni_dev, CMD_PRI_LOW, priv->token, - DPNI_QUEUE_RX, &layout); - if (ret) { - PMD_INIT_LOG(ERR, "Err(%d) in setting rx buffer layout", ret); - return -1; + PMD_INIT_LOG(ERR, "DPNI get mac address failed:Err Code = %d\n", + ret); + goto init_err; } /* ... tx buffer layout ... */ @@ -886,9 +1442,9 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev) ret = dpni_set_buffer_layout(dpni_dev, CMD_PRI_LOW, priv->token, DPNI_QUEUE_TX, &layout); if (ret) { - PMD_INIT_LOG(ERR, "Error (%d) in setting tx buffer" - " layout", ret); - return -1; + PMD_INIT_LOG(ERR, "Error (%d) in setting tx buffer layout", + ret); + goto init_err; } /* ... tx-conf and error buffer layout ... */ @@ -898,19 +1454,21 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev) ret = dpni_set_buffer_layout(dpni_dev, CMD_PRI_LOW, priv->token, DPNI_QUEUE_TX_CONFIRM, &layout); if (ret) { - PMD_INIT_LOG(ERR, "Error (%d) in setting tx-conf buffer" - " layout", ret); - return -1; + PMD_INIT_LOG(ERR, "Error (%d) in setting tx-conf buffer layout", + ret); + goto init_err; } eth_dev->dev_ops = &dpaa2_ethdev_ops; - eth_dev->data->drv_name = rte_dpaa2_pmd.driver.name; - eth_dev->rx_pkt_burst = dpaa2_dev_rx; + eth_dev->rx_pkt_burst = dpaa2_dev_prefetch_rx; eth_dev->tx_pkt_burst = dpaa2_dev_tx; rte_fslmc_vfio_dmamap(); return 0; +init_err: + dpaa2_dev_uninit(eth_dev); + return ret; } static int @@ -924,7 +1482,7 @@ dpaa2_dev_uninit(struct rte_eth_dev *eth_dev) PMD_INIT_FUNC_TRACE(); if (rte_eal_process_type() != RTE_PROC_PRIMARY) - return -EPERM; + return 0; if (!dpni) { PMD_INIT_LOG(WARNING, "Already closed or not started"); @@ -945,22 +1503,23 @@ dpaa2_dev_uninit(struct rte_eth_dev *eth_dev) priv->rx_vq[0] = NULL; } - /* Allocate memory for storing MAC addresses */ + /* free memory for storing MAC addresses */ if (eth_dev->data->mac_addrs) { rte_free(eth_dev->data->mac_addrs); eth_dev->data->mac_addrs = NULL; } - /*Close the device at underlying layer*/ + /* Close the device at underlying layer*/ ret = dpni_close(dpni, CMD_PRI_LOW, priv->token); if (ret) { - PMD_INIT_LOG(ERR, "Failure closing dpni device with" - " error code %d\n", ret); + PMD_INIT_LOG(ERR, + "Failure closing dpni device with err code %d\n", + ret); } - /*Free the allocated memory for ethernet private data and dpni*/ + /* Free the allocated memory for ethernet private data and dpni*/ priv->hw = NULL; - free(dpni); + rte_free(dpni); eth_dev->dev_ops = NULL; eth_dev->rx_pkt_burst = NULL; @@ -970,21 +1529,16 @@ dpaa2_dev_uninit(struct rte_eth_dev *eth_dev) } static int -rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused, +rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv, struct rte_dpaa2_device *dpaa2_dev) { struct rte_eth_dev *eth_dev; - char ethdev_name[RTE_ETH_NAME_MAX_LEN]; - int diag; - sprintf(ethdev_name, "dpni-%d", dpaa2_dev->object_id); - - eth_dev = rte_eth_dev_allocate(ethdev_name); - if (eth_dev == NULL) - return -ENOMEM; - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + eth_dev = rte_eth_dev_allocate(dpaa2_dev->device.name); + if (!eth_dev) + return -ENODEV; eth_dev->data->dev_private = rte_zmalloc( "ethdev private structure", sizeof(struct dpaa2_dev_priv), @@ -995,8 +1549,15 @@ rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused, rte_eth_dev_release_port(eth_dev); return -ENOMEM; } + } else { + eth_dev = rte_eth_dev_attach_secondary(dpaa2_dev->device.name); + if (!eth_dev) + return -ENODEV; } + eth_dev->device = &dpaa2_dev->device; + eth_dev->device->driver = &dpaa2_drv->driver; + dpaa2_dev->eth_dev = eth_dev; eth_dev->data->rx_mbuf_alloc_failed = 0; diff --git a/drivers/net/dpaa2/dpaa2_ethdev.h b/drivers/net/dpaa2/dpaa2_ethdev.h index 7196398f..a2902da2 100644 --- a/drivers/net/dpaa2/dpaa2_ethdev.h +++ b/drivers/net/dpaa2/dpaa2_ethdev.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -47,9 +47,32 @@ /*default tc to be used for ,congestion, distribution etc configuration. */ #define DPAA2_DEF_TC 0 +/* Threshold for a Tx queue to *Enter* Congestion state. + */ +#define CONG_ENTER_TX_THRESHOLD 512 + +/* Threshold for a queue to *Exit* Congestion state. + */ +#define CONG_EXIT_TX_THRESHOLD 480 + +#define CONG_RETRY_COUNT 18000 + +/* RX queue tail drop threshold + * currently considering 32 KB packets + */ +#define CONG_THRESHOLD_RX_Q (64 * 1024) + /* Size of the input SMMU mapped memory required by MC */ #define DIST_PARAM_IOVA_SIZE 256 +/* Enable TX Congestion control support + * default is disable + */ +#define DPAA2_TX_CGR_OFF 0x01 + +/* Disable RX tail drop, default is enable */ +#define DPAA2_RX_TAILDROP_OFF 0x04 + struct dpaa2_dev_priv { void *hw; int32_t hw_id; @@ -62,7 +85,6 @@ struct dpaa2_dev_priv { struct dpaa2_bp_list *bp_list; /**<Attached buffer pool list */ uint32_t options; - uint16_t num_dist_per_tc[MAX_TCS]; uint8_t max_mac_filters; uint8_t max_vlan_filters; uint8_t num_tc; @@ -77,7 +99,8 @@ int dpaa2_remove_flow_dist(struct rte_eth_dev *eth_dev, int dpaa2_attach_bp_list(struct dpaa2_dev_priv *priv, void *blist); -uint16_t dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts); +uint16_t dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, + uint16_t nb_pkts); uint16_t dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts); - +uint16_t dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts); #endif /* _DPAA2_ETHDEV_H */ diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c index c5d49cbe..3c057a39 100644 --- a/drivers/net/dpaa2/dpaa2_rxtx.c +++ b/drivers/net/dpaa2/dpaa2_rxtx.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -40,7 +40,6 @@ #include <rte_memcpy.h> #include <rte_string_fns.h> #include <rte_dev.h> -#include <rte_ethdev.h> #include <fslmc_logs.h> #include <fslmc_vfio.h> @@ -133,6 +132,66 @@ dpaa2_dev_rx_offload(uint64_t hw_annot_addr, struct rte_mbuf *mbuf) } static inline struct rte_mbuf *__attribute__((hot)) +eth_sg_fd_to_mbuf(const struct qbman_fd *fd) +{ + struct qbman_sge *sgt, *sge; + dma_addr_t sg_addr; + int i = 0; + uint64_t fd_addr; + struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp; + + fd_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); + + /* Get Scatter gather table address */ + sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd)); + + sge = &sgt[i++]; + sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge)); + + /* First Scatter gather entry */ + first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); + /* Prepare all the metadata for first segment */ + first_seg->buf_addr = (uint8_t *)sg_addr; + first_seg->ol_flags = 0; + first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); + first_seg->data_len = sge->length & 0x1FFFF; + first_seg->pkt_len = DPAA2_GET_FD_LEN(fd); + first_seg->nb_segs = 1; + first_seg->next = NULL; + + first_seg->packet_type = dpaa2_dev_rx_parse( + (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + + DPAA2_FD_PTA_SIZE); + dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR( + DPAA2_GET_FD_ADDR(fd)) + + DPAA2_FD_PTA_SIZE, first_seg); + rte_mbuf_refcnt_set(first_seg, 1); + cur_seg = first_seg; + while (!DPAA2_SG_IS_FINAL(sge)) { + sge = &sgt[i++]; + sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR( + DPAA2_GET_FLE_ADDR(sge)); + next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, + rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size); + next_seg->buf_addr = (uint8_t *)sg_addr; + next_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); + next_seg->data_len = sge->length & 0x1FFFF; + first_seg->nb_segs += 1; + rte_mbuf_refcnt_set(next_seg, 1); + cur_seg->next = next_seg; + next_seg->next = NULL; + cur_seg = next_seg; + } + temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr, + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); + rte_mbuf_refcnt_set(temp, 1); + rte_pktmbuf_free_seg(temp); + + return (void *)first_seg; +} + +static inline struct rte_mbuf *__attribute__((hot)) eth_fd_to_mbuf(const struct qbman_fd *fd) { struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF( @@ -171,6 +230,80 @@ eth_fd_to_mbuf(const struct qbman_fd *fd) return mbuf; } +static int __attribute__ ((noinline)) __attribute__((hot)) +eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp; + struct qbman_sge *sgt, *sge = NULL; + int i; + + /* First Prepare FD to be transmited*/ + /* Resetting the buffer pool id and offset field*/ + fd->simple.bpid_offset = 0; + + temp = rte_pktmbuf_alloc(mbuf->pool); + if (temp == NULL) { + PMD_TX_LOG(ERR, "No memory to allocate S/G table"); + return -ENOMEM; + } + + DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp)); + DPAA2_SET_FD_LEN(fd, mbuf->pkt_len); + DPAA2_SET_FD_OFFSET(fd, temp->data_off); + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL); + DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg); + /*Set Scatter gather table and Scatter gather entries*/ + sgt = (struct qbman_sge *)( + (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + + DPAA2_GET_FD_OFFSET(fd)); + + for (i = 0; i < mbuf->nb_segs; i++) { + sge = &sgt[i]; + /*Resetting the buffer pool id and offset field*/ + sge->fin_bpid_offset = 0; + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg)); + DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off); + sge->length = cur_seg->data_len; + if (RTE_MBUF_DIRECT(cur_seg)) { + if (rte_mbuf_refcnt_read(cur_seg) > 1) { + /* If refcnt > 1, invalid bpid is set to ensure + * buffer is not freed by HW + */ + DPAA2_SET_FLE_IVP(sge); + rte_mbuf_refcnt_update(cur_seg, -1); + } else + DPAA2_SET_FLE_BPID(sge, + mempool_to_bpid(cur_seg->pool)); + cur_seg = cur_seg->next; + } else { + /* Get owner MBUF from indirect buffer */ + mi = rte_mbuf_from_indirect(cur_seg); + if (rte_mbuf_refcnt_read(mi) > 1) { + /* If refcnt > 1, invalid bpid is set to ensure + * owner buffer is not freed by HW + */ + DPAA2_SET_FLE_IVP(sge); + } else { + DPAA2_SET_FLE_BPID(sge, + mempool_to_bpid(mi->pool)); + rte_mbuf_refcnt_update(mi, 1); + } + prev_seg = cur_seg; + cur_seg = cur_seg->next; + prev_seg->next = NULL; + rte_pktmbuf_free(prev_seg); + } + } + DPAA2_SG_SET_FINAL(sge, true); + return 0; +} + +static void +eth_mbuf_to_fd(struct rte_mbuf *mbuf, + struct qbman_fd *fd, uint16_t bpid) __attribute__((unused)); + static void __attribute__ ((noinline)) __attribute__((hot)) eth_mbuf_to_fd(struct rte_mbuf *mbuf, struct qbman_fd *fd, uint16_t bpid) @@ -190,8 +323,22 @@ eth_mbuf_to_fd(struct rte_mbuf *mbuf, DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); -} + if (RTE_MBUF_DIRECT(mbuf)) { + if (rte_mbuf_refcnt_read(mbuf) > 1) { + DPAA2_SET_FD_IVP(fd); + rte_mbuf_refcnt_update(mbuf, -1); + } + } else { + struct rte_mbuf *mi; + mi = rte_mbuf_from_indirect(mbuf); + if (rte_mbuf_refcnt_read(mi) > 1) + DPAA2_SET_FD_IVP(fd); + else + rte_mbuf_refcnt_update(mi, 1); + rte_pktmbuf_free(mbuf); + } +} static inline int __attribute__((hot)) eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, @@ -242,17 +389,18 @@ eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, } uint16_t -dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { - /* Function is responsible to receive frames for a given device and VQ*/ + /* Function receive frames for a given device and VQ*/ struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; struct qbman_result *dq_storage; uint32_t fqid = dpaa2_q->fqid; int ret, num_rx = 0; uint8_t is_last = 0, status; struct qbman_swp *swp; - const struct qbman_fd *fd; + const struct qbman_fd *fd[DPAA2_DQRR_RING_SIZE]; struct qbman_pull_desc pulldesc; + struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; struct rte_eth_dev *dev = dpaa2_q->dev; if (unlikely(!DPAA2_PER_LCORE_DPIO)) { @@ -263,44 +411,51 @@ dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) } } swp = DPAA2_PER_LCORE_PORTAL; - dq_storage = dpaa2_q->q_storage->dq_storage[0]; - - qbman_pull_desc_clear(&pulldesc); - qbman_pull_desc_set_numframes(&pulldesc, - (nb_pkts > DPAA2_DQRR_RING_SIZE) ? - DPAA2_DQRR_RING_SIZE : nb_pkts); - qbman_pull_desc_set_fq(&pulldesc, fqid); - /* todo optimization - we can have dq_storage_phys available*/ - qbman_pull_desc_set_storage(&pulldesc, dq_storage, + if (!q_storage->active_dqs) { + q_storage->toggle = 0; + dq_storage = q_storage->dq_storage[q_storage->toggle]; + qbman_pull_desc_clear(&pulldesc); + qbman_pull_desc_set_numframes(&pulldesc, + (nb_pkts > DPAA2_DQRR_RING_SIZE) ? + DPAA2_DQRR_RING_SIZE : nb_pkts); + qbman_pull_desc_set_fq(&pulldesc, fqid); + qbman_pull_desc_set_storage(&pulldesc, dq_storage, (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); - - /*Issue a volatile dequeue command. */ - while (1) { - if (qbman_swp_pull(swp, &pulldesc)) { - PMD_RX_LOG(ERR, "VDQ command is not issued." - "QBMAN is busy\n"); - /* Portal was busy, try again */ - continue; + if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { + while (!qbman_check_command_complete(swp, + get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index))) + ; + clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); } - break; - }; - - /* Receive the packets till Last Dequeue entry is found with - * respect to the above issues PULL command. + while (1) { + if (qbman_swp_pull(swp, &pulldesc)) { + PMD_RX_LOG(WARNING, "VDQ command is not issued." + "QBMAN is busy\n"); + /* Portal was busy, try again */ + continue; + } + break; + } + q_storage->active_dqs = dq_storage; + q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; + set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage); + } + dq_storage = q_storage->active_dqs; + /* Check if the previous issued command is completed. + * Also seems like the SWP is shared between the Ethernet Driver + * and the SEC driver. */ + while (!qbman_check_command_complete(swp, dq_storage)) + ; + if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) + clear_swp_active_dqs(q_storage->active_dpio_id); while (!is_last) { - struct rte_mbuf *mbuf; - /*Check if the previous issued command is completed. - * Also seems like the SWP is shared between the - * Ethernet Driver and the SEC driver. - */ - while (!qbman_check_command_complete(swp, dq_storage)) - ; /* Loop until the dq_storage is updated with * new token by QBMAN */ while (!qbman_result_has_new_result(swp, dq_storage)) ; + rte_prefetch0((void *)((uint64_t)(dq_storage + 1))); /* Check whether Last Pull command is Expired and * setting Condition for Loop termination */ @@ -311,27 +466,54 @@ dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) continue; } + fd[num_rx] = qbman_result_DQ_fd(dq_storage); - fd = qbman_result_DQ_fd(dq_storage); - mbuf = (struct rte_mbuf *)DPAA2_IOVA_TO_VADDR( - DPAA2_GET_FD_ADDR(fd) - - rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); - /* Prefeth mbuf */ - rte_prefetch0(mbuf); /* Prefetch Annotation address for the parse results */ - rte_prefetch0((void *)((uint64_t)DPAA2_GET_FD_ADDR(fd) - + DPAA2_FD_PTA_SIZE + 16)); + rte_prefetch0((void *)((uint64_t)DPAA2_GET_FD_ADDR(fd[num_rx]) + + DPAA2_FD_PTA_SIZE + 16)); - bufs[num_rx] = eth_fd_to_mbuf(fd); + if (unlikely(DPAA2_FD_GET_FORMAT(fd[num_rx]) == qbman_fd_sg)) + bufs[num_rx] = eth_sg_fd_to_mbuf(fd[num_rx]); + else + bufs[num_rx] = eth_fd_to_mbuf(fd[num_rx]); bufs[num_rx]->port = dev->data->port_id; - num_rx++; + if (dev->data->dev_conf.rxmode.hw_vlan_strip) + rte_vlan_strip(bufs[num_rx]); + dq_storage++; - } /* End of Packet Rx loop */ + num_rx++; + } + + if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { + while (!qbman_check_command_complete(swp, + get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index))) + ; + clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); + } + q_storage->toggle ^= 1; + dq_storage = q_storage->dq_storage[q_storage->toggle]; + qbman_pull_desc_clear(&pulldesc); + qbman_pull_desc_set_numframes(&pulldesc, DPAA2_DQRR_RING_SIZE); + qbman_pull_desc_set_fq(&pulldesc, fqid); + qbman_pull_desc_set_storage(&pulldesc, dq_storage, + (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); + /* Issue a volatile dequeue command. */ + while (1) { + if (qbman_swp_pull(swp, &pulldesc)) { + PMD_RX_LOG(WARNING, "VDQ command is not issued." + "QBMAN is busy\n"); + continue; + } + break; + } + q_storage->active_dqs = dq_storage; + q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; + set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage); dpaa2_q->rx_pkts += num_rx; - /*Return the total number of packets received to DPAA2 app*/ + /* Return the total number of packets received to DPAA2 app */ return num_rx; } @@ -342,9 +524,10 @@ uint16_t dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { /* Function to transmit the frames to given device and VQ*/ - uint32_t loop; + uint32_t loop, retry_count; int32_t ret; struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; + struct rte_mbuf *mi; uint32_t frames_to_send; struct rte_mempool *mp; struct qbman_eq_desc eqdesc; @@ -375,14 +558,32 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) /*Clear the unused FD fields before sending*/ while (nb_pkts) { + /*Check if the queue is congested*/ + retry_count = 0; + if (qbman_result_SCN_state_in_mem(dpaa2_q->cscn)) { + retry_count++; + /* Retry for some time before giving up */ + if (retry_count > CONG_RETRY_COUNT) + goto skip_tx; + } + frames_to_send = (nb_pkts >> 3) ? MAX_TX_RING_SLOTS : nb_pkts; for (loop = 0; loop < frames_to_send; loop++) { fd_arr[loop].simple.frc = 0; DPAA2_RESET_FD_CTRL((&fd_arr[loop])); DPAA2_SET_FD_FLC((&fd_arr[loop]), NULL); - mp = (*bufs)->pool; + if (RTE_MBUF_DIRECT(*bufs)) { + mp = (*bufs)->pool; + } else { + mi = rte_mbuf_from_indirect(*bufs); + mp = mi->pool; + } /* Not a hw_pkt pool allocated frame */ + if (!mp) { + PMD_TX_LOG(ERR, "err: no bpool attached"); + goto skip_tx; + } if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { PMD_TX_LOG(ERR, "non hw offload bufffer "); /* alloc should be from the default buffer pool @@ -391,11 +592,16 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) if (priv->bp_list) { bpid = priv->bp_list->buf_pool.bpid; } else { - PMD_TX_LOG(ERR, "errr: why no bpool" - " attached"); + PMD_TX_LOG(ERR, + "err: no bpool attached"); num_tx = 0; goto skip_tx; } + if (unlikely((*bufs)->nb_segs > 1)) { + PMD_TX_LOG(ERR, "S/G support not added" + " for non hw offload buffer"); + goto skip_tx; + } if (eth_copy_mbuf_to_fd(*bufs, &fd_arr[loop], bpid)) { bufs++; @@ -403,7 +609,14 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) } } else { bpid = mempool_to_bpid(mp); - eth_mbuf_to_fd(*bufs, &fd_arr[loop], bpid); + if (unlikely((*bufs)->nb_segs > 1)) { + if (eth_mbuf_to_sg_fd(*bufs, + &fd_arr[loop], bpid)) + goto skip_tx; + } else { + eth_mbuf_to_fd(*bufs, + &fd_arr[loop], bpid); + } } bufs++; } @@ -420,3 +633,28 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) skip_tx: return num_tx; } + +/** + * Dummy DPDK callback for TX. + * + * This function is used to temporarily replace the real callback during + * unsafe control operations on the queue, or in case of error. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + (void)queue; + (void)bufs; + (void)nb_pkts; + return 0; +} diff --git a/drivers/net/dpaa2/mc/dpni.c b/drivers/net/dpaa2/mc/dpni.c index 33306140..c2d39691 100644 --- a/drivers/net/dpaa2/mc/dpni.c +++ b/drivers/net/dpaa2/mc/dpni.c @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -444,6 +444,24 @@ int dpni_get_qdid(struct fsl_mc_io *mc_io, return 0; } + +int dpni_set_link_cfg(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpni_link_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_LINK_CFG, + cmd_flags, + token); + DPNI_CMD_SET_LINK_CFG(cmd, cfg); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + int dpni_get_link_state(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, @@ -509,6 +527,47 @@ int dpni_get_max_frame_length(struct fsl_mc_io *mc_io, return 0; } +int dpni_set_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_MCAST_PROMISC, + cmd_flags, + token); + DPNI_CMD_SET_MULTICAST_PROMISC(cmd, en); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_get_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_MCAST_PROMISC, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPNI_RSP_GET_MULTICAST_PROMISC(cmd, *en); + + return 0; +} + int dpni_set_unicast_promisc(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, @@ -591,6 +650,148 @@ int dpni_get_primary_mac_addr(struct fsl_mc_io *mc_io, return 0; } +int dpni_add_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_ADD_MAC_ADDR, + cmd_flags, + token); + DPNI_CMD_ADD_MAC_ADDR(cmd, mac_addr); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_remove_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_REMOVE_MAC_ADDR, + cmd_flags, + token); + DPNI_CMD_REMOVE_MAC_ADDR(cmd, mac_addr); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_clear_mac_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int unicast, + int multicast) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_CLR_MAC_FILTERS, + cmd_flags, + token); + DPNI_CMD_CLEAR_MAC_FILTERS(cmd, unicast, multicast); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_get_port_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_PORT_MAC_ADDR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPNI_RSP_GET_PORT_MAC_ADDR(cmd, mac_addr); + + return 0; +} + +int dpni_enable_vlan_filter(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_ENABLE_VLAN_FILTER, + cmd_flags, + token); + DPNI_CMD_ENABLE_VLAN_FILTER(cmd, en); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_add_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_ADD_VLAN_ID, + cmd_flags, + token); + DPNI_CMD_ADD_VLAN_ID(cmd, vlan_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_remove_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_REMOVE_VLAN_ID, + cmd_flags, + token); + DPNI_CMD_REMOVE_VLAN_ID(cmd, vlan_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_clear_vlan_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_CLR_VLAN_FILTERS, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + int dpni_set_rx_tc_dist(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, @@ -626,6 +827,54 @@ int dpni_set_tx_confirmation_mode(struct fsl_mc_io *mc_io, return mc_send_command(mc_io, &cmd); } +int dpni_set_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + const struct dpni_congestion_notification_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPNI_CMDID_SET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + DPNI_CMD_SET_CONGESTION_NOTIFICATION(cmd, qtype, tc_id, cfg); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_get_congestion_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + struct dpni_congestion_notification_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPNI_CMDID_GET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + DPNI_CMD_GET_CONGESTION_NOTIFICATION(cmd, qtype, tc_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + DPNI_RSP_GET_CONGESTION_NOTIFICATION(cmd, cfg); + + return 0; +} + int dpni_get_api_version(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t *major_ver, @@ -737,3 +986,53 @@ int dpni_reset_statistics(struct fsl_mc_io *mc_io, /* send command to mc*/ return mc_send_command(mc_io, &cmd); } + +int dpni_set_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type q_type, + uint8_t tc, + uint8_t q_index, + struct dpni_taildrop *taildrop) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_TAILDROP, + cmd_flags, + token); + DPNI_CMD_SET_TAILDROP(cmd, cg_point, q_type, tc, q_index, taildrop); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpni_get_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type q_type, + uint8_t tc, + uint8_t q_index, + struct dpni_taildrop *taildrop) +{ + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_TAILDROP, + cmd_flags, + token); + DPNI_CMD_GET_TAILDROP(cmd, cg_point, q_type, tc, q_index); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + DPNI_RSP_GET_TAILDROP(cmd, taildrop); + + return 0; +} diff --git a/drivers/net/dpaa2/mc/fsl_dpkg.h b/drivers/net/dpaa2/mc/fsl_dpkg.h index 3e0f4b0e..2391e401 100644 --- a/drivers/net/dpaa2/mc/fsl_dpkg.h +++ b/drivers/net/dpaa2/mc/fsl_dpkg.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2015 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/net/dpaa2/mc/fsl_dpni.h b/drivers/net/dpaa2/mc/fsl_dpni.h index ef14f858..64db70dc 100644 --- a/drivers/net/dpaa2/mc/fsl_dpni.h +++ b/drivers/net/dpaa2/mc/fsl_dpni.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -72,10 +72,7 @@ struct fsl_mc_io; * All flows within traffic class considered; see dpni_set_queue() */ #define DPNI_ALL_TC_FLOWS (uint16_t)(-1) -/** - * Generate new flow ID; see dpni_set_queue() - */ -#define DPNI_NEW_FLOW_ID (uint16_t)(-1) + /** * Tx traffic is always released to a buffer pool on transmit, there are no * resources allocated to have the frames confirmed back to the source after @@ -743,6 +740,30 @@ union dpni_statistics { #define DPNI_LINK_OPT_ASYM_PAUSE 0x0000000000000008ULL /** + * struct - Structure representing DPNI link configuration + * @rate: Rate + * @options: Mask of available options; use 'DPNI_LINK_OPT_<X>' values + */ +struct dpni_link_cfg { + uint32_t rate; + uint64_t options; +}; + +/** + * dpni_set_link_cfg() - set the link configuration. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cfg: Link configuration + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_link_cfg(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpni_link_cfg *cfg); + +/** * struct dpni_link_state - Structure representing DPNI link state * @rate: Rate * @options: Mask of available options; use 'DPNI_LINK_OPT_<X>' values @@ -800,6 +821,33 @@ int dpni_get_max_frame_length(struct fsl_mc_io *mc_io, uint16_t token, uint16_t *max_frame_length); +/** + * dpni_set_multicast_promisc() - Enable/disable multicast promiscuous mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Set to '1' to enable; '0' to disable + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en); + +/** + * dpni_get_multicast_promisc() - Get multicast promiscuous mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Returns '1' if enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); /** * dpni_set_unicast_promisc() - Enable/disable unicast promiscuous mode @@ -857,6 +905,51 @@ int dpni_get_primary_mac_addr(struct fsl_mc_io *mc_io, uint16_t token, uint8_t mac_addr[6]); +/** + * dpni_add_mac_addr() - Add MAC address filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mac_addr: MAC address to add + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_add_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]); + +/** + * dpni_remove_mac_addr() - Remove MAC address filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mac_addr: MAC address to remove + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_remove_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]); + +/** + * dpni_clear_mac_filters() - Clear all unicast and/or multicast MAC filters + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @unicast: Set to '1' to clear unicast addresses + * @multicast: Set to '1' to clear multicast addresses + * + * The primary MAC address is not cleared by this operation. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_clear_mac_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int unicast, + int multicast); /** * dpni_get_port_mac_addr() - Retrieve MAC address associated to the physical @@ -876,6 +969,60 @@ int dpni_get_port_mac_addr(struct fsl_mc_io *mc_io, uint8_t mac_addr[6]); /** + * dpni_enable_vlan_filter() - Enable/disable VLAN filtering mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Set to '1' to enable; '0' to disable + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_enable_vlan_filter(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en); + +/** + * dpni_add_vlan_id() - Add VLAN ID filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @vlan_id: VLAN ID to add + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_add_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id); + +/** + * dpni_remove_vlan_id() - Remove VLAN ID filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @vlan_id: VLAN ID to remove + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_remove_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id); + +/** + * dpni_clear_vlan_filters() - Clear all VLAN filters + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_clear_vlan_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** * enum dpni_dist_mode - DPNI distribution mode * @DPNI_DIST_MODE_NONE: No distribution * @DPNI_DIST_MODE_HASH: Use hash distribution; only relevant if @@ -961,6 +1108,16 @@ int dpni_set_rx_tc_dist(struct fsl_mc_io *mc_io, uint16_t token, uint8_t tc_id, const struct dpni_rx_tc_dist_cfg *cfg); +/** + * enum dpni_congestion_unit - DPNI congestion units + * @DPNI_CONGESTION_UNIT_BYTES: bytes units + * @DPNI_CONGESTION_UNIT_FRAMES: frames units + */ +enum dpni_congestion_unit { + DPNI_CONGESTION_UNIT_BYTES = 0, + DPNI_CONGESTION_UNIT_FRAMES +}; + /** * enum dpni_dest - DPNI destination types @@ -981,6 +1138,119 @@ enum dpni_dest { DPNI_DEST_DPCON = 2 }; +/** + * struct dpni_dest_cfg - Structure representing DPNI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that + * channel; not relevant for 'DPNI_DEST_NONE' option + */ +struct dpni_dest_cfg { + enum dpni_dest dest_type; + int dest_id; + uint8_t priority; +}; + +/* DPNI congestion options */ + +/** + * CSCN message is written to message_iova once entering a + * congestion state (see 'threshold_entry') + */ +#define DPNI_CONG_OPT_WRITE_MEM_ON_ENTER 0x00000001 +/** + * CSCN message is written to message_iova once exiting a + * congestion state (see 'threshold_exit') + */ +#define DPNI_CONG_OPT_WRITE_MEM_ON_EXIT 0x00000002 +/** + * CSCN write will attempt to allocate into a cache (coherent write); + * valid only if 'DPNI_CONG_OPT_WRITE_MEM_<X>' is selected + */ +#define DPNI_CONG_OPT_COHERENT_WRITE 0x00000004 +/** + * if 'dest_cfg.dest_type != DPNI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once entering a congestion state + * (see 'threshold_entry') + */ +#define DPNI_CONG_OPT_NOTIFY_DEST_ON_ENTER 0x00000008 +/** + * if 'dest_cfg.dest_type != DPNI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once exiting a congestion state + * (see 'threshold_exit') + */ +#define DPNI_CONG_OPT_NOTIFY_DEST_ON_EXIT 0x00000010 +/** + * if 'dest_cfg.dest_type != DPNI_DEST_NONE' when the CSCN is written to the + * sw-portal's DQRR, the DQRI interrupt is asserted immediately (if enabled) + */ +#define DPNI_CONG_OPT_INTR_COALESCING_DISABLED 0x00000020 + +/** + * struct dpni_congestion_notification_cfg - congestion notification + * configuration + * @units: units type + * @threshold_entry: above this threshold we enter a congestion state. + * set it to '0' to disable it + * @threshold_exit: below this threshold we exit the congestion state. + * @message_ctx: The context that will be part of the CSCN message + * @message_iova: I/O virtual address (must be in DMA-able memory), + * must be 16B aligned; valid only if 'DPNI_CONG_OPT_WRITE_MEM_<X>' is + * contained in 'options' + * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel + * @notification_mode: Mask of available options; use 'DPNI_CONG_OPT_<X>' values + */ + +struct dpni_congestion_notification_cfg { + enum dpni_congestion_unit units; + uint32_t threshold_entry; + uint32_t threshold_exit; + uint64_t message_ctx; + uint64_t message_iova; + struct dpni_dest_cfg dest_cfg; + uint16_t notification_mode; +}; + +/** + * dpni_set_congestion_notification() - Set traffic class congestion + * notification configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue - Rx, Tx and Tx confirm types are supported + * @tc_id: Traffic class selection (0-7) + * @cfg: congestion notification configuration + * + * Return: '0' on Success; error code otherwise. + */ +int dpni_set_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + const struct dpni_congestion_notification_cfg *cfg); + +/** + * dpni_get_congestion_notification() - Get traffic class congestion + * notification configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue - Rx, Tx and Tx confirm types are supported + * @tc_id: Traffic class selection (0-7) + * @cfg: congestion notification configuration + * + * Return: '0' on Success; error code otherwise. + */ +int dpni_get_congestion_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + struct dpni_congestion_notification_cfg *cfg); + /** * struct dpni_queue - Queue structure @@ -1077,6 +1347,8 @@ enum dpni_confirmation_mode { * Calling this function with 'mode' set to DPNI_CONF_SINGLE switches all * Tx confirmations to a shared Tx conf queue. The ID of the queue when * calling dpni_set/get_queue is -1. + * Tx confirmation mode can only be changed while the DPNI is disabled. + * Executing this command while the DPNI is enabled will return an error. * * Return: '0' on Success; Error code otherwise. */ @@ -1214,4 +1486,89 @@ int dpni_reset_statistics(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token); +/** + * enum dpni_congestion_point - Structure representing congestion point + * @DPNI_CP_QUEUE: Set taildrop per queue, identified by QUEUE_TYPE, TC and + * QUEUE_INDEX + * @DPNI_CP_GROUP: Set taildrop per queue group. Depending on options used + * to define the DPNI this can be either per + * TC (default) or per interface + * (DPNI_OPT_SHARED_CONGESTION set at DPNI create). + * QUEUE_INDEX is ignored if this type is used. + */ +enum dpni_congestion_point { + DPNI_CP_QUEUE, + DPNI_CP_GROUP, +}; + +/** + * struct dpni_taildrop - Structure representing the taildrop + * @enable: Indicates whether the taildrop is active or not. + * @units: Indicates the unit of THRESHOLD. Queue taildrop only + * supports byte units, this field is ignored and + * assumed = 0 if CONGESTION_POINT is 0. + * @threshold: Threshold value, in units identified by UNITS field. Value 0 + * cannot be used as a valid taildrop threshold, + * THRESHOLD must be > 0 if the taildrop is + * enabled. + */ +struct dpni_taildrop { + char enable; + enum dpni_congestion_unit units; + uint32_t threshold; +}; + +/** + * dpni_set_taildrop() - Set taildrop per queue or TC + * + * Setting a per-TC taildrop (cg_point = DPNI_CP_GROUP) will reset any current + * congestion notification or early drop (WRED) configuration previously applied + * to the same TC. + * + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cg_point: Congestion point. DPNI_CP_QUEUE is only supported in + * combination with DPNI_QUEUE_RX. + * @q_type: Queue type, can be DPNI_QUEUE_RX or DPNI_QUEUE_TX. + * @tc: Traffic class to apply this taildrop to + * @q_index: Index of the queue if the DPNI supports multiple queues for + * traffic distribution. + * Ignored if CONGESTION_POINT is not DPNI_CP_QUEUE. + * @taildrop: Taildrop structure + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type q_type, + uint8_t tc, + uint8_t q_index, + struct dpni_taildrop *taildrop); + +/** + * dpni_get_taildrop() - Get taildrop information + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cg_point: Congestion point + * @q_type: + * @tc: Traffic class to apply this taildrop to + * @q_index: Index of the queue if the DPNI supports multiple queues for + * traffic distribution. Ignored if CONGESTION_POINT + * is not 0. + * @taildrop: Taildrop structure + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type q_type, + uint8_t tc, + uint8_t q_index, + struct dpni_taildrop *taildrop); #endif /* __FSL_DPNI_H */ diff --git a/drivers/net/dpaa2/mc/fsl_dpni_cmd.h b/drivers/net/dpaa2/mc/fsl_dpni_cmd.h index bb92ea89..2ac397cd 100644 --- a/drivers/net/dpaa2/mc/fsl_dpni_cmd.h +++ b/drivers/net/dpaa2/mc/fsl_dpni_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -64,11 +64,22 @@ #define DPNI_CMDID_GET_LINK_STATE ((0x215 << 4) | (0x1)) #define DPNI_CMDID_SET_MAX_FRAME_LENGTH ((0x216 << 4) | (0x1)) #define DPNI_CMDID_GET_MAX_FRAME_LENGTH ((0x217 << 4) | (0x1)) +#define DPNI_CMDID_SET_LINK_CFG ((0x21a << 4) | (0x1)) +#define DPNI_CMDID_SET_MCAST_PROMISC ((0x220 << 4) | (0x1)) +#define DPNI_CMDID_GET_MCAST_PROMISC ((0x221 << 4) | (0x1)) #define DPNI_CMDID_SET_UNICAST_PROMISC ((0x222 << 4) | (0x1)) #define DPNI_CMDID_GET_UNICAST_PROMISC ((0x223 << 4) | (0x1)) #define DPNI_CMDID_SET_PRIM_MAC ((0x224 << 4) | (0x1)) #define DPNI_CMDID_GET_PRIM_MAC ((0x225 << 4) | (0x1)) +#define DPNI_CMDID_ADD_MAC_ADDR ((0x226 << 4) | (0x1)) +#define DPNI_CMDID_REMOVE_MAC_ADDR ((0x227 << 4) | (0x1)) +#define DPNI_CMDID_CLR_MAC_FILTERS ((0x228 << 4) | (0x1)) + +#define DPNI_CMDID_ENABLE_VLAN_FILTER ((0x230 << 4) | (0x1)) +#define DPNI_CMDID_ADD_VLAN_ID ((0x231 << 4) | (0x1)) +#define DPNI_CMDID_REMOVE_VLAN_ID ((0x232 << 4) | (0x1)) +#define DPNI_CMDID_CLR_VLAN_FILTERS ((0x233 << 4) | (0x1)) #define DPNI_CMDID_SET_RX_TC_DIST ((0x235 << 4) | (0x1)) @@ -76,12 +87,16 @@ #define DPNI_CMDID_RESET_STATISTICS ((0x25E << 4) | (0x1)) #define DPNI_CMDID_GET_QUEUE ((0x25F << 4) | (0x1)) #define DPNI_CMDID_SET_QUEUE ((0x260 << 4) | (0x1)) +#define DPNI_CMDID_GET_TAILDROP ((0x261 << 4) | (0x1)) +#define DPNI_CMDID_SET_TAILDROP ((0x262 << 4) | (0x1)) #define DPNI_CMDID_GET_PORT_MAC_ADDR ((0x263 << 4) | (0x1)) #define DPNI_CMDID_GET_BUFFER_LAYOUT ((0x264 << 4) | (0x1)) #define DPNI_CMDID_SET_BUFFER_LAYOUT ((0x265 << 4) | (0x1)) +#define DPNI_CMDID_SET_CONGESTION_NOTIFICATION ((0x267 << 4) | (0x1)) +#define DPNI_CMDID_GET_CONGESTION_NOTIFICATION ((0x268 << 4) | (0x1)) #define DPNI_CMDID_GET_OFFLOAD ((0x26B << 4) | (0x1)) #define DPNI_CMDID_SET_OFFLOAD ((0x26C << 4) | (0x1)) #define DPNI_CMDID_SET_TX_CONFIRMATION_MODE ((0x266 << 4) | (0x1)) @@ -224,6 +239,13 @@ do { \ } while (0) /* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_SET_LINK_CFG(cmd, cfg) \ +do { \ + MC_CMD_OP(cmd, 1, 0, 32, uint32_t, cfg->rate);\ + MC_CMD_OP(cmd, 2, 0, 64, uint64_t, cfg->options);\ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ #define DPNI_RSP_GET_LINK_STATE(cmd, state) \ do { \ MC_RSP_OP(cmd, 0, 32, 1, int, state->up);\ @@ -240,6 +262,14 @@ do { \ MC_RSP_OP(cmd, 0, 0, 16, uint16_t, max_frame_length) /* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_SET_MULTICAST_PROMISC(cmd, en) \ + MC_CMD_OP(cmd, 0, 0, 1, int, en) + +/* cmd, param, offset, width, type, arg_name */ +#define DPNI_RSP_GET_MULTICAST_PROMISC(cmd, en) \ + MC_RSP_OP(cmd, 0, 0, 1, int, en) + +/* cmd, param, offset, width, type, arg_name */ #define DPNI_CMD_SET_UNICAST_PROMISC(cmd, en) \ MC_CMD_OP(cmd, 0, 0, 1, int, en) @@ -269,6 +299,57 @@ do { \ MC_RSP_OP(cmd, 0, 56, 8, uint8_t, mac_addr[0]); \ } while (0) +#define DPNI_RSP_GET_PORT_MAC_ADDR(cmd, mac_addr) \ +do { \ + MC_RSP_OP(cmd, 0, 16, 8, uint8_t, mac_addr[5]); \ + MC_RSP_OP(cmd, 0, 24, 8, uint8_t, mac_addr[4]); \ + MC_RSP_OP(cmd, 0, 32, 8, uint8_t, mac_addr[3]); \ + MC_RSP_OP(cmd, 0, 40, 8, uint8_t, mac_addr[2]); \ + MC_RSP_OP(cmd, 0, 48, 8, uint8_t, mac_addr[1]); \ + MC_RSP_OP(cmd, 0, 56, 8, uint8_t, mac_addr[0]); \ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_ADD_MAC_ADDR(cmd, mac_addr) \ +do { \ + MC_CMD_OP(cmd, 0, 16, 8, uint8_t, mac_addr[5]); \ + MC_CMD_OP(cmd, 0, 24, 8, uint8_t, mac_addr[4]); \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, mac_addr[3]); \ + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, mac_addr[2]); \ + MC_CMD_OP(cmd, 0, 48, 8, uint8_t, mac_addr[1]); \ + MC_CMD_OP(cmd, 0, 56, 8, uint8_t, mac_addr[0]); \ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_REMOVE_MAC_ADDR(cmd, mac_addr) \ +do { \ + MC_CMD_OP(cmd, 0, 16, 8, uint8_t, mac_addr[5]); \ + MC_CMD_OP(cmd, 0, 24, 8, uint8_t, mac_addr[4]); \ + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, mac_addr[3]); \ + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, mac_addr[2]); \ + MC_CMD_OP(cmd, 0, 48, 8, uint8_t, mac_addr[1]); \ + MC_CMD_OP(cmd, 0, 56, 8, uint8_t, mac_addr[0]); \ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_CLEAR_MAC_FILTERS(cmd, unicast, multicast) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 1, int, unicast); \ + MC_CMD_OP(cmd, 0, 1, 1, int, multicast); \ +} while (0) + +/* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_ENABLE_VLAN_FILTER(cmd, en) \ + MC_CMD_OP(cmd, 0, 0, 1, int, en) + +/* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_ADD_VLAN_ID(cmd, vlan_id) \ + MC_CMD_OP(cmd, 0, 32, 16, uint16_t, vlan_id) + +/* cmd, param, offset, width, type, arg_name */ +#define DPNI_CMD_REMOVE_VLAN_ID(cmd, vlan_id) \ + MC_CMD_OP(cmd, 0, 32, 16, uint16_t, vlan_id) + /* cmd, param, offset, width, type, arg_name */ #define DPNI_CMD_SET_RX_TC_DIST(cmd, tc_id, cfg) \ @@ -324,6 +405,33 @@ do { \ MC_RSP_OP(cmd, 0, 16, 16, uint16_t, minor);\ } while (0) +#define DPNI_CMD_GET_TAILDROP(cmd, cp, q_type, tc, q_index) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 8, enum dpni_congestion_point, cp); \ + MC_CMD_OP(cmd, 0, 8, 8, enum dpni_queue_type, q_type); \ + MC_CMD_OP(cmd, 0, 16, 8, uint8_t, tc); \ + MC_CMD_OP(cmd, 0, 24, 8, uint8_t, q_index); \ +} while (0) + +#define DPNI_RSP_GET_TAILDROP(cmd, taildrop) \ +do { \ + MC_RSP_OP(cmd, 1, 0, 1, char, (taildrop)->enable); \ + MC_RSP_OP(cmd, 1, 16, 8, enum dpni_congestion_unit, \ + (taildrop)->units); \ + MC_RSP_OP(cmd, 1, 32, 32, uint32_t, (taildrop)->threshold); \ +} while (0) + +#define DPNI_CMD_SET_TAILDROP(cmd, cp, q_type, tc, q_index, taildrop) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 8, enum dpni_congestion_point, cp); \ + MC_CMD_OP(cmd, 0, 8, 8, enum dpni_queue_type, q_type); \ + MC_CMD_OP(cmd, 0, 16, 8, uint8_t, tc); \ + MC_CMD_OP(cmd, 0, 24, 8, uint8_t, q_index); \ + MC_CMD_OP(cmd, 1, 0, 1, char, (taildrop)->enable); \ + MC_CMD_OP(cmd, 1, 16, 8, enum dpni_congestion_unit, \ + (taildrop)->units); \ + MC_CMD_OP(cmd, 1, 32, 32, uint32_t, (taildrop)->threshold); \ +} while (0) #define DPNI_CMD_SET_TX_CONFIRMATION_MODE(cmd, mode) \ MC_CMD_OP(cmd, 0, 32, 8, enum dpni_confirmation_mode, mode) @@ -331,4 +439,38 @@ do { \ #define DPNI_RSP_GET_TX_CONFIRMATION_MODE(cmd, mode) \ MC_RSP_OP(cmd, 0, 32, 8, enum dpni_confirmation_mode, mode) +#define DPNI_CMD_SET_CONGESTION_NOTIFICATION(cmd, qtype, tc, cfg) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 8, enum dpni_queue_type, qtype); \ + MC_CMD_OP(cmd, 0, 8, 8, uint8_t, tc); \ + MC_CMD_OP(cmd, 1, 0, 32, uint32_t, (cfg)->dest_cfg.dest_id); \ + MC_CMD_OP(cmd, 1, 32, 16, uint16_t, (cfg)->notification_mode); \ + MC_CMD_OP(cmd, 1, 48, 8, uint8_t, (cfg)->dest_cfg.priority); \ + MC_CMD_OP(cmd, 1, 56, 4, enum dpni_dest, (cfg)->dest_cfg.dest_type); \ + MC_CMD_OP(cmd, 1, 60, 2, enum dpni_congestion_unit, (cfg)->units); \ + MC_CMD_OP(cmd, 2, 0, 64, uint64_t, (cfg)->message_iova); \ + MC_CMD_OP(cmd, 3, 0, 64, uint64_t, (cfg)->message_ctx); \ + MC_CMD_OP(cmd, 4, 0, 32, uint32_t, (cfg)->threshold_entry); \ + MC_CMD_OP(cmd, 4, 32, 32, uint32_t, (cfg)->threshold_exit); \ +} while (0) + +#define DPNI_CMD_GET_CONGESTION_NOTIFICATION(cmd, qtype, tc) \ +do { \ + MC_CMD_OP(cmd, 0, 0, 8, enum dpni_queue_type, qtype); \ + MC_CMD_OP(cmd, 0, 8, 8, uint8_t, tc); \ +} while (0) + +#define DPNI_RSP_GET_CONGESTION_NOTIFICATION(cmd, cfg) \ +do { \ + MC_RSP_OP(cmd, 1, 0, 32, uint32_t, (cfg)->dest_cfg.dest_id); \ + MC_RSP_OP(cmd, 1, 0, 16, uint16_t, (cfg)->notification_mode); \ + MC_RSP_OP(cmd, 1, 48, 8, uint8_t, (cfg)->dest_cfg.priority); \ + MC_RSP_OP(cmd, 1, 56, 4, enum dpni_dest, (cfg)->dest_cfg.dest_type); \ + MC_RSP_OP(cmd, 1, 60, 2, enum dpni_congestion_unit, (cfg)->units); \ + MC_RSP_OP(cmd, 2, 0, 64, uint64_t, (cfg)->message_iova); \ + MC_RSP_OP(cmd, 3, 0, 64, uint64_t, (cfg)->message_ctx); \ + MC_RSP_OP(cmd, 4, 0, 32, uint32_t, (cfg)->threshold_entry); \ + MC_RSP_OP(cmd, 4, 32, 32, uint32_t, (cfg)->threshold_exit); \ +} while (0) + #endif /* _FSL_DPNI_CMD_H */ diff --git a/drivers/net/e1000/Makefile b/drivers/net/e1000/Makefile index b5592d6b..ffdf36d3 100644 --- a/drivers/net/e1000/Makefile +++ b/drivers/net/e1000/Makefile @@ -96,6 +96,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += e1000_vf.c SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_pf.c +SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_flow.c SRCS-$(CONFIG_RTE_LIBRTE_EM_PMD) += em_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_EM_PMD) += em_rxtx.c diff --git a/drivers/net/e1000/e1000_ethdev.h b/drivers/net/e1000/e1000_ethdev.h index 8352d0a7..5668910c 100644 --- a/drivers/net/e1000/e1000_ethdev.h +++ b/drivers/net/e1000/e1000_ethdev.h @@ -34,6 +34,7 @@ #ifndef _E1000_ETHDEV_H_ #define _E1000_ETHDEV_H_ #include <rte_time.h> +#include <rte_pci.h> #define E1000_INTEL_VENDOR_ID 0x8086 @@ -82,7 +83,7 @@ #define E1000_MAX_FLEX_FILTER_DWDS \ (E1000_MAX_FLEX_FILTER_LEN / sizeof(uint32_t)) #define E1000_FLEX_FILTERS_MASK_SIZE \ - (E1000_MAX_FLEX_FILTER_DWDS / 4) + (E1000_MAX_FLEX_FILTER_DWDS / 2) #define E1000_FHFT_QUEUEING_LEN 0x0000007F #define E1000_FHFT_QUEUEING_QUEUE 0x00000700 #define E1000_FHFT_QUEUEING_PRIO 0x00070000 @@ -143,6 +144,19 @@ #define EM_TX_MAX_SEG UINT8_MAX #define EM_TX_MAX_MTU_SEG UINT8_MAX +#define MAC_TYPE_FILTER_SUP(type) do {\ + if ((type) != e1000_82580 && (type) != e1000_i350 &&\ + (type) != e1000_82576 && (type) != e1000_i210 &&\ + (type) != e1000_i211)\ + return -ENOTSUP;\ +} while (0) + +#define MAC_TYPE_FILTER_SUP_EXT(type) do {\ + if ((type) != e1000_82580 && (type) != e1000_i350 &&\ + (type) != e1000_i210 && (type) != e1000_i211)\ + return -ENOTSUP; \ +} while (0) + /* structure for interrupt relative data */ struct e1000_interrupt { uint32_t flags; @@ -237,13 +251,19 @@ struct e1000_2tuple_filter { uint16_t queue; /* rx queue assigned to */ }; +/* ethertype filter structure */ +struct igb_ethertype_filter { + uint16_t ethertype; + uint32_t etqf; +}; + /* - * Structure to store filters' info. + * Structure to store filters'info. */ struct e1000_filter_info { uint8_t ethertype_mask; /* Bit mask for every used ethertype filter */ /* store used ethertype filters*/ - uint16_t ethertype_filters[E1000_MAX_ETQF_FILTERS]; + struct igb_ethertype_filter ethertype_filters[E1000_MAX_ETQF_FILTERS]; uint8_t flex_mask; /* Bit mask for every used flex filter */ struct e1000_flex_filter_list flex_list; /* Bit mask for every used 5tuple filter */ @@ -252,6 +272,8 @@ struct e1000_filter_info { /* Bit mask for every used 2tuple filter */ uint8_t twotuple_mask; struct e1000_2tuple_filter_list twotuple_list; + /* store the SYN filter info */ + uint32_t syn_info; }; /* @@ -291,8 +313,55 @@ struct e1000_adapter { #define E1000_DEV_PRIVATE_TO_FILTER_INFO(adapter) \ (&((struct e1000_adapter *)adapter)->filter) -#define E1000_DEV_TO_PCI(eth_dev) \ - RTE_DEV_TO_PCI((eth_dev)->device) +struct rte_flow { + enum rte_filter_type filter_type; + void *rule; +}; + +/* ntuple filter list structure */ +struct igb_ntuple_filter_ele { + TAILQ_ENTRY(igb_ntuple_filter_ele) entries; + struct rte_eth_ntuple_filter filter_info; +}; + +/* ethertype filter list structure */ +struct igb_ethertype_filter_ele { + TAILQ_ENTRY(igb_ethertype_filter_ele) entries; + struct rte_eth_ethertype_filter filter_info; +}; + +/* syn filter list structure */ +struct igb_eth_syn_filter_ele { + TAILQ_ENTRY(igb_eth_syn_filter_ele) entries; + struct rte_eth_syn_filter filter_info; +}; + +/* flex filter list structure */ +struct igb_flex_filter_ele { + TAILQ_ENTRY(igb_flex_filter_ele) entries; + struct rte_eth_flex_filter filter_info; +}; + +/* igb_flow memory list structure */ +struct igb_flow_mem { + TAILQ_ENTRY(igb_flow_mem) entries; + struct rte_flow *flow; + struct rte_eth_dev *dev; +}; + +TAILQ_HEAD(igb_ntuple_filter_list, igb_ntuple_filter_ele); +struct igb_ntuple_filter_list igb_filter_ntuple_list; +TAILQ_HEAD(igb_ethertype_filter_list, igb_ethertype_filter_ele); +struct igb_ethertype_filter_list igb_filter_ethertype_list; +TAILQ_HEAD(igb_syn_filter_list, igb_eth_syn_filter_ele); +struct igb_syn_filter_list igb_filter_syn_list; +TAILQ_HEAD(igb_flex_filter_list, igb_flex_filter_ele); +struct igb_flex_filter_list igb_filter_flex_list; +TAILQ_HEAD(igb_flow_mem_list, igb_flow_mem); +struct igb_flow_mem_list igb_flow_list; + +extern const struct rte_flow_ops igb_flow_ops; + /* * RX/TX IGB function prototypes */ @@ -411,4 +480,24 @@ void em_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, void igb_pf_host_uninit(struct rte_eth_dev *dev); +void igb_filterlist_flush(struct rte_eth_dev *dev); +int igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, + struct e1000_5tuple_filter *filter); +int igb_delete_2tuple_filter(struct rte_eth_dev *dev, + struct e1000_2tuple_filter *filter); +void igb_remove_flex_filter(struct rte_eth_dev *dev, + struct e1000_flex_filter *filter); +int igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, + uint8_t idx); +int igb_add_del_ntuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter, bool add); +int igb_add_del_ethertype_filter(struct rte_eth_dev *dev, + struct rte_eth_ethertype_filter *filter, + bool add); +int eth_igb_syn_filter_set(struct rte_eth_dev *dev, + struct rte_eth_syn_filter *filter, + bool add); +int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, + struct rte_eth_flex_filter *filter, + bool add); #endif /* _E1000_ETHDEV_H_ */ diff --git a/drivers/net/e1000/em_ethdev.c b/drivers/net/e1000/em_ethdev.c index a9bd92bc..3d4ab936 100644 --- a/drivers/net/e1000/em_ethdev.c +++ b/drivers/net/e1000/em_ethdev.c @@ -316,7 +316,7 @@ eth_em_dev_is_ich8(struct e1000_hw *hw) static int eth_em_dev_init(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); @@ -390,7 +390,7 @@ eth_em_dev_init(struct rte_eth_dev *eth_dev) static int eth_em_dev_uninit(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; @@ -593,8 +593,7 @@ eth_em_start(struct rte_eth_dev *dev) E1000_DEV_PRIVATE(dev->data->dev_private); struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = - E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int ret, mask; uint32_t intr_vector = 0; @@ -777,7 +776,7 @@ eth_em_stop(struct rte_eth_dev *dev) { struct rte_eth_link link; struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; em_rxq_intr_disable(hw); @@ -1041,7 +1040,7 @@ static int eth_em_rx_queue_intr_enable(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; em_rxq_intr_enable(hw); @@ -1091,7 +1090,7 @@ eth_em_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ dev_info->max_rx_pktlen = em_get_max_pktlen(hw); dev_info->max_mac_addrs = hw->mac.rar_entry_count; @@ -1598,7 +1597,7 @@ static int eth_em_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) { - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_interrupt *intr = @@ -1671,7 +1670,7 @@ eth_em_interrupt_handler(void *param) eth_em_interrupt_get_status(dev); eth_em_interrupt_action(dev, dev->intr_handle); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } static int diff --git a/drivers/net/e1000/igb_ethdev.c b/drivers/net/e1000/igb_ethdev.c index d18dd48e..e4f7a9fa 100644 --- a/drivers/net/e1000/igb_ethdev.c +++ b/drivers/net/e1000/igb_ethdev.c @@ -138,7 +138,7 @@ static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); -static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev); +static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); static int eth_igb_interrupt_action(struct rte_eth_dev *dev, @@ -213,9 +213,6 @@ static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size); -static int eth_igb_syn_filter_set(struct rte_eth_dev *dev, - struct rte_eth_syn_filter *filter, - bool add); static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, struct rte_eth_syn_filter *filter); static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, @@ -225,9 +222,6 @@ static int igb_add_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); -static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, - struct rte_eth_flex_filter *filter, - bool add); static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, struct rte_eth_flex_filter *filter); static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, @@ -237,17 +231,11 @@ static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); -static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev, - struct rte_eth_ntuple_filter *filter, - bool add); static int igb_get_ntuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *filter); static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); -static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev, - struct rte_eth_ethertype_filter *filter, - bool add); static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); @@ -291,6 +279,7 @@ static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); static void eth_igbvf_interrupt_handler(void *param); static void igbvf_mbx_process(struct rte_eth_dev *dev); +static int igb_filter_restore(struct rte_eth_dev *dev); /* * Define VF Stats MACRO for Non "cleared on read" register @@ -756,11 +745,51 @@ igb_reset_swfw_lock(struct e1000_hw *hw) return E1000_SUCCESS; } +/* Remove all ntuple filters of the device */ +static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); + struct e1000_5tuple_filter *p_5tuple; + struct e1000_2tuple_filter *p_2tuple; + + while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { + TAILQ_REMOVE(&filter_info->fivetuple_list, + p_5tuple, entries); + rte_free(p_5tuple); + } + filter_info->fivetuple_mask = 0; + while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { + TAILQ_REMOVE(&filter_info->twotuple_list, + p_2tuple, entries); + rte_free(p_2tuple); + } + filter_info->twotuple_mask = 0; + + return 0; +} + +/* Remove all flex filters of the device */ +static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); + struct e1000_flex_filter *p_flex; + + while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { + TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); + rte_free(p_flex); + } + filter_info->flex_mask = 0; + + return 0; +} + static int eth_igb_dev_init(struct rte_eth_dev *eth_dev) { int error = 0; - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); struct e1000_vfta * shadow_vfta = @@ -906,12 +935,19 @@ eth_igb_dev_init(struct rte_eth_dev *eth_dev) /* enable support intr */ igb_intr_enable(eth_dev); + /* initialize filter info */ + memset(filter_info, 0, + sizeof(struct e1000_filter_info)); + TAILQ_INIT(&filter_info->flex_list); - filter_info->flex_mask = 0; TAILQ_INIT(&filter_info->twotuple_list); - filter_info->twotuple_mask = 0; TAILQ_INIT(&filter_info->fivetuple_list); - filter_info->fivetuple_mask = 0; + + TAILQ_INIT(&igb_filter_ntuple_list); + TAILQ_INIT(&igb_filter_ethertype_list); + TAILQ_INIT(&igb_filter_syn_list); + TAILQ_INIT(&igb_filter_flex_list); + TAILQ_INIT(&igb_flow_list); return 0; @@ -929,6 +965,8 @@ eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) struct e1000_hw *hw; struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); PMD_INIT_FUNC_TRACE(); @@ -936,7 +974,7 @@ eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) return -EPERM; hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); - pci_dev = E1000_DEV_TO_PCI(eth_dev); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); intr_handle = &pci_dev->intr_handle; if (adapter->stopped == 0) @@ -960,6 +998,23 @@ eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) rte_intr_callback_unregister(intr_handle, eth_igb_interrupt_handler, eth_dev); + /* clear the SYN filter info */ + filter_info->syn_info = 0; + + /* clear the ethertype filters info */ + filter_info->ethertype_mask = 0; + memset(filter_info->ethertype_filters, 0, + E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); + + /* remove all ntuple filters of the device */ + igb_ntuple_filter_uninit(eth_dev); + + /* remove all flex filters of the device */ + igb_flex_filter_uninit(eth_dev); + + /* clear all the filters list */ + igb_filterlist_flush(eth_dev); + return 0; } @@ -994,7 +1049,7 @@ eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) return 0; } - pci_dev = E1000_DEV_TO_PCI(eth_dev); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; @@ -1071,7 +1126,7 @@ eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) { struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); PMD_INIT_FUNC_TRACE(); @@ -1252,7 +1307,7 @@ eth_igb_start(struct rte_eth_dev *dev) E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_adapter *adapter = E1000_DEV_PRIVATE(dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int ret, mask; uint32_t intr_vector = 0; @@ -1417,7 +1472,9 @@ eth_igb_start(struct rte_eth_dev *dev) if (rte_intr_allow_others(intr_handle)) { /* check if lsc interrupt is enabled */ if (dev->data->dev_conf.intr_conf.lsc != 0) - eth_igb_lsc_interrupt_setup(dev); + eth_igb_lsc_interrupt_setup(dev, TRUE); + else + eth_igb_lsc_interrupt_setup(dev, FALSE); } else { rte_intr_callback_unregister(intr_handle, eth_igb_interrupt_handler, @@ -1438,6 +1495,9 @@ eth_igb_start(struct rte_eth_dev *dev) /* resume enabled intr since hw reset */ igb_intr_enable(dev); + /* restore all types filter */ + igb_filter_restore(dev); + PMD_INIT_LOG(DEBUG, "<<"); return 0; @@ -1459,13 +1519,8 @@ static void eth_igb_stop(struct rte_eth_dev *dev) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct e1000_filter_info *filter_info = - E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_eth_link link; - struct e1000_flex_filter *p_flex; - struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next; - struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next; struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; igb_intr_disable(hw); @@ -1494,31 +1549,6 @@ eth_igb_stop(struct rte_eth_dev *dev) memset(&link, 0, sizeof(link)); rte_igb_dev_atomic_write_link_status(dev, &link); - /* Remove all flex filters of the device */ - while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { - TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); - rte_free(p_flex); - } - filter_info->flex_mask = 0; - - /* Remove all ntuple filters of the device */ - for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list); - p_5tuple != NULL; p_5tuple = p_5tuple_next) { - p_5tuple_next = TAILQ_NEXT(p_5tuple, entries); - TAILQ_REMOVE(&filter_info->fivetuple_list, - p_5tuple, entries); - rte_free(p_5tuple); - } - filter_info->fivetuple_mask = 0; - for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list); - p_2tuple != NULL; p_2tuple = p_2tuple_next) { - p_2tuple_next = TAILQ_NEXT(p_2tuple, entries); - TAILQ_REMOVE(&filter_info->twotuple_list, - p_2tuple, entries); - rte_free(p_2tuple); - } - filter_info->twotuple_mask = 0; - if (!rte_intr_allow_others(intr_handle)) /* resume to the default handler */ rte_intr_callback_register(intr_handle, @@ -1566,7 +1596,7 @@ eth_igb_close(struct rte_eth_dev *dev) struct e1000_adapter *adapter = E1000_DEV_PRIVATE(dev->data->dev_private); struct rte_eth_link link; - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; eth_igb_stop(dev); @@ -2152,7 +2182,7 @@ eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ dev_info->max_mac_addrs = hw->mac.rar_entry_count; @@ -2281,7 +2311,7 @@ eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ dev_info->max_mac_addrs = hw->mac.rar_entry_count; @@ -2716,18 +2746,23 @@ eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) * * @param dev * Pointer to struct rte_eth_dev. + * @param on + * Enable or Disable * * @return * - On success, zero. * - On failure, a negative value. */ static int -eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev) +eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) { struct e1000_interrupt *intr = E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); - intr->mask |= E1000_ICR_LSC; + if (on) + intr->mask |= E1000_ICR_LSC; + else + intr->mask &= ~E1000_ICR_LSC; return 0; } @@ -2813,7 +2848,7 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev, E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_interrupt *intr = E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); uint32_t tctl, rctl; struct rte_eth_link link; int ret; @@ -2870,7 +2905,8 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev, E1000_WRITE_REG(hw, E1000_TCTL, tctl); E1000_WRITE_REG(hw, E1000_RCTL, rctl); E1000_WRITE_FLUSH(hw); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } return 0; @@ -2929,7 +2965,8 @@ void igbvf_mbx_process(struct rte_eth_dev *dev) /* PF reset VF event */ if (in_msg == E1000_PF_CONTROL_MSG) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, NULL); } static int @@ -3081,7 +3118,7 @@ eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) #define E1000_RAH_POOLSEL_SHIFT (18) static int eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, - uint32_t index, __rte_unused uint32_t pool) + uint32_t index, uint32_t pool) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t rah; @@ -3229,7 +3266,7 @@ igbvf_dev_start(struct rte_eth_dev *dev) E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_adapter *adapter = E1000_DEV_PRIVATE(dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int ret; uint32_t intr_vector = 0; @@ -3285,7 +3322,7 @@ igbvf_dev_start(struct rte_eth_dev *dev) static void igbvf_dev_stop(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; PMD_INIT_FUNC_TRACE(); @@ -3549,18 +3586,14 @@ eth_igb_rss_reta_query(struct rte_eth_dev *dev, return 0; } -#define MAC_TYPE_FILTER_SUP(type) do {\ - if ((type) != e1000_82580 && (type) != e1000_i350 &&\ - (type) != e1000_82576)\ - return -ENOTSUP;\ -} while (0) - -static int +int eth_igb_syn_filter_set(struct rte_eth_dev *dev, struct rte_eth_syn_filter *filter, bool add) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); uint32_t synqf, rfctl; if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) @@ -3588,6 +3621,7 @@ eth_igb_syn_filter_set(struct rte_eth_dev *dev, synqf = 0; } + filter_info->syn_info = synqf; E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); E1000_WRITE_FLUSH(hw); return 0; @@ -3655,11 +3689,6 @@ eth_igb_syn_filter_handle(struct rte_eth_dev *dev, return ret; } -#define MAC_TYPE_FILTER_SUP_EXT(type) do {\ - if ((type) != e1000_82580 && (type) != e1000_i350)\ - return -ENOSYS; \ -} while (0) - /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ static inline int ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, @@ -3722,6 +3751,54 @@ igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, return NULL; } +/* inject a igb 2tuple filter to HW */ +static inline void +igb_inject_2uple_filter(struct rte_eth_dev *dev, + struct e1000_2tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t ttqf = E1000_TTQF_DISABLE_MASK; + uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; + int i; + + i = filter->index; + imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); + if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ + imir |= E1000_IMIR_PORT_BP; + else + imir &= ~E1000_IMIR_PORT_BP; + + imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + + ttqf |= E1000_TTQF_QUEUE_ENABLE; + ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); + ttqf |= (uint32_t)(filter->filter_info.proto & + E1000_TTQF_PROTOCOL_MASK); + if (filter->filter_info.proto_mask == 0) + ttqf &= ~E1000_TTQF_MASK_ENABLE; + + /* tcp flags bits setting. */ + if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { + if (filter->filter_info.tcp_flags & TCP_URG_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_URG; + if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_ACK; + if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_PSH; + if (filter->filter_info.tcp_flags & TCP_RST_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_RST; + if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_SYN; + if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_FIN; + } else { + imir_ext |= E1000_IMIREXT_CTRL_BP; + } + E1000_WRITE_REG(hw, E1000_IMIR(i), imir); + E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); + E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); +} + /* * igb_add_2tuple_filter - add a 2tuple filter * @@ -3737,12 +3814,9 @@ static int igb_add_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_2tuple_filter *filter; - uint32_t ttqf = E1000_TTQF_DISABLE_MASK; - uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; int i, ret; filter = rte_zmalloc("e1000_2tuple_filter", @@ -3784,39 +3858,25 @@ igb_add_2tuple_filter(struct rte_eth_dev *dev, return -ENOSYS; } - imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); - if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ - imir |= E1000_IMIR_PORT_BP; - else - imir &= ~E1000_IMIR_PORT_BP; + igb_inject_2uple_filter(dev, filter); + return 0; +} - imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; +int +igb_delete_2tuple_filter(struct rte_eth_dev *dev, + struct e1000_2tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); - ttqf |= E1000_TTQF_QUEUE_ENABLE; - ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); - ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK); - if (filter->filter_info.proto_mask == 0) - ttqf &= ~E1000_TTQF_MASK_ENABLE; + filter_info->twotuple_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); + rte_free(filter); - /* tcp flags bits setting. */ - if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { - if (filter->filter_info.tcp_flags & TCP_URG_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_URG; - if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_ACK; - if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_PSH; - if (filter->filter_info.tcp_flags & TCP_RST_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_RST; - if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_SYN; - if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_FIN; - } else - imir_ext |= E1000_IMIREXT_CTRL_BP; - E1000_WRITE_REG(hw, E1000_IMIR(i), imir); - E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); - E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); + E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); + E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } @@ -3835,7 +3895,6 @@ static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_2tuple_filter_info filter_2tuple; @@ -3855,16 +3914,50 @@ igb_remove_2tuple_filter(struct rte_eth_dev *dev, return -ENOENT; } - filter_info->twotuple_mask &= ~(1 << filter->index); - TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); - rte_free(filter); + igb_delete_2tuple_filter(dev, filter); - E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); - E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); - E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } +/* inject a igb flex filter to HW */ +static inline void +igb_inject_flex_filter(struct rte_eth_dev *dev, + struct e1000_flex_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t wufc, queueing; + uint32_t reg_off; + uint8_t i, j = 0; + + wufc = E1000_READ_REG(hw, E1000_WUFC); + if (filter->index < E1000_MAX_FHFT) + reg_off = E1000_FHFT(filter->index); + else + reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); + + E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | + (E1000_WUFC_FLX0 << filter->index)); + queueing = filter->filter_info.len | + (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | + (filter->filter_info.priority << + E1000_FHFT_QUEUEING_PRIO_SHIFT); + E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, + queueing); + + for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { + E1000_WRITE_REG(hw, reg_off, + filter->filter_info.dwords[j]); + reg_off += sizeof(uint32_t); + E1000_WRITE_REG(hw, reg_off, + filter->filter_info.dwords[++j]); + reg_off += sizeof(uint32_t); + E1000_WRITE_REG(hw, reg_off, + (uint32_t)filter->filter_info.mask[i]); + reg_off += sizeof(uint32_t) * 2; + ++j; + } +} + static inline struct e1000_flex_filter * eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, struct e1000_flex_filter_info *key) @@ -3880,18 +3973,48 @@ eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, return NULL; } -static int +/* remove a flex byte filter + * @param + * dev: Pointer to struct rte_eth_dev. + * filter: the pointer of the filter will be removed. + */ +void +igb_remove_flex_filter(struct rte_eth_dev *dev, + struct e1000_flex_filter *filter) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t wufc, i; + uint32_t reg_off; + + wufc = E1000_READ_REG(hw, E1000_WUFC); + if (filter->index < E1000_MAX_FHFT) + reg_off = E1000_FHFT(filter->index); + else + reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); + + for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) + E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); + + E1000_WRITE_REG(hw, E1000_WUFC, wufc & + (~(E1000_WUFC_FLX0 << filter->index))); + + filter_info->flex_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->flex_list, filter, entries); + rte_free(filter); +} + +int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, struct rte_eth_flex_filter *filter, bool add) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_flex_filter *flex_filter, *it; - uint32_t wufc, queueing, mask; - uint32_t reg_off; - uint8_t shift, i, j = 0; + uint32_t mask; + uint8_t shift, i; flex_filter = rte_zmalloc("e1000_flex_filter", sizeof(struct e1000_flex_filter), 0); @@ -3911,15 +4034,20 @@ eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, flex_filter->filter_info.mask[i] = mask; } - wufc = E1000_READ_REG(hw, E1000_WUFC); + it = eth_igb_flex_filter_lookup(&filter_info->flex_list, + &flex_filter->filter_info); + if (it == NULL && !add) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + rte_free(flex_filter); + return -ENOENT; + } + if (it != NULL && add) { + PMD_DRV_LOG(ERR, "filter exists."); + rte_free(flex_filter); + return -EEXIST; + } if (add) { - if (eth_igb_flex_filter_lookup(&filter_info->flex_list, - &flex_filter->filter_info) != NULL) { - PMD_DRV_LOG(ERR, "filter exists."); - rte_free(flex_filter); - return -EEXIST; - } flex_filter->queue = filter->queue; /* * look for an unused flex filter index @@ -3941,52 +4069,10 @@ eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, return -ENOSYS; } - if (flex_filter->index < E1000_MAX_FHFT) - reg_off = E1000_FHFT(flex_filter->index); - else - reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT); - - E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | - (E1000_WUFC_FLX0 << flex_filter->index)); - queueing = filter->len | - (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | - (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT); - E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, - queueing); - for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { - E1000_WRITE_REG(hw, reg_off, - flex_filter->filter_info.dwords[j]); - reg_off += sizeof(uint32_t); - E1000_WRITE_REG(hw, reg_off, - flex_filter->filter_info.dwords[++j]); - reg_off += sizeof(uint32_t); - E1000_WRITE_REG(hw, reg_off, - (uint32_t)flex_filter->filter_info.mask[i]); - reg_off += sizeof(uint32_t) * 2; - ++j; - } - } else { - it = eth_igb_flex_filter_lookup(&filter_info->flex_list, - &flex_filter->filter_info); - if (it == NULL) { - PMD_DRV_LOG(ERR, "filter doesn't exist."); - rte_free(flex_filter); - return -ENOENT; - } - - if (it->index < E1000_MAX_FHFT) - reg_off = E1000_FHFT(it->index); - else - reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); + igb_inject_flex_filter(dev, flex_filter); - for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) - E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); - E1000_WRITE_REG(hw, E1000_WUFC, wufc & - (~(E1000_WUFC_FLX0 << it->index))); - - filter_info->flex_mask &= ~(1 << it->index); - TAILQ_REMOVE(&filter_info->flex_list, it, entries); - rte_free(it); + } else { + igb_remove_flex_filter(dev, it); rte_free(flex_filter); } @@ -4190,6 +4276,64 @@ igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, return NULL; } +/* inject a igb 5-tuple filter to HW */ +static inline void +igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, + struct e1000_5tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; + uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; + uint8_t i; + + i = filter->index; + ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; + if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ + ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; + if (filter->filter_info.dst_ip_mask == 0) + ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; + if (filter->filter_info.src_port_mask == 0) + ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; + if (filter->filter_info.proto_mask == 0) + ftqf &= ~E1000_FTQF_MASK_PROTO_BP; + ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & + E1000_FTQF_QUEUE_MASK; + ftqf |= E1000_FTQF_QUEUE_ENABLE; + E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); + E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); + E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); + + spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; + E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); + + imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); + if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ + imir |= E1000_IMIR_PORT_BP; + else + imir &= ~E1000_IMIR_PORT_BP; + imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + + /* tcp flags bits setting. */ + if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { + if (filter->filter_info.tcp_flags & TCP_URG_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_URG; + if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_ACK; + if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_PSH; + if (filter->filter_info.tcp_flags & TCP_RST_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_RST; + if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_SYN; + if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_FIN; + } else { + imir_ext |= E1000_IMIREXT_CTRL_BP; + } + E1000_WRITE_REG(hw, E1000_IMIR(i), imir); + E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); +} + /* * igb_add_5tuple_filter_82576 - add a 5tuple filter * @@ -4205,12 +4349,9 @@ static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_5tuple_filter *filter; - uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; - uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; uint8_t i; int ret; @@ -4254,50 +4395,29 @@ igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, return -ENOSYS; } - ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; - if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ - ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; - if (filter->filter_info.dst_ip_mask == 0) - ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; - if (filter->filter_info.src_port_mask == 0) - ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; - if (filter->filter_info.proto_mask == 0) - ftqf &= ~E1000_FTQF_MASK_PROTO_BP; - ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & - E1000_FTQF_QUEUE_MASK; - ftqf |= E1000_FTQF_QUEUE_ENABLE; - E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); - E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); - E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); + igb_inject_5tuple_filter_82576(dev, filter); + return 0; +} - spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; - E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); +int +igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, + struct e1000_5tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); - imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); - if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ - imir |= E1000_IMIR_PORT_BP; - else - imir &= ~E1000_IMIR_PORT_BP; - imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + filter_info->fivetuple_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); + rte_free(filter); - /* tcp flags bits setting. */ - if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { - if (filter->filter_info.tcp_flags & TCP_URG_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_URG; - if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_ACK; - if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_PSH; - if (filter->filter_info.tcp_flags & TCP_RST_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_RST; - if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_SYN; - if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_FIN; - } else - imir_ext |= E1000_IMIREXT_CTRL_BP; - E1000_WRITE_REG(hw, E1000_IMIR(i), imir); - E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); + E1000_WRITE_REG(hw, E1000_FTQF(filter->index), + E1000_FTQF_VF_BP | E1000_FTQF_MASK); + E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } @@ -4316,7 +4436,6 @@ static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_5tuple_filter_info filter_5tuple; @@ -4336,17 +4455,8 @@ igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, return -ENOENT; } - filter_info->fivetuple_mask &= ~(1 << filter->index); - TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); - rte_free(filter); + igb_delete_5tuple_filter_82576(dev, filter); - E1000_WRITE_REG(hw, E1000_FTQF(filter->index), - E1000_FTQF_VF_BP | E1000_FTQF_MASK); - E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); - E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); - E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); - E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); - E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } @@ -4412,7 +4522,7 @@ eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) * - On success, zero. * - On failure, a negative value. */ -static int +int igb_add_del_ntuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter, bool add) @@ -4434,7 +4544,9 @@ igb_add_del_ntuple_filter(struct rte_eth_dev *dev, break; case RTE_2TUPLE_FLAGS: case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): - if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) + if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && + hw->mac.type != e1000_i210 && + hw->mac.type != e1000_i211) return -ENOTSUP; if (add) ret = igb_add_2tuple_filter(dev, ntuple_filter); @@ -4576,7 +4688,7 @@ igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, int i; for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { - if (filter_info->ethertype_filters[i] == ethertype && + if (filter_info->ethertype_filters[i].ethertype == ethertype && (filter_info->ethertype_mask & (1 << i))) return i; } @@ -4585,33 +4697,35 @@ igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, static inline int igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, - uint16_t ethertype) + uint16_t ethertype, uint32_t etqf) { int i; for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { if (!(filter_info->ethertype_mask & (1 << i))) { filter_info->ethertype_mask |= 1 << i; - filter_info->ethertype_filters[i] = ethertype; + filter_info->ethertype_filters[i].ethertype = ethertype; + filter_info->ethertype_filters[i].etqf = etqf; return i; } } return -1; } -static inline int +int igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, uint8_t idx) { if (idx >= E1000_MAX_ETQF_FILTERS) return -1; filter_info->ethertype_mask &= ~(1 << idx); - filter_info->ethertype_filters[idx] = 0; + filter_info->ethertype_filters[idx].ethertype = 0; + filter_info->ethertype_filters[idx].etqf = 0; return idx; } -static int +int igb_add_del_ethertype_filter(struct rte_eth_dev *dev, struct rte_eth_ethertype_filter *filter, bool add) @@ -4651,16 +4765,15 @@ igb_add_del_ethertype_filter(struct rte_eth_dev *dev, } if (add) { + etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; + etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); + etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; ret = igb_ethertype_filter_insert(filter_info, - filter->ether_type); + filter->ether_type, etqf); if (ret < 0) { PMD_DRV_LOG(ERR, "ethertype filters are full."); return -ENOSYS; } - - etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; - etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); - etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; } else { ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); if (ret < 0) @@ -4755,7 +4868,7 @@ eth_igb_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = -EINVAL; + int ret = 0; switch (filter_type) { case RTE_ETH_FILTER_NTUPLE: @@ -4770,6 +4883,11 @@ eth_igb_filter_ctrl(struct rte_eth_dev *dev, case RTE_ETH_FILTER_FLEXIBLE: ret = eth_igb_flex_filter_handle(dev, filter_op, arg); break; + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &igb_flow_ops; + break; default: PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", filter_type); @@ -5277,7 +5395,7 @@ eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t mask = 1 << queue_id; uint32_t regval; @@ -5350,7 +5468,7 @@ eth_igb_configure_msix_intr(struct rte_eth_dev *dev) uint32_t vec = E1000_MISC_VEC_ID; uint32_t base = E1000_MISC_VEC_ID; uint32_t misc_shift = 0; - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; /* won't configure msix register if no mapping is done @@ -5422,6 +5540,84 @@ eth_igb_configure_msix_intr(struct rte_eth_dev *dev) E1000_WRITE_FLUSH(hw); } +/* restore n-tuple filter */ +static inline void +igb_ntuple_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_5tuple_filter *p_5tuple; + struct e1000_2tuple_filter *p_2tuple; + + TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { + igb_inject_5tuple_filter_82576(dev, p_5tuple); + } + + TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { + igb_inject_2uple_filter(dev, p_2tuple); + } +} + +/* restore SYN filter */ +static inline void +igb_syn_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + uint32_t synqf; + + synqf = filter_info->syn_info; + + if (synqf & E1000_SYN_FILTER_ENABLE) { + E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); + E1000_WRITE_FLUSH(hw); + } +} + +/* restore ethernet type filter */ +static inline void +igb_ethertype_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + int i; + + for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_mask & (1 << i)) { + E1000_WRITE_REG(hw, E1000_ETQF(i), + filter_info->ethertype_filters[i].etqf); + E1000_WRITE_FLUSH(hw); + } + } +} + +/* restore flex byte filter */ +static inline void +igb_flex_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_flex_filter *flex_filter; + + TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { + igb_inject_flex_filter(dev, flex_filter); + } +} + +/* restore all types filter */ +static int +igb_filter_restore(struct rte_eth_dev *dev) +{ + igb_ntuple_filter_restore(dev); + igb_ethertype_filter_restore(dev); + igb_syn_filter_restore(dev); + igb_flex_filter_restore(dev); + + return 0; +} + RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/drivers/net/e1000/igb_flow.c b/drivers/net/e1000/igb_flow.c new file mode 100644 index 00000000..ed2ecc40 --- /dev/null +++ b/drivers/net/e1000/igb_flow.c @@ -0,0 +1,1707 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/queue.h> +#include <stdio.h> +#include <errno.h> +#include <stdint.h> +#include <stdarg.h> + +#include <rte_common.h> +#include <rte_interrupts.h> +#include <rte_byteorder.h> +#include <rte_log.h> +#include <rte_debug.h> +#include <rte_pci.h> +#include <rte_ether.h> +#include <rte_ethdev.h> +#include <rte_ethdev_pci.h> +#include <rte_memory.h> +#include <rte_memzone.h> +#include <rte_eal.h> +#include <rte_atomic.h> +#include <rte_malloc.h> +#include <rte_dev.h> +#include <rte_flow.h> +#include <rte_flow_driver.h> + +#include "e1000_logs.h" +#include "base/e1000_api.h" +#include "e1000_ethdev.h" + +#define NEXT_ITEM_OF_PATTERN(item, pattern, index) \ + do { \ + item = (pattern) + (index); \ + while (item->type == RTE_FLOW_ITEM_TYPE_VOID) { \ + (index)++; \ + item = (pattern) + (index); \ + } \ + } while (0) + +#define NEXT_ITEM_OF_ACTION(act, actions, index) \ + do { \ + act = (actions) + (index); \ + while (act->type == RTE_FLOW_ACTION_TYPE_VOID) {\ + (index)++; \ + act = (actions) + (index); \ + } \ + } while (0) + +#define IGB_FLEX_RAW_NUM 12 + +/** + * Please aware there's an asumption for all the parsers. + * rte_flow_item is using big endian, rte_flow_attr and + * rte_flow_action are using CPU order. + * Because the pattern is used to describe the packets, + * normally the packets should use network order. + */ + +/** + * Parse the rule to see if it is a n-tuple rule. + * And get the n-tuple filter info BTW. + * pattern: + * The first not void item can be ETH or IPV4. + * The second not void item must be IPV4 if the first one is ETH. + * The third not void item must be UDP or TCP or SCTP + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4 src_addr 192.168.1.20 0xFFFFFFFF + * dst_addr 192.167.3.50 0xFFFFFFFF + * next_proto_id 17 0xFF + * UDP/TCP/ src_port 80 0xFFFF + * SCTP dst_port 80 0xFFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_ntuple_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_ipv4 *ipv4_spec; + const struct rte_flow_item_ipv4 *ipv4_mask; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_item_udp *udp_spec; + const struct rte_flow_item_udp *udp_mask; + const struct rte_flow_item_sctp *sctp_spec; + const struct rte_flow_item_sctp *sctp_mask; + uint32_t index; + + if (!pattern) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* parse pattern */ + index = 0; + + /* the first not void item can be MAC or IPv4 */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + /* Skip Ethernet */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, + EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + /* if the first item is MAC, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + /* check if the next not void item is IPv4 */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + } + + /* get the IPv4 info */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ntuple mask"); + return -rte_errno; + } + /* Not supported last point for range */ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + ipv4_mask = (const struct rte_flow_item_ipv4 *)item->mask; + /** + * Only support src & dst addresses, protocol, + * others should be masked. + */ + + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.type_of_service || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.time_to_live || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_ip_mask = ipv4_mask->hdr.dst_addr; + filter->src_ip_mask = ipv4_mask->hdr.src_addr; + filter->proto_mask = ipv4_mask->hdr.next_proto_id; + + ipv4_spec = (const struct rte_flow_item_ipv4 *)item->spec; + filter->dst_ip = ipv4_spec->hdr.dst_addr; + filter->src_ip = ipv4_spec->hdr.src_addr; + filter->proto = ipv4_spec->hdr.next_proto_id; + + /* check if the next not void item is TCP or UDP or SCTP */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + /* Not supported last point for range */ + if (item->last) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* get the TCP/UDP/SCTP info */ + if (item->type == RTE_FLOW_ITEM_TYPE_TCP) { + if (item->spec && item->mask) { + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + + /** + * Only support src & dst ports, tcp flags, + * others should be masked. + */ + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = tcp_mask->hdr.dst_port; + filter->src_port_mask = tcp_mask->hdr.src_port; + if (tcp_mask->hdr.tcp_flags == 0xFF) { + filter->flags |= RTE_NTUPLE_FLAGS_TCP_FLAG; + } else if (!tcp_mask->hdr.tcp_flags) { + filter->flags &= ~RTE_NTUPLE_FLAGS_TCP_FLAG; + } else { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + filter->dst_port = tcp_spec->hdr.dst_port; + filter->src_port = tcp_spec->hdr.src_port; + filter->tcp_flags = tcp_spec->hdr.tcp_flags; + } + } else if (item->type == RTE_FLOW_ITEM_TYPE_UDP) { + if (item->spec && item->mask) { + udp_mask = (const struct rte_flow_item_udp *)item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = udp_mask->hdr.dst_port; + filter->src_port_mask = udp_mask->hdr.src_port; + + udp_spec = (const struct rte_flow_item_udp *)item->spec; + filter->dst_port = udp_spec->hdr.dst_port; + filter->src_port = udp_spec->hdr.src_port; + } + } else { + if (item->spec && item->mask) { + sctp_mask = (const struct rte_flow_item_sctp *) + item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (sctp_mask->hdr.tag || + sctp_mask->hdr.cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = sctp_mask->hdr.dst_port; + filter->src_port_mask = sctp_mask->hdr.src_port; + + sctp_spec = (const struct rte_flow_item_sctp *) + item->spec; + filter->dst_port = sctp_spec->hdr.dst_port; + filter->src_port = sctp_spec->hdr.src_port; + } + } + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + /* parse action */ + index = 0; + + /** + * n-tuple only supports forwarding, + * check if the first not void action is QUEUE. + */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + item, "Not supported action."); + return -rte_errno; + } + filter->queue = + ((const struct rte_flow_action_queue *)act->conf)->index; + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + if (attr->priority > 0xFFFF) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Error priority."); + return -rte_errno; + } + filter->priority = (uint16_t)attr->priority; + + return 0; +} + +/* a specific function for igb because the flags is specific */ +static int +igb_parse_ntuple_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_ntuple_filter(attr, pattern, actions, filter, error); + + if (ret) + return ret; + + /* Igb doesn't support many priorities. */ + if (filter->priority > E1000_2TUPLE_MAX_PRI) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "Priority not supported by ntuple filter"); + return -rte_errno; + } + + if (hw->mac.type == e1000_82576) { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by ntuple filter"); + return -rte_errno; + } + filter->flags |= RTE_5TUPLE_FLAGS; + } else { + if (filter->src_ip_mask || filter->dst_ip_mask || + filter->src_port_mask) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "only two tuple are " + "supported by this filter"); + return -rte_errno; + } + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by ntuple filter"); + return -rte_errno; + } + filter->flags |= RTE_2TUPLE_FLAGS; + } + + return 0; +} + +/** + * Parse the rule to see if it is a ethertype rule. + * And get the ethertype filter info BTW. + * pattern: + * The first not void item can be ETH. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH type 0x0807 0xFFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_ethertype_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item *pattern, + const struct rte_flow_action *actions, + struct rte_eth_ethertype_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_action_queue *act_q; + uint32_t index; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* Parse pattern */ + index = 0; + + /* The first non-void item should be MAC. */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter"); + return -rte_errno; + } + + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Get the MAC info. */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter"); + return -rte_errno; + } + + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Mask bits of source MAC address must be full of 0. + * Mask bits of destination MAC address must be full + * of 1 or full of 0. + */ + if (!is_zero_ether_addr(ð_mask->src) || + (!is_zero_ether_addr(ð_mask->dst) && + !is_broadcast_ether_addr(ð_mask->dst))) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ether address mask"); + return -rte_errno; + } + + if ((eth_mask->type & UINT16_MAX) != UINT16_MAX) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ethertype mask"); + return -rte_errno; + } + + /* If mask bits of destination MAC address + * are full of 1, set RTE_ETHTYPE_FLAGS_MAC. + */ + if (is_broadcast_ether_addr(ð_mask->dst)) { + filter->mac_addr = eth_spec->dst; + filter->flags |= RTE_ETHTYPE_FLAGS_MAC; + } else { + filter->flags &= ~RTE_ETHTYPE_FLAGS_MAC; + } + filter->ether_type = rte_be_to_cpu_16(eth_spec->type); + + /* Check if the next non-void item is END. */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter."); + return -rte_errno; + } + + /* Parse action */ + + index = 0; + /* Check if the first non-void action is QUEUE or DROP. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && + act->type != RTE_FLOW_ACTION_TYPE_DROP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + if (act->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + } else { + filter->flags |= RTE_ETHTYPE_FLAGS_DROP; + } + + /* Check if the next non-void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* Parse attr */ + /* Must be input direction */ + if (!attr->ingress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->egress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->priority) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + /* Not supported */ + if (attr->group) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + attr, "Not support group."); + return -rte_errno; + } + + return 0; +} + +static int +igb_parse_ethertype_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ethertype_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_ethertype_filter(attr, pattern, + actions, filter, error); + + if (ret) + return ret; + + if (hw->mac.type == e1000_82576) { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) { + memset(filter, 0, sizeof( + struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not supported " + "by ethertype filter"); + return -rte_errno; + } + } else { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof( + struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not supported " + "by ethertype filter"); + return -rte_errno; + } + } + + if (filter->ether_type == ETHER_TYPE_IPv4 || + filter->ether_type == ETHER_TYPE_IPv6) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "IPv4/IPv6 not supported by ethertype filter"); + return -rte_errno; + } + + if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "mac compare is unsupported"); + return -rte_errno; + } + + if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "drop option is unsupported"); + return -rte_errno; + } + + return 0; +} + +/** + * Parse the rule to see if it is a TCP SYN rule. + * And get the TCP SYN filter info BTW. + * pattern: + * The first not void item must be ETH. + * The second not void item must be IPV4 or IPV6. + * The third not void item must be TCP. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4/IPV6 NULL NULL + * TCP tcp_flags 0x02 0xFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_syn_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_syn_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_action_queue *act_q; + uint32_t index; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* parse pattern */ + index = 0; + + /* the first not void item should be MAC or IPv4 or IPv6 or TCP */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6 && + item->type != RTE_FLOW_ITEM_TYPE_TCP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Skip Ethernet */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /* if the item is MAC, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN address mask"); + return -rte_errno; + } + + /* check if the next not void item is IPv4 or IPv6 */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + } + + /* Skip IP */ + if (item->type == RTE_FLOW_ITEM_TYPE_IPV4 || + item->type == RTE_FLOW_ITEM_TYPE_IPV6) { + /* if the item is IP, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN mask"); + return -rte_errno; + } + + /* check if the next not void item is TCP */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + } + + /* Get the TCP info. Only support SYN. */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN mask"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + if (!(tcp_spec->hdr.tcp_flags & TCP_SYN_FLAG) || + tcp_mask->hdr.src_port || + tcp_mask->hdr.dst_port || + tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags != TCP_SYN_FLAG || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + + /* parse action */ + index = 0; + + /* check if the first not void action is QUEUE. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Support 2 priorities, the lowest or highest. */ + if (!attr->priority) { + filter->hig_pri = 0; + } else if (attr->priority == (uint32_t)~0U) { + filter->hig_pri = 1; + } else { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + return 0; +} + +static int +igb_parse_syn_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_syn_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_syn_filter(attr, pattern, + actions, filter, error); + + if (hw->mac.type == e1000_82576) { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by syn filter"); + return -rte_errno; + } + } else { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by syn filter"); + return -rte_errno; + } + } + + if (ret) + return ret; + + return 0; +} + +/** + * Parse the rule to see if it is a flex byte rule. + * And get the flex byte filter info BTW. + * pattern: + * The first not void item must be RAW. + * The second not void item can be RAW or END. + * The third not void item can be RAW or END. + * The last not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * RAW relative 0 0x1 + * offset 0 0xFFFFFFFF + * pattern {0x08, 0x06} {0xFF, 0xFF} + * RAW relative 1 0x1 + * offset 100 0xFFFFFFFF + * pattern {0x11, 0x22, 0x33} {0xFF, 0xFF, 0xFF} + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_flex_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_flex_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_raw *raw_spec; + const struct rte_flow_item_raw *raw_mask; + const struct rte_flow_action_queue *act_q; + uint32_t index, i, offset, total_offset; + uint32_t max_offset = 0; + int32_t shift, j, raw_index = 0; + int32_t relative[IGB_FLEX_RAW_NUM] = {0}; + int32_t raw_offset[IGB_FLEX_RAW_NUM] = {0}; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* parse pattern */ + index = 0; + +item_loop: + + /* the first not void item should be RAW */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + raw_spec = (const struct rte_flow_item_raw *)item->spec; + raw_mask = (const struct rte_flow_item_raw *)item->mask; + + if (!raw_mask->length || + !raw_mask->relative) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + + if (raw_mask->offset) + offset = raw_spec->offset; + else + offset = 0; + + for (j = 0; j < raw_spec->length; j++) { + if (raw_mask->pattern[j] != 0xFF) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + } + + total_offset = 0; + + if (raw_spec->relative) { + for (j = raw_index; j > 0; j--) { + total_offset += raw_offset[j - 1]; + if (!relative[j - 1]) + break; + } + if (total_offset + raw_spec->length + offset > max_offset) + max_offset = total_offset + raw_spec->length + offset; + } else { + if (raw_spec->length + offset > max_offset) + max_offset = raw_spec->length + offset; + } + + if ((raw_spec->length + offset + total_offset) > + RTE_FLEX_FILTER_MAXLEN) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + + if (raw_spec->relative == 0) { + for (j = 0; j < raw_spec->length; j++) + filter->bytes[offset + j] = + raw_spec->pattern[j]; + j = offset / CHAR_BIT; + shift = offset % CHAR_BIT; + } else { + for (j = 0; j < raw_spec->length; j++) + filter->bytes[total_offset + offset + j] = + raw_spec->pattern[j]; + j = (total_offset + offset) / CHAR_BIT; + shift = (total_offset + offset) % CHAR_BIT; + } + + i = 0; + + for ( ; shift < CHAR_BIT; shift++) { + filter->mask[j] |= (0x80 >> shift); + i++; + if (i == raw_spec->length) + break; + if (shift == (CHAR_BIT - 1)) { + j++; + shift = -1; + } + } + + relative[raw_index] = raw_spec->relative; + raw_offset[raw_index] = offset + raw_spec->length; + raw_index++; + + /* check if the next not void item is RAW */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + + /* go back to parser */ + if (item->type == RTE_FLOW_ITEM_TYPE_RAW) { + /* if the item is RAW, the content should be parse */ + goto item_loop; + } + + filter->len = RTE_ALIGN(max_offset, 8); + + /* parse action */ + index = 0; + + /* check if the first not void action is QUEUE. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + if (attr->priority > 0xFFFF) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Error priority."); + return -rte_errno; + } + + filter->priority = (uint16_t)attr->priority; + + return 0; +} + +static int +igb_parse_flex_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_flex_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); + + ret = cons_parse_flex_filter(attr, pattern, + actions, filter, error); + + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not supported by flex filter"); + return -rte_errno; + } + + if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN || + filter->len % sizeof(uint64_t) != 0) { + PMD_DRV_LOG(ERR, "filter's length is out of range"); + return -EINVAL; + } + + if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { + PMD_DRV_LOG(ERR, "filter's priority is out of range"); + return -EINVAL; + } + + if (ret) + return ret; + + return 0; +} + +/** + * Create a flow rule. + * Theorically one rule can match more than one filters. + * We will let it use the filter which it hitt first. + * So, the sequence matters. + */ +static struct rte_flow * +igb_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + int ret; + struct rte_eth_ntuple_filter ntuple_filter; + struct rte_eth_ethertype_filter ethertype_filter; + struct rte_eth_syn_filter syn_filter; + struct rte_eth_flex_filter flex_filter; + struct rte_flow *flow = NULL; + struct igb_ntuple_filter_ele *ntuple_filter_ptr; + struct igb_ethertype_filter_ele *ethertype_filter_ptr; + struct igb_eth_syn_filter_ele *syn_filter_ptr; + struct igb_flex_filter_ele *flex_filter_ptr; + struct igb_flow_mem *igb_flow_mem_ptr; + + flow = rte_zmalloc("igb_rte_flow", sizeof(struct rte_flow), 0); + if (!flow) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + return (struct rte_flow *)flow; + } + igb_flow_mem_ptr = rte_zmalloc("igb_flow_mem", + sizeof(struct igb_flow_mem), 0); + if (!igb_flow_mem_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + rte_free(flow); + return NULL; + } + igb_flow_mem_ptr->flow = flow; + igb_flow_mem_ptr->dev = dev; + TAILQ_INSERT_TAIL(&igb_flow_list, + igb_flow_mem_ptr, entries); + + memset(&ntuple_filter, 0, sizeof(struct rte_eth_ntuple_filter)); + ret = igb_parse_ntuple_filter(dev, attr, pattern, + actions, &ntuple_filter, error); + if (!ret) { + ret = igb_add_del_ntuple_filter(dev, &ntuple_filter, TRUE); + if (!ret) { + ntuple_filter_ptr = rte_zmalloc("igb_ntuple_filter", + sizeof(struct igb_ntuple_filter_ele), 0); + (void)rte_memcpy(&ntuple_filter_ptr->filter_info, + &ntuple_filter, + sizeof(struct rte_eth_ntuple_filter)); + TAILQ_INSERT_TAIL(&igb_filter_ntuple_list, + ntuple_filter_ptr, entries); + flow->rule = ntuple_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_NTUPLE; + return flow; + } + goto out; + } + + memset(ðertype_filter, 0, sizeof(struct rte_eth_ethertype_filter)); + ret = igb_parse_ethertype_filter(dev, attr, pattern, + actions, ðertype_filter, error); + if (!ret) { + ret = igb_add_del_ethertype_filter(dev, + ðertype_filter, TRUE); + if (!ret) { + ethertype_filter_ptr = rte_zmalloc( + "igb_ethertype_filter", + sizeof(struct igb_ethertype_filter_ele), 0); + (void)rte_memcpy(ðertype_filter_ptr->filter_info, + ðertype_filter, + sizeof(struct rte_eth_ethertype_filter)); + TAILQ_INSERT_TAIL(&igb_filter_ethertype_list, + ethertype_filter_ptr, entries); + flow->rule = ethertype_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_ETHERTYPE; + return flow; + } + goto out; + } + + memset(&syn_filter, 0, sizeof(struct rte_eth_syn_filter)); + ret = igb_parse_syn_filter(dev, attr, pattern, + actions, &syn_filter, error); + if (!ret) { + ret = eth_igb_syn_filter_set(dev, &syn_filter, TRUE); + if (!ret) { + syn_filter_ptr = rte_zmalloc("igb_syn_filter", + sizeof(struct igb_eth_syn_filter_ele), 0); + (void)rte_memcpy(&syn_filter_ptr->filter_info, + &syn_filter, + sizeof(struct rte_eth_syn_filter)); + TAILQ_INSERT_TAIL(&igb_filter_syn_list, + syn_filter_ptr, + entries); + flow->rule = syn_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_SYN; + return flow; + } + goto out; + } + + memset(&flex_filter, 0, sizeof(struct rte_eth_flex_filter)); + ret = igb_parse_flex_filter(dev, attr, pattern, + actions, &flex_filter, error); + if (!ret) { + ret = eth_igb_add_del_flex_filter(dev, &flex_filter, TRUE); + if (!ret) { + flex_filter_ptr = rte_zmalloc("igb_flex_filter", + sizeof(struct igb_flex_filter_ele), 0); + (void)rte_memcpy(&flex_filter_ptr->filter_info, + &flex_filter, + sizeof(struct rte_eth_flex_filter)); + TAILQ_INSERT_TAIL(&igb_filter_flex_list, + flex_filter_ptr, entries); + flow->rule = flex_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_FLEXIBLE; + return flow; + } + } + +out: + TAILQ_REMOVE(&igb_flow_list, + igb_flow_mem_ptr, entries); + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to create flow."); + rte_free(igb_flow_mem_ptr); + rte_free(flow); + return NULL; +} + +/** + * Check if the flow rule is supported by igb. + * It only checkes the format. Don't guarantee the rule can be programmed into + * the HW. Because there can be no enough room for the rule. + */ +static int +igb_flow_validate(__rte_unused struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_eth_ntuple_filter ntuple_filter; + struct rte_eth_ethertype_filter ethertype_filter; + struct rte_eth_syn_filter syn_filter; + struct rte_eth_flex_filter flex_filter; + int ret; + + memset(&ntuple_filter, 0, sizeof(struct rte_eth_ntuple_filter)); + ret = igb_parse_ntuple_filter(dev, attr, pattern, + actions, &ntuple_filter, error); + if (!ret) + return 0; + + memset(ðertype_filter, 0, sizeof(struct rte_eth_ethertype_filter)); + ret = igb_parse_ethertype_filter(dev, attr, pattern, + actions, ðertype_filter, error); + if (!ret) + return 0; + + memset(&syn_filter, 0, sizeof(struct rte_eth_syn_filter)); + ret = igb_parse_syn_filter(dev, attr, pattern, + actions, &syn_filter, error); + if (!ret) + return 0; + + memset(&flex_filter, 0, sizeof(struct rte_eth_flex_filter)); + ret = igb_parse_flex_filter(dev, attr, pattern, + actions, &flex_filter, error); + + return ret; +} + +/* Destroy a flow rule on igb. */ +static int +igb_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int ret; + struct rte_flow *pmd_flow = flow; + enum rte_filter_type filter_type = pmd_flow->filter_type; + struct igb_ntuple_filter_ele *ntuple_filter_ptr; + struct igb_ethertype_filter_ele *ethertype_filter_ptr; + struct igb_eth_syn_filter_ele *syn_filter_ptr; + struct igb_flex_filter_ele *flex_filter_ptr; + struct igb_flow_mem *igb_flow_mem_ptr; + + switch (filter_type) { + case RTE_ETH_FILTER_NTUPLE: + ntuple_filter_ptr = (struct igb_ntuple_filter_ele *) + pmd_flow->rule; + ret = igb_add_del_ntuple_filter(dev, + &ntuple_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_ntuple_list, + ntuple_filter_ptr, entries); + rte_free(ntuple_filter_ptr); + } + break; + case RTE_ETH_FILTER_ETHERTYPE: + ethertype_filter_ptr = (struct igb_ethertype_filter_ele *) + pmd_flow->rule; + ret = igb_add_del_ethertype_filter(dev, + ðertype_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_ethertype_list, + ethertype_filter_ptr, entries); + rte_free(ethertype_filter_ptr); + } + break; + case RTE_ETH_FILTER_SYN: + syn_filter_ptr = (struct igb_eth_syn_filter_ele *) + pmd_flow->rule; + ret = eth_igb_syn_filter_set(dev, + &syn_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_syn_list, + syn_filter_ptr, entries); + rte_free(syn_filter_ptr); + } + break; + case RTE_ETH_FILTER_FLEXIBLE: + flex_filter_ptr = (struct igb_flex_filter_ele *) + pmd_flow->rule; + ret = eth_igb_add_del_flex_filter(dev, + &flex_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_flex_list, + flex_filter_ptr, entries); + rte_free(flex_filter_ptr); + } + break; + default: + PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", + filter_type); + ret = -EINVAL; + break; + } + + if (ret) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failed to destroy flow"); + return ret; + } + + TAILQ_FOREACH(igb_flow_mem_ptr, &igb_flow_list, entries) { + if (igb_flow_mem_ptr->flow == pmd_flow) { + TAILQ_REMOVE(&igb_flow_list, + igb_flow_mem_ptr, entries); + rte_free(igb_flow_mem_ptr); + } + } + rte_free(flow); + + return ret; +} + +/* remove all the n-tuple filters */ +static void +igb_clear_all_ntuple_filter(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_5tuple_filter *p_5tuple; + struct e1000_2tuple_filter *p_2tuple; + + while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) + igb_delete_5tuple_filter_82576(dev, p_5tuple); + + while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) + igb_delete_2tuple_filter(dev, p_2tuple); +} + +/* remove all the ether type filters */ +static void +igb_clear_all_ethertype_filter(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + int i; + + for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_mask & (1 << i)) { + (void)igb_ethertype_filter_remove(filter_info, + (uint8_t)i); + E1000_WRITE_REG(hw, E1000_ETQF(i), 0); + E1000_WRITE_FLUSH(hw); + } + } +} + +/* remove the SYN filter */ +static void +igb_clear_syn_filter(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + + if (filter_info->syn_info & E1000_SYN_FILTER_ENABLE) { + filter_info->syn_info = 0; + E1000_WRITE_REG(hw, E1000_SYNQF(0), 0); + E1000_WRITE_FLUSH(hw); + } +} + +/* remove all the flex filters */ +static void +igb_clear_all_flex_filter(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_flex_filter *flex_filter; + + while ((flex_filter = TAILQ_FIRST(&filter_info->flex_list))) + igb_remove_flex_filter(dev, flex_filter); +} + +void +igb_filterlist_flush(struct rte_eth_dev *dev) +{ + struct igb_ntuple_filter_ele *ntuple_filter_ptr; + struct igb_ethertype_filter_ele *ethertype_filter_ptr; + struct igb_eth_syn_filter_ele *syn_filter_ptr; + struct igb_flex_filter_ele *flex_filter_ptr; + struct igb_flow_mem *igb_flow_mem_ptr; + enum rte_filter_type filter_type; + struct rte_flow *pmd_flow; + + TAILQ_FOREACH(igb_flow_mem_ptr, &igb_flow_list, entries) { + if (igb_flow_mem_ptr->dev == dev) { + pmd_flow = igb_flow_mem_ptr->flow; + filter_type = pmd_flow->filter_type; + + switch (filter_type) { + case RTE_ETH_FILTER_NTUPLE: + ntuple_filter_ptr = + (struct igb_ntuple_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_ntuple_list, + ntuple_filter_ptr, entries); + rte_free(ntuple_filter_ptr); + break; + case RTE_ETH_FILTER_ETHERTYPE: + ethertype_filter_ptr = + (struct igb_ethertype_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_ethertype_list, + ethertype_filter_ptr, entries); + rte_free(ethertype_filter_ptr); + break; + case RTE_ETH_FILTER_SYN: + syn_filter_ptr = + (struct igb_eth_syn_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_syn_list, + syn_filter_ptr, entries); + rte_free(syn_filter_ptr); + break; + case RTE_ETH_FILTER_FLEXIBLE: + flex_filter_ptr = + (struct igb_flex_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_flex_list, + flex_filter_ptr, entries); + rte_free(flex_filter_ptr); + break; + default: + PMD_DRV_LOG(WARNING, "Filter type" + "(%d) not supported", filter_type); + break; + } + TAILQ_REMOVE(&igb_flow_list, + igb_flow_mem_ptr, + entries); + rte_free(igb_flow_mem_ptr->flow); + rte_free(igb_flow_mem_ptr); + } + } +} + +/* Destroy all flow rules associated with a port on igb. */ +static int +igb_flow_flush(struct rte_eth_dev *dev, + __rte_unused struct rte_flow_error *error) +{ + igb_clear_all_ntuple_filter(dev); + igb_clear_all_ethertype_filter(dev); + igb_clear_syn_filter(dev); + igb_clear_all_flex_filter(dev); + igb_filterlist_flush(dev); + + return 0; +} + +const struct rte_flow_ops igb_flow_ops = { + .validate = igb_flow_validate, + .create = igb_flow_create, + .destroy = igb_flow_destroy, + .flush = igb_flow_flush, +}; diff --git a/drivers/net/e1000/igb_pf.c b/drivers/net/e1000/igb_pf.c index 923c78a1..6809d30c 100644 --- a/drivers/net/e1000/igb_pf.c +++ b/drivers/net/e1000/igb_pf.c @@ -57,7 +57,7 @@ static inline uint16_t dev_num_vf(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); return pci_dev->max_vfs; } diff --git a/drivers/net/ena/base/ena_plat_dpdk.h b/drivers/net/ena/base/ena_plat_dpdk.h index 7eaebf40..71a8c1e2 100644 --- a/drivers/net/ena/base/ena_plat_dpdk.h +++ b/drivers/net/ena/base/ena_plat_dpdk.h @@ -207,6 +207,7 @@ typedef uint64_t dma_addr_t; snprintf(z_name, sizeof(z_name), \ "ena_alloc_%d", ena_alloc_cnt++); \ mz = rte_memzone_reserve(z_name, size, node, 0); \ + memset(mz->addr, 0, size); \ virt = mz->addr; \ phys = mz->phys_addr; \ } while (0) @@ -219,6 +220,7 @@ typedef uint64_t dma_addr_t; snprintf(z_name, sizeof(z_name), \ "ena_alloc_%d", ena_alloc_cnt++); \ mz = rte_memzone_reserve(z_name, size, node, 0); \ + memset(mz->addr, 0, size); \ virt = mz->addr; \ } while (0) diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c index 806073ca..80ce1f35 100644 --- a/drivers/net/ena/ena_ethdev.c +++ b/drivers/net/ena/ena_ethdev.c @@ -213,12 +213,12 @@ static void ena_tx_queue_release(void *queue); static void ena_rx_queue_release_bufs(struct ena_ring *ring); static void ena_tx_queue_release_bufs(struct ena_ring *ring); static int ena_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); + int wait_to_complete); static int ena_queue_restart(struct ena_ring *ring); static int ena_queue_restart_all(struct rte_eth_dev *dev, enum ena_ring_type ring_type); static void ena_stats_restart(struct rte_eth_dev *dev); -static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, +static void ena_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int ena_rss_reta_update(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, @@ -689,11 +689,10 @@ static void ena_rx_queue_release_bufs(struct ena_ring *ring) static void ena_tx_queue_release_bufs(struct ena_ring *ring) { - unsigned int ring_mask = ring->ring_size - 1; + unsigned int i; - while (ring->next_to_clean != ring->next_to_use) { - struct ena_tx_buffer *tx_buf = - &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; + for (i = 0; i < ring->ring_size; ++i) { + struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; if (tx_buf->mbuf) rte_pktmbuf_free(tx_buf->mbuf); @@ -1289,7 +1288,7 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); adapter->pdev = pci_dev; PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", @@ -1451,7 +1450,7 @@ static void ena_infos_get(struct rte_eth_dev *dev, ena_dev = &adapter->ena_dev; ena_assert_msg(ena_dev != NULL, "Uninitialized device"); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->speed_capa = ETH_LINK_SPEED_1G | @@ -1772,6 +1771,7 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, /* Free whole mbuf chain */ mbuf = tx_info->mbuf; rte_pktmbuf_free(mbuf); + tx_info->mbuf = NULL; /* Put back descriptor to the ring for reuse */ tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile index 2c7496dc..db48ff2d 100644 --- a/drivers/net/enic/Makefile +++ b/drivers/net/enic/Makefile @@ -56,6 +56,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c +SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_flow.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_wq.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_dev.c diff --git a/drivers/net/enic/base/cq_enet_desc.h b/drivers/net/enic/base/cq_enet_desc.h index f9822a45..e8410563 100644 --- a/drivers/net/enic/base/cq_enet_desc.h +++ b/drivers/net/enic/base/cq_enet_desc.h @@ -71,6 +71,19 @@ struct cq_enet_rq_desc { u8 type_color; }; +/* Completion queue descriptor: Ethernet receive queue, 16B */ +struct cq_enet_rq_clsf_desc { + __le16 completed_index_flags; + __le16 q_number_rss_type_flags; + __le16 filter_id; + __le16 lif; + __le16 bytes_written_flags; + __le16 vlan; + __le16 checksum_fcoe; + u8 flags; + u8 type_color; +}; + #define CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT (0x1 << 12) #define CQ_ENET_RQ_DESC_FLAGS_FCOE (0x1 << 13) #define CQ_ENET_RQ_DESC_FLAGS_EOP (0x1 << 14) diff --git a/drivers/net/enic/base/vnic_dev.c b/drivers/net/enic/base/vnic_dev.c index 1cd031ac..49b36555 100644 --- a/drivers/net/enic/base/vnic_dev.c +++ b/drivers/net/enic/base/vnic_dev.c @@ -387,17 +387,24 @@ static int _vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, static int vnic_dev_cmd_proxy(struct vnic_dev *vdev, enum vnic_devcmd_cmd proxy_cmd, enum vnic_devcmd_cmd cmd, - u64 *a0, u64 *a1, int wait) + u64 *args, int nargs, int wait) { u32 status; int err; + /* + * Proxy command consumes 2 arguments. One for proxy index, + * the other is for command to be proxied + */ + if (nargs > VNIC_DEVCMD_NARGS - 2) { + pr_err("number of args %d exceeds the maximum\n", nargs); + return -EINVAL; + } memset(vdev->args, 0, sizeof(vdev->args)); vdev->args[0] = vdev->proxy_index; vdev->args[1] = cmd; - vdev->args[2] = *a0; - vdev->args[3] = *a1; + memcpy(&vdev->args[2], args, nargs * sizeof(args[0])); err = _vnic_dev_cmd(vdev, proxy_cmd, wait); if (err) @@ -412,24 +419,26 @@ static int vnic_dev_cmd_proxy(struct vnic_dev *vdev, return err; } - *a0 = vdev->args[1]; - *a1 = vdev->args[2]; + memcpy(args, &vdev->args[1], nargs * sizeof(args[0])); return 0; } static int vnic_dev_cmd_no_proxy(struct vnic_dev *vdev, - enum vnic_devcmd_cmd cmd, u64 *a0, u64 *a1, int wait) + enum vnic_devcmd_cmd cmd, u64 *args, int nargs, int wait) { int err; - vdev->args[0] = *a0; - vdev->args[1] = *a1; + if (nargs > VNIC_DEVCMD_NARGS) { + pr_err("number of args %d exceeds the maximum\n", nargs); + return -EINVAL; + } + memset(vdev->args, 0, sizeof(vdev->args)); + memcpy(vdev->args, args, nargs * sizeof(args[0])); err = _vnic_dev_cmd(vdev, cmd, wait); - *a0 = vdev->args[0]; - *a1 = vdev->args[1]; + memcpy(args, vdev->args, nargs * sizeof(args[0])); return err; } @@ -455,24 +464,64 @@ void vnic_dev_cmd_proxy_end(struct vnic_dev *vdev) int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, u64 *a0, u64 *a1, int wait) { + u64 args[2]; + int err; + + args[0] = *a0; + args[1] = *a1; memset(vdev->args, 0, sizeof(vdev->args)); switch (vdev->proxy) { case PROXY_BY_INDEX: + err = vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_INDEX, cmd, + args, ARRAY_SIZE(args), wait); + break; + case PROXY_BY_BDF: + err = vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_BDF, cmd, + args, ARRAY_SIZE(args), wait); + break; + case PROXY_NONE: + default: + err = vnic_dev_cmd_no_proxy(vdev, cmd, args, 2, wait); + break; + } + + if (err == 0) { + *a0 = args[0]; + *a1 = args[1]; + } + + return err; +} + +int vnic_dev_cmd_args(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, + u64 *args, int nargs, int wait) +{ + switch (vdev->proxy) { + case PROXY_BY_INDEX: return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_INDEX, cmd, - a0, a1, wait); + args, nargs, wait); case PROXY_BY_BDF: return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_BDF, cmd, - a0, a1, wait); + args, nargs, wait); case PROXY_NONE: default: - return vnic_dev_cmd_no_proxy(vdev, cmd, a0, a1, wait); + return vnic_dev_cmd_no_proxy(vdev, cmd, args, nargs, wait); } } +static int vnic_dev_advanced_filters_cap(struct vnic_dev *vdev, u64 *args, + int nargs) +{ + memset(args, 0, nargs * sizeof(*args)); + args[0] = CMD_ADD_ADV_FILTER; + args[1] = FILTER_CAP_MODE_V1_FLAG; + return vnic_dev_cmd_args(vdev, CMD_CAPABILITY, args, nargs, 1000); +} + int vnic_dev_capable_adv_filters(struct vnic_dev *vdev) { - u64 a0 = (u32)CMD_ADD_ADV_FILTER, a1 = 0; + u64 a0 = CMD_ADD_ADV_FILTER, a1 = 0; int wait = 1000; int err; @@ -482,7 +531,65 @@ int vnic_dev_capable_adv_filters(struct vnic_dev *vdev) return (a1 >= (u32)FILTER_DPDK_1); } -static int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd) +/* Determine the "best" filtering mode VIC is capaible of. Returns one of 3 + * value or 0 on error: + * FILTER_DPDK_1- advanced filters availabile + * FILTER_USNIC_IP_FLAG - advanced filters but with the restriction that + * the IP layer must explicitly specified. I.e. cannot have a UDP + * filter that matches both IPv4 and IPv6. + * FILTER_IPV4_5TUPLE - fallback if either of the 2 above aren't available. + * all other filter types are not available. + * Retrun true in filter_tags if supported + */ +int vnic_dev_capable_filter_mode(struct vnic_dev *vdev, u32 *mode, + u8 *filter_tags) +{ + u64 args[4]; + int err; + u32 max_level = 0; + + err = vnic_dev_advanced_filters_cap(vdev, args, 4); + + /* determine if filter tags are available */ + if (err) + *filter_tags = 0; + if ((args[2] == FILTER_CAP_MODE_V1) && + (args[3] & FILTER_ACTION_FILTER_ID_FLAG)) + *filter_tags = 1; + else + *filter_tags = 0; + + if (err || ((args[0] == 1) && (args[1] == 0))) { + /* Adv filter Command not supported or adv filters available but + * not enabled. Try the normal filter capability command. + */ + args[0] = CMD_ADD_FILTER; + args[1] = 0; + err = vnic_dev_cmd_args(vdev, CMD_CAPABILITY, args, 2, 1000); + if (err) + return err; + max_level = args[1]; + goto parse_max_level; + } else if (args[2] == FILTER_CAP_MODE_V1) { + /* parse filter capability mask in args[1] */ + if (args[1] & FILTER_DPDK_1_FLAG) + *mode = FILTER_DPDK_1; + else if (args[1] & FILTER_USNIC_IP_FLAG) + *mode = FILTER_USNIC_IP; + else if (args[1] & FILTER_IPV4_5TUPLE_FLAG) + *mode = FILTER_IPV4_5TUPLE; + return 0; + } + max_level = args[1]; +parse_max_level: + if (max_level >= (u32)FILTER_USNIC_IP) + *mode = FILTER_USNIC_IP; + else + *mode = FILTER_IPV4_5TUPLE; + return 0; +} + +int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd) { u64 a0 = (u32)cmd, a1 = 0; int wait = 1000; @@ -1017,32 +1124,30 @@ int vnic_dev_set_mac_addr(struct vnic_dev *vdev, u8 *mac_addr) * In case of DEL filter, the caller passes the RQ number. Return * value is irrelevant. * @data: filter data + * @action: action data */ int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry, - struct filter_v2 *data) + struct filter_v2 *data, struct filter_action_v2 *action_v2) { u64 a0 = 0, a1 = 0; int wait = 1000; dma_addr_t tlv_pa; int ret = -EINVAL; struct filter_tlv *tlv, *tlv_va; - struct filter_action *action; u64 tlv_size; - u32 filter_size; + u32 filter_size, action_size; static unsigned int unique_id; char z_name[RTE_MEMZONE_NAMESIZE]; enum vnic_devcmd_cmd dev_cmd; - if (cmd == CLSF_ADD) { - if (data->type == FILTER_DPDK_1) - dev_cmd = CMD_ADD_ADV_FILTER; - else - dev_cmd = CMD_ADD_FILTER; + dev_cmd = (data->type >= FILTER_DPDK_1) ? + CMD_ADD_ADV_FILTER : CMD_ADD_FILTER; filter_size = vnic_filter_size(data); - tlv_size = filter_size + - sizeof(struct filter_action) + + action_size = vnic_action_size(action_v2); + + tlv_size = filter_size + action_size + 2*sizeof(struct filter_tlv); snprintf((char *)z_name, sizeof(z_name), "vnic_clsf_%d", unique_id++); @@ -1063,11 +1168,8 @@ int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry, filter_size); tlv->type = CLSF_TLV_ACTION; - tlv->length = sizeof(struct filter_action); - action = (struct filter_action *)&tlv->val; - action->type = FILTER_ACTION_RQ_STEERING; - action->u.rq_idx = *entry; - + tlv->length = action_size; + memcpy(&tlv->val, (void *)action_v2, action_size); ret = vnic_dev_cmd(vdev, dev_cmd, &a0, &a1, wait); *entry = (u16)a0; vdev->free_consistent(vdev->priv, tlv_size, tlv_va, tlv_pa); diff --git a/drivers/net/enic/base/vnic_dev.h b/drivers/net/enic/base/vnic_dev.h index 06ebd4ce..9a9e6917 100644 --- a/drivers/net/enic/base/vnic_dev.h +++ b/drivers/net/enic/base/vnic_dev.h @@ -135,6 +135,9 @@ void vnic_dev_cmd_proxy_end(struct vnic_dev *vdev); int vnic_dev_fw_info(struct vnic_dev *vdev, struct vnic_devcmd_fw_info **fw_info); int vnic_dev_capable_adv_filters(struct vnic_dev *vdev); +int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd); +int vnic_dev_capable_filter_mode(struct vnic_dev *vdev, u32 *mode, + u8 *filter_tags); int vnic_dev_asic_info(struct vnic_dev *vdev, u16 *asic_type, u16 *asic_rev); int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, size_t size, void *value); @@ -202,7 +205,7 @@ int vnic_dev_enable2_done(struct vnic_dev *vdev, int *status); int vnic_dev_deinit_done(struct vnic_dev *vdev, int *status); int vnic_dev_set_mac_addr(struct vnic_dev *vdev, u8 *mac_addr); int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry, - struct filter_v2 *data); + struct filter_v2 *data, struct filter_action_v2 *action_v2); #ifdef ENIC_VXLAN int vnic_dev_overlay_offload_enable_disable(struct vnic_dev *vdev, u8 overlay, u8 config); diff --git a/drivers/net/enic/base/vnic_devcmd.h b/drivers/net/enic/base/vnic_devcmd.h index 785fd6fd..05d87b91 100644 --- a/drivers/net/enic/base/vnic_devcmd.h +++ b/drivers/net/enic/base/vnic_devcmd.h @@ -92,6 +92,8 @@ #define _CMD_VTYPE(cmd) (((cmd) >> _CMD_VTYPESHIFT) & _CMD_VTYPEMASK) #define _CMD_N(cmd) (((cmd) >> _CMD_NSHIFT) & _CMD_NMASK) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + enum vnic_devcmd_cmd { CMD_NONE = _CMDC(_CMD_DIR_NONE, _CMD_VTYPE_NONE, 0), @@ -598,12 +600,29 @@ enum vnic_devcmd_cmd { * out: (u32) a0=filter identifier * * Capability query: - * out: (u64) a0= 1 if capabliity query supported - * (u64) a1= MAX filter type supported + * in: (u64) a1= supported filter capability exchange modes + * out: (u64) a0= 1 if capability query supported + * if (u64) a1 = 0: a1 = MAX filter type supported + * if (u64) a1 & FILTER_CAP_MODE_V1_FLAG: + * a1 = bitmask of supported filters + * a2 = FILTER_CAP_MODE_V1 + * a3 = bitmask of supported actions */ CMD_ADD_ADV_FILTER = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 77), }; +/* Modes for exchanging advanced filter capabilities. The modes supported by + * the driver are passed in the CMD_ADD_ADV_FILTER capability command and the + * mode selected is returned. + * V0: the maximum filter type supported is returned + * V1: bitmasks of supported filters and actions are returned + */ +enum filter_cap_mode { + FILTER_CAP_MODE_V0 = 0, /* Must always be 0 for legacy drivers */ + FILTER_CAP_MODE_V1 = 1, +}; +#define FILTER_CAP_MODE_V1_FLAG (1 << FILTER_CAP_MODE_V1) + /* CMD_ENABLE2 flags */ #define CMD_ENABLE2_STANDBY 0x0 #define CMD_ENABLE2_ACTIVE 0x1 @@ -837,6 +856,7 @@ struct filter_generic_1 { /* Specifies the filter_action type. */ enum { FILTER_ACTION_RQ_STEERING = 0, + FILTER_ACTION_V2 = 1, FILTER_ACTION_MAX }; @@ -847,6 +867,22 @@ struct filter_action { } u; } __attribute__((packed)); +#define FILTER_ACTION_RQ_STEERING_FLAG (1 << 0) +#define FILTER_ACTION_FILTER_ID_FLAG (1 << 1) +#define FILTER_ACTION_V2_ALL (FILTER_ACTION_RQ_STEERING_FLAG \ + | FILTER_ACTION_FILTER_ID_FLAG) + +/* Version 2 of filter action must be a strict extension of struct filter_action + * where the first fields exactly match in size and meaning. + */ +struct filter_action_v2 { + u32 type; + u32 rq_idx; + u32 flags; /* use FILTER_ACTION_XXX_FLAG defines */ + u16 filter_id; + u_int8_t reserved[32]; /* for future expansion */ +} __attribute__((packed)); + /* Specifies the filter type. */ enum filter_type { FILTER_USNIC_ID = 0, @@ -859,6 +895,21 @@ enum filter_type { FILTER_MAX }; +#define FILTER_USNIC_ID_FLAG (1 << FILTER_USNIC_ID) +#define FILTER_IPV4_5TUPLE_FLAG (1 << FILTER_IPV4_5TUPLE) +#define FILTER_MAC_VLAN_FLAG (1 << FILTER_MAC_VLAN) +#define FILTER_VLAN_IP_3TUPLE_FLAG (1 << FILTER_VLAN_IP_3TUPLE) +#define FILTER_NVGRE_VMQ_FLAG (1 << FILTER_NVGRE_VMQ) +#define FILTER_USNIC_IP_FLAG (1 << FILTER_USNIC_IP) +#define FILTER_DPDK_1_FLAG (1 << FILTER_DPDK_1) +#define FILTER_V1_ALL (FILTER_USNIC_ID_FLAG | \ + FILTER_IPV4_5TUPLE_FLAG | \ + FILTER_MAC_VLAN_FLAG | \ + FILTER_VLAN_IP_3TUPLE_FLAG | \ + FILTER_NVGRE_VMQ_FLAG | \ + FILTER_USNIC_IP_FLAG | \ + FILTER_DPDK_1_FLAG) + struct filter { u32 type; union { @@ -903,7 +954,7 @@ struct filter_tlv { /* Data for CMD_ADD_FILTER is 2 TLV and filter + action structs */ #define FILTER_MAX_BUF_SIZE 100 #define FILTER_V2_MAX_BUF_SIZE (sizeof(struct filter_v2) + \ - sizeof(struct filter_action) + \ + sizeof(struct filter_action_v2) + \ (2 * sizeof(struct filter_tlv))) /* @@ -949,6 +1000,30 @@ enum { }; /* + * Get the action structure size given action type. To be "future-proof," + * drivers should use this instead of "sizeof (struct filter_action_v2)" + * when computing length for TLV. + */ +static inline u_int32_t +vnic_action_size(struct filter_action_v2 *fap) +{ + u_int32_t size; + + switch (fap->type) { + case FILTER_ACTION_RQ_STEERING: + size = sizeof(struct filter_action); + break; + case FILTER_ACTION_V2: + size = sizeof(struct filter_action_v2); + break; + default: + size = sizeof(struct filter_action); + break; + } + return size; +} + +/* * Writing cmd register causes STAT_BUSY to get set in status register. * When cmd completes, STAT_BUSY will be cleared. * diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h index d17a35f4..e28f2235 100644 --- a/drivers/net/enic/enic.h +++ b/drivers/net/enic/enic.h @@ -80,6 +80,8 @@ #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */ #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */ +/* Special Filter id for non-specific packet flagging. Don't change value */ +#define ENIC_MAGIC_FILTER_ID 0xffff #define ENICPMD_FDIR_MAX 64 @@ -111,6 +113,12 @@ struct enic_memzone_entry { LIST_ENTRY(enic_memzone_entry) entries; }; +struct rte_flow { + LIST_ENTRY(rte_flow) next; + u16 enic_filter_id; + struct filter_v2 enic_filter; +}; + /* Per-instance private data structure */ struct enic { struct enic *next; @@ -135,7 +143,9 @@ struct enic { int link_status; u8 hw_ip_checksum; u16 max_mtu; - u16 adv_filters; + u8 adv_filters; + u32 flow_filter_mode; + u8 filter_tags; unsigned int flags; unsigned int priv_flags; @@ -170,6 +180,8 @@ struct enic { rte_spinlock_t memzone_list_lock; rte_spinlock_t mtu_lock; + LIST_HEAD(enic_flows, rte_flow) flows; + rte_spinlock_t flows_lock; }; /* Get the CQ index from a Start of Packet(SOP) RQ index */ @@ -293,9 +305,9 @@ extern int enic_clsf_init(struct enic *enic); extern void enic_clsf_destroy(struct enic *enic); uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); -uint16_t enic_dummy_recv_pkts(__rte_unused void *rx_queue, - __rte_unused struct rte_mbuf **rx_pkts, - __rte_unused uint16_t nb_pkts); +uint16_t enic_dummy_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); int enic_set_mtu(struct enic *enic, uint16_t new_mtu); @@ -303,7 +315,8 @@ int enic_link_update(struct enic *enic); void enic_fdir_info(struct enic *enic); void enic_fdir_info_get(struct enic *enic, struct rte_eth_fdir_info *stats); void copy_fltr_v1(struct filter_v2 *fltr, struct rte_eth_fdir_input *input, - __rte_unused struct rte_eth_fdir_masks *masks); + struct rte_eth_fdir_masks *masks); void copy_fltr_v2(struct filter_v2 *fltr, struct rte_eth_fdir_input *input, struct rte_eth_fdir_masks *masks); +extern const struct rte_flow_ops enic_flow_ops; #endif /* _ENIC_H_ */ diff --git a/drivers/net/enic/enic_clsf.c b/drivers/net/enic/enic_clsf.c index 487f8044..9b461424 100644 --- a/drivers/net/enic/enic_clsf.c +++ b/drivers/net/enic/enic_clsf.c @@ -57,7 +57,7 @@ #include "vnic_intr.h" #include "vnic_nic.h" -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef RTE_ARCH_X86 #include <rte_hash_crc.h> #define DEFAULT_HASH_FUNC rte_hash_crc #else @@ -345,7 +345,7 @@ int enic_fdir_del_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) /* Delete the filter */ vnic_dev_classifier(enic->vdev, CLSF_DEL, - &key->fltr_id, NULL); + &key->fltr_id, NULL, NULL); rte_free(key); enic->fdir.nodes[pos] = NULL; enic->fdir.stats.free++; @@ -365,8 +365,10 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) u32 flowtype_supported; u16 flex_bytes; u16 queue; + struct filter_action_v2 action; memset(&fltr, 0, sizeof(fltr)); + memset(&action, 0, sizeof(action)); flowtype_supported = enic->fdir.types_mask & (1 << params->input.flow_type); @@ -439,7 +441,7 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) * Delete the filter and add the modified one later */ vnic_dev_classifier(enic->vdev, CLSF_DEL, - &key->fltr_id, NULL); + &key->fltr_id, NULL, NULL); enic->fdir.stats.free++; } @@ -451,8 +453,11 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) enic->fdir.copy_fltr_fn(&fltr, ¶ms->input, &enic->rte_dev->data->dev_conf.fdir_conf.mask); + action.type = FILTER_ACTION_RQ_STEERING; + action.rq_idx = queue; - if (!vnic_dev_classifier(enic->vdev, CLSF_ADD, &queue, &fltr)) { + if (!vnic_dev_classifier(enic->vdev, CLSF_ADD, &queue, &fltr, + &action)) { key->fltr_id = queue; } else { dev_err(enic, "Add classifier entry failed\n"); @@ -462,7 +467,8 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) } if (do_free) - vnic_dev_classifier(enic->vdev, CLSF_DEL, &old_fltr_id, NULL); + vnic_dev_classifier(enic->vdev, CLSF_DEL, &old_fltr_id, NULL, + NULL); else{ enic->fdir.stats.free--; enic->fdir.stats.add++; @@ -488,7 +494,7 @@ void enic_clsf_destroy(struct enic *enic) key = enic->fdir.nodes[index]; if (key) { vnic_dev_classifier(enic->vdev, CLSF_DEL, - &key->fltr_id, NULL); + &key->fltr_id, NULL, NULL); rte_free(key); enic->fdir.nodes[index] = NULL; } diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c index 331cd5ef..da8fec2d 100644 --- a/drivers/net/enic/enic_ethdev.c +++ b/drivers/net/enic/enic_ethdev.c @@ -116,13 +116,25 @@ enicpmd_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = -EINVAL; + int ret = 0; + + ENICPMD_FUNC_TRACE(); - if (RTE_ETH_FILTER_FDIR == filter_type) + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &enic_flow_ops; + break; + case RTE_ETH_FILTER_FDIR: ret = enicpmd_fdir_ctrl_func(dev, filter_op, arg); - else + break; + default: dev_warning(enic, "Filter type (%d) not supported", filter_type); + ret = -EINVAL; + break; + } return ret; } @@ -455,7 +467,7 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, struct enic *enic = pmd_priv(eth_dev); ENICPMD_FUNC_TRACE(); - device_info->pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + device_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); /* Scattered Rx uses two receive queues per rx queue exposed to dpdk */ device_info->max_rx_queues = enic->conf_rq_count / 2; device_info->max_tx_queues = enic->conf_wq_count; @@ -619,7 +631,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) eth_dev->rx_pkt_burst = &enic_recv_pkts; eth_dev->tx_pkt_burst = &enic_xmit_pkts; - pdev = RTE_DEV_TO_PCI(eth_dev->device); + pdev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pdev); enic->pdev = pdev; addr = &pdev->addr; diff --git a/drivers/net/enic/enic_flow.c b/drivers/net/enic/enic_flow.c new file mode 100644 index 00000000..a728d077 --- /dev/null +++ b/drivers/net/enic/enic_flow.c @@ -0,0 +1,1544 @@ +/* + * Copyright (c) 2017, Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <errno.h> +#include <rte_log.h> +#include <rte_ethdev.h> +#include <rte_flow_driver.h> +#include <rte_ether.h> +#include <rte_ip.h> +#include <rte_udp.h> + +#include "enic_compat.h" +#include "enic.h" +#include "vnic_dev.h" +#include "vnic_nic.h" + +#ifdef RTE_LIBRTE_ENIC_DEBUG_FLOW +#define FLOW_TRACE() \ + RTE_LOG(DEBUG, PMD, "%s()\n", __func__) +#define FLOW_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, fmt, ## args) +#else +#define FLOW_TRACE() do { } while (0) +#define FLOW_LOG(level, fmt, args...) do { } while (0) +#endif + +/** Info about how to copy items into enic filters. */ +struct enic_items { + /** Function for copying and validating an item. */ + int (*copy_item)(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst); + /** List of valid previous items. */ + const enum rte_flow_item_type * const prev_items; + /** True if it's OK for this item to be the first item. For some NIC + * versions, it's invalid to start the stack above layer 3. + */ + const u8 valid_start_item; +}; + +/** Filtering capabilities for various NIC and firmware versions. */ +struct enic_filter_cap { + /** list of valid items and their handlers and attributes. */ + const struct enic_items *item_info; +}; + +/* functions for copying flow actions into enic actions */ +typedef int (copy_action_fn)(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action); + +/* functions for copying items into enic filters */ +typedef int(enic_copy_item_fn)(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst); + +/** Action capabilities for various NICs. */ +struct enic_action_cap { + /** list of valid actions */ + const enum rte_flow_action_type *actions; + /** copy function for a particular NIC */ + int (*copy_fn)(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action); +}; + +/* Forward declarations */ +static enic_copy_item_fn enic_copy_item_ipv4_v1; +static enic_copy_item_fn enic_copy_item_udp_v1; +static enic_copy_item_fn enic_copy_item_tcp_v1; +static enic_copy_item_fn enic_copy_item_eth_v2; +static enic_copy_item_fn enic_copy_item_vlan_v2; +static enic_copy_item_fn enic_copy_item_ipv4_v2; +static enic_copy_item_fn enic_copy_item_ipv6_v2; +static enic_copy_item_fn enic_copy_item_udp_v2; +static enic_copy_item_fn enic_copy_item_tcp_v2; +static enic_copy_item_fn enic_copy_item_sctp_v2; +static enic_copy_item_fn enic_copy_item_sctp_v2; +static enic_copy_item_fn enic_copy_item_vxlan_v2; +static copy_action_fn enic_copy_action_v1; +static copy_action_fn enic_copy_action_v2; + +/** + * Legacy NICs or NICs with outdated firmware. Only 5-tuple perfect match + * is supported. + */ +static const struct enic_items enic_items_v1[] = { + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .copy_item = enic_copy_item_ipv4_v1, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .copy_item = enic_copy_item_udp_v1, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .copy_item = enic_copy_item_tcp_v1, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** + * NICs have Advanced Filters capability but they are disabled. This means + * that layer 3 must be specified. + */ +static const struct enic_items enic_items_v2[] = { + [RTE_FLOW_ITEM_TYPE_ETH] = { + .copy_item = enic_copy_item_eth_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .copy_item = enic_copy_item_vlan_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .copy_item = enic_copy_item_ipv4_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV6] = { + .copy_item = enic_copy_item_ipv6_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .copy_item = enic_copy_item_udp_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .copy_item = enic_copy_item_tcp_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_SCTP] = { + .copy_item = enic_copy_item_sctp_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VXLAN] = { + .copy_item = enic_copy_item_vxlan_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** NICs with Advanced filters enabled */ +static const struct enic_items enic_items_v3[] = { + [RTE_FLOW_ITEM_TYPE_ETH] = { + .copy_item = enic_copy_item_eth_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .copy_item = enic_copy_item_vlan_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .copy_item = enic_copy_item_ipv4_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV6] = { + .copy_item = enic_copy_item_ipv6_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .copy_item = enic_copy_item_udp_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .copy_item = enic_copy_item_tcp_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_SCTP] = { + .copy_item = enic_copy_item_sctp_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VXLAN] = { + .copy_item = enic_copy_item_vxlan_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** Filtering capabilities indexed this NICs supported filter type. */ +static const struct enic_filter_cap enic_filter_cap[] = { + [FILTER_IPV4_5TUPLE] = { + .item_info = enic_items_v1, + }, + [FILTER_USNIC_IP] = { + .item_info = enic_items_v2, + }, + [FILTER_DPDK_1] = { + .item_info = enic_items_v3, + }, +}; + +/** Supported actions for older NICs */ +static const enum rte_flow_action_type enic_supported_actions_v1[] = { + RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_END, +}; + +/** Supported actions for newer NICs */ +static const enum rte_flow_action_type enic_supported_actions_v2[] = { + RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_MARK, + RTE_FLOW_ACTION_TYPE_FLAG, + RTE_FLOW_ACTION_TYPE_END, +}; + +/** Action capabilities indexed by NIC version information */ +static const struct enic_action_cap enic_action_cap[] = { + [FILTER_ACTION_RQ_STEERING_FLAG] = { + .actions = enic_supported_actions_v1, + .copy_fn = enic_copy_action_v1, + }, + [FILTER_ACTION_V2_ALL] = { + .actions = enic_supported_actions_v2, + .copy_fn = enic_copy_action_v2, + }, +}; + +static int +mask_exact_match(const u8 *supported, const u8 *supplied, + unsigned int size) +{ + unsigned int i; + for (i = 0; i < size; i++) { + if (supported[i] != supplied[i]) + return 0; + } + return 1; +} + +/** + * Copy IPv4 item into version 1 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Should always be 0 for version 1. + */ +static int +enic_copy_item_ipv4_v1(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = item->mask; + struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; + struct ipv4_hdr supported_mask = { + .src_addr = 0xffffffff, + .dst_addr = 0xffffffff, + }; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + if (!mask) + mask = &rte_flow_item_ipv4_mask; + + /* This is an exact match filter, both fields must be set */ + if (!spec || !spec->hdr.src_addr || !spec->hdr.dst_addr) { + FLOW_LOG(ERR, "IPv4 exact match src/dst addr"); + return ENOTSUP; + } + + /* check that the suppied mask exactly matches capabilty */ + if (!mask_exact_match((const u8 *)&supported_mask, + (const u8 *)item->mask, sizeof(*mask))) { + FLOW_LOG(ERR, "IPv4 exact match mask"); + return ENOTSUP; + } + + enic_filter->u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; + enic_5tup->src_addr = spec->hdr.src_addr; + enic_5tup->dst_addr = spec->hdr.dst_addr; + + return 0; +} + +/** + * Copy UDP item into version 1 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Should always be 0 for version 1. + */ +static int +enic_copy_item_udp_v1(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = item->mask; + struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; + struct udp_hdr supported_mask = { + .src_port = 0xffff, + .dst_port = 0xffff, + }; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + if (!mask) + mask = &rte_flow_item_udp_mask; + + /* This is an exact match filter, both ports must be set */ + if (!spec || !spec->hdr.src_port || !spec->hdr.dst_port) { + FLOW_LOG(ERR, "UDP exact match src/dst addr"); + return ENOTSUP; + } + + /* check that the suppied mask exactly matches capabilty */ + if (!mask_exact_match((const u8 *)&supported_mask, + (const u8 *)item->mask, sizeof(*mask))) { + FLOW_LOG(ERR, "UDP exact match mask"); + return ENOTSUP; + } + + enic_filter->u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; + enic_5tup->src_port = spec->hdr.src_port; + enic_5tup->dst_port = spec->hdr.dst_port; + enic_5tup->protocol = PROTO_UDP; + + return 0; +} + +/** + * Copy TCP item into version 1 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Should always be 0 for version 1. + */ +static int +enic_copy_item_tcp_v1(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = item->mask; + struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; + struct tcp_hdr supported_mask = { + .src_port = 0xffff, + .dst_port = 0xffff, + }; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + if (!mask) + mask = &rte_flow_item_tcp_mask; + + /* This is an exact match filter, both ports must be set */ + if (!spec || !spec->hdr.src_port || !spec->hdr.dst_port) { + FLOW_LOG(ERR, "TCPIPv4 exact match src/dst addr"); + return ENOTSUP; + } + + /* check that the suppied mask exactly matches capabilty */ + if (!mask_exact_match((const u8 *)&supported_mask, + (const u8 *)item->mask, sizeof(*mask))) { + FLOW_LOG(ERR, "TCP exact match mask"); + return ENOTSUP; + } + + enic_filter->u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; + enic_5tup->src_port = spec->hdr.src_port; + enic_5tup->dst_port = spec->hdr.dst_port; + enic_5tup->protocol = PROTO_TCP; + + return 0; +} + +/** + * Copy ETH item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * If zero, this is an outer header. If non-zero, this is the offset into L5 + * where the header begins. + */ +static int +enic_copy_item_eth_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + struct ether_hdr enic_spec; + struct ether_hdr enic_mask; + const struct rte_flow_item_eth *spec = item->spec; + const struct rte_flow_item_eth *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_eth_mask; + + memcpy(enic_spec.d_addr.addr_bytes, spec->dst.addr_bytes, + ETHER_ADDR_LEN); + memcpy(enic_spec.s_addr.addr_bytes, spec->src.addr_bytes, + ETHER_ADDR_LEN); + + memcpy(enic_mask.d_addr.addr_bytes, mask->dst.addr_bytes, + ETHER_ADDR_LEN); + memcpy(enic_mask.s_addr.addr_bytes, mask->src.addr_bytes, + ETHER_ADDR_LEN); + enic_spec.ether_type = spec->type; + enic_mask.ether_type = mask->type; + + if (*inner_ofst == 0) { + /* outer header */ + memcpy(gp->layer[FILTER_GENERIC_1_L2].mask, &enic_mask, + sizeof(struct ether_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L2].val, &enic_spec, + sizeof(struct ether_hdr)); + } else { + /* inner header */ + if ((*inner_ofst + sizeof(struct ether_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + /* Offset into L5 where inner Ethernet header goes */ + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + &enic_mask, sizeof(struct ether_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + &enic_spec, sizeof(struct ether_hdr)); + *inner_ofst += sizeof(struct ether_hdr); + } + return 0; +} + +/** + * Copy VLAN item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * If zero, this is an outer header. If non-zero, this is the offset into L5 + * where the header begins. + */ +static int +enic_copy_item_vlan_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_vlan *spec = item->spec; + const struct rte_flow_item_vlan *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match all if no spec */ + if (!spec) + return 0; + + /* Don't support filtering in tpid */ + if (mask) { + if (mask->tpid != 0) + return ENOTSUP; + } else { + mask = &rte_flow_item_vlan_mask; + RTE_ASSERT(mask->tpid == 0); + } + + if (*inner_ofst == 0) { + /* Outer header. Use the vlan mask/val fields */ + gp->mask_vlan = mask->tci; + gp->val_vlan = spec->tci; + } else { + /* Inner header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct vlan_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct vlan_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct vlan_hdr)); + *inner_ofst += sizeof(struct vlan_hdr); + } + return 0; +} + +/** + * Copy IPv4 item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner IPv4 filtering. + */ +static int +enic_copy_item_ipv4_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + if (*inner_ofst == 0) { + /* Match IPv4 */ + gp->mask_flags |= FILTER_GENERIC_1_IPV4; + gp->val_flags |= FILTER_GENERIC_1_IPV4; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_ipv4_mask; + + memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, + sizeof(struct ipv4_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, + sizeof(struct ipv4_hdr)); + } else { + /* Inner IPv4 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct ipv4_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct ipv4_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct ipv4_hdr)); + *inner_ofst += sizeof(struct ipv4_hdr); + } + return 0; +} + +/** + * Copy IPv6 item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner IPv6 filtering. + */ +static int +enic_copy_item_ipv6_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_ipv6 *spec = item->spec; + const struct rte_flow_item_ipv6 *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match IPv6 */ + gp->mask_flags |= FILTER_GENERIC_1_IPV6; + gp->val_flags |= FILTER_GENERIC_1_IPV6; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_ipv6_mask; + + if (*inner_ofst == 0) { + memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, + sizeof(struct ipv6_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, + sizeof(struct ipv6_hdr)); + } else { + /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct ipv6_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct ipv6_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct ipv6_hdr)); + *inner_ofst += sizeof(struct ipv6_hdr); + } + return 0; +} + +/** + * Copy UDP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner UDP filtering. + */ +static int +enic_copy_item_udp_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match UDP */ + gp->mask_flags |= FILTER_GENERIC_1_UDP; + gp->val_flags |= FILTER_GENERIC_1_UDP; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_udp_mask; + + if (*inner_ofst == 0) { + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct udp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct udp_hdr)); + } else { + /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct udp_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct udp_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct udp_hdr)); + *inner_ofst += sizeof(struct udp_hdr); + } + return 0; +} + +/** + * Copy TCP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner TCP filtering. + */ +static int +enic_copy_item_tcp_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match TCP */ + gp->mask_flags |= FILTER_GENERIC_1_TCP; + gp->val_flags |= FILTER_GENERIC_1_TCP; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + return ENOTSUP; + + if (*inner_ofst == 0) { + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct tcp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct tcp_hdr)); + } else { + /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct tcp_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct tcp_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct tcp_hdr)); + *inner_ofst += sizeof(struct tcp_hdr); + } + return 0; +} + +/** + * Copy SCTP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner SCTP filtering. + */ +static int +enic_copy_item_sctp_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_sctp *spec = item->spec; + const struct rte_flow_item_sctp *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_sctp_mask; + + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct sctp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct sctp_hdr)); + return 0; +} + +/** + * Copy UDP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. VxLAN headers always start at the beginning of L5. + */ +static int +enic_copy_item_vxlan_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_vxlan *spec = item->spec; + const struct rte_flow_item_vxlan *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + if (*inner_ofst) + return EINVAL; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_vxlan_mask; + + memcpy(gp->layer[FILTER_GENERIC_1_L5].mask, mask, + sizeof(struct vxlan_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L5].val, spec, + sizeof(struct vxlan_hdr)); + + *inner_ofst = sizeof(struct vxlan_hdr); + return 0; +} + +/** + * Return 1 if current item is valid on top of the previous one. + * + * @param prev_item[in] + * The item before this one in the pattern or RTE_FLOW_ITEM_TYPE_END if this + * is the first item. + * @param item_info[in] + * Info about this item, like valid previous items. + * @param is_first[in] + * True if this the first item in the pattern. + */ +static int +item_stacking_valid(enum rte_flow_item_type prev_item, + const struct enic_items *item_info, u8 is_first_item) +{ + enum rte_flow_item_type const *allowed_items = item_info->prev_items; + + FLOW_TRACE(); + + for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) { + if (prev_item == *allowed_items) + return 1; + } + + /* This is the first item in the stack. Check if that's cool */ + if (is_first_item && item_info->valid_start_item) + return 1; + + return 0; +} + +/** + * Build the intenal enic filter structure from the provided pattern. The + * pattern is validated as the items are copied. + * + * @param pattern[in] + * @param items_info[in] + * Info about this NICs item support, like valid previous items. + * @param enic_filter[out] + * NIC specfilc filters derived from the pattern. + * @param error[out] + */ +static int +enic_copy_filter(const struct rte_flow_item pattern[], + const struct enic_items *items_info, + struct filter_v2 *enic_filter, + struct rte_flow_error *error) +{ + int ret; + const struct rte_flow_item *item = pattern; + u8 inner_ofst = 0; /* If encapsulated, ofst into L5 */ + enum rte_flow_item_type prev_item; + const struct enic_items *item_info; + + u8 is_first_item = 1; + + FLOW_TRACE(); + + prev_item = 0; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + /* Get info about how to validate and copy the item. If NULL + * is returned the nic does not support the item. + */ + if (item->type == RTE_FLOW_ITEM_TYPE_VOID) + continue; + + item_info = &items_info[item->type]; + + /* check to see if item stacking is valid */ + if (!item_stacking_valid(prev_item, item_info, is_first_item)) + goto stacking_error; + + ret = item_info->copy_item(item, enic_filter, &inner_ofst); + if (ret) + goto item_not_supported; + prev_item = item->type; + is_first_item = 0; + } + return 0; + +item_not_supported: + rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "enic type error"); + return -rte_errno; + +stacking_error: + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "stacking error"); + return -rte_errno; +} + +/** + * Build the intenal version 1 NIC action structure from the provided pattern. + * The pattern is validated as the items are copied. + * + * @param actions[in] + * @param enic_action[out] + * NIC specfilc actions derived from the actions. + * @param error[out] + */ +static int +enic_copy_action_v1(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action) +{ + FLOW_TRACE(); + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) + continue; + + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: { + const struct rte_flow_action_queue *queue = + (const struct rte_flow_action_queue *) + actions->conf; + enic_action->rq_idx = + enic_rte_rq_idx_to_sop_idx(queue->index); + break; + } + default: + RTE_ASSERT(0); + break; + } + } + enic_action->type = FILTER_ACTION_RQ_STEERING; + return 0; +} + +/** + * Build the intenal version 2 NIC action structure from the provided pattern. + * The pattern is validated as the items are copied. + * + * @param actions[in] + * @param enic_action[out] + * NIC specfilc actions derived from the actions. + * @param error[out] + */ +static int +enic_copy_action_v2(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action) +{ + FLOW_TRACE(); + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: { + const struct rte_flow_action_queue *queue = + (const struct rte_flow_action_queue *) + actions->conf; + enic_action->rq_idx = + enic_rte_rq_idx_to_sop_idx(queue->index); + enic_action->flags |= FILTER_ACTION_RQ_STEERING_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_MARK: { + const struct rte_flow_action_mark *mark = + (const struct rte_flow_action_mark *) + actions->conf; + + /* ENIC_MAGIC_FILTER_ID is reserved and is the highest + * in the range of allows mark ids. + */ + if (mark->id >= ENIC_MAGIC_FILTER_ID) + return EINVAL; + enic_action->filter_id = mark->id; + enic_action->flags |= FILTER_ACTION_FILTER_ID_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_FLAG: { + enic_action->filter_id = ENIC_MAGIC_FILTER_ID; + enic_action->flags |= FILTER_ACTION_FILTER_ID_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_VOID: + continue; + default: + RTE_ASSERT(0); + break; + } + } + enic_action->type = FILTER_ACTION_V2; + return 0; +} + +/** Check if the action is supported */ +static int +enic_match_action(const struct rte_flow_action *action, + const enum rte_flow_action_type *supported_actions) +{ + for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END; + supported_actions++) { + if (action->type == *supported_actions) + return 1; + } + return 0; +} + +/** Get the NIC filter capabilties structure */ +static const struct enic_filter_cap * +enic_get_filter_cap(struct enic *enic) +{ + if (enic->flow_filter_mode) + return &enic_filter_cap[enic->flow_filter_mode]; + + return NULL; +} + +/** Get the actions for this NIC version. */ +static const struct enic_action_cap * +enic_get_action_cap(struct enic *enic) +{ + static const struct enic_action_cap *ea; + + if (enic->filter_tags) + ea = &enic_action_cap[FILTER_ACTION_V2_ALL]; + else + ea = &enic_action_cap[FILTER_ACTION_RQ_STEERING_FLAG]; + return ea; +} + +/* Debug function to dump internal NIC action structure. */ +static void +enic_dump_actions(const struct filter_action_v2 *ea) +{ + if (ea->type == FILTER_ACTION_RQ_STEERING) { + FLOW_LOG(INFO, "Action(V1), queue: %u\n", ea->rq_idx); + } else if (ea->type == FILTER_ACTION_V2) { + FLOW_LOG(INFO, "Actions(V2)\n"); + if (ea->flags & FILTER_ACTION_RQ_STEERING_FLAG) + FLOW_LOG(INFO, "\tqueue: %u\n", + enic_sop_rq_idx_to_rte_idx(ea->rq_idx)); + if (ea->flags & FILTER_ACTION_FILTER_ID_FLAG) + FLOW_LOG(INFO, "\tfilter_id: %u\n", ea->filter_id); + } +} + +/* Debug function to dump internal NIC filter structure. */ +static void +enic_dump_filter(const struct filter_v2 *filt) +{ + const struct filter_generic_1 *gp; + int i, j, mbyte; + char buf[128], *bp; + char ip4[16], ip6[16], udp[16], tcp[16], tcpudp[16], ip4csum[16]; + char l4csum[16], ipfrag[16]; + + switch (filt->type) { + case FILTER_IPV4_5TUPLE: + FLOW_LOG(INFO, "FILTER_IPV4_5TUPLE\n"); + break; + case FILTER_USNIC_IP: + case FILTER_DPDK_1: + /* FIXME: this should be a loop */ + gp = &filt->u.generic_1; + FLOW_LOG(INFO, "Filter: vlan: 0x%04x, mask: 0x%04x\n", + gp->val_vlan, gp->mask_vlan); + + if (gp->mask_flags & FILTER_GENERIC_1_IPV4) + sprintf(ip4, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPV4) + ? "ip4(y)" : "ip4(n)"); + else + sprintf(ip4, "%s ", "ip4(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IPV6) + sprintf(ip6, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPV4) + ? "ip6(y)" : "ip6(n)"); + else + sprintf(ip6, "%s ", "ip6(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_UDP) + sprintf(udp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_UDP) + ? "udp(y)" : "udp(n)"); + else + sprintf(udp, "%s ", "udp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_TCP) + sprintf(tcp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_TCP) + ? "tcp(y)" : "tcp(n)"); + else + sprintf(tcp, "%s ", "tcp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_TCP_OR_UDP) + sprintf(tcpudp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_TCP_OR_UDP) + ? "tcpudp(y)" : "tcpudp(n)"); + else + sprintf(tcpudp, "%s ", "tcpudp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IP4SUM_OK) + sprintf(ip4csum, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IP4SUM_OK) + ? "ip4csum(y)" : "ip4csum(n)"); + else + sprintf(ip4csum, "%s ", "ip4csum(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_L4SUM_OK) + sprintf(l4csum, "%s ", + (gp->val_flags & FILTER_GENERIC_1_L4SUM_OK) + ? "l4csum(y)" : "l4csum(n)"); + else + sprintf(l4csum, "%s ", "l4csum(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IPFRAG) + sprintf(ipfrag, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPFRAG) + ? "ipfrag(y)" : "ipfrag(n)"); + else + sprintf(ipfrag, "%s ", "ipfrag(x)"); + FLOW_LOG(INFO, "\tFlags: %s%s%s%s%s%s%s%s\n", ip4, ip6, udp, + tcp, tcpudp, ip4csum, l4csum, ipfrag); + + for (i = 0; i < FILTER_GENERIC_1_NUM_LAYERS; i++) { + mbyte = FILTER_GENERIC_1_KEY_LEN - 1; + while (mbyte && !gp->layer[i].mask[mbyte]) + mbyte--; + if (mbyte == 0) + continue; + + bp = buf; + for (j = 0; j <= mbyte; j++) { + sprintf(bp, "%02x", + gp->layer[i].mask[j]); + bp += 2; + } + *bp = '\0'; + FLOW_LOG(INFO, "\tL%u mask: %s\n", i + 2, buf); + bp = buf; + for (j = 0; j <= mbyte; j++) { + sprintf(bp, "%02x", + gp->layer[i].val[j]); + bp += 2; + } + *bp = '\0'; + FLOW_LOG(INFO, "\tL%u val: %s\n", i + 2, buf); + } + break; + default: + FLOW_LOG(INFO, "FILTER UNKNOWN\n"); + break; + } +} + +/* Debug function to dump internal NIC flow structures. */ +static void +enic_dump_flow(const struct filter_action_v2 *ea, const struct filter_v2 *filt) +{ + enic_dump_filter(filt); + enic_dump_actions(ea); +} + + +/** + * Internal flow parse/validate function. + * + * @param dev[in] + * This device pointer. + * @param pattern[in] + * @param actions[in] + * @param error[out] + * @param enic_filter[out] + * Internal NIC filter structure pointer. + * @param enic_action[out] + * Internal NIC action structure pointer. + */ +static int +enic_flow_parse(struct rte_eth_dev *dev, + const struct rte_flow_attr *attrs, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct filter_v2 *enic_filter, + struct filter_action_v2 *enic_action) +{ + unsigned int ret = 0; + struct enic *enic = pmd_priv(dev); + const struct enic_filter_cap *enic_filter_cap; + const struct enic_action_cap *enic_action_cap; + const struct rte_flow_action *action; + + FLOW_TRACE(); + + memset(enic_filter, 0, sizeof(*enic_filter)); + memset(enic_action, 0, sizeof(*enic_action)); + + if (!pattern) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "No pattern specified"); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "No action specified"); + return -rte_errno; + } + + if (attrs) { + if (attrs->group) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, + "priority groups are not supported"); + return -rte_errno; + } else if (attrs->priority) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, + "priorities are not supported"); + return -rte_errno; + } else if (attrs->egress) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "egress is not supported"); + return -rte_errno; + } else if (!attrs->ingress) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, + "only ingress is supported"); + return -rte_errno; + } + + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "No attribute specified"); + return -rte_errno; + } + + /* Verify Actions. */ + enic_action_cap = enic_get_action_cap(enic); + for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END; + action++) { + if (action->type == RTE_FLOW_ACTION_TYPE_VOID) + continue; + else if (!enic_match_action(action, enic_action_cap->actions)) + break; + } + if (action->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION, + action, "Invalid action."); + return -rte_errno; + } + ret = enic_action_cap->copy_fn(actions, enic_action); + if (ret) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Unsupported action."); + return -rte_errno; + } + + /* Verify Flow items. If copying the filter from flow format to enic + * format fails, the flow is not supported + */ + enic_filter_cap = enic_get_filter_cap(enic); + if (enic_filter_cap == NULL) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Flow API not available"); + return -rte_errno; + } + enic_filter->type = enic->flow_filter_mode; + ret = enic_copy_filter(pattern, enic_filter_cap->item_info, + enic_filter, error); + return ret; +} + +/** + * Push filter/action to the NIC. + * + * @param enic[in] + * Device structure pointer. + * @param enic_filter[in] + * Internal NIC filter structure pointer. + * @param enic_action[in] + * Internal NIC action structure pointer. + * @param error[out] + */ +static struct rte_flow * +enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter, + struct filter_action_v2 *enic_action, + struct rte_flow_error *error) +{ + struct rte_flow *flow; + int ret; + u16 entry; + + FLOW_TRACE(); + + flow = rte_calloc(__func__, 1, sizeof(*flow), 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "cannot allocate flow memory"); + return NULL; + } + + /* entry[in] is the queue id, entry[out] is the filter Id for delete */ + entry = enic_action->rq_idx; + ret = vnic_dev_classifier(enic->vdev, CLSF_ADD, &entry, enic_filter, + enic_action); + if (!ret) { + flow->enic_filter_id = entry; + flow->enic_filter = *enic_filter; + } else { + rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "vnic_dev_classifier error"); + rte_free(flow); + return NULL; + } + return flow; +} + +/** + * Remove filter/action from the NIC. + * + * @param enic[in] + * Device structure pointer. + * @param filter_id[in] + * Id of NIC filter. + * @param enic_action[in] + * Internal NIC action structure pointer. + * @param error[out] + */ +static int +enic_flow_del_filter(struct enic *enic, u16 filter_id, + struct rte_flow_error *error) +{ + int ret; + + FLOW_TRACE(); + + ret = vnic_dev_classifier(enic->vdev, CLSF_DEL, &filter_id, NULL, NULL); + if (!ret) + rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "vnic_dev_classifier failed"); + return ret; +} + +/* + * The following functions are callbacks for Generic flow API. + */ + +/** + * Validate a flow supported by the NIC. + * + * @see rte_flow_validate() + * @see rte_flow_ops + */ +static int +enic_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attrs, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct filter_v2 enic_filter; + struct filter_action_v2 enic_action; + int ret; + + FLOW_TRACE(); + + ret = enic_flow_parse(dev, attrs, pattern, actions, error, + &enic_filter, &enic_action); + if (!ret) + enic_dump_flow(&enic_action, &enic_filter); + return ret; +} + +/** + * Create a flow supported by the NIC. + * + * @see rte_flow_create() + * @see rte_flow_ops + */ +static struct rte_flow * +enic_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attrs, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + int ret; + struct filter_v2 enic_filter; + struct filter_action_v2 enic_action; + struct rte_flow *flow; + struct enic *enic = pmd_priv(dev); + + FLOW_TRACE(); + + ret = enic_flow_parse(dev, attrs, pattern, actions, error, &enic_filter, + &enic_action); + if (ret < 0) + return NULL; + + rte_spinlock_lock(&enic->flows_lock); + flow = enic_flow_add_filter(enic, &enic_filter, &enic_action, + error); + if (flow) + LIST_INSERT_HEAD(&enic->flows, flow, next); + rte_spinlock_unlock(&enic->flows_lock); + + return flow; +} + +/** + * Destroy a flow supported by the NIC. + * + * @see rte_flow_destroy() + * @see rte_flow_ops + */ +static int +enic_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, + __rte_unused struct rte_flow_error *error) +{ + struct enic *enic = pmd_priv(dev); + + FLOW_TRACE(); + + rte_spinlock_lock(&enic->flows_lock); + enic_flow_del_filter(enic, flow->enic_filter_id, error); + LIST_REMOVE(flow, next); + rte_spinlock_unlock(&enic->flows_lock); + return 0; +} + +/** + * Flush all flows on the device. + * + * @see rte_flow_flush() + * @see rte_flow_ops + */ +static int +enic_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct rte_flow *flow; + struct enic *enic = pmd_priv(dev); + + FLOW_TRACE(); + + rte_spinlock_lock(&enic->flows_lock); + + while (!LIST_EMPTY(&enic->flows)) { + flow = LIST_FIRST(&enic->flows); + enic_flow_del_filter(enic, flow->enic_filter_id, error); + LIST_REMOVE(flow, next); + } + rte_spinlock_unlock(&enic->flows_lock); + return 0; +} + +/** + * Flow callback registration. + * + * @see rte_flow_ops + */ +const struct rte_flow_ops enic_flow_ops = { + .validate = enic_flow_validate, + .create = enic_flow_create, + .destroy = enic_flow_destroy, + .flush = enic_flow_flush, +}; diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c index d0262418..40dbec7f 100644 --- a/drivers/net/enic/enic_main.c +++ b/drivers/net/enic/enic_main.c @@ -430,7 +430,7 @@ enic_intr_handler(void *arg) vnic_intr_return_all_credits(&enic->intr); enic_link_update(enic); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); enic_log_q_error(enic); } @@ -1225,7 +1225,7 @@ int enic_set_mtu(struct enic *enic, uint16_t new_mtu) } } - /* replace Rx funciton with a no-op to avoid getting stale pkts */ + /* replace Rx function with a no-op to avoid getting stale pkts */ eth_dev->rx_pkt_burst = enic_dummy_recv_pkts; rte_mb(); @@ -1314,6 +1314,9 @@ static int enic_dev_init(struct enic *enic) vnic_dev_set_reset_flag(enic->vdev, 0); + LIST_INIT(&enic->flows); + rte_spinlock_init(&enic->flows_lock); + /* set up link status checking */ vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ diff --git a/drivers/net/enic/enic_res.c b/drivers/net/enic/enic_res.c index 867bd25c..e4b80d49 100644 --- a/drivers/net/enic/enic_res.c +++ b/drivers/net/enic/enic_res.c @@ -104,6 +104,21 @@ int enic_get_vnic_config(struct enic *enic) dev_info(enic, "Advanced Filters %savailable\n", ((enic->adv_filters) ? "" : "not ")); + err = vnic_dev_capable_filter_mode(enic->vdev, &enic->flow_filter_mode, + &enic->filter_tags); + if (err) { + dev_err(enic_get_dev(enic), + "Error getting filter modes, %d\n", err); + return err; + } + + dev_info(enic, "Flow api filter mode: %s, Filter tagging %savailable\n", + ((enic->flow_filter_mode == FILTER_DPDK_1) ? "DPDK" : + ((enic->flow_filter_mode == FILTER_USNIC_IP) ? "USNIC" : + ((enic->flow_filter_mode == FILTER_IPV4_5TUPLE) ? "5TUPLE" : + "NONE"))), + ((enic->filter_tags) ? "" : "not ")); + c->wq_desc_count = min_t(u32, ENIC_MAX_WQ_DESCS, max_t(u32, ENIC_MIN_WQ_DESCS, diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c index ba0cfd01..a39172f1 100644 --- a/drivers/net/enic/enic_rxtx.c +++ b/drivers/net/enic/enic_rxtx.c @@ -253,8 +253,20 @@ enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) } mbuf->vlan_tci = vlan_tci; - /* RSS flag */ - if (enic_cq_rx_desc_rss_type(cqrd)) { + if ((cqd->type_color & CQ_DESC_TYPE_MASK) == CQ_DESC_TYPE_CLASSIFIER) { + struct cq_enet_rq_clsf_desc *clsf_cqd; + uint16_t filter_id; + clsf_cqd = (struct cq_enet_rq_clsf_desc *)cqd; + filter_id = clsf_cqd->filter_id; + if (filter_id) { + pkt_flags |= PKT_RX_FDIR; + if (filter_id != ENIC_MAGIC_FILTER_ID) { + mbuf->hash.fdir.hi = clsf_cqd->filter_id; + pkt_flags |= PKT_RX_FDIR_ID; + } + } + } else if (enic_cq_rx_desc_rss_type(cqrd)) { + /* RSS flag */ pkt_flags |= PKT_RX_RSS_HASH; mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); } @@ -491,7 +503,8 @@ static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) tail_idx = enic_ring_incr(desc_count, tail_idx); } - rte_mempool_put_bulk(pool, (void **)free, nb_free); + if (nb_free > 0) + rte_mempool_put_bulk(pool, (void **)free, nb_free); wq->tail_idx = tail_idx; wq->ring.desc_avail += nb_to_free; diff --git a/drivers/net/failsafe/Makefile b/drivers/net/failsafe/Makefile new file mode 100644 index 00000000..d516d362 --- /dev/null +++ b/drivers/net/failsafe/Makefile @@ -0,0 +1,62 @@ +# BSD LICENSE +# +# Copyright 2017 6WIND S.A. +# Copyright 2017 Mellanox. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# Library name +LIB = librte_pmd_failsafe.a + +EXPORT_MAP := rte_pmd_failsafe_version.map + +LIBABIVER := 1 + +# Sources are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_args.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_eal.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_ops.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_rxtx.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_ether.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_flow.c + +# No exported include files + +# Basic CFLAGS: +CFLAGS += -std=gnu99 -Wextra +CFLAGS += -O3 +CFLAGS += -I. +CFLAGS += -D_DEFAULT_SOURCE +CFLAGS += -D_XOPEN_SOURCE=700 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -Wno-strict-prototypes +CFLAGS += -pedantic + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/net/failsafe/failsafe.c b/drivers/net/failsafe/failsafe.c new file mode 100644 index 00000000..6006bef8 --- /dev/null +++ b/drivers/net/failsafe/failsafe.c @@ -0,0 +1,299 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_alarm.h> +#include <rte_malloc.h> +#include <rte_ethdev.h> +#include <rte_ethdev_vdev.h> +#include <rte_devargs.h> +#include <rte_kvargs.h> +#include <rte_vdev.h> + +#include "failsafe_private.h" + +const char pmd_failsafe_driver_name[] = FAILSAFE_DRIVER_NAME; +static const struct rte_eth_link eth_link = { + .link_speed = ETH_SPEED_NUM_10G, + .link_duplex = ETH_LINK_FULL_DUPLEX, + .link_status = ETH_LINK_UP, + .link_autoneg = ETH_LINK_SPEED_AUTONEG, +}; + +static int +fs_sub_device_alloc(struct rte_eth_dev *dev, + const char *params) +{ + uint8_t nb_subs; + int ret; + + ret = failsafe_args_count_subdevice(dev, params); + if (ret) + return ret; + if (PRIV(dev)->subs_tail > FAILSAFE_MAX_ETHPORTS) { + ERROR("Cannot allocate more than %d ports", + FAILSAFE_MAX_ETHPORTS); + return -ENOSPC; + } + nb_subs = PRIV(dev)->subs_tail; + PRIV(dev)->subs = rte_zmalloc(NULL, + sizeof(struct sub_device) * nb_subs, + RTE_CACHE_LINE_SIZE); + if (PRIV(dev)->subs == NULL) { + ERROR("Could not allocate sub_devices"); + return -ENOMEM; + } + return 0; +} + +static void +fs_sub_device_free(struct rte_eth_dev *dev) +{ + rte_free(PRIV(dev)->subs); +} + +static void fs_hotplug_alarm(void *arg); + +int +failsafe_hotplug_alarm_install(struct rte_eth_dev *dev) +{ + int ret; + + if (dev == NULL) + return -EINVAL; + if (PRIV(dev)->pending_alarm) + return 0; + ret = rte_eal_alarm_set(hotplug_poll * 1000, + fs_hotplug_alarm, + dev); + if (ret) { + ERROR("Could not set up plug-in event detection"); + return ret; + } + PRIV(dev)->pending_alarm = 1; + return 0; +} + +int +failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev) +{ + int ret = 0; + + if (PRIV(dev)->pending_alarm) { + rte_errno = 0; + rte_eal_alarm_cancel(fs_hotplug_alarm, dev); + if (rte_errno) { + ERROR("rte_eal_alarm_cancel failed (errno: %s)", + strerror(rte_errno)); + ret = -rte_errno; + } else { + PRIV(dev)->pending_alarm = 0; + } + } + return ret; +} + +static void +fs_hotplug_alarm(void *arg) +{ + struct rte_eth_dev *dev = arg; + struct sub_device *sdev; + int ret; + uint8_t i; + + if (!PRIV(dev)->pending_alarm) + return; + PRIV(dev)->pending_alarm = 0; + FOREACH_SUBDEV(sdev, i, dev) + if (sdev->state != PRIV(dev)->state) + break; + /* if we have non-probed device */ + if (i != PRIV(dev)->subs_tail) { + ret = failsafe_eth_dev_state_sync(dev); + if (ret) + ERROR("Unable to synchronize sub_device state"); + } + failsafe_dev_remove(dev); + ret = failsafe_hotplug_alarm_install(dev); + if (ret) + ERROR("Unable to set up next alarm"); +} + +static int +fs_eth_dev_create(struct rte_vdev_device *vdev) +{ + struct rte_eth_dev *dev; + struct ether_addr *mac; + struct fs_priv *priv; + struct sub_device *sdev; + const char *params; + unsigned int socket_id; + uint8_t i; + int ret; + + dev = NULL; + priv = NULL; + socket_id = rte_socket_id(); + INFO("Creating fail-safe device on NUMA socket %u", socket_id); + params = rte_vdev_device_args(vdev); + if (params == NULL) { + ERROR("This PMD requires sub-devices, none provided"); + return -1; + } + dev = rte_eth_vdev_allocate(vdev, sizeof(*priv)); + if (dev == NULL) { + ERROR("Unable to allocate rte_eth_dev"); + return -1; + } + priv = PRIV(dev); + priv->dev = dev; + dev->dev_ops = &failsafe_ops; + dev->data->mac_addrs = &PRIV(dev)->mac_addrs[0]; + dev->data->dev_link = eth_link; + PRIV(dev)->nb_mac_addr = 1; + TAILQ_INIT(&PRIV(dev)->flow_list); + dev->rx_pkt_burst = (eth_rx_burst_t)&failsafe_rx_burst; + dev->tx_pkt_burst = (eth_tx_burst_t)&failsafe_tx_burst; + ret = fs_sub_device_alloc(dev, params); + if (ret) { + ERROR("Could not allocate sub_devices"); + goto free_dev; + } + ret = failsafe_args_parse(dev, params); + if (ret) + goto free_subs; + ret = failsafe_eal_init(dev); + if (ret) + goto free_args; + ret = failsafe_hotplug_alarm_install(dev); + if (ret) { + ERROR("Could not set up plug-in event detection"); + goto free_args; + } + mac = &dev->data->mac_addrs[0]; + if (mac_from_arg) { + /* + * If MAC address was provided as a parameter, + * apply to all probed slaves. + */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), + mac); + if (ret) { + ERROR("Failed to set default MAC address"); + goto free_args; + } + } + } else { + /* + * Use the ether_addr from first probed + * device, either preferred or fallback. + */ + FOREACH_SUBDEV(sdev, i, dev) + if (sdev->state >= DEV_PROBED) { + ether_addr_copy(Ð(sdev)->data->mac_addrs[0], + mac); + break; + } + /* + * If no device has been probed and no ether_addr + * has been provided on the command line, use a random + * valid one. + * It will be applied during future slave state syncs to + * probed slaves. + */ + if (i == priv->subs_tail) + eth_random_addr(&mac->addr_bytes[0]); + } + INFO("MAC address is %02x:%02x:%02x:%02x:%02x:%02x", + mac->addr_bytes[0], mac->addr_bytes[1], + mac->addr_bytes[2], mac->addr_bytes[3], + mac->addr_bytes[4], mac->addr_bytes[5]); + dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; + return 0; +free_args: + failsafe_args_free(dev); +free_subs: + fs_sub_device_free(dev); +free_dev: + rte_free(PRIV(dev)); + rte_eth_dev_release_port(dev); + return -1; +} + +static int +fs_rte_eth_free(const char *name) +{ + struct rte_eth_dev *dev; + int ret; + + dev = rte_eth_dev_allocated(name); + if (dev == NULL) + return -ENODEV; + ret = failsafe_eal_uninit(dev); + if (ret) + ERROR("Error while uninitializing sub-EAL"); + failsafe_args_free(dev); + fs_sub_device_free(dev); + rte_free(PRIV(dev)); + rte_eth_dev_release_port(dev); + return ret; +} + +static int +rte_pmd_failsafe_probe(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + INFO("Initializing " FAILSAFE_DRIVER_NAME " for %s", + name); + return fs_eth_dev_create(vdev); +} + +static int +rte_pmd_failsafe_remove(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + INFO("Uninitializing " FAILSAFE_DRIVER_NAME " for %s", name); + return fs_rte_eth_free(name); +} + +static struct rte_vdev_driver failsafe_drv = { + .probe = rte_pmd_failsafe_probe, + .remove = rte_pmd_failsafe_remove, +}; + +RTE_PMD_REGISTER_VDEV(net_failsafe, failsafe_drv); +RTE_PMD_REGISTER_PARAM_STRING(net_failsafe, PMD_FAILSAFE_PARAM_STRING); diff --git a/drivers/net/failsafe/failsafe_args.c b/drivers/net/failsafe/failsafe_args.c new file mode 100644 index 00000000..1f22416f --- /dev/null +++ b/drivers/net/failsafe/failsafe_args.c @@ -0,0 +1,474 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <errno.h> + +#include <rte_debug.h> +#include <rte_devargs.h> +#include <rte_malloc.h> +#include <rte_kvargs.h> + +#include "failsafe_private.h" + +#define DEVARGS_MAXLEN 4096 + +/* Callback used when a new device is found in devargs */ +typedef int (parse_cb)(struct rte_eth_dev *dev, const char *params, + uint8_t head); + +uint64_t hotplug_poll = FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS; +int mac_from_arg = 0; + +const char *pmd_failsafe_init_parameters[] = { + PMD_FAILSAFE_HOTPLUG_POLL_KVARG, + PMD_FAILSAFE_MAC_KVARG, + NULL, +}; + +/* + * input: text. + * output: 0: if text[0] != '(', + * 0: if there are no corresponding ')' + * n: distance to corresponding ')' otherwise + */ +static size_t +closing_paren(const char *text) +{ + int nb_open = 0; + size_t i = 0; + + while (text[i] != '\0') { + if (text[i] == '(') + nb_open++; + if (text[i] == ')') + nb_open--; + if (nb_open == 0) + return i; + i++; + } + return 0; +} + +static int +fs_parse_device(struct sub_device *sdev, char *args) +{ + struct rte_devargs *d; + int ret; + + d = &sdev->devargs; + DEBUG("%s", args); + ret = rte_eal_devargs_parse(args, d); + if (ret) { + DEBUG("devargs parsing failed with code %d", ret); + return ret; + } + sdev->bus = d->bus; + sdev->state = DEV_PARSED; + return 0; +} + +static void +fs_sanitize_cmdline(char *args) +{ + char *nl; + + nl = strrchr(args, '\n'); + if (nl) + nl[0] = '\0'; +} + +static int +fs_execute_cmd(struct sub_device *sdev, char *cmdline) +{ + FILE *fp; + /* store possible newline as well */ + char output[DEVARGS_MAXLEN + 1]; + size_t len; + int old_err; + int ret, pclose_ret; + + RTE_ASSERT(cmdline != NULL || sdev->cmdline != NULL); + if (sdev->cmdline == NULL) { + size_t i; + + len = strlen(cmdline) + 1; + sdev->cmdline = calloc(1, len); + if (sdev->cmdline == NULL) { + ERROR("Command line allocation failed"); + return -ENOMEM; + } + snprintf(sdev->cmdline, len, "%s", cmdline); + /* Replace all commas in the command line by spaces */ + for (i = 0; i < len; i++) + if (sdev->cmdline[i] == ',') + sdev->cmdline[i] = ' '; + } + DEBUG("'%s'", sdev->cmdline); + old_err = errno; + fp = popen(sdev->cmdline, "r"); + if (fp == NULL) { + ret = errno; + ERROR("popen: %s", strerror(errno)); + errno = old_err; + return ret; + } + /* We only read one line */ + if (fgets(output, sizeof(output) - 1, fp) == NULL) { + DEBUG("Could not read command output"); + ret = -ENODEV; + goto ret_pclose; + } + fs_sanitize_cmdline(output); + if (output[0] == '\0') { + ret = -ENODEV; + goto ret_pclose; + } + ret = fs_parse_device(sdev, output); + if (ret) { + ERROR("Parsing device '%s' failed", output); + goto ret_pclose; + } +ret_pclose: + pclose_ret = pclose(fp); + if (pclose_ret) { + pclose_ret = errno; + ERROR("pclose: %s", strerror(errno)); + errno = old_err; + return pclose_ret; + } + return ret; +} + +static int +fs_parse_device_param(struct rte_eth_dev *dev, const char *param, + uint8_t head) +{ + struct fs_priv *priv; + struct sub_device *sdev; + char *args = NULL; + size_t a, b; + int ret; + + priv = PRIV(dev); + a = 0; + b = 0; + ret = 0; + while (param[b] != '(' && + param[b] != '\0') + b++; + a = b; + b += closing_paren(¶m[b]); + if (a == b) { + ERROR("Dangling parenthesis"); + return -EINVAL; + } + a += 1; + args = strndup(¶m[a], b - a); + if (args == NULL) { + ERROR("Not enough memory for parameter parsing"); + return -ENOMEM; + } + sdev = &priv->subs[head]; + if (strncmp(param, "dev", 3) == 0) { + ret = fs_parse_device(sdev, args); + if (ret) + goto free_args; + } else if (strncmp(param, "exec", 4) == 0) { + ret = fs_execute_cmd(sdev, args); + if (ret == -ENODEV) { + DEBUG("Reading device info from command line failed"); + ret = 0; + } + if (ret) + goto free_args; + } else { + ERROR("Unrecognized device type: %.*s", (int)b, param); + return -EINVAL; + } +free_args: + free(args); + return ret; +} + +static int +fs_parse_sub_devices(parse_cb *cb, + struct rte_eth_dev *dev, const char *params) +{ + size_t a, b; + uint8_t head; + int ret; + + a = 0; + head = 0; + ret = 0; + while (params[a] != '\0') { + b = a; + while (params[b] != '(' && + params[b] != ',' && + params[b] != '\0') + b++; + if (b == a) { + ERROR("Invalid parameter"); + return -EINVAL; + } + if (params[b] == ',') { + a = b + 1; + continue; + } + if (params[b] == '(') { + size_t start = b; + + b += closing_paren(¶ms[b]); + if (b == start) { + ERROR("Dangling parenthesis"); + return -EINVAL; + } + ret = (*cb)(dev, ¶ms[a], head); + if (ret) + return ret; + head += 1; + b += 1; + if (params[b] == '\0') + return 0; + } + a = b + 1; + } + return 0; +} + +static int +fs_remove_sub_devices_definition(char params[DEVARGS_MAXLEN]) +{ + char buffer[DEVARGS_MAXLEN] = {0}; + size_t a, b; + int i; + + a = 0; + i = 0; + while (params[a] != '\0') { + b = a; + while (params[b] != '(' && + params[b] != ',' && + params[b] != '\0') + b++; + if (b == a) { + ERROR("Invalid parameter"); + return -EINVAL; + } + if (params[b] == ',' || params[b] == '\0') + i += snprintf(&buffer[i], b - a + 1, "%s", ¶ms[a]); + if (params[b] == '(') { + size_t start = b; + b += closing_paren(¶ms[b]); + if (b == start) + return -EINVAL; + b += 1; + if (params[b] == '\0') + goto out; + } + a = b + 1; + } +out: + snprintf(params, DEVARGS_MAXLEN, "%s", buffer); + return 0; +} + +static int +fs_get_u64_arg(const char *key __rte_unused, + const char *value, void *out) +{ + uint64_t *u64 = out; + char *endptr = NULL; + + if ((value == NULL) || (out == NULL)) + return -EINVAL; + errno = 0; + *u64 = strtoull(value, &endptr, 0); + if (errno != 0) + return -errno; + if (endptr == value) + return -1; + return 0; +} + +static int +fs_get_mac_addr_arg(const char *key __rte_unused, + const char *value, void *out) +{ + struct ether_addr *ea = out; + int ret; + + if ((value == NULL) || (out == NULL)) + return -EINVAL; + ret = sscanf(value, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", + &ea->addr_bytes[0], &ea->addr_bytes[1], + &ea->addr_bytes[2], &ea->addr_bytes[3], + &ea->addr_bytes[4], &ea->addr_bytes[5]); + return ret != ETHER_ADDR_LEN; +} + +int +failsafe_args_parse(struct rte_eth_dev *dev, const char *params) +{ + struct fs_priv *priv; + char mut_params[DEVARGS_MAXLEN] = ""; + struct rte_kvargs *kvlist = NULL; + unsigned int arg_count; + size_t n; + int ret; + + priv = PRIV(dev); + ret = 0; + priv->subs_tx = FAILSAFE_MAX_ETHPORTS; + /* default parameters */ + n = snprintf(mut_params, sizeof(mut_params), "%s", params); + if (n >= sizeof(mut_params)) { + ERROR("Parameter string too long (>=%zu)", + sizeof(mut_params)); + return -ENOMEM; + } + ret = fs_parse_sub_devices(fs_parse_device_param, + dev, params); + if (ret < 0) + return ret; + ret = fs_remove_sub_devices_definition(mut_params); + if (ret < 0) + return ret; + if (strnlen(mut_params, sizeof(mut_params)) > 0) { + kvlist = rte_kvargs_parse(mut_params, + pmd_failsafe_init_parameters); + if (kvlist == NULL) { + ERROR("Error parsing parameters, usage:\n" + PMD_FAILSAFE_PARAM_STRING); + return -1; + } + /* PLUG_IN event poll timer */ + arg_count = rte_kvargs_count(kvlist, + PMD_FAILSAFE_HOTPLUG_POLL_KVARG); + if (arg_count == 1) { + ret = rte_kvargs_process(kvlist, + PMD_FAILSAFE_HOTPLUG_POLL_KVARG, + &fs_get_u64_arg, &hotplug_poll); + if (ret < 0) + goto free_kvlist; + } + /* MAC addr */ + arg_count = rte_kvargs_count(kvlist, + PMD_FAILSAFE_MAC_KVARG); + if (arg_count > 0) { + ret = rte_kvargs_process(kvlist, + PMD_FAILSAFE_MAC_KVARG, + &fs_get_mac_addr_arg, + &dev->data->mac_addrs[0]); + if (ret < 0) + goto free_kvlist; + mac_from_arg = 1; + } + } + PRIV(dev)->state = DEV_PARSED; +free_kvlist: + rte_kvargs_free(kvlist); + return ret; +} + +void +failsafe_args_free(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV(sdev, i, dev) { + rte_free(sdev->cmdline); + sdev->cmdline = NULL; + free(sdev->devargs.args); + sdev->devargs.args = NULL; + } +} + +static int +fs_count_device(struct rte_eth_dev *dev, const char *param, + uint8_t head __rte_unused) +{ + size_t b = 0; + + while (param[b] != '(' && + param[b] != '\0') + b++; + if (strncmp(param, "dev", b) != 0 && + strncmp(param, "exec", b) != 0) { + ERROR("Unrecognized device type: %.*s", (int)b, param); + return -EINVAL; + } + PRIV(dev)->subs_tail += 1; + return 0; +} + +int +failsafe_args_count_subdevice(struct rte_eth_dev *dev, + const char *params) +{ + return fs_parse_sub_devices(fs_count_device, + dev, params); +} + +static int +fs_parse_sub_device(struct sub_device *sdev) +{ + struct rte_devargs *da; + char devstr[DEVARGS_MAXLEN] = ""; + + da = &sdev->devargs; + snprintf(devstr, sizeof(devstr), "%s,%s", da->name, da->args); + return fs_parse_device(sdev, devstr); +} + +int +failsafe_args_parse_subs(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret = 0; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state >= DEV_PARSED) + continue; + if (sdev->cmdline) + ret = fs_execute_cmd(sdev, sdev->cmdline); + else + ret = fs_parse_sub_device(sdev); + if (ret == 0) + sdev->state = DEV_PARSED; + } + return 0; +} diff --git a/drivers/net/failsafe/failsafe_eal.c b/drivers/net/failsafe/failsafe_eal.c new file mode 100644 index 00000000..c8f4318e --- /dev/null +++ b/drivers/net/failsafe/failsafe_eal.c @@ -0,0 +1,118 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_malloc.h> + +#include "failsafe_private.h" + +static int +fs_bus_init(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + struct rte_devargs *da; + uint8_t i; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state != DEV_PARSED) + continue; + da = &sdev->devargs; + ret = rte_eal_hotplug_add(da->bus->name, + da->name, + da->args); + if (ret) { + ERROR("sub_device %d probe failed %s%s%s", i, + rte_errno ? "(" : "", + rte_errno ? strerror(rte_errno) : "", + rte_errno ? ")" : ""); + continue; + } + ETH(sdev) = rte_eth_dev_allocated(da->name); + if (ETH(sdev) == NULL) { + ERROR("sub_device %d init went wrong", i); + return -ENODEV; + } + SUB_ID(sdev) = i; + sdev->fs_dev = dev; + sdev->dev = ETH(sdev)->device; + ETH(sdev)->state = RTE_ETH_DEV_DEFERRED; + sdev->state = DEV_PROBED; + } + return 0; +} + +int +failsafe_eal_init(struct rte_eth_dev *dev) +{ + int ret; + + ret = fs_bus_init(dev); + if (ret) + return ret; + if (PRIV(dev)->state < DEV_PROBED) + PRIV(dev)->state = DEV_PROBED; + fs_switch_dev(dev, NULL); + return 0; +} + +static int +fs_bus_uninit(struct rte_eth_dev *dev) +{ + struct sub_device *sdev = NULL; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + ret = rte_eal_hotplug_remove(sdev->bus->name, + sdev->dev->name); + if (ret) { + ERROR("Failed to remove requested device %s", + sdev->dev->name); + continue; + } + sdev->state = DEV_PROBED - 1; + } + return 0; +} + +int +failsafe_eal_uninit(struct rte_eth_dev *dev) +{ + int ret; + + ret = fs_bus_uninit(dev); + if (ret) + return ret; + PRIV(dev)->state = DEV_PROBED - 1; + return 0; +} diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c new file mode 100644 index 00000000..a3a8cce9 --- /dev/null +++ b/drivers/net/failsafe/failsafe_ether.c @@ -0,0 +1,437 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <unistd.h> + +#include <rte_flow.h> +#include <rte_flow_driver.h> + +#include "failsafe_private.h" + +/** Print a message out of a flow error. */ +static int +fs_flow_complain(struct rte_flow_error *error) +{ + static const char *const errstrlist[] = { + [RTE_FLOW_ERROR_TYPE_NONE] = "no error", + [RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified", + [RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)", + [RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field", + [RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field", + [RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field", + [RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field", + [RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure", + [RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length", + [RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item", + [RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions", + [RTE_FLOW_ERROR_TYPE_ACTION] = "specific action", + }; + const char *errstr; + char buf[32]; + int err = rte_errno; + + if ((unsigned int)error->type >= RTE_DIM(errstrlist) || + !errstrlist[error->type]) + errstr = "unknown type"; + else + errstr = errstrlist[error->type]; + ERROR("Caught error type %d (%s): %s%s\n", + error->type, errstr, + error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ", + error->cause), buf) : "", + error->message ? error->message : "(no stated reason)"); + return -err; +} + +static int +eth_dev_flow_isolate_set(struct rte_eth_dev *dev, + struct sub_device *sdev) +{ + struct rte_flow_error ferror; + int ret; + + if (!PRIV(dev)->flow_isolated) { + DEBUG("Flow isolation already disabled"); + } else { + DEBUG("Enabling flow isolation"); + ret = rte_flow_isolate(PORT_ID(sdev), + PRIV(dev)->flow_isolated, + &ferror); + if (ret) { + fs_flow_complain(&ferror); + return ret; + } + } + return 0; +} + +static int +fs_eth_dev_conf_apply(struct rte_eth_dev *dev, + struct sub_device *sdev) +{ + struct rte_eth_dev *edev; + struct rte_vlan_filter_conf *vfc1; + struct rte_vlan_filter_conf *vfc2; + struct rte_flow *flow; + struct rte_flow_error ferror; + uint32_t i; + int ret; + + edev = ETH(sdev); + /* RX queue setup */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct rxq *rxq; + + rxq = dev->data->rx_queues[i]; + ret = rte_eth_rx_queue_setup(PORT_ID(sdev), i, + rxq->info.nb_desc, rxq->socket_id, + &rxq->info.conf, rxq->info.mp); + if (ret) { + ERROR("rx_queue_setup failed"); + return ret; + } + } + /* TX queue setup */ + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct txq *txq; + + txq = dev->data->tx_queues[i]; + ret = rte_eth_tx_queue_setup(PORT_ID(sdev), i, + txq->info.nb_desc, txq->socket_id, + &txq->info.conf); + if (ret) { + ERROR("tx_queue_setup failed"); + return ret; + } + } + /* dev_link.link_status */ + if (dev->data->dev_link.link_status != + edev->data->dev_link.link_status) { + DEBUG("Configuring link_status"); + if (dev->data->dev_link.link_status) + ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); + else + ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); + if (ret) { + ERROR("Failed to apply link_status"); + return ret; + } + } else { + DEBUG("link_status already set"); + } + /* promiscuous */ + if (dev->data->promiscuous != edev->data->promiscuous) { + DEBUG("Configuring promiscuous"); + if (dev->data->promiscuous) + rte_eth_promiscuous_enable(PORT_ID(sdev)); + else + rte_eth_promiscuous_disable(PORT_ID(sdev)); + } else { + DEBUG("promiscuous already set"); + } + /* all_multicast */ + if (dev->data->all_multicast != edev->data->all_multicast) { + DEBUG("Configuring all_multicast"); + if (dev->data->all_multicast) + rte_eth_allmulticast_enable(PORT_ID(sdev)); + else + rte_eth_allmulticast_disable(PORT_ID(sdev)); + } else { + DEBUG("all_multicast already set"); + } + /* MTU */ + if (dev->data->mtu != edev->data->mtu) { + DEBUG("Configuring MTU"); + ret = rte_eth_dev_set_mtu(PORT_ID(sdev), dev->data->mtu); + if (ret) { + ERROR("Failed to apply MTU"); + return ret; + } + } else { + DEBUG("MTU already set"); + } + /* default MAC */ + DEBUG("Configuring default MAC address"); + ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), + &dev->data->mac_addrs[0]); + if (ret) { + ERROR("Setting default MAC address failed"); + return ret; + } + /* additional MAC */ + if (PRIV(dev)->nb_mac_addr > 1) + DEBUG("Configure additional MAC address%s", + (PRIV(dev)->nb_mac_addr > 2 ? "es" : "")); + for (i = 1; i < PRIV(dev)->nb_mac_addr; i++) { + struct ether_addr *ea; + + ea = &dev->data->mac_addrs[i]; + ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), ea, + PRIV(dev)->mac_addr_pool[i]); + if (ret) { + char ea_fmt[ETHER_ADDR_FMT_SIZE]; + + ether_format_addr(ea_fmt, ETHER_ADDR_FMT_SIZE, ea); + ERROR("Adding MAC address %s failed", ea_fmt); + } + } + /* VLAN filter */ + vfc1 = &dev->data->vlan_filter_conf; + vfc2 = &edev->data->vlan_filter_conf; + if (memcmp(vfc1, vfc2, sizeof(struct rte_vlan_filter_conf))) { + uint64_t vbit; + uint64_t ids; + size_t i; + uint16_t vlan_id; + + DEBUG("Configuring VLAN filter"); + for (i = 0; i < RTE_DIM(vfc1->ids); i++) { + if (vfc1->ids[i] == 0) + continue; + ids = vfc1->ids[i]; + while (ids) { + vlan_id = 64 * i; + /* count trailing zeroes */ + vbit = ~ids & (ids - 1); + /* clear least significant bit set */ + ids ^= (ids ^ (ids - 1)) ^ vbit; + for (; vbit; vlan_id++) + vbit >>= 1; + ret = rte_eth_dev_vlan_filter( + PORT_ID(sdev), vlan_id, 1); + if (ret) { + ERROR("Failed to apply VLAN filter %hu", + vlan_id); + return ret; + } + } + } + } else { + DEBUG("VLAN filter already set"); + } + /* rte_flow */ + if (TAILQ_EMPTY(&PRIV(dev)->flow_list)) { + DEBUG("rte_flow already set"); + } else { + DEBUG("Resetting rte_flow configuration"); + ret = rte_flow_flush(PORT_ID(sdev), &ferror); + if (ret) { + fs_flow_complain(&ferror); + return ret; + } + i = 0; + rte_errno = 0; + DEBUG("Configuring rte_flow"); + TAILQ_FOREACH(flow, &PRIV(dev)->flow_list, next) { + DEBUG("Creating flow #%" PRIu32, i++); + flow->flows[SUB_ID(sdev)] = + rte_flow_create(PORT_ID(sdev), + &flow->fd->attr, + flow->fd->items, + flow->fd->actions, + &ferror); + ret = rte_errno; + if (ret) + break; + } + if (ret) { + fs_flow_complain(&ferror); + return ret; + } + } + return 0; +} + +static void +fs_dev_remove(struct sub_device *sdev) +{ + int ret; + + if (sdev == NULL) + return; + switch (sdev->state) { + case DEV_STARTED: + rte_eth_dev_stop(PORT_ID(sdev)); + sdev->state = DEV_ACTIVE; + /* fallthrough */ + case DEV_ACTIVE: + rte_eth_dev_close(PORT_ID(sdev)); + sdev->state = DEV_PROBED; + /* fallthrough */ + case DEV_PROBED: + ret = rte_eal_hotplug_remove(sdev->bus->name, + sdev->dev->name); + if (ret) { + ERROR("Bus detach failed for sub_device %u", + SUB_ID(sdev)); + } else { + ETH(sdev)->state = RTE_ETH_DEV_UNUSED; + } + sdev->state = DEV_PARSED; + /* fallthrough */ + case DEV_PARSED: + case DEV_UNDEFINED: + sdev->state = DEV_UNDEFINED; + /* the end */ + break; + } + failsafe_hotplug_alarm_install(sdev->fs_dev); +} + +static inline int +fs_rxtx_clean(struct sub_device *sdev) +{ + uint16_t i; + + for (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++) + if (FS_ATOMIC_RX(sdev, i)) + return 0; + for (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++) + if (FS_ATOMIC_TX(sdev, i)) + return 0; + return 1; +} + +void +failsafe_dev_remove(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + if (sdev->remove && fs_rxtx_clean(sdev)) + fs_dev_remove(sdev); +} + +int +failsafe_eth_dev_state_sync(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint32_t inactive; + int ret; + uint8_t i; + + if (PRIV(dev)->state < DEV_PARSED) + return 0; + + ret = failsafe_args_parse_subs(dev); + if (ret) + goto err_remove; + + if (PRIV(dev)->state < DEV_PROBED) + return 0; + ret = failsafe_eal_init(dev); + if (ret) + goto err_remove; + if (PRIV(dev)->state < DEV_ACTIVE) + return 0; + inactive = 0; + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state == DEV_PROBED) { + inactive |= UINT32_C(1) << i; + ret = eth_dev_flow_isolate_set(dev, sdev); + if (ret) { + ERROR("Could not apply configuration to sub_device %d", + i); + goto err_remove; + } + } + } + ret = dev->dev_ops->dev_configure(dev); + if (ret) + goto err_remove; + FOREACH_SUBDEV(sdev, i, dev) { + if (inactive & (UINT32_C(1) << i)) { + ret = fs_eth_dev_conf_apply(dev, sdev); + if (ret) { + ERROR("Could not apply configuration to sub_device %d", + i); + goto err_remove; + } + } + } + /* + * If new devices have been configured, check if + * the link state has changed. + */ + if (inactive) + dev->dev_ops->link_update(dev, 1); + if (PRIV(dev)->state < DEV_STARTED) + return 0; + ret = dev->dev_ops->dev_start(dev); + if (ret) + goto err_remove; + return 0; +err_remove: + FOREACH_SUBDEV(sdev, i, dev) + if (sdev->state != PRIV(dev)->state) + sdev->remove = 1; + return ret; +} + +int +failsafe_eth_rmv_event_callback(uint8_t port_id __rte_unused, + enum rte_eth_event_type event __rte_unused, + void *cb_arg, void *out __rte_unused) +{ + struct sub_device *sdev = cb_arg; + + /* Switch as soon as possible tx_dev. */ + fs_switch_dev(sdev->fs_dev, sdev); + /* Use safe bursts in any case. */ + set_burst_fn(sdev->fs_dev, 1); + /* + * Async removal, the sub-PMD will try to unregister + * the callback at the source of the current thread context. + */ + sdev->remove = 1; + return 0; +} + +int +failsafe_eth_lsc_event_callback(uint8_t port_id __rte_unused, + enum rte_eth_event_type event __rte_unused, + void *cb_arg, void *out __rte_unused) +{ + struct rte_eth_dev *dev = cb_arg; + int ret; + + ret = dev->dev_ops->link_update(dev, 0); + /* We must pass on the LSC event */ + if (ret) + return _rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + else + return 0; +} diff --git a/drivers/net/failsafe/failsafe_flow.c b/drivers/net/failsafe/failsafe_flow.c new file mode 100644 index 00000000..153ceeed --- /dev/null +++ b/drivers/net/failsafe/failsafe_flow.c @@ -0,0 +1,244 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/queue.h> + +#include <rte_malloc.h> +#include <rte_tailq.h> +#include <rte_flow.h> +#include <rte_flow_driver.h> + +#include "failsafe_private.h" + +static struct rte_flow * +fs_flow_allocate(const struct rte_flow_attr *attr, + const struct rte_flow_item *items, + const struct rte_flow_action *actions) +{ + struct rte_flow *flow; + size_t fdsz; + + fdsz = rte_flow_copy(NULL, 0, attr, items, actions); + flow = rte_zmalloc(NULL, + sizeof(struct rte_flow) + fdsz, + RTE_CACHE_LINE_SIZE); + if (flow == NULL) { + ERROR("Could not allocate new flow"); + return NULL; + } + flow->fd = (void *)((uintptr_t)flow + sizeof(*flow)); + if (rte_flow_copy(flow->fd, fdsz, attr, items, actions) != fdsz) { + ERROR("Failed to copy flow description"); + rte_free(flow); + return NULL; + } + return flow; +} + +static void +fs_flow_release(struct rte_flow **flow) +{ + rte_free(*flow); + *flow = NULL; +} + +static int +fs_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item patterns[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_flow_validate on sub_device %d", i); + ret = rte_flow_validate(PORT_ID(sdev), + attr, patterns, actions, error); + if (ret) { + ERROR("Operation rte_flow_validate failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static struct rte_flow * +fs_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item patterns[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct sub_device *sdev; + struct rte_flow *flow; + uint8_t i; + + flow = fs_flow_allocate(attr, patterns, actions); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + flow->flows[i] = rte_flow_create(PORT_ID(sdev), + attr, patterns, actions, error); + if (flow->flows[i] == NULL) { + ERROR("Failed to create flow on sub_device %d", + i); + goto err; + } + } + TAILQ_INSERT_TAIL(&PRIV(dev)->flow_list, flow, next); + return flow; +err: + FOREACH_SUBDEV(sdev, i, dev) { + if (flow->flows[i] != NULL) + rte_flow_destroy(PORT_ID(sdev), + flow->flows[i], error); + } + fs_flow_release(&flow); + return NULL; +} + +static int +fs_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + if (flow == NULL) { + ERROR("Invalid flow"); + return -EINVAL; + } + ret = 0; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + int local_ret; + + if (flow->flows[i] == NULL) + continue; + local_ret = rte_flow_destroy(PORT_ID(sdev), + flow->flows[i], error); + if (local_ret) { + ERROR("Failed to destroy flow on sub_device %d: %d", + i, local_ret); + if (ret == 0) + ret = local_ret; + } + } + TAILQ_REMOVE(&PRIV(dev)->flow_list, flow, next); + fs_flow_release(&flow); + return ret; +} + +static int +fs_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + struct rte_flow *flow; + void *tmp; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_flow_flush on sub_device %d", i); + ret = rte_flow_flush(PORT_ID(sdev), error); + if (ret) { + ERROR("Operation rte_flow_flush failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + TAILQ_FOREACH_SAFE(flow, &PRIV(dev)->flow_list, next, tmp) { + TAILQ_REMOVE(&PRIV(dev)->flow_list, flow, next); + fs_flow_release(&flow); + } + return 0; +} + +static int +fs_flow_query(struct rte_eth_dev *dev, + struct rte_flow *flow, + enum rte_flow_action_type type, + void *arg, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + + sdev = TX_SUBDEV(dev); + if (sdev != NULL) { + return rte_flow_query(PORT_ID(sdev), + flow->flows[SUB_ID(sdev)], type, arg, error); + } + WARN("No active sub_device to query about its flow"); + return -1; +} + +static int +fs_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state < DEV_PROBED) + continue; + DEBUG("Calling rte_flow_isolate on sub_device %d", i); + if (PRIV(dev)->flow_isolated != sdev->flow_isolated) + WARN("flow isolation mode of sub_device %d in incoherent state.", + i); + ret = rte_flow_isolate(PORT_ID(sdev), set, error); + if (ret) { + ERROR("Operation rte_flow_isolate failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + sdev->flow_isolated = set; + } + PRIV(dev)->flow_isolated = set; + return 0; +} + +const struct rte_flow_ops fs_flow_ops = { + .validate = fs_flow_validate, + .create = fs_flow_create, + .destroy = fs_flow_destroy, + .flush = fs_flow_flush, + .query = fs_flow_query, + .isolate = fs_flow_isolate, +}; diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c new file mode 100644 index 00000000..ff9ad155 --- /dev/null +++ b/drivers/net/failsafe/failsafe_ops.c @@ -0,0 +1,867 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdint.h> + +#include <rte_debug.h> +#include <rte_atomic.h> +#include <rte_ethdev.h> +#include <rte_malloc.h> +#include <rte_flow.h> + +#include "failsafe_private.h" + +static struct rte_eth_dev_info default_infos = { + /* Max possible number of elements */ + .max_rx_pktlen = UINT32_MAX, + .max_rx_queues = RTE_MAX_QUEUES_PER_PORT, + .max_tx_queues = RTE_MAX_QUEUES_PER_PORT, + .max_mac_addrs = FAILSAFE_MAX_ETHADDR, + .max_hash_mac_addrs = UINT32_MAX, + .max_vfs = UINT16_MAX, + .max_vmdq_pools = UINT16_MAX, + .rx_desc_lim = { + .nb_max = UINT16_MAX, + .nb_min = 0, + .nb_align = 1, + .nb_seg_max = UINT16_MAX, + .nb_mtu_seg_max = UINT16_MAX, + }, + .tx_desc_lim = { + .nb_max = UINT16_MAX, + .nb_min = 0, + .nb_align = 1, + .nb_seg_max = UINT16_MAX, + .nb_mtu_seg_max = UINT16_MAX, + }, + /* + * Set of capabilities that can be verified upon + * configuring a sub-device. + */ + .rx_offload_capa = + DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_QINQ_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_TCP_LRO, + .tx_offload_capa = 0x0, + .flow_type_rss_offloads = 0x0, +}; + +/** + * Check whether a specific offloading capability + * is supported by a sub_device. + * + * @return + * 0: all requested capabilities are supported by the sub_device + * positive value: This flag at least is not supported by the sub_device + */ +static int +fs_port_offload_validate(struct rte_eth_dev *dev, + struct sub_device *sdev) +{ + struct rte_eth_dev_info infos = {0}; + struct rte_eth_conf *cf; + uint32_t cap; + + cf = &dev->data->dev_conf; + SUBOPS(sdev, dev_infos_get)(ETH(sdev), &infos); + /* RX capabilities */ + cap = infos.rx_offload_capa; + if (cf->rxmode.hw_vlan_strip && + ((cap & DEV_RX_OFFLOAD_VLAN_STRIP) == 0)) { + WARN("VLAN stripping offload requested but not supported by sub_device %d", + SUB_ID(sdev)); + return DEV_RX_OFFLOAD_VLAN_STRIP; + } + if (cf->rxmode.hw_ip_checksum && + ((cap & (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM)) != + (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM))) { + WARN("IP checksum offload requested but not supported by sub_device %d", + SUB_ID(sdev)); + return DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + } + if (cf->rxmode.enable_lro && + ((cap & DEV_RX_OFFLOAD_TCP_LRO) == 0)) { + WARN("TCP LRO offload requested but not supported by sub_device %d", + SUB_ID(sdev)); + return DEV_RX_OFFLOAD_TCP_LRO; + } + if (cf->rxmode.hw_vlan_extend && + ((cap & DEV_RX_OFFLOAD_QINQ_STRIP) == 0)) { + WARN("Stacked VLAN stripping offload requested but not supported by sub_device %d", + SUB_ID(sdev)); + return DEV_RX_OFFLOAD_QINQ_STRIP; + } + /* TX capabilities */ + /* Nothing to do, no tx capa supported */ + return 0; +} + +/* + * Disable the dev_conf flag related to an offload capability flag + * within an ethdev configuration. + */ +static int +fs_port_disable_offload(struct rte_eth_conf *cf, + uint32_t ol_cap) +{ + switch (ol_cap) { + case DEV_RX_OFFLOAD_VLAN_STRIP: + INFO("Disabling VLAN stripping offload"); + cf->rxmode.hw_vlan_strip = 0; + break; + case DEV_RX_OFFLOAD_IPV4_CKSUM: + case DEV_RX_OFFLOAD_UDP_CKSUM: + case DEV_RX_OFFLOAD_TCP_CKSUM: + case (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM): + INFO("Disabling IP checksum offload"); + cf->rxmode.hw_ip_checksum = 0; + break; + case DEV_RX_OFFLOAD_TCP_LRO: + INFO("Disabling TCP LRO offload"); + cf->rxmode.enable_lro = 0; + break; + case DEV_RX_OFFLOAD_QINQ_STRIP: + INFO("Disabling stacked VLAN stripping offload"); + cf->rxmode.hw_vlan_extend = 0; + break; + default: + DEBUG("Unable to disable offload capability: %" PRIx32, + ol_cap); + return -1; + } + return 0; +} + +static int +fs_dev_configure(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int capa_flag; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state != DEV_PROBED) + continue; + DEBUG("Checking capabilities for sub_device %d", i); + while ((capa_flag = fs_port_offload_validate(dev, sdev))) { + /* + * Refuse to change configuration if multiple devices + * are present and we already have configured at least + * some of them. + */ + if (PRIV(dev)->state >= DEV_ACTIVE && + PRIV(dev)->subs_tail > 1) { + ERROR("device already configured, cannot fix live configuration"); + return -1; + } + ret = fs_port_disable_offload(&dev->data->dev_conf, + capa_flag); + if (ret) { + ERROR("Unable to disable offload capability"); + return ret; + } + } + } + FOREACH_SUBDEV(sdev, i, dev) { + int rmv_interrupt = 0; + int lsc_interrupt = 0; + int lsc_enabled; + + if (sdev->state != DEV_PROBED) + continue; + + rmv_interrupt = ETH(sdev)->data->dev_flags & + RTE_ETH_DEV_INTR_RMV; + if (rmv_interrupt) { + DEBUG("Enabling RMV interrupts for sub_device %d", i); + dev->data->dev_conf.intr_conf.rmv = 1; + } else { + DEBUG("sub_device %d does not support RMV event", i); + } + lsc_enabled = dev->data->dev_conf.intr_conf.lsc; + lsc_interrupt = lsc_enabled && + (ETH(sdev)->data->dev_flags & + RTE_ETH_DEV_INTR_LSC); + if (lsc_interrupt) { + DEBUG("Enabling LSC interrupts for sub_device %d", i); + dev->data->dev_conf.intr_conf.lsc = 1; + } else if (lsc_enabled && !lsc_interrupt) { + DEBUG("Disabling LSC interrupts for sub_device %d", i); + dev->data->dev_conf.intr_conf.lsc = 0; + } + DEBUG("Configuring sub-device %d", i); + sdev->remove = 0; + ret = rte_eth_dev_configure(PORT_ID(sdev), + dev->data->nb_rx_queues, + dev->data->nb_tx_queues, + &dev->data->dev_conf); + if (ret) { + ERROR("Could not configure sub_device %d", i); + return ret; + } + if (rmv_interrupt) { + ret = rte_eth_dev_callback_register(PORT_ID(sdev), + RTE_ETH_EVENT_INTR_RMV, + failsafe_eth_rmv_event_callback, + sdev); + if (ret) + WARN("Failed to register RMV callback for sub_device %d", + SUB_ID(sdev)); + } + dev->data->dev_conf.intr_conf.rmv = 0; + if (lsc_interrupt) { + ret = rte_eth_dev_callback_register(PORT_ID(sdev), + RTE_ETH_EVENT_INTR_LSC, + failsafe_eth_lsc_event_callback, + dev); + if (ret) + WARN("Failed to register LSC callback for sub_device %d", + SUB_ID(sdev)); + } + dev->data->dev_conf.intr_conf.lsc = lsc_enabled; + sdev->state = DEV_ACTIVE; + } + if (PRIV(dev)->state < DEV_ACTIVE) + PRIV(dev)->state = DEV_ACTIVE; + return 0; +} + +static int +fs_dev_start(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state != DEV_ACTIVE) + continue; + DEBUG("Starting sub_device %d", i); + ret = rte_eth_dev_start(PORT_ID(sdev)); + if (ret) + return ret; + sdev->state = DEV_STARTED; + } + if (PRIV(dev)->state < DEV_STARTED) + PRIV(dev)->state = DEV_STARTED; + fs_switch_dev(dev, NULL); + return 0; +} + +static void +fs_dev_stop(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + PRIV(dev)->state = DEV_STARTED - 1; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) { + rte_eth_dev_stop(PORT_ID(sdev)); + sdev->state = DEV_STARTED - 1; + } +} + +static int +fs_dev_set_link_up(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i); + ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); + if (ret) { + ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static int +fs_dev_set_link_down(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i); + ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); + if (ret) { + ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static void fs_dev_free_queues(struct rte_eth_dev *dev); +static void +fs_dev_close(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + failsafe_hotplug_alarm_cancel(dev); + if (PRIV(dev)->state == DEV_STARTED) + dev->dev_ops->dev_stop(dev); + PRIV(dev)->state = DEV_ACTIVE - 1; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Closing sub_device %d", i); + rte_eth_dev_close(PORT_ID(sdev)); + sdev->state = DEV_ACTIVE - 1; + } + fs_dev_free_queues(dev); +} + +static void +fs_rx_queue_release(void *queue) +{ + struct rte_eth_dev *dev; + struct sub_device *sdev; + uint8_t i; + struct rxq *rxq; + + if (queue == NULL) + return; + rxq = queue; + dev = rxq->priv->dev; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + SUBOPS(sdev, rx_queue_release) + (ETH(sdev)->data->rx_queues[rxq->qid]); + dev->data->rx_queues[rxq->qid] = NULL; + rte_free(rxq); +} + +static int +fs_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool) +{ + struct sub_device *sdev; + struct rxq *rxq; + uint8_t i; + int ret; + + rxq = dev->data->rx_queues[rx_queue_id]; + if (rxq != NULL) { + fs_rx_queue_release(rxq); + dev->data->rx_queues[rx_queue_id] = NULL; + } + rxq = rte_zmalloc(NULL, + sizeof(*rxq) + + sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, + RTE_CACHE_LINE_SIZE); + if (rxq == NULL) + return -ENOMEM; + FOREACH_SUBDEV(sdev, i, dev) + rte_atomic64_init(&rxq->refcnt[i]); + rxq->qid = rx_queue_id; + rxq->socket_id = socket_id; + rxq->info.mp = mb_pool; + rxq->info.conf = *rx_conf; + rxq->info.nb_desc = nb_rx_desc; + rxq->priv = PRIV(dev); + dev->data->rx_queues[rx_queue_id] = rxq; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_rx_queue_setup(PORT_ID(sdev), + rx_queue_id, + nb_rx_desc, socket_id, + rx_conf, mb_pool); + if (ret) { + ERROR("RX queue setup failed for sub_device %d", i); + goto free_rxq; + } + } + return 0; +free_rxq: + fs_rx_queue_release(rxq); + return ret; +} + +static void +fs_tx_queue_release(void *queue) +{ + struct rte_eth_dev *dev; + struct sub_device *sdev; + uint8_t i; + struct txq *txq; + + if (queue == NULL) + return; + txq = queue; + dev = txq->priv->dev; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + SUBOPS(sdev, tx_queue_release) + (ETH(sdev)->data->tx_queues[txq->qid]); + dev->data->tx_queues[txq->qid] = NULL; + rte_free(txq); +} + +static int +fs_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf) +{ + struct sub_device *sdev; + struct txq *txq; + uint8_t i; + int ret; + + txq = dev->data->tx_queues[tx_queue_id]; + if (txq != NULL) { + fs_tx_queue_release(txq); + dev->data->tx_queues[tx_queue_id] = NULL; + } + txq = rte_zmalloc("ethdev TX queue", + sizeof(*txq) + + sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, + RTE_CACHE_LINE_SIZE); + if (txq == NULL) + return -ENOMEM; + FOREACH_SUBDEV(sdev, i, dev) + rte_atomic64_init(&txq->refcnt[i]); + txq->qid = tx_queue_id; + txq->socket_id = socket_id; + txq->info.conf = *tx_conf; + txq->info.nb_desc = nb_tx_desc; + txq->priv = PRIV(dev); + dev->data->tx_queues[tx_queue_id] = txq; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_tx_queue_setup(PORT_ID(sdev), + tx_queue_id, + nb_tx_desc, socket_id, + tx_conf); + if (ret) { + ERROR("TX queue setup failed for sub_device %d", i); + goto free_txq; + } + } + return 0; +free_txq: + fs_tx_queue_release(txq); + return ret; +} + +static void +fs_dev_free_queues(struct rte_eth_dev *dev) +{ + uint16_t i; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + fs_rx_queue_release(dev->data->rx_queues[i]); + dev->data->rx_queues[i] = NULL; + } + dev->data->nb_rx_queues = 0; + for (i = 0; i < dev->data->nb_tx_queues; i++) { + fs_tx_queue_release(dev->data->tx_queues[i]); + dev->data->tx_queues[i] = NULL; + } + dev->data->nb_tx_queues = 0; +} + +static void +fs_promiscuous_enable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_promiscuous_enable(PORT_ID(sdev)); +} + +static void +fs_promiscuous_disable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_promiscuous_disable(PORT_ID(sdev)); +} + +static void +fs_allmulticast_enable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_allmulticast_enable(PORT_ID(sdev)); +} + +static void +fs_allmulticast_disable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_allmulticast_disable(PORT_ID(sdev)); +} + +static int +fs_link_update(struct rte_eth_dev *dev, + int wait_to_complete) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling link_update on sub_device %d", i); + ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete); + if (ret && ret != -1) { + ERROR("Link update failed for sub_device %d with error %d", + i, ret); + return ret; + } + } + if (TX_SUBDEV(dev)) { + struct rte_eth_link *l1; + struct rte_eth_link *l2; + + l1 = &dev->data->dev_link; + l2 = Ð(TX_SUBDEV(dev))->data->dev_link; + if (memcmp(l1, l2, sizeof(*l1))) { + *l1 = *l2; + return 0; + } + } + return -1; +} + +static void +fs_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats) +{ + if (TX_SUBDEV(dev) == NULL) + return; + rte_eth_stats_get(PORT_ID(TX_SUBDEV(dev)), stats); +} + +static void +fs_stats_reset(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_stats_reset(PORT_ID(sdev)); +} + +/** + * Fail-safe dev_infos_get rules: + * + * No sub_device: + * Numerables: + * Use the maximum possible values for any field, so as not + * to impede any further configuration effort. + * Capabilities: + * Limits capabilities to those that are understood by the + * fail-safe PMD. This understanding stems from the fail-safe + * being capable of verifying that the related capability is + * expressed within the device configuration (struct rte_eth_conf). + * + * At least one probed sub_device: + * Numerables: + * Uses values from the active probed sub_device + * The rationale here is that if any sub_device is less capable + * (for example concerning the number of queues) than the active + * sub_device, then its subsequent configuration will fail. + * It is impossible to foresee this failure when the failing sub_device + * is supposed to be plugged-in later on, so the configuration process + * is the single point of failure and error reporting. + * Capabilities: + * Uses a logical AND of RX capabilities among + * all sub_devices and the default capabilities. + * Uses a logical AND of TX capabilities among + * the active probed sub_device and the default capabilities. + * + */ +static void +fs_dev_infos_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *infos) +{ + struct sub_device *sdev; + uint8_t i; + + sdev = TX_SUBDEV(dev); + if (sdev == NULL) { + DEBUG("No probed device, using default infos"); + rte_memcpy(&PRIV(dev)->infos, &default_infos, + sizeof(default_infos)); + } else { + uint32_t rx_offload_capa; + + rx_offload_capa = default_infos.rx_offload_capa; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + rte_eth_dev_info_get(PORT_ID(sdev), + &PRIV(dev)->infos); + rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa; + } + sdev = TX_SUBDEV(dev); + rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos); + PRIV(dev)->infos.rx_offload_capa = rx_offload_capa; + PRIV(dev)->infos.tx_offload_capa &= + default_infos.tx_offload_capa; + PRIV(dev)->infos.flow_type_rss_offloads &= + default_infos.flow_type_rss_offloads; + } + rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos)); +} + +static const uint32_t * +fs_dev_supported_ptypes_get(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + struct rte_eth_dev *edev; + + sdev = TX_SUBDEV(dev); + if (sdev == NULL) + return NULL; + edev = ETH(sdev); + /* ENOTSUP: counts as no supported ptypes */ + if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) + return NULL; + /* + * The API does not permit to do a clean AND of all ptypes, + * It is also incomplete by design and we do not really care + * to have a best possible value in this context. + * We just return the ptypes of the device of highest + * priority, usually the PREFERRED device. + */ + return SUBOPS(sdev, dev_supported_ptypes_get)(edev); +} + +static int +fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i); + ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu); + if (ret) { + ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d", + i, ret); + return ret; + } + } + return 0; +} + +static int +fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i); + ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on); + if (ret) { + ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static int +fs_flow_ctrl_get(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf) +{ + struct sub_device *sdev; + + sdev = TX_SUBDEV(dev); + if (sdev == NULL) + return 0; + if (SUBOPS(sdev, flow_ctrl_get) == NULL) + return -ENOTSUP; + return SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf); +} + +static int +fs_flow_ctrl_set(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i); + ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf); + if (ret) { + ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static void +fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) +{ + struct sub_device *sdev; + uint8_t i; + + /* No check: already done within the rte_eth_dev_mac_addr_remove + * call for the fail-safe device. + */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_dev_mac_addr_remove(PORT_ID(sdev), + &dev->data->mac_addrs[index]); + PRIV(dev)->mac_addr_pool[index] = 0; +} + +static int +fs_mac_addr_add(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + uint32_t index, + uint32_t vmdq) +{ + struct sub_device *sdev; + int ret; + uint8_t i; + + RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq); + if (ret) { + ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %" + PRIu8 " with error %d", i, ret); + return ret; + } + } + if (index >= PRIV(dev)->nb_mac_addr) { + DEBUG("Growing mac_addrs array"); + PRIV(dev)->nb_mac_addr = index; + } + PRIV(dev)->mac_addr_pool[index] = vmdq; + return 0; +} + +static void +fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr); +} + +static int +fs_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type type, + enum rte_filter_op op, + void *arg) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + if (type == RTE_ETH_FILTER_GENERIC && + op == RTE_ETH_FILTER_GET) { + *(const void **)arg = &fs_flow_ops; + return 0; + } + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i); + ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg); + if (ret) { + ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +const struct eth_dev_ops failsafe_ops = { + .dev_configure = fs_dev_configure, + .dev_start = fs_dev_start, + .dev_stop = fs_dev_stop, + .dev_set_link_down = fs_dev_set_link_down, + .dev_set_link_up = fs_dev_set_link_up, + .dev_close = fs_dev_close, + .promiscuous_enable = fs_promiscuous_enable, + .promiscuous_disable = fs_promiscuous_disable, + .allmulticast_enable = fs_allmulticast_enable, + .allmulticast_disable = fs_allmulticast_disable, + .link_update = fs_link_update, + .stats_get = fs_stats_get, + .stats_reset = fs_stats_reset, + .dev_infos_get = fs_dev_infos_get, + .dev_supported_ptypes_get = fs_dev_supported_ptypes_get, + .mtu_set = fs_mtu_set, + .vlan_filter_set = fs_vlan_filter_set, + .rx_queue_setup = fs_rx_queue_setup, + .tx_queue_setup = fs_tx_queue_setup, + .rx_queue_release = fs_rx_queue_release, + .tx_queue_release = fs_tx_queue_release, + .flow_ctrl_get = fs_flow_ctrl_get, + .flow_ctrl_set = fs_flow_ctrl_set, + .mac_addr_remove = fs_mac_addr_remove, + .mac_addr_add = fs_mac_addr_add, + .mac_addr_set = fs_mac_addr_set, + .filter_ctrl = fs_filter_ctrl, +}; diff --git a/drivers/net/failsafe/failsafe_private.h b/drivers/net/failsafe/failsafe_private.h new file mode 100644 index 00000000..0361cf43 --- /dev/null +++ b/drivers/net/failsafe/failsafe_private.h @@ -0,0 +1,359 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ETH_FAILSAFE_PRIVATE_H_ +#define _RTE_ETH_FAILSAFE_PRIVATE_H_ + +#include <sys/queue.h> + +#include <rte_atomic.h> +#include <rte_dev.h> +#include <rte_ethdev.h> +#include <rte_devargs.h> + +#define FAILSAFE_DRIVER_NAME "Fail-safe PMD" + +#define PMD_FAILSAFE_MAC_KVARG "mac" +#define PMD_FAILSAFE_HOTPLUG_POLL_KVARG "hotplug_poll" +#define PMD_FAILSAFE_PARAM_STRING \ + "dev(<ifc>)," \ + "exec(<shell command>)," \ + "mac=mac_addr," \ + "hotplug_poll=u64" \ + "" + +#define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000 + +#define FAILSAFE_MAX_ETHPORTS 2 +#define FAILSAFE_MAX_ETHADDR 128 + +/* TYPES */ + +struct rxq { + struct fs_priv *priv; + uint16_t qid; + /* id of last sub_device polled */ + uint8_t last_polled; + unsigned int socket_id; + struct rte_eth_rxq_info info; + rte_atomic64_t refcnt[]; +}; + +struct txq { + struct fs_priv *priv; + uint16_t qid; + unsigned int socket_id; + struct rte_eth_txq_info info; + rte_atomic64_t refcnt[]; +}; + +struct rte_flow { + TAILQ_ENTRY(rte_flow) next; + /* sub_flows */ + struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS]; + /* flow description for synchronization */ + struct rte_flow_desc *fd; +}; + +enum dev_state { + DEV_UNDEFINED, + DEV_PARSED, + DEV_PROBED, + DEV_ACTIVE, + DEV_STARTED, +}; + +struct sub_device { + /* Exhaustive DPDK device description */ + struct rte_devargs devargs; + struct rte_bus *bus; + struct rte_device *dev; + struct rte_eth_dev *edev; + uint8_t sid; + /* Device state machine */ + enum dev_state state; + /* Some device are defined as a command line */ + char *cmdline; + /* fail-safe device backreference */ + struct rte_eth_dev *fs_dev; + /* flag calling for recollection */ + volatile unsigned int remove:1; + /* flow isolation state */ + int flow_isolated:1; +}; + +struct fs_priv { + struct rte_eth_dev *dev; + /* + * Set of sub_devices. + * subs[0] is the preferred device + * any other is just another slave + */ + struct sub_device *subs; + uint8_t subs_head; /* if head == tail, no subs */ + uint8_t subs_tail; /* first invalid */ + uint8_t subs_tx; /* current emitting device */ + uint8_t current_probed; + /* flow mapping */ + TAILQ_HEAD(sub_flows, rte_flow) flow_list; + /* current number of mac_addr slots allocated. */ + uint32_t nb_mac_addr; + struct ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR]; + uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR]; + /* current capabilities */ + struct rte_eth_dev_info infos; + /* + * Fail-safe state machine. + * This level will be tracking state of the EAL and eth + * layer at large as defined by the user application. + * It will then steer the sub_devices toward the same + * synchronized state. + */ + enum dev_state state; + unsigned int pending_alarm:1; /* An alarm is pending */ + /* flow isolation state */ + int flow_isolated:1; +}; + +/* MISC */ + +int failsafe_hotplug_alarm_install(struct rte_eth_dev *dev); +int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev); + +/* RX / TX */ + +void set_burst_fn(struct rte_eth_dev *dev, int force_safe); + +uint16_t failsafe_rx_burst(void *rxq, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t failsafe_tx_burst(void *txq, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts); + +uint16_t failsafe_rx_burst_fast(void *rxq, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t failsafe_tx_burst_fast(void *txq, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts); + +/* ARGS */ + +int failsafe_args_parse(struct rte_eth_dev *dev, const char *params); +void failsafe_args_free(struct rte_eth_dev *dev); +int failsafe_args_count_subdevice(struct rte_eth_dev *dev, const char *params); +int failsafe_args_parse_subs(struct rte_eth_dev *dev); + +/* EAL */ + +int failsafe_eal_init(struct rte_eth_dev *dev); +int failsafe_eal_uninit(struct rte_eth_dev *dev); + +/* ETH_DEV */ + +int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev); +void failsafe_dev_remove(struct rte_eth_dev *dev); +int failsafe_eth_rmv_event_callback(uint8_t port_id, + enum rte_eth_event_type type, + void *arg, void *out); +int failsafe_eth_lsc_event_callback(uint8_t port_id, + enum rte_eth_event_type event, + void *cb_arg, void *out); + +/* GLOBALS */ + +extern const char pmd_failsafe_driver_name[]; +extern const struct eth_dev_ops failsafe_ops; +extern const struct rte_flow_ops fs_flow_ops; +extern uint64_t hotplug_poll; +extern int mac_from_arg; + +/* HELPERS */ + +/* dev: (struct rte_eth_dev *) fail-safe device */ +#define PRIV(dev) \ + ((struct fs_priv *)(dev)->data->dev_private) + +/* sdev: (struct sub_device *) */ +#define ETH(sdev) \ + ((sdev)->edev) + +/* sdev: (struct sub_device *) */ +#define PORT_ID(sdev) \ + (ETH(sdev)->data->port_id) + +/* sdev: (struct sub_device *) */ +#define SUB_ID(sdev) \ + ((sdev)->sid) + +/** + * Stateful iterator construct over fail-safe sub-devices: + * s: (struct sub_device *), iterator + * i: (uint8_t), increment + * dev: (struct rte_eth_dev *), fail-safe ethdev + * state: (enum dev_state), minimum acceptable device state + */ +#define FOREACH_SUBDEV_STATE(s, i, dev, state) \ + for (i = fs_find_next((dev), 0, state); \ + i < PRIV(dev)->subs_tail && (s = &PRIV(dev)->subs[i]); \ + i = fs_find_next((dev), i + 1, state)) + +/** + * Iterator construct over fail-safe sub-devices: + * s: (struct sub_device *), iterator + * i: (uint8_t), increment + * dev: (struct rte_eth_dev *), fail-safe ethdev + */ +#define FOREACH_SUBDEV(s, i, dev) \ + FOREACH_SUBDEV_STATE(s, i, dev, DEV_UNDEFINED) + +/* dev: (struct rte_eth_dev *) fail-safe device */ +#define PREFERRED_SUBDEV(dev) \ + (&PRIV(dev)->subs[0]) + +/* dev: (struct rte_eth_dev *) fail-safe device */ +#define TX_SUBDEV(dev) \ + (PRIV(dev)->subs_tx >= PRIV(dev)->subs_tail ? NULL \ + : (PRIV(dev)->subs[PRIV(dev)->subs_tx].state < DEV_PROBED ? NULL \ + : &PRIV(dev)->subs[PRIV(dev)->subs_tx])) + +/** + * s: (struct sub_device *) + * ops: (struct eth_dev_ops) member + */ +#define SUBOPS(s, ops) \ + (ETH(s)->dev_ops->ops) + +/** + * Atomic guard + */ + +/** + * a: (rte_atomic64_t) + */ +#define FS_ATOMIC_P(a) \ + rte_atomic64_add(&(a), 1) + +/** + * a: (rte_atomic64_t) + */ +#define FS_ATOMIC_V(a) \ + rte_atomic64_sub(&(a), 1) + +/** + * s: (struct sub_device *) + * i: uint16_t qid + */ +#define FS_ATOMIC_RX(s, i) \ + rte_atomic64_read( \ + &((struct rxq *)((s)->fs_dev->data->rx_queues[i]))->refcnt[(s)->sid] \ + ) +/** + * s: (struct sub_device *) + * i: uint16_t qid + */ +#define FS_ATOMIC_TX(s, i) \ + rte_atomic64_read( \ + &((struct txq *)((s)->fs_dev->data->tx_queues[i]))->refcnt[(s)->sid] \ + ) + +#define LOG__(level, m, ...) \ + RTE_LOG(level, PMD, "net_failsafe: " m "%c", __VA_ARGS__) +#define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n') +#define DEBUG(...) LOG_(DEBUG, __VA_ARGS__) +#define INFO(...) LOG_(INFO, __VA_ARGS__) +#define WARN(...) LOG_(WARNING, __VA_ARGS__) +#define ERROR(...) LOG_(ERR, __VA_ARGS__) + +/* inlined functions */ + +static inline uint8_t +fs_find_next(struct rte_eth_dev *dev, uint8_t sid, + enum dev_state min_state) +{ + while (sid < PRIV(dev)->subs_tail) { + if (PRIV(dev)->subs[sid].state >= min_state) + break; + sid++; + } + if (sid >= PRIV(dev)->subs_tail) + return PRIV(dev)->subs_tail; + return sid; +} + +/* + * Switch emitting device. + * If banned is set, banned must not be considered for + * the role of emitting device. + */ +static inline void +fs_switch_dev(struct rte_eth_dev *dev, + struct sub_device *banned) +{ + struct sub_device *txd; + enum dev_state req_state; + + req_state = PRIV(dev)->state; + txd = TX_SUBDEV(dev); + if (PREFERRED_SUBDEV(dev)->state >= req_state && + PREFERRED_SUBDEV(dev) != banned) { + if (txd != PREFERRED_SUBDEV(dev) && + (txd == NULL || + (req_state == DEV_STARTED) || + (txd && txd->state < DEV_STARTED))) { + DEBUG("Switching tx_dev to preferred sub_device"); + PRIV(dev)->subs_tx = 0; + } + } else if ((txd && txd->state < req_state) || + txd == NULL || + txd == banned) { + struct sub_device *sdev; + uint8_t i; + + /* Using acceptable device */ + FOREACH_SUBDEV_STATE(sdev, i, dev, req_state) { + if (sdev == banned) + continue; + DEBUG("Switching tx_dev to sub_device %d", + i); + PRIV(dev)->subs_tx = i; + break; + } + } else if (txd && txd->state < req_state) { + DEBUG("No device ready, deactivating tx_dev"); + PRIV(dev)->subs_tx = PRIV(dev)->subs_tail; + } else { + return; + } + set_burst_fn(dev, 0); + rte_wmb(); +} + +#endif /* _RTE_ETH_FAILSAFE_PRIVATE_H_ */ diff --git a/drivers/net/failsafe/failsafe_rxtx.c b/drivers/net/failsafe/failsafe_rxtx.c new file mode 100644 index 00000000..73114215 --- /dev/null +++ b/drivers/net/failsafe/failsafe_rxtx.c @@ -0,0 +1,203 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_atomic.h> +#include <rte_debug.h> +#include <rte_mbuf.h> +#include <rte_ethdev.h> + +#include "failsafe_private.h" + +static inline int +fs_rx_unsafe(struct sub_device *sdev) +{ + return (ETH(sdev) == NULL) || + (ETH(sdev)->rx_pkt_burst == NULL) || + (sdev->state != DEV_STARTED); +} + +static inline int +fs_tx_unsafe(struct sub_device *sdev) +{ + return (sdev == NULL) || + (ETH(sdev) == NULL) || + (ETH(sdev)->tx_pkt_burst == NULL) || + (sdev->state != DEV_STARTED); +} + +void +set_burst_fn(struct rte_eth_dev *dev, int force_safe) +{ + struct sub_device *sdev; + uint8_t i; + int need_safe; + int safe_set; + + need_safe = force_safe; + FOREACH_SUBDEV(sdev, i, dev) + need_safe |= fs_rx_unsafe(sdev); + safe_set = (dev->rx_pkt_burst == &failsafe_rx_burst); + if (need_safe && !safe_set) { + DEBUG("Using safe RX bursts%s", + (force_safe ? " (forced)" : "")); + dev->rx_pkt_burst = &failsafe_rx_burst; + } else if (!need_safe && safe_set) { + DEBUG("Using fast RX bursts"); + dev->rx_pkt_burst = &failsafe_rx_burst_fast; + } + need_safe = force_safe || fs_tx_unsafe(TX_SUBDEV(dev)); + safe_set = (dev->tx_pkt_burst == &failsafe_tx_burst); + if (need_safe && !safe_set) { + DEBUG("Using safe TX bursts%s", + (force_safe ? " (forced)" : "")); + dev->tx_pkt_burst = &failsafe_tx_burst; + } else if (!need_safe && safe_set) { + DEBUG("Using fast TX bursts"); + dev->tx_pkt_burst = &failsafe_tx_burst_fast; + } + rte_wmb(); +} + +uint16_t +failsafe_rx_burst(void *queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct fs_priv *priv; + struct sub_device *sdev; + struct rxq *rxq; + void *sub_rxq; + uint16_t nb_rx; + uint8_t nb_polled, nb_subs; + uint8_t i; + + rxq = queue; + priv = rxq->priv; + nb_subs = priv->subs_tail - priv->subs_head; + nb_polled = 0; + for (i = rxq->last_polled; nb_polled < nb_subs; nb_polled++) { + i++; + if (i == priv->subs_tail) + i = priv->subs_head; + sdev = &priv->subs[i]; + if (unlikely(fs_rx_unsafe(sdev))) + continue; + sub_rxq = ETH(sdev)->data->rx_queues[rxq->qid]; + FS_ATOMIC_P(rxq->refcnt[sdev->sid]); + nb_rx = ETH(sdev)-> + rx_pkt_burst(sub_rxq, rx_pkts, nb_pkts); + FS_ATOMIC_V(rxq->refcnt[sdev->sid]); + if (nb_rx) { + rxq->last_polled = i; + return nb_rx; + } + } + return 0; +} + +uint16_t +failsafe_rx_burst_fast(void *queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct fs_priv *priv; + struct sub_device *sdev; + struct rxq *rxq; + void *sub_rxq; + uint16_t nb_rx; + uint8_t nb_polled, nb_subs; + uint8_t i; + + rxq = queue; + priv = rxq->priv; + nb_subs = priv->subs_tail - priv->subs_head; + nb_polled = 0; + for (i = rxq->last_polled; nb_polled < nb_subs; nb_polled++) { + i++; + if (i == priv->subs_tail) + i = priv->subs_head; + sdev = &priv->subs[i]; + RTE_ASSERT(!fs_rx_unsafe(sdev)); + sub_rxq = ETH(sdev)->data->rx_queues[rxq->qid]; + FS_ATOMIC_P(rxq->refcnt[sdev->sid]); + nb_rx = ETH(sdev)-> + rx_pkt_burst(sub_rxq, rx_pkts, nb_pkts); + FS_ATOMIC_V(rxq->refcnt[sdev->sid]); + if (nb_rx) { + rxq->last_polled = i; + return nb_rx; + } + } + return 0; +} + +uint16_t +failsafe_tx_burst(void *queue, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct sub_device *sdev; + struct txq *txq; + void *sub_txq; + uint16_t nb_tx; + + txq = queue; + sdev = TX_SUBDEV(txq->priv->dev); + if (unlikely(fs_tx_unsafe(sdev))) + return 0; + sub_txq = ETH(sdev)->data->tx_queues[txq->qid]; + FS_ATOMIC_P(txq->refcnt[sdev->sid]); + nb_tx = ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts); + FS_ATOMIC_V(txq->refcnt[sdev->sid]); + return nb_tx; +} + +uint16_t +failsafe_tx_burst_fast(void *queue, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct sub_device *sdev; + struct txq *txq; + void *sub_txq; + uint16_t nb_tx; + + txq = queue; + sdev = TX_SUBDEV(txq->priv->dev); + RTE_ASSERT(!fs_tx_unsafe(sdev)); + sub_txq = ETH(sdev)->data->tx_queues[txq->qid]; + FS_ATOMIC_P(txq->refcnt[sdev->sid]); + nb_tx = ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts); + FS_ATOMIC_V(txq->refcnt[sdev->sid]); + return nb_tx; +} diff --git a/drivers/net/failsafe/rte_pmd_failsafe_version.map b/drivers/net/failsafe/rte_pmd_failsafe_version.map new file mode 100644 index 00000000..b6d2840b --- /dev/null +++ b/drivers/net/failsafe/rte_pmd_failsafe_version.map @@ -0,0 +1,4 @@ +DPDK_17.08 { + + local: *; +}; diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c index 7363defa..e60d3a36 100644 --- a/drivers/net/fm10k/fm10k_ethdev.c +++ b/drivers/net/fm10k/fm10k_ethdev.c @@ -84,6 +84,7 @@ static void fm10k_rx_queue_release(void *queue); static void fm10k_set_rx_function(struct rte_eth_dev *dev); static void fm10k_set_tx_function(struct rte_eth_dev *dev); static int fm10k_check_ftag(struct rte_devargs *devargs); +static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete); struct fm10k_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; @@ -712,7 +713,7 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_macvlan_filter_info *macvlan; - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; int i, ret; struct fm10k_rx_queue *rxq; @@ -1166,6 +1167,8 @@ fm10k_dev_start(struct rte_eth_dev *dev) if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)) fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); + fm10k_link_update(dev, 0); + return 0; } @@ -1173,7 +1176,7 @@ static void fm10k_dev_stop(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; int i; @@ -1393,7 +1396,7 @@ fm10k_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); PMD_INIT_FUNC_TRACE(); @@ -1588,7 +1591,7 @@ fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) } static void -fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask) +fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask) { if (mask & ETH_VLAN_STRIP_MASK) { if (!dev->data->dev_conf.rxmode.hw_vlan_strip) @@ -2345,7 +2348,7 @@ static int fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); /* Enable ITR */ if (hw->mac.type == fm10k_mac_pf) @@ -2362,7 +2365,7 @@ static int fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); /* Disable ITR */ if (hw->mac.type == fm10k_mac_pf) @@ -2378,7 +2381,7 @@ static int fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; uint32_t intr_vector, vec; uint16_t queue_id; @@ -2882,7 +2885,7 @@ static int eth_fm10k_dev_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; int diag, i; struct fm10k_macvlan_filter_info *macvlan; @@ -3069,7 +3072,7 @@ static int eth_fm10k_dev_uninit(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; PMD_INIT_FUNC_TRACE(); diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c index 411bc445..d23bfe9b 100644 --- a/drivers/net/fm10k/fm10k_rxtx_vec.c +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c @@ -331,6 +331,8 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq) *(uint64_t *)p1 = rxq->mbuf_initializer; /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) != + offsetof(struct rte_mbuf, buf_addr) + 8); vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); @@ -448,6 +450,19 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* skip high 16 bits pkt_type */ 0xFF, 0xFF /* Skip pkt_type field in shuffle operation */ ); + /* + * Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); /* Cache is empty -> need to scan the buffer rings, but first move * the next 'n' mbufs into the cache @@ -738,7 +753,7 @@ vtx(volatile struct fm10k_tx_desc *txdp, vtx1(txdp, *pkt, flags); } -static inline int __attribute__((always_inline)) +static __rte_always_inline int fm10k_tx_free_bufs(struct fm10k_tx_queue *txq) { struct rte_mbuf **txep; @@ -794,7 +809,7 @@ fm10k_tx_free_bufs(struct fm10k_tx_queue *txq) return txq->rs_thresh; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void tx_backlog_entry(struct rte_mbuf **txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { diff --git a/drivers/net/i40e/Makefile b/drivers/net/i40e/Makefile index 56f210d6..55c79a60 100644 --- a/drivers/net/i40e/Makefile +++ b/drivers/net/i40e/Makefile @@ -109,11 +109,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_pf.c SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_fdir.c SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_flow.c SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += rte_pmd_i40e.c - -# vector PMD driver needs SSE4.1 support -ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),) -CFLAGS_i40e_rxtx_vec_sse.o += -msse4.1 -endif +SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_tm.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_I40E_PMD)-include := rte_pmd_i40e.h diff --git a/drivers/net/i40e/base/README b/drivers/net/i40e/base/README index 0da9f674..59e76c21 100644 --- a/drivers/net/i40e/base/README +++ b/drivers/net/i40e/base/README @@ -34,7 +34,7 @@ Intel® I40E driver ================== This directory contains source code of FreeBSD i40e driver of version -cid-i40e.2017.03.21.tar.gz released by the team which develops +cid-i40e.2017.06.23.tar.gz released by the team which develops basic drivers for any i40e NIC. The directory of base/ contains the original source package. This driver is valid for the product(s) listed below diff --git a/drivers/net/i40e/base/i40e_adminq.c b/drivers/net/i40e/base/i40e_adminq.c index a60292a3..8cc8c5ec 100644 --- a/drivers/net/i40e/base/i40e_adminq.c +++ b/drivers/net/i40e/base/i40e_adminq.c @@ -682,6 +682,18 @@ enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw) &oem_lo); hw->nvm.oem_ver = ((u32)oem_hi << 16) | oem_lo; + /* The ability to RX (not drop) 802.1ad frames was added in API 1.7 */ + if ((hw->aq.api_maj_ver > 1) || + ((hw->aq.api_maj_ver == 1) && + (hw->aq.api_min_ver >= 7))) + hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE; + + if (hw->mac.type == I40E_MAC_XL710 && + hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { + hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE; + } + if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) { ret_code = I40E_ERR_FIRMWARE_API_VERSION; goto init_adminq_free_arq; diff --git a/drivers/net/i40e/base/i40e_adminq_cmd.h b/drivers/net/i40e/base/i40e_adminq_cmd.h index 09f5bf5c..c36da2a3 100644 --- a/drivers/net/i40e/base/i40e_adminq_cmd.h +++ b/drivers/net/i40e/base/i40e_adminq_cmd.h @@ -41,7 +41,15 @@ POSSIBILITY OF SUCH DAMAGE. */ #define I40E_FW_API_VERSION_MAJOR 0x0001 -#define I40E_FW_API_VERSION_MINOR 0x0005 +#define I40E_FW_API_VERSION_MINOR_X722 0x0005 +#define I40E_FW_API_VERSION_MINOR_X710 0x0007 + +#define I40E_FW_MINOR_VERSION(_h) ((_h)->mac.type == I40E_MAC_XL710 ? \ + I40E_FW_API_VERSION_MINOR_X710 : \ + I40E_FW_API_VERSION_MINOR_X722) + +/* API version 1.7 implements additional link and PHY-specific APIs */ +#define I40E_MINOR_VER_GET_LINK_INFO_XL710 0x0007 struct i40e_aq_desc { __le16 flags; @@ -245,6 +253,8 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_set_phy_debug = 0x0622, i40e_aqc_opc_upload_ext_phy_fm = 0x0625, i40e_aqc_opc_run_phy_activity = 0x0626, + i40e_aqc_opc_set_phy_register = 0x0628, + i40e_aqc_opc_get_phy_register = 0x0629, /* NVM commands */ i40e_aqc_opc_nvm_read = 0x0701, @@ -777,7 +787,22 @@ struct i40e_aqc_set_switch_config { #define I40E_AQ_SET_SWITCH_CFG_L2_FILTER 0x0002 #define I40E_AQ_SET_SWITCH_CFG_HW_ATR_EVICT 0x0004 __le16 valid_flags; - u8 reserved[12]; + /* The ethertype in switch_tag is dropped on ingress and used + * internally by the switch. Set this to zero for the default + * of 0x88a8 (802.1ad). Should be zero for firmware API + * versions lower than 1.7. + */ + __le16 switch_tag; + /* The ethertypes in first_tag and second_tag are used to + * match the outer and inner VLAN tags (respectively) when HW + * double VLAN tagging is enabled via the set port parameters + * AQ command. Otherwise these are both ignored. Set them to + * zero for their defaults of 0x8100 (802.1Q). Should be zero + * for firmware API versions lower than 1.7. + */ + __le16 first_tag; + __le16 second_tag; + u8 reserved[6]; }; I40E_CHECK_CMD_LENGTH(i40e_aqc_set_switch_config); @@ -1829,6 +1854,8 @@ enum i40e_aq_phy_type { I40E_PHY_TYPE_10GBASE_CR1_CU = 0xB, I40E_PHY_TYPE_10GBASE_AOC = 0xC, I40E_PHY_TYPE_40GBASE_AOC = 0xD, + I40E_PHY_TYPE_UNRECOGNIZED = 0xE, + I40E_PHY_TYPE_UNSUPPORTED = 0xF, I40E_PHY_TYPE_100BASE_TX = 0x11, I40E_PHY_TYPE_1000BASE_T = 0x12, I40E_PHY_TYPE_10GBASE_T = 0x13, @@ -1847,7 +1874,11 @@ enum i40e_aq_phy_type { I40E_PHY_TYPE_25GBASE_CR = 0x20, I40E_PHY_TYPE_25GBASE_SR = 0x21, I40E_PHY_TYPE_25GBASE_LR = 0x22, - I40E_PHY_TYPE_MAX + I40E_PHY_TYPE_25GBASE_AOC = 0x23, + I40E_PHY_TYPE_25GBASE_ACC = 0x24, + I40E_PHY_TYPE_MAX, + I40E_PHY_TYPE_EMPTY = 0xFE, + I40E_PHY_TYPE_DEFAULT = 0xFF, }; #define I40E_LINK_SPEED_100MB_SHIFT 0x1 @@ -1904,6 +1935,8 @@ struct i40e_aq_get_phy_abilities_resp { #define I40E_AQ_PHY_TYPE_EXT_25G_CR 0x02 #define I40E_AQ_PHY_TYPE_EXT_25G_SR 0x04 #define I40E_AQ_PHY_TYPE_EXT_25G_LR 0x08 +#define I40E_AQ_PHY_TYPE_EXT_25G_AOC 0x10 +#define I40E_AQ_PHY_TYPE_EXT_25G_ACC 0x20 u8 fec_cfg_curr_mod_ext_info; #define I40E_AQ_ENABLE_FEC_KR 0x01 #define I40E_AQ_ENABLE_FEC_RS 0x02 @@ -2033,19 +2066,31 @@ struct i40e_aqc_get_link_status { #define I40E_AQ_25G_SERDES_UCODE_ERR 0X04 #define I40E_AQ_25G_NIMB_UCODE_ERR 0X05 u8 loopback; /* use defines from i40e_aqc_set_lb_mode */ +/* Since firmware API 1.7 loopback field keeps power class info as well */ +#define I40E_AQ_LOOPBACK_MASK 0x07 +#define I40E_AQ_PWR_CLASS_SHIFT_LB 6 +#define I40E_AQ_PWR_CLASS_MASK_LB (0x03 << I40E_AQ_PWR_CLASS_SHIFT_LB) __le16 max_frame_size; u8 config; #define I40E_AQ_CONFIG_FEC_KR_ENA 0x01 #define I40E_AQ_CONFIG_FEC_RS_ENA 0x02 #define I40E_AQ_CONFIG_CRC_ENA 0x04 #define I40E_AQ_CONFIG_PACING_MASK 0x78 - u8 power_desc; + union { + struct { + u8 power_desc; #define I40E_AQ_LINK_POWER_CLASS_1 0x00 #define I40E_AQ_LINK_POWER_CLASS_2 0x01 #define I40E_AQ_LINK_POWER_CLASS_3 0x02 #define I40E_AQ_LINK_POWER_CLASS_4 0x03 #define I40E_AQ_PWR_CLASS_MASK 0x03 - u8 reserved[4]; + u8 reserved[4]; + }; + struct { + u8 link_type[4]; + u8 link_type_ext; + }; + }; }; I40E_CHECK_CMD_LENGTH(i40e_aqc_get_link_status); @@ -2128,6 +2173,22 @@ struct i40e_aqc_run_phy_activity { I40E_CHECK_CMD_LENGTH(i40e_aqc_run_phy_activity); +/* Set PHY Register command (0x0628) */ +/* Get PHY Register command (0x0629) */ +struct i40e_aqc_phy_register_access { + u8 phy_interface; +#define I40E_AQ_PHY_REG_ACCESS_INTERNAL 0 +#define I40E_AQ_PHY_REG_ACCESS_EXTERNAL 1 +#define I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE 2 + u8 dev_addres; + u8 reserved1[2]; + u32 reg_address; + u32 reg_value; + u8 reserved2[4]; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_phy_register_access); + /* NVM Read command (indirect 0x0701) * NVM Erase commands (direct 0x0702) * NVM Update commands (indirect 0x0703) diff --git a/drivers/net/i40e/base/i40e_common.c b/drivers/net/i40e/base/i40e_common.c index 03e94bc8..900d379c 100644 --- a/drivers/net/i40e/base/i40e_common.c +++ b/drivers/net/i40e/base/i40e_common.c @@ -34,7 +34,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "i40e_type.h" #include "i40e_adminq.h" #include "i40e_prototype.h" -#include "i40e_virtchnl.h" +#include "virtchnl.h" /** @@ -93,6 +93,7 @@ STATIC enum i40e_status_code i40e_set_mac_type(struct i40e_hw *hw) #if defined(INTEGRATED_VF) || defined(VF_DRIVER) case I40E_DEV_ID_VF: case I40E_DEV_ID_VF_HV: + case I40E_DEV_ID_ADAPTIVE_VF: hw->mac.type = I40E_MAC_VF; break; #endif @@ -329,13 +330,15 @@ void i40e_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc, void *buffer, u16 buf_len) { struct i40e_aq_desc *aq_desc = (struct i40e_aq_desc *)desc; - u16 len = LE16_TO_CPU(aq_desc->datalen); u8 *buf = (u8 *)buffer; + u16 len; u16 i = 0; if ((!(mask & hw->debug_mask)) || (desc == NULL)) return; + len = LE16_TO_CPU(aq_desc->datalen); + i40e_debug(hw, mask, "AQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n", LE16_TO_CPU(aq_desc->opcode), @@ -1295,6 +1298,8 @@ STATIC enum i40e_media_type i40e_get_media_type(struct i40e_hw *hw) case I40E_PHY_TYPE_40GBASE_AOC: case I40E_PHY_TYPE_10GBASE_AOC: case I40E_PHY_TYPE_25GBASE_CR: + case I40E_PHY_TYPE_25GBASE_AOC: + case I40E_PHY_TYPE_25GBASE_ACC: media = I40E_MEDIA_TYPE_DA; break; case I40E_PHY_TYPE_1000BASE_KX: @@ -1377,6 +1382,8 @@ enum i40e_status_code i40e_pf_reset(struct i40e_hw *hw) * we don't need to do the PF Reset */ if (!cnt) { + u32 reg2 = 0; + reg = rd32(hw, I40E_PFGEN_CTRL); wr32(hw, I40E_PFGEN_CTRL, (reg | I40E_PFGEN_CTRL_PFSWR_MASK)); @@ -1384,6 +1391,12 @@ enum i40e_status_code i40e_pf_reset(struct i40e_hw *hw) reg = rd32(hw, I40E_PFGEN_CTRL); if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) break; + reg2 = rd32(hw, I40E_GLGEN_RSTAT); + if (reg2 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { + DEBUGOUT("Core reset upcoming.\n"); + DEBUGOUT1("I40E_GLGEN_RSTAT = 0x%x\n", reg2); + return I40E_ERR_NOT_READY; + } i40e_msec_delay(1); } if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { @@ -1689,8 +1702,15 @@ enum i40e_status_code i40e_aq_get_phy_capabilities(struct i40e_hw *hw, status = I40E_ERR_UNKNOWN_PHY; if (report_init) { - hw->phy.phy_types = LE32_TO_CPU(abilities->phy_type); - hw->phy.phy_types |= ((u64)abilities->phy_type_ext << 32); + if (hw->mac.type == I40E_MAC_XL710 && + hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { + status = i40e_aq_get_link_info(hw, true, NULL, NULL); + } else { + hw->phy.phy_types = LE32_TO_CPU(abilities->phy_type); + hw->phy.phy_types |= + ((u64)abilities->phy_type_ext << 32); + } } return status; @@ -1952,7 +1972,7 @@ enum i40e_status_code i40e_aq_get_link_info(struct i40e_hw *hw, hw_link_info->fec_info = resp->config & (I40E_AQ_CONFIG_FEC_KR_ENA | I40E_AQ_CONFIG_FEC_RS_ENA); hw_link_info->ext_info = resp->ext_info; - hw_link_info->loopback = resp->loopback; + hw_link_info->loopback = resp->loopback & I40E_AQ_LOOPBACK_MASK; hw_link_info->max_frame_size = LE16_TO_CPU(resp->max_frame_size); hw_link_info->pacing = resp->config & I40E_AQ_CONFIG_PACING_MASK; @@ -1983,6 +2003,12 @@ enum i40e_status_code i40e_aq_get_link_info(struct i40e_hw *hw, hw->aq.fw_min_ver < 40)) && hw_link_info->phy_type == 0xE) hw_link_info->phy_type = I40E_PHY_TYPE_10GBASE_SFPP_CU; + if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= 7) { + hw->phy.phy_types = LE32_TO_CPU(*(__le32 *)resp->link_type); + hw->phy.phy_types |= ((u64)resp->link_type_ext << 32); + } + /* save link status information */ if (link) i40e_memcpy(link, hw_link_info, sizeof(*hw_link_info), @@ -2679,7 +2705,11 @@ enum i40e_status_code i40e_aq_set_switch_config(struct i40e_hw *hw, i40e_aqc_opc_set_switch_config); scfg->flags = CPU_TO_LE16(flags); scfg->valid_flags = CPU_TO_LE16(valid_flags); - + if (hw->flags & I40E_HW_FLAG_802_1AD_CAPABLE) { + scfg->switch_tag = CPU_TO_LE16(hw->switch_tag); + scfg->first_tag = CPU_TO_LE16(hw->first_tag); + scfg->second_tag = CPU_TO_LE16(hw->second_tag); + } status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); return status; @@ -2825,6 +2855,10 @@ enum i40e_status_code i40e_update_link_info(struct i40e_hw *hw) if (status) return status; + hw->phy.link_info.req_fec_info = + abilities.fec_cfg_curr_mod_ext_info & + (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS); + i40e_memcpy(hw->phy.link_info.module_type, &abilities.module_type, sizeof(hw->phy.link_info.module_type), I40E_NONDMA_TO_NONDMA); } @@ -5523,7 +5557,7 @@ enum i40e_status_code i40e_aq_add_rem_control_packet_filter(struct i40e_hw *hw, } if (mac_addr) - i40e_memcpy(cmd->mac, mac_addr, I40E_ETH_LENGTH_OF_ADDRESS, + i40e_memcpy(cmd->mac, mac_addr, ETH_ALEN, I40E_NONDMA_TO_NONDMA); cmd->etype = CPU_TO_LE16(ethtype); @@ -6654,24 +6688,38 @@ enum i40e_status_code i40e_led_get_phy(struct i40e_hw *hw, u16 *led_addr, u16 temp_addr; u8 port_num; u32 i; - - temp_addr = I40E_PHY_LED_PROV_REG_1; - i = rd32(hw, I40E_PFGEN_PORTNUM); - port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); - phy_addr = i40e_get_phy_address(hw, port_num); - - for (gpio_led_port = 0; gpio_led_port < 3; gpio_led_port++, - temp_addr++) { - status = i40e_read_phy_register_clause45(hw, + u32 reg_val_aq; + + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_get_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + ®_val_aq, NULL); + if (status) + return status; + *val = (u16)reg_val_aq; + } else { + temp_addr = I40E_PHY_LED_PROV_REG_1; + i = rd32(hw, I40E_PFGEN_PORTNUM); + port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); + phy_addr = i40e_get_phy_address(hw, port_num); + + for (gpio_led_port = 0; gpio_led_port < 3; gpio_led_port++, + temp_addr++) { + status = + i40e_read_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, temp_addr, phy_addr, ®_val); - if (status) - return status; - *val = reg_val; - if (reg_val & I40E_PHY_LED_LINK_MODE_MASK) { - *led_addr = temp_addr; - break; + if (status) + return status; + *val = reg_val; + if (reg_val & I40E_PHY_LED_LINK_MODE_MASK) { + *led_addr = temp_addr; + break; + } } } return status; @@ -6689,51 +6737,115 @@ enum i40e_status_code i40e_led_set_phy(struct i40e_hw *hw, bool on, u16 led_addr, u32 mode) { enum i40e_status_code status = I40E_SUCCESS; - u16 led_ctl = 0; - u16 led_reg = 0; + u32 led_ctl = 0; + u32 led_reg = 0; u8 phy_addr = 0; u8 port_num; u32 i; - i = rd32(hw, I40E_PFGEN_PORTNUM); - port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); - phy_addr = i40e_get_phy_address(hw, port_num); - status = i40e_read_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, &led_reg); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_get_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + &led_reg, NULL); + } else { + i = rd32(hw, I40E_PFGEN_PORTNUM); + port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); + phy_addr = i40e_get_phy_address(hw, port_num); + status = i40e_read_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16 *)&led_reg); + } if (status) return status; led_ctl = led_reg; if (led_reg & I40E_PHY_LED_LINK_MODE_MASK) { led_reg = 0; - status = i40e_write_phy_register_clause45(hw, - I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, - led_reg); + if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { + status = i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_reg, NULL); + } else { + status = i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_reg); + } if (status) return status; } - status = i40e_read_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, &led_reg); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_get_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + &led_reg, NULL); + } else { + status = i40e_read_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16 *)&led_reg); + } if (status) goto restore_config; if (on) led_reg = I40E_PHY_LED_MANUAL_ON; else led_reg = 0; - status = i40e_write_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, led_reg); + + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_reg, NULL); + } else { + status = + i40e_write_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_reg); + } if (status) goto restore_config; if (mode & I40E_PHY_LED_MODE_ORIG) { led_ctl = (mode & I40E_PHY_LED_MODE_MASK); - status = i40e_write_phy_register_clause45(hw, - I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, led_ctl); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_ctl, NULL); + } else { + status = i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_ctl); + } } return status; restore_config: - status = i40e_write_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, led_ctl); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_ctl, NULL); + } else { + status = + i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_ctl); + } return status; } #endif /* PF_DRIVER */ @@ -6863,6 +6975,76 @@ do_retry: if (status || use_register) wr32(hw, reg_addr, reg_val); } + +/** + * i40e_aq_set_phy_register + * @hw: pointer to the hw struct + * @phy_select: select which phy should be accessed + * @dev_addr: PHY device address + * @reg_addr: PHY register address + * @reg_val: new register value + * @cmd_details: pointer to command details structure or NULL + * + * Write the external PHY register. + **/ +enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 reg_val, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_phy_register_access *cmd = + (struct i40e_aqc_phy_register_access *)&desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_phy_register); + + cmd->phy_interface = phy_select; + cmd->dev_addres = dev_addr; + cmd->reg_address = reg_addr; + cmd->reg_value = reg_val; + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + return status; +} + +/** + * i40e_aq_get_phy_register + * @hw: pointer to the hw struct + * @phy_select: select which phy should be accessed + * @dev_addr: PHY device address + * @reg_addr: PHY register address + * @reg_val: read register value + * @cmd_details: pointer to command details structure or NULL + * + * Read the external PHY register. + **/ +enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 *reg_val, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_phy_register_access *cmd = + (struct i40e_aqc_phy_register_access *)&desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_get_phy_register); + + cmd->phy_interface = phy_select; + cmd->dev_addres = dev_addr; + cmd->reg_address = reg_addr; + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + if (!status) + *reg_val = cmd->reg_value; + + return status; +} + #ifdef VF_DRIVER /** @@ -6879,7 +7061,7 @@ do_retry: * completion before returning. **/ enum i40e_status_code i40e_aq_send_msg_to_pf(struct i40e_hw *hw, - enum i40e_virtchnl_ops v_opcode, + enum virtchnl_ops v_opcode, enum i40e_status_code v_retval, u8 *msg, u16 msglen, struct i40e_asq_cmd_details *cmd_details) @@ -6918,9 +7100,9 @@ enum i40e_status_code i40e_aq_send_msg_to_pf(struct i40e_hw *hw, * with appropriate information. **/ void i40e_vf_parse_hw_config(struct i40e_hw *hw, - struct i40e_virtchnl_vf_resource *msg) + struct virtchnl_vf_resource *msg) { - struct i40e_virtchnl_vsi_resource *vsi_res; + struct virtchnl_vsi_resource *vsi_res; int i; vsi_res = &msg->vsi_res[0]; @@ -6930,19 +7112,17 @@ void i40e_vf_parse_hw_config(struct i40e_hw *hw, hw->dev_caps.num_tx_qp = msg->num_queue_pairs; hw->dev_caps.num_msix_vectors_vf = msg->max_vectors; hw->dev_caps.dcb = msg->vf_offload_flags & - I40E_VIRTCHNL_VF_OFFLOAD_L2; - hw->dev_caps.fcoe = (msg->vf_offload_flags & - I40E_VIRTCHNL_VF_OFFLOAD_FCOE) ? 1 : 0; + VIRTCHNL_VF_OFFLOAD_L2; hw->dev_caps.iwarp = (msg->vf_offload_flags & - I40E_VIRTCHNL_VF_OFFLOAD_IWARP) ? 1 : 0; + VIRTCHNL_VF_OFFLOAD_IWARP) ? 1 : 0; for (i = 0; i < msg->num_vsis; i++) { - if (vsi_res->vsi_type == I40E_VSI_SRIOV) { + if (vsi_res->vsi_type == VIRTCHNL_VSI_SRIOV) { i40e_memcpy(hw->mac.perm_addr, vsi_res->default_mac_addr, - I40E_ETH_LENGTH_OF_ADDRESS, + ETH_ALEN, I40E_NONDMA_TO_NONDMA); i40e_memcpy(hw->mac.addr, vsi_res->default_mac_addr, - I40E_ETH_LENGTH_OF_ADDRESS, + ETH_ALEN, I40E_NONDMA_TO_NONDMA); } vsi_res++; @@ -6959,7 +7139,7 @@ void i40e_vf_parse_hw_config(struct i40e_hw *hw, **/ enum i40e_status_code i40e_vf_reset(struct i40e_hw *hw) { - return i40e_aq_send_msg_to_pf(hw, I40E_VIRTCHNL_OP_RESET_VF, + return i40e_aq_send_msg_to_pf(hw, VIRTCHNL_OP_RESET_VF, I40E_SUCCESS, NULL, 0, NULL); } #endif /* VF_DRIVER */ @@ -7261,6 +7441,165 @@ i40e_find_segment_in_package(u32 segment_type, return NULL; } +/* Get section table in profile */ +#define I40E_SECTION_TABLE(profile, sec_tbl) \ + do { \ + struct i40e_profile_segment *p = (profile); \ + u32 count; \ + u32 *nvm; \ + count = p->device_table_count; \ + nvm = (u32 *)&p->device_table[count]; \ + sec_tbl = (struct i40e_section_table *)&nvm[nvm[0] + 1]; \ + } while (0) + +/* Get section header in profile */ +#define I40E_SECTION_HEADER(profile, offset) \ + (struct i40e_profile_section_header *)((u8 *)(profile) + (offset)) + +/** + * i40e_find_section_in_profile + * @section_type: the section type to search for (i.e., SECTION_TYPE_NOTE) + * @profile: pointer to the i40e segment header to be searched + * + * This function searches i40e segment for a particular section type. On + * success it returns a pointer to the section header, otherwise it will + * return NULL. + **/ +struct i40e_profile_section_header * +i40e_find_section_in_profile(u32 section_type, + struct i40e_profile_segment *profile) +{ + struct i40e_profile_section_header *sec; + struct i40e_section_table *sec_tbl; + u32 sec_off; + u32 i; + + if (profile->header.type != SEGMENT_TYPE_I40E) + return NULL; + + I40E_SECTION_TABLE(profile, sec_tbl); + + for (i = 0; i < sec_tbl->section_count; i++) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + if (sec->section.type == section_type) + return sec; + } + + return NULL; +} + +/** + * i40e_ddp_exec_aq_section - Execute generic AQ for DDP + * @hw: pointer to the hw struct + * @aq: command buffer containing all data to execute AQ + **/ +STATIC enum +i40e_status_code i40e_ddp_exec_aq_section(struct i40e_hw *hw, + struct i40e_profile_aq_section *aq) +{ + enum i40e_status_code status; + struct i40e_aq_desc desc; + u8 *msg = NULL; + u16 msglen; + + i40e_fill_default_direct_cmd_desc(&desc, aq->opcode); + desc.flags |= CPU_TO_LE16(aq->flags); + i40e_memcpy(desc.params.raw, aq->param, sizeof(desc.params.raw), + I40E_NONDMA_TO_NONDMA); + + msglen = aq->datalen; + if (msglen) { + desc.flags |= CPU_TO_LE16((u16)(I40E_AQ_FLAG_BUF | + I40E_AQ_FLAG_RD)); + if (msglen > I40E_AQ_LARGE_BUF) + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_LB); + desc.datalen = CPU_TO_LE16(msglen); + msg = &aq->data[0]; + } + + status = i40e_asq_send_command(hw, &desc, msg, msglen, NULL); + + if (status != I40E_SUCCESS) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "unable to exec DDP AQ opcode %u, error %d\n", + aq->opcode, status); + return status; + } + + /* copy returned desc to aq_buf */ + i40e_memcpy(aq->param, desc.params.raw, sizeof(desc.params.raw), + I40E_NONDMA_TO_NONDMA); + + return I40E_SUCCESS; +} + +/** + * i40e_validate_profile + * @hw: pointer to the hardware structure + * @profile: pointer to the profile segment of the package to be validated + * @track_id: package tracking id + * @rollback: flag if the profile is for rollback. + * + * Validates supported devices and profile's sections. + */ +STATIC enum i40e_status_code +i40e_validate_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, + u32 track_id, bool rollback) +{ + struct i40e_profile_section_header *sec = NULL; + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_section_table *sec_tbl; + u32 vendor_dev_id; + u32 dev_cnt; + u32 sec_off; + u32 i; + + if (track_id == I40E_DDP_TRACKID_INVALID) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, "Invalid track_id\n"); + return I40E_NOT_SUPPORTED; + } + + dev_cnt = profile->device_table_count; + for (i = 0; i < dev_cnt; i++) { + vendor_dev_id = profile->device_table[i].vendor_dev_id; + if ((vendor_dev_id >> 16) == I40E_INTEL_VENDOR_ID && + hw->device_id == (vendor_dev_id & 0xFFFF)) + break; + } + if (dev_cnt && (i == dev_cnt)) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Device doesn't support DDP\n"); + return I40E_ERR_DEVICE_NOT_SUPPORTED; + } + + I40E_SECTION_TABLE(profile, sec_tbl); + + /* Validate sections types */ + for (i = 0; i < sec_tbl->section_count; i++) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + if (rollback) { + if (sec->section.type == SECTION_TYPE_MMIO || + sec->section.type == SECTION_TYPE_AQ || + sec->section.type == SECTION_TYPE_RB_AQ) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Not a roll-back package\n"); + return I40E_NOT_SUPPORTED; + } + } else { + if (sec->section.type == SECTION_TYPE_RB_AQ || + sec->section.type == SECTION_TYPE_RB_MMIO) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Not an original package\n"); + return I40E_NOT_SUPPORTED; + } + } + } + + return status; +} + /** * i40e_write_profile * @hw: pointer to the hardware structure @@ -7276,52 +7615,99 @@ i40e_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, enum i40e_status_code status = I40E_SUCCESS; struct i40e_section_table *sec_tbl; struct i40e_profile_section_header *sec = NULL; - u32 dev_cnt; - u32 vendor_dev_id; - u32 *nvm; + struct i40e_profile_aq_section *ddp_aq; u32 section_size = 0; u32 offset = 0, info = 0; + u32 sec_off; u32 i; - if (!track_id) { - i40e_debug(hw, I40E_DEBUG_PACKAGE, "Track_id can't be 0."); - return I40E_NOT_SUPPORTED; - } + status = i40e_validate_profile(hw, profile, track_id, false); + if (status) + return status; - dev_cnt = profile->device_table_count; + I40E_SECTION_TABLE(profile, sec_tbl); - for (i = 0; i < dev_cnt; i++) { - vendor_dev_id = profile->device_table[i].vendor_dev_id; - if ((vendor_dev_id >> 16) == I40E_INTEL_VENDOR_ID) - if (hw->device_id == (vendor_dev_id & 0xFFFF)) + for (i = 0; i < sec_tbl->section_count; i++) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + /* Process generic admin command */ + if (sec->section.type == SECTION_TYPE_AQ) { + ddp_aq = (struct i40e_profile_aq_section *)&sec[1]; + status = i40e_ddp_exec_aq_section(hw, ddp_aq); + if (status) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Failed to execute aq: section %d, opcode %u\n", + i, ddp_aq->opcode); break; + } + sec->section.type = SECTION_TYPE_RB_AQ; + } + + /* Skip any non-mmio sections */ + if (sec->section.type != SECTION_TYPE_MMIO) + continue; + + section_size = sec->section.size + + sizeof(struct i40e_profile_section_header); + + /* Write MMIO section */ + status = i40e_aq_write_ddp(hw, (void *)sec, (u16)section_size, + track_id, &offset, &info, NULL); + if (status) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Failed to write profile: section %d, offset %d, info %d\n", + i, offset, info); + break; + } } - if (i == dev_cnt) { - i40e_debug(hw, I40E_DEBUG_PACKAGE, "Device doesn't support DDP"); - return I40E_ERR_DEVICE_NOT_SUPPORTED; - } + return status; +} - nvm = (u32 *)&profile->device_table[dev_cnt]; - sec_tbl = (struct i40e_section_table *)&nvm[nvm[0] + 1]; +/** + * i40e_rollback_profile + * @hw: pointer to the hardware structure + * @profile: pointer to the profile segment of the package to be removed + * @track_id: package tracking id + * + * Rolls back previously loaded package. + */ +enum i40e_status_code +i40e_rollback_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, + u32 track_id) +{ + struct i40e_profile_section_header *sec = NULL; + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_section_table *sec_tbl; + u32 offset = 0, info = 0; + u32 section_size = 0; + u32 sec_off; + int i; - for (i = 0; i < sec_tbl->section_count; i++) { - sec = (struct i40e_profile_section_header *)((u8 *)profile + - sec_tbl->section_offset[i]); + status = i40e_validate_profile(hw, profile, track_id, true); + if (status) + return status; - /* Skip 'AQ', 'note' and 'name' sections */ - if (sec->section.type != SECTION_TYPE_MMIO) + I40E_SECTION_TABLE(profile, sec_tbl); + + /* For rollback write sections in reverse */ + for (i = sec_tbl->section_count - 1; i >= 0; i--) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + + /* Skip any non-rollback sections */ + if (sec->section.type != SECTION_TYPE_RB_MMIO) continue; section_size = sec->section.size + sizeof(struct i40e_profile_section_header); - /* Write profile */ + /* Write roll-back MMIO section */ status = i40e_aq_write_ddp(hw, (void *)sec, (u16)section_size, track_id, &offset, &info, NULL); if (status) { i40e_debug(hw, I40E_DEBUG_PACKAGE, - "Failed to write profile: offset %d, info %d", - offset, info); + "Failed to write profile: section %d, offset %d, info %d\n", + i, offset, info); break; } } @@ -7359,9 +7745,10 @@ i40e_add_pinfo_to_list(struct i40e_hw *hw, pinfo->track_id = track_id; pinfo->version = profile->version; pinfo->op = I40E_DDP_ADD_TRACKID; - memcpy(pinfo->name, profile->name, I40E_DDP_NAME_SIZE); + i40e_memcpy(pinfo->name, profile->name, I40E_DDP_NAME_SIZE, + I40E_NONDMA_TO_NONDMA); status = i40e_aq_write_ddp(hw, (void *)sec, sec->data_end, - track_id, &offset, &info, NULL); + track_id, &offset, &info, NULL); return status; } diff --git a/drivers/net/i40e/base/i40e_devids.h b/drivers/net/i40e/base/i40e_devids.h index 4546689a..f4a87842 100644 --- a/drivers/net/i40e/base/i40e_devids.h +++ b/drivers/net/i40e/base/i40e_devids.h @@ -54,6 +54,7 @@ POSSIBILITY OF SUCH DAMAGE. #if defined(INTEGRATED_VF) || defined(VF_DRIVER) || defined(I40E_NDIS_SUPPORT) #define I40E_DEV_ID_VF 0x154C #define I40E_DEV_ID_VF_HV 0x1571 +#define I40E_DEV_ID_ADAPTIVE_VF 0x1889 #endif /* VF_DRIVER */ #ifdef X722_A0_SUPPORT #define I40E_DEV_ID_X722_A0 0x374C diff --git a/drivers/net/i40e/base/i40e_nvm.c b/drivers/net/i40e/base/i40e_nvm.c index e8965024..a1e78300 100644 --- a/drivers/net/i40e/base/i40e_nvm.c +++ b/drivers/net/i40e/base/i40e_nvm.c @@ -749,12 +749,18 @@ enum i40e_status_code i40e_validate_nvm_checksum(struct i40e_hw *hw, DEBUGFUNC("i40e_validate_nvm_checksum"); - if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) - ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); + /* acquire_nvm provides exclusive NVM lock to synchronize access across + * PFs. X710 uses i40e_read_nvm_word_srctl which polls for done bit + * twice (first time to be able to write address to I40E_GLNVM_SRCTL + * register, second to read data from I40E_GLNVM_SRDATA. One PF can see + * done bit and try to write address, while another one will interpret + * it as a good time to read data. It will cause invalid data to be + * read. + */ + ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); if (!ret_code) { ret_code = i40e_calc_nvm_checksum(hw, &checksum_local); - if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) - i40e_release_nvm(hw); + i40e_release_nvm(hw); if (ret_code != I40E_SUCCESS) goto i40e_validate_nvm_checksum_exit; } else { @@ -899,6 +905,11 @@ enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; } + /* Acquire lock to prevent race condition where adminq_task + * can execute after i40e_nvmupd_nvm_read/write but before state + * variables (nvm_wait_opcode, nvm_release_on_done) are updated + */ + i40e_acquire_spinlock(&hw->aq.arq_spinlock); switch (hw->nvmupd_state) { case I40E_NVMUPD_STATE_INIT: status = i40e_nvmupd_state_init(hw, cmd, bytes, perrno); @@ -934,6 +945,7 @@ enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, *perrno = -ESRCH; break; } + i40e_release_spinlock(&hw->aq.arq_spinlock); return status; } diff --git a/drivers/net/i40e/base/i40e_prototype.h b/drivers/net/i40e/base/i40e_prototype.h index 4bd589e7..acb2023f 100644 --- a/drivers/net/i40e/base/i40e_prototype.h +++ b/drivers/net/i40e/base/i40e_prototype.h @@ -36,7 +36,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "i40e_type.h" #include "i40e_alloc.h" -#include "i40e_virtchnl.h" +#include "virtchnl.h" /* Prototypes for shared code functions that are not in * the standard function pointer structures. These are @@ -504,10 +504,10 @@ void i40e_destroy_spinlock(struct i40e_spinlock *sp); /* i40e_common for VF drivers*/ void i40e_vf_parse_hw_config(struct i40e_hw *hw, - struct i40e_virtchnl_vf_resource *msg); + struct virtchnl_vf_resource *msg); enum i40e_status_code i40e_vf_reset(struct i40e_hw *hw); enum i40e_status_code i40e_aq_send_msg_to_pf(struct i40e_hw *hw, - enum i40e_virtchnl_ops v_opcode, + enum virtchnl_ops v_opcode, enum i40e_status_code v_retval, u8 *msg, u16 msglen, struct i40e_asq_cmd_details *cmd_details); @@ -533,6 +533,15 @@ enum i40e_status_code i40e_aq_rx_ctl_write_register(struct i40e_hw *hw, u32 reg_addr, u32 reg_val, struct i40e_asq_cmd_details *cmd_details); void i40e_write_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val); +enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 reg_val, + struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 *reg_val, + struct i40e_asq_cmd_details *cmd_details); + enum i40e_status_code i40e_aq_set_arp_proxy_config(struct i40e_hw *hw, struct i40e_aqc_arp_proxy_data *proxy_config, struct i40e_asq_cmd_details *cmd_details); @@ -566,19 +575,27 @@ u8 i40e_get_phy_address(struct i40e_hw *hw, u8 dev_num); enum i40e_status_code i40e_blink_phy_link_led(struct i40e_hw *hw, u32 time, u32 interval); enum i40e_status_code i40e_aq_write_ddp(struct i40e_hw *hw, void *buff, - u16 buff_size, u32 track_id, - u32 *error_offset, u32 *error_info, - struct i40e_asq_cmd_details *cmd_details); + u16 buff_size, u32 track_id, + u32 *error_offset, u32 *error_info, + struct i40e_asq_cmd_details * + cmd_details); enum i40e_status_code i40e_aq_get_ddp_list(struct i40e_hw *hw, void *buff, - u16 buff_size, u8 flags, - struct i40e_asq_cmd_details *cmd_details); + u16 buff_size, u8 flags, + struct i40e_asq_cmd_details * + cmd_details); struct i40e_generic_seg_header * i40e_find_segment_in_package(u32 segment_type, struct i40e_package_header *pkg_header); +struct i40e_profile_section_header * +i40e_find_section_in_profile(u32 section_type, + struct i40e_profile_segment *profile); enum i40e_status_code i40e_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *i40e_seg, u32 track_id); enum i40e_status_code +i40e_rollback_profile(struct i40e_hw *hw, struct i40e_profile_segment *i40e_seg, + u32 track_id); +enum i40e_status_code i40e_add_pinfo_to_list(struct i40e_hw *hw, struct i40e_profile_segment *profile, u8 *profile_info_sec, u32 track_id); diff --git a/drivers/net/i40e/base/i40e_register.h b/drivers/net/i40e/base/i40e_register.h index b150fbdf..a482ab90 100644 --- a/drivers/net/i40e/base/i40e_register.h +++ b/drivers/net/i40e/base/i40e_register.h @@ -2805,7 +2805,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_GLV_RUPP_MAX_INDEX 383 #define I40E_GLV_RUPP_RUPP_SHIFT 0 #define I40E_GLV_RUPP_RUPP_MASK I40E_MASK(0xFFFFFFFF, I40E_GLV_RUPP_RUPP_SHIFT) -#define I40E_GLV_TEPC(_VSI) (0x00344000 + ((_VSI) * 8)) /* _i=0...383 */ /* Reset: CORER */ +#define I40E_GLV_TEPC(_i) (0x00344000 + ((_i) * 8)) /* _i=0...383 */ /* Reset: CORER */ #define I40E_GLV_TEPC_MAX_INDEX 383 #define I40E_GLV_TEPC_TEPC_SHIFT 0 #define I40E_GLV_TEPC_TEPC_MASK I40E_MASK(0xFFFFFFFF, I40E_GLV_TEPC_TEPC_SHIFT) diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h index 84d57576..dca725af 100644 --- a/drivers/net/i40e/base/i40e_type.h +++ b/drivers/net/i40e/base/i40e_type.h @@ -92,7 +92,9 @@ POSSIBILITY OF SUCH DAMAGE. struct i40e_hw; typedef void (*I40E_ADMINQ_CALLBACK)(struct i40e_hw *, struct i40e_aq_desc *); -#define I40E_ETH_LENGTH_OF_ADDRESS 6 +#ifndef ETH_ALEN +#define ETH_ALEN 6 +#endif /* Data type manipulation macros. */ #define I40E_HI_DWORD(x) ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF)) #define I40E_LO_DWORD(x) ((u32)((x) & 0xFFFFFFFF)) @@ -197,10 +199,6 @@ enum i40e_memcpy_type { I40E_DMA_TO_NONDMA }; -#define I40E_FW_API_VERSION_MINOR_X722 0x0005 -#define I40E_FW_API_VERSION_MINOR_X710 0x0005 - - /* These are structs for managing the hardware information and the operations. * The structures of function pointers are filled out at init time when we * know for sure exactly which hardware we're working with. This gives us the @@ -269,6 +267,7 @@ struct i40e_link_status { enum i40e_aq_link_speed link_speed; u8 link_info; u8 an_info; + u8 req_fec_info; u8 fec_info; u8 ext_info; u8 loopback; @@ -439,10 +438,10 @@ struct i40e_hw_capabilities { struct i40e_mac_info { enum i40e_mac_type type; - u8 addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 perm_addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 san_addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 port_addr[I40E_ETH_LENGTH_OF_ADDRESS]; + u8 addr[ETH_ALEN]; + u8 perm_addr[ETH_ALEN]; + u8 san_addr[ETH_ALEN]; + u8 port_addr[ETH_ALEN]; u16 max_fcoeq; }; @@ -701,8 +700,15 @@ struct i40e_hw { u16 wol_proxy_vsi_seid; #define I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE BIT_ULL(0) +#define I40E_HW_FLAG_802_1AD_CAPABLE BIT_ULL(1) +#define I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE BIT_ULL(2) u64 flags; + /* Used in set switch config AQ command */ + u16 switch_tag; + u16 first_tag; + u16 second_tag; + /* debug mask */ u32 debug_mask; char err_str[16]; @@ -1912,8 +1918,10 @@ struct i40e_generic_seg_header { struct i40e_metadata_segment { struct i40e_generic_seg_header header; struct i40e_ddp_version version; +#define I40E_DDP_TRACKID_RDONLY 0 +#define I40E_DDP_TRACKID_INVALID 0xFFFFFFFF u32 track_id; - char name[I40E_DDP_NAME_SIZE]; + char name[I40E_DDP_NAME_SIZE]; }; struct i40e_device_id_entry { @@ -1940,15 +1948,36 @@ struct i40e_profile_section_header { struct { #define SECTION_TYPE_INFO 0x00000010 #define SECTION_TYPE_MMIO 0x00000800 +#define SECTION_TYPE_RB_MMIO 0x00001800 #define SECTION_TYPE_AQ 0x00000801 +#define SECTION_TYPE_RB_AQ 0x00001801 #define SECTION_TYPE_NOTE 0x80000000 #define SECTION_TYPE_NAME 0x80000001 +#define SECTION_TYPE_PROTO 0x80000002 +#define SECTION_TYPE_PCTYPE 0x80000003 +#define SECTION_TYPE_PTYPE 0x80000004 u32 type; u32 offset; u32 size; } section; }; +struct i40e_profile_tlv_section_record { + u8 rtype; + u8 type; + u16 len; + u8 data[12]; +}; + +/* Generic AQ section in proflie */ +struct i40e_profile_aq_section { + u16 opcode; + u16 flags; + u8 param[16]; + u16 datalen; + u8 data[1]; +}; + struct i40e_profile_info { u32 track_id; struct i40e_ddp_version version; diff --git a/drivers/net/i40e/base/i40e_virtchnl.h b/drivers/net/i40e/base/i40e_virtchnl.h deleted file mode 100644 index 7a24c0f1..00000000 --- a/drivers/net/i40e/base/i40e_virtchnl.h +++ /dev/null @@ -1,445 +0,0 @@ -/******************************************************************************* - -Copyright (c) 2013 - 2015, Intel Corporation -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -***************************************************************************/ - -#ifndef _I40E_VIRTCHNL_H_ -#define _I40E_VIRTCHNL_H_ - -#include "i40e_type.h" - -/* Description: - * This header file describes the VF-PF communication protocol used - * by the various i40e drivers. - * - * Admin queue buffer usage: - * desc->opcode is always i40e_aqc_opc_send_msg_to_pf - * flags, retval, datalen, and data addr are all used normally. - * Firmware copies the cookie fields when sending messages between the PF and - * VF, but uses all other fields internally. Due to this limitation, we - * must send all messages as "indirect", i.e. using an external buffer. - * - * All the vsi indexes are relative to the VF. Each VF can have maximum of - * three VSIs. All the queue indexes are relative to the VSI. Each VF can - * have a maximum of sixteen queues for all of its VSIs. - * - * The PF is required to return a status code in v_retval for all messages - * except RESET_VF, which does not require any response. The return value is of - * i40e_status_code type, defined in the i40e_type.h. - * - * In general, VF driver initialization should roughly follow the order of these - * opcodes. The VF driver must first validate the API version of the PF driver, - * then request a reset, then get resources, then configure queues and - * interrupts. After these operations are complete, the VF driver may start - * its queues, optionally add MAC and VLAN filters, and process traffic. - */ - -/* Opcodes for VF-PF communication. These are placed in the v_opcode field - * of the virtchnl_msg structure. - */ -enum i40e_virtchnl_ops { -/* The PF sends status change events to VFs using - * the I40E_VIRTCHNL_OP_EVENT opcode. - * VFs send requests to the PF using the other ops. - */ - I40E_VIRTCHNL_OP_UNKNOWN = 0, - I40E_VIRTCHNL_OP_VERSION = 1, /* must ALWAYS be 1 */ - I40E_VIRTCHNL_OP_RESET_VF = 2, - I40E_VIRTCHNL_OP_GET_VF_RESOURCES = 3, - I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE = 4, - I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE = 5, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES = 6, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP = 7, - I40E_VIRTCHNL_OP_ENABLE_QUEUES = 8, - I40E_VIRTCHNL_OP_DISABLE_QUEUES = 9, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS = 10, - I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS = 11, - I40E_VIRTCHNL_OP_ADD_VLAN = 12, - I40E_VIRTCHNL_OP_DEL_VLAN = 13, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE = 14, - I40E_VIRTCHNL_OP_GET_STATS = 15, - I40E_VIRTCHNL_OP_FCOE = 16, - I40E_VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */ -#ifdef I40E_SOL_VF_SUPPORT - I40E_VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG = 19, -#endif - I40E_VIRTCHNL_OP_CONFIG_RSS_KEY = 23, - I40E_VIRTCHNL_OP_CONFIG_RSS_LUT = 24, - I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS = 25, - I40E_VIRTCHNL_OP_SET_RSS_HENA = 26, - -}; - -/* Virtual channel message descriptor. This overlays the admin queue - * descriptor. All other data is passed in external buffers. - */ - -struct i40e_virtchnl_msg { - u8 pad[8]; /* AQ flags/opcode/len/retval fields */ - enum i40e_virtchnl_ops v_opcode; /* avoid confusion with desc->opcode */ - enum i40e_status_code v_retval; /* ditto for desc->retval */ - u32 vfid; /* used by PF when sending to VF */ -}; - -/* Message descriptions and data structures.*/ - -/* I40E_VIRTCHNL_OP_VERSION - * VF posts its version number to the PF. PF responds with its version number - * in the same format, along with a return code. - * Reply from PF has its major/minor versions also in param0 and param1. - * If there is a major version mismatch, then the VF cannot operate. - * If there is a minor version mismatch, then the VF can operate but should - * add a warning to the system log. - * - * This enum element MUST always be specified as == 1, regardless of other - * changes in the API. The PF must always respond to this message without - * error regardless of version mismatch. - */ -#define I40E_VIRTCHNL_VERSION_MAJOR 1 -#define I40E_VIRTCHNL_VERSION_MINOR 1 -#define I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS 0 - -struct i40e_virtchnl_version_info { - u32 major; - u32 minor; -}; - -/* I40E_VIRTCHNL_OP_RESET_VF - * VF sends this request to PF with no parameters - * PF does NOT respond! VF driver must delay then poll VFGEN_RSTAT register - * until reset completion is indicated. The admin queue must be reinitialized - * after this operation. - * - * When reset is complete, PF must ensure that all queues in all VSIs associated - * with the VF are stopped, all queue configurations in the HMC are set to 0, - * and all MAC and VLAN filters (except the default MAC address) on all VSIs - * are cleared. - */ - -/* I40E_VIRTCHNL_OP_GET_VF_RESOURCES - * Version 1.0 VF sends this request to PF with no parameters - * Version 1.1 VF sends this request to PF with u32 bitmap of its capabilities - * PF responds with an indirect message containing - * i40e_virtchnl_vf_resource and one or more - * i40e_virtchnl_vsi_resource structures. - */ - -struct i40e_virtchnl_vsi_resource { - u16 vsi_id; - u16 num_queue_pairs; - enum i40e_vsi_type vsi_type; - u16 qset_handle; - u8 default_mac_addr[I40E_ETH_LENGTH_OF_ADDRESS]; -}; -/* VF offload flags */ -#define I40E_VIRTCHNL_VF_OFFLOAD_L2 0x00000001 -#define I40E_VIRTCHNL_VF_OFFLOAD_IWARP 0x00000002 -#define I40E_VIRTCHNL_VF_OFFLOAD_FCOE 0x00000004 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ 0x00000008 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG 0x00000010 -#define I40E_VIRTCHNL_VF_OFFLOAD_WB_ON_ITR 0x00000020 -#define I40E_VIRTCHNL_VF_OFFLOAD_VLAN 0x00010000 -#define I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING 0x00020000 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 0x00040000 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF 0X00080000 -#define I40E_VIRTCHNL_VF_OFFLOAD_ENCAP 0X00100000 -#define I40E_VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM 0X00200000 -#define I40E_VIRTCHNL_VF_OFFLOAD_RX_ENCAP_CSUM 0X00400000 - -#define I40E_VF_BASE_MODE_OFFLOADS (I40E_VIRTCHNL_VF_OFFLOAD_L2 | \ - I40E_VIRTCHNL_VF_OFFLOAD_VLAN | \ - I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF) - -struct i40e_virtchnl_vf_resource { - u16 num_vsis; - u16 num_queue_pairs; - u16 max_vectors; - u16 max_mtu; - - u32 vf_offload_flags; - u32 rss_key_size; - u32 rss_lut_size; - - struct i40e_virtchnl_vsi_resource vsi_res[1]; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE - * VF sends this message to set up parameters for one TX queue. - * External data buffer contains one instance of i40e_virtchnl_txq_info. - * PF configures requested queue and returns a status code. - */ - -/* Tx queue config info */ -struct i40e_virtchnl_txq_info { - u16 vsi_id; - u16 queue_id; - u16 ring_len; /* number of descriptors, multiple of 8 */ - u16 headwb_enabled; - u64 dma_ring_addr; - u64 dma_headwb_addr; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE - * VF sends this message to set up parameters for one RX queue. - * External data buffer contains one instance of i40e_virtchnl_rxq_info. - * PF configures requested queue and returns a status code. - */ - -/* Rx queue config info */ -struct i40e_virtchnl_rxq_info { - u16 vsi_id; - u16 queue_id; - u32 ring_len; /* number of descriptors, multiple of 32 */ - u16 hdr_size; - u16 splithdr_enabled; - u32 databuffer_size; - u32 max_pkt_size; - u64 dma_ring_addr; - enum i40e_hmc_obj_rx_hsplit_0 rx_split_pos; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES - * VF sends this message to set parameters for all active TX and RX queues - * associated with the specified VSI. - * PF configures queues and returns status. - * If the number of queues specified is greater than the number of queues - * associated with the VSI, an error is returned and no queues are configured. - */ -struct i40e_virtchnl_queue_pair_info { - /* NOTE: vsi_id and queue_id should be identical for both queues. */ - struct i40e_virtchnl_txq_info txq; - struct i40e_virtchnl_rxq_info rxq; -}; - -struct i40e_virtchnl_vsi_queue_config_info { - u16 vsi_id; - u16 num_queue_pairs; - struct i40e_virtchnl_queue_pair_info qpair[1]; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP - * VF uses this message to map vectors to queues. - * The rxq_map and txq_map fields are bitmaps used to indicate which queues - * are to be associated with the specified vector. - * The "other" causes are always mapped to vector 0. - * PF configures interrupt mapping and returns status. - */ -struct i40e_virtchnl_vector_map { - u16 vsi_id; - u16 vector_id; - u16 rxq_map; - u16 txq_map; - u16 rxitr_idx; - u16 txitr_idx; -}; - -struct i40e_virtchnl_irq_map_info { - u16 num_vectors; - struct i40e_virtchnl_vector_map vecmap[1]; -}; - -/* I40E_VIRTCHNL_OP_ENABLE_QUEUES - * I40E_VIRTCHNL_OP_DISABLE_QUEUES - * VF sends these message to enable or disable TX/RX queue pairs. - * The queues fields are bitmaps indicating which queues to act upon. - * (Currently, we only support 16 queues per VF, but we make the field - * u32 to allow for expansion.) - * PF performs requested action and returns status. - */ -struct i40e_virtchnl_queue_select { - u16 vsi_id; - u16 pad; - u32 rx_queues; - u32 tx_queues; -}; - -/* I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS - * VF sends this message in order to add one or more unicast or multicast - * address filters for the specified VSI. - * PF adds the filters and returns status. - */ - -/* I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS - * VF sends this message in order to remove one or more unicast or multicast - * filters for the specified VSI. - * PF removes the filters and returns status. - */ - -struct i40e_virtchnl_ether_addr { - u8 addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 pad[2]; -}; - -struct i40e_virtchnl_ether_addr_list { - u16 vsi_id; - u16 num_elements; - struct i40e_virtchnl_ether_addr list[1]; -}; - -#ifdef I40E_SOL_VF_SUPPORT -/* I40E_VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG - * VF sends this message to get the default MTU and list of additional ethernet - * addresses it is allowed to use. - * PF responds with an indirect message containing - * i40e_virtchnl_addnl_solaris_config with zero or more - * i40e_virtchnl_ether_addr structures. - * - * It is expected that this operation will only ever be needed for Solaris VFs - * running under a Solaris PF. - */ -struct i40e_virtchnl_addnl_solaris_config { - u16 default_mtu; - struct i40e_virtchnl_ether_addr_list al; -}; - -#endif -/* I40E_VIRTCHNL_OP_ADD_VLAN - * VF sends this message to add one or more VLAN tag filters for receives. - * PF adds the filters and returns status. - * If a port VLAN is configured by the PF, this operation will return an - * error to the VF. - */ - -/* I40E_VIRTCHNL_OP_DEL_VLAN - * VF sends this message to remove one or more VLAN tag filters for receives. - * PF removes the filters and returns status. - * If a port VLAN is configured by the PF, this operation will return an - * error to the VF. - */ - -struct i40e_virtchnl_vlan_filter_list { - u16 vsi_id; - u16 num_elements; - u16 vlan_id[1]; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE - * VF sends VSI id and flags. - * PF returns status code in retval. - * Note: we assume that broadcast accept mode is always enabled. - */ -struct i40e_virtchnl_promisc_info { - u16 vsi_id; - u16 flags; -}; - -#define I40E_FLAG_VF_UNICAST_PROMISC 0x00000001 -#define I40E_FLAG_VF_MULTICAST_PROMISC 0x00000002 - -/* I40E_VIRTCHNL_OP_GET_STATS - * VF sends this message to request stats for the selected VSI. VF uses - * the i40e_virtchnl_queue_select struct to specify the VSI. The queue_id - * field is ignored by the PF. - * - * PF replies with struct i40e_eth_stats in an external buffer. - */ - -/* I40E_VIRTCHNL_OP_CONFIG_RSS_KEY - * I40E_VIRTCHNL_OP_CONFIG_RSS_LUT - * VF sends these messages to configure RSS. Only supported if both PF - * and VF drivers set the I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF bit during - * configuration negotiation. If this is the case, then the rss fields in - * the vf resource struct are valid. - * Both the key and LUT are initialized to 0 by the PF, meaning that - * RSS is effectively disabled until set up by the VF. - */ -struct i40e_virtchnl_rss_key { - u16 vsi_id; - u16 key_len; - u8 key[1]; /* RSS hash key, packed bytes */ -}; - -struct i40e_virtchnl_rss_lut { - u16 vsi_id; - u16 lut_entries; - u8 lut[1]; /* RSS lookup table*/ -}; - -/* I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS - * I40E_VIRTCHNL_OP_SET_RSS_HENA - * VF sends these messages to get and set the hash filter enable bits for RSS. - * By default, the PF sets these to all possible traffic types that the - * hardware supports. The VF can query this value if it wants to change the - * traffic types that are hashed by the hardware. - * Traffic types are defined in the i40e_filter_pctype enum in i40e_type.h - */ -struct i40e_virtchnl_rss_hena { - u64 hena; -}; - -/* I40E_VIRTCHNL_OP_EVENT - * PF sends this message to inform the VF driver of events that may affect it. - * No direct response is expected from the VF, though it may generate other - * messages in response to this one. - */ -enum i40e_virtchnl_event_codes { - I40E_VIRTCHNL_EVENT_UNKNOWN = 0, - I40E_VIRTCHNL_EVENT_LINK_CHANGE, - I40E_VIRTCHNL_EVENT_RESET_IMPENDING, - I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE, -}; -#define I40E_PF_EVENT_SEVERITY_INFO 0 -#define I40E_PF_EVENT_SEVERITY_ATTENTION 1 -#define I40E_PF_EVENT_SEVERITY_ACTION_REQUIRED 2 -#define I40E_PF_EVENT_SEVERITY_CERTAIN_DOOM 255 - -struct i40e_virtchnl_pf_event { - enum i40e_virtchnl_event_codes event; - union { - struct { - enum i40e_aq_link_speed link_speed; - bool link_status; - } link_event; - } event_data; - - int severity; -}; - -/* VF reset states - these are written into the RSTAT register: - * I40E_VFGEN_RSTAT1 on the PF - * I40E_VFGEN_RSTAT on the VF - * When the PF initiates a reset, it writes 0 - * When the reset is complete, it writes 1 - * When the PF detects that the VF has recovered, it writes 2 - * VF checks this register periodically to determine if a reset has occurred, - * then polls it to know when the reset is complete. - * If either the PF or VF reads the register while the hardware - * is in a reset state, it will return DEADBEEF, which, when masked - * will result in 3. - */ -enum i40e_vfr_states { - I40E_VFR_INPROGRESS = 0, - I40E_VFR_COMPLETED, - I40E_VFR_VFACTIVE, - I40E_VFR_UNKNOWN, -}; - -#endif /* _I40E_VIRTCHNL_H_ */ diff --git a/drivers/net/i40e/base/virtchnl.h b/drivers/net/i40e/base/virtchnl.h new file mode 100644 index 00000000..f00dd360 --- /dev/null +++ b/drivers/net/i40e/base/virtchnl.h @@ -0,0 +1,772 @@ +/******************************************************************************* + +Copyright (c) 2013 - 2015, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +***************************************************************************/ + +#ifndef _VIRTCHNL_H_ +#define _VIRTCHNL_H_ + +/* Description: + * This header file describes the VF-PF communication protocol used + * by the drivers for all devices starting from our 40G product line + * + * Admin queue buffer usage: + * desc->opcode is always aqc_opc_send_msg_to_pf + * flags, retval, datalen, and data addr are all used normally. + * The Firmware copies the cookie fields when sending messages between the + * PF and VF, but uses all other fields internally. Due to this limitation, + * we must send all messages as "indirect", i.e. using an external buffer. + * + * All the VSI indexes are relative to the VF. Each VF can have maximum of + * three VSIs. All the queue indexes are relative to the VSI. Each VF can + * have a maximum of sixteen queues for all of its VSIs. + * + * The PF is required to return a status code in v_retval for all messages + * except RESET_VF, which does not require any response. The return value + * is of status_code type, defined in the shared type.h. + * + * In general, VF driver initialization should roughly follow the order of + * these opcodes. The VF driver must first validate the API version of the + * PF driver, then request a reset, then get resources, then configure + * queues and interrupts. After these operations are complete, the VF + * driver may start its queues, optionally add MAC and VLAN filters, and + * process traffic. + */ + +/* START GENERIC DEFINES + * Need to ensure the following enums and defines hold the same meaning and + * value in current and future projects + */ + +/* Error Codes */ +enum virtchnl_status_code { + VIRTCHNL_STATUS_SUCCESS = 0, + VIRTCHNL_ERR_PARAM = -5, + VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH = -38, + VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR = -39, + VIRTCHNL_STATUS_ERR_INVALID_VF_ID = -40, + VIRTCHNL_STATUS_NOT_SUPPORTED = -64, +}; + +#define VIRTCHNL_LINK_SPEED_100MB_SHIFT 0x1 +#define VIRTCHNL_LINK_SPEED_1000MB_SHIFT 0x2 +#define VIRTCHNL_LINK_SPEED_10GB_SHIFT 0x3 +#define VIRTCHNL_LINK_SPEED_40GB_SHIFT 0x4 +#define VIRTCHNL_LINK_SPEED_20GB_SHIFT 0x5 +#define VIRTCHNL_LINK_SPEED_25GB_SHIFT 0x6 + +enum virtchnl_link_speed { + VIRTCHNL_LINK_SPEED_UNKNOWN = 0, + VIRTCHNL_LINK_SPEED_100MB = BIT(VIRTCHNL_LINK_SPEED_100MB_SHIFT), + VIRTCHNL_LINK_SPEED_1GB = BIT(VIRTCHNL_LINK_SPEED_1000MB_SHIFT), + VIRTCHNL_LINK_SPEED_10GB = BIT(VIRTCHNL_LINK_SPEED_10GB_SHIFT), + VIRTCHNL_LINK_SPEED_40GB = BIT(VIRTCHNL_LINK_SPEED_40GB_SHIFT), + VIRTCHNL_LINK_SPEED_20GB = BIT(VIRTCHNL_LINK_SPEED_20GB_SHIFT), + VIRTCHNL_LINK_SPEED_25GB = BIT(VIRTCHNL_LINK_SPEED_25GB_SHIFT), +}; + +/* for hsplit_0 field of Rx HMC context */ +/* deprecated with AVF 1.0 */ +enum virtchnl_rx_hsplit { + VIRTCHNL_RX_HSPLIT_NO_SPLIT = 0, + VIRTCHNL_RX_HSPLIT_SPLIT_L2 = 1, + VIRTCHNL_RX_HSPLIT_SPLIT_IP = 2, + VIRTCHNL_RX_HSPLIT_SPLIT_TCP_UDP = 4, + VIRTCHNL_RX_HSPLIT_SPLIT_SCTP = 8, +}; + +#define VIRTCHNL_ETH_LENGTH_OF_ADDRESS 6 +/* END GENERIC DEFINES */ + +/* Opcodes for VF-PF communication. These are placed in the v_opcode field + * of the virtchnl_msg structure. + */ +enum virtchnl_ops { +/* The PF sends status change events to VFs using + * the VIRTCHNL_OP_EVENT opcode. + * VFs send requests to the PF using the other ops. + * Use of "advanced opcode" features must be negotiated as part of capabilities + * exchange and are not considered part of base mode feature set. + */ + VIRTCHNL_OP_UNKNOWN = 0, + VIRTCHNL_OP_VERSION = 1, /* must ALWAYS be 1 */ + VIRTCHNL_OP_RESET_VF = 2, + VIRTCHNL_OP_GET_VF_RESOURCES = 3, + VIRTCHNL_OP_CONFIG_TX_QUEUE = 4, + VIRTCHNL_OP_CONFIG_RX_QUEUE = 5, + VIRTCHNL_OP_CONFIG_VSI_QUEUES = 6, + VIRTCHNL_OP_CONFIG_IRQ_MAP = 7, + VIRTCHNL_OP_ENABLE_QUEUES = 8, + VIRTCHNL_OP_DISABLE_QUEUES = 9, + VIRTCHNL_OP_ADD_ETH_ADDR = 10, + VIRTCHNL_OP_DEL_ETH_ADDR = 11, + VIRTCHNL_OP_ADD_VLAN = 12, + VIRTCHNL_OP_DEL_VLAN = 13, + VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE = 14, + VIRTCHNL_OP_GET_STATS = 15, + VIRTCHNL_OP_RSVD = 16, + VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */ +#ifdef VIRTCHNL_SOL_VF_SUPPORT + VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG = 19, +#endif +#ifdef VIRTCHNL_IWARP + VIRTCHNL_OP_IWARP = 20, /* advanced opcode */ + VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP = 21, /* advanced opcode */ + VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP = 22, /* advanced opcode */ +#endif + VIRTCHNL_OP_CONFIG_RSS_KEY = 23, + VIRTCHNL_OP_CONFIG_RSS_LUT = 24, + VIRTCHNL_OP_GET_RSS_HENA_CAPS = 25, + VIRTCHNL_OP_SET_RSS_HENA = 26, + VIRTCHNL_OP_ENABLE_VLAN_STRIPPING = 27, + VIRTCHNL_OP_DISABLE_VLAN_STRIPPING = 28, + VIRTCHNL_OP_REQUEST_QUEUES = 29, + +}; + +/* This macro is used to generate a compilation error if a structure + * is not exactly the correct length. It gives a divide by zero error if the + * structure is not of the correct size, otherwise it creates an enum that is + * never used. + */ +#define VIRTCHNL_CHECK_STRUCT_LEN(n, X) enum virtchnl_static_assert_enum_##X \ + {virtchnl_static_assert_##X = (n) / ((sizeof(struct X) == (n)) ? 1 : 0)} + +/* Virtual channel message descriptor. This overlays the admin queue + * descriptor. All other data is passed in external buffers. + */ + +struct virtchnl_msg { + u8 pad[8]; /* AQ flags/opcode/len/retval fields */ + enum virtchnl_ops v_opcode; /* avoid confusion with desc->opcode */ + enum virtchnl_status_code v_retval; /* ditto for desc->retval */ + u32 vfid; /* used by PF when sending to VF */ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(20, virtchnl_msg); + +/* Message descriptions and data structures.*/ + +/* VIRTCHNL_OP_VERSION + * VF posts its version number to the PF. PF responds with its version number + * in the same format, along with a return code. + * Reply from PF has its major/minor versions also in param0 and param1. + * If there is a major version mismatch, then the VF cannot operate. + * If there is a minor version mismatch, then the VF can operate but should + * add a warning to the system log. + * + * This enum element MUST always be specified as == 1, regardless of other + * changes in the API. The PF must always respond to this message without + * error regardless of version mismatch. + */ +#define VIRTCHNL_VERSION_MAJOR 1 +#define VIRTCHNL_VERSION_MINOR 1 +#define VIRTCHNL_VERSION_MINOR_NO_VF_CAPS 0 + +struct virtchnl_version_info { + u32 major; + u32 minor; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_version_info); + +#define VF_IS_V10(_v) (((_v)->major == 1) && ((_v)->minor == 0)) +#define VF_IS_V11(_ver) (((_ver)->major == 1) && ((_ver)->minor == 1)) + +/* VIRTCHNL_OP_RESET_VF + * VF sends this request to PF with no parameters + * PF does NOT respond! VF driver must delay then poll VFGEN_RSTAT register + * until reset completion is indicated. The admin queue must be reinitialized + * after this operation. + * + * When reset is complete, PF must ensure that all queues in all VSIs associated + * with the VF are stopped, all queue configurations in the HMC are set to 0, + * and all MAC and VLAN filters (except the default MAC address) on all VSIs + * are cleared. + */ + +/* VSI types that use VIRTCHNL interface for VF-PF communication. VSI_SRIOV + * vsi_type should always be 6 for backward compatibility. Add other fields + * as needed. + */ +enum virtchnl_vsi_type { + VIRTCHNL_VSI_TYPE_INVALID = 0, + VIRTCHNL_VSI_SRIOV = 6, +}; + +/* VIRTCHNL_OP_GET_VF_RESOURCES + * Version 1.0 VF sends this request to PF with no parameters + * Version 1.1 VF sends this request to PF with u32 bitmap of its capabilities + * PF responds with an indirect message containing + * virtchnl_vf_resource and one or more + * virtchnl_vsi_resource structures. + */ + +struct virtchnl_vsi_resource { + u16 vsi_id; + u16 num_queue_pairs; + enum virtchnl_vsi_type vsi_type; + u16 qset_handle; + u8 default_mac_addr[VIRTCHNL_ETH_LENGTH_OF_ADDRESS]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource); + +/* VF offload flags + * VIRTCHNL_VF_OFFLOAD_L2 flag is inclusive of base mode L2 offloads including + * TX/RX Checksum offloading and TSO for non-tunnelled packets. + */ +#define VIRTCHNL_VF_OFFLOAD_L2 0x00000001 +#define VIRTCHNL_VF_OFFLOAD_IWARP 0x00000002 +#define VIRTCHNL_VF_OFFLOAD_RSVD 0x00000004 +#define VIRTCHNL_VF_OFFLOAD_RSS_AQ 0x00000008 +#define VIRTCHNL_VF_OFFLOAD_RSS_REG 0x00000010 +#define VIRTCHNL_VF_OFFLOAD_WB_ON_ITR 0x00000020 +#define VIRTCHNL_VF_OFFLOAD_REQ_QUEUES 0x00000040 +#define VIRTCHNL_VF_OFFLOAD_VLAN 0x00010000 +#define VIRTCHNL_VF_OFFLOAD_RX_POLLING 0x00020000 +#define VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 0x00040000 +#define VIRTCHNL_VF_OFFLOAD_RSS_PF 0X00080000 +#define VIRTCHNL_VF_OFFLOAD_ENCAP 0X00100000 +#define VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM 0X00200000 +#define VIRTCHNL_VF_OFFLOAD_RX_ENCAP_CSUM 0X00400000 + +#define VF_BASE_MODE_OFFLOADS (VIRTCHNL_VF_OFFLOAD_L2 | \ + VIRTCHNL_VF_OFFLOAD_VLAN | \ + VIRTCHNL_VF_OFFLOAD_RSS_PF) + +struct virtchnl_vf_resource { + u16 num_vsis; + u16 num_queue_pairs; + u16 max_vectors; + u16 max_mtu; + + u32 vf_offload_flags; + u32 rss_key_size; + u32 rss_lut_size; + + struct virtchnl_vsi_resource vsi_res[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(36, virtchnl_vf_resource); + +/* VIRTCHNL_OP_CONFIG_TX_QUEUE + * VF sends this message to set up parameters for one TX queue. + * External data buffer contains one instance of virtchnl_txq_info. + * PF configures requested queue and returns a status code. + */ + +/* Tx queue config info */ +struct virtchnl_txq_info { + u16 vsi_id; + u16 queue_id; + u16 ring_len; /* number of descriptors, multiple of 8 */ + u16 headwb_enabled; /* deprecated with AVF 1.0 */ + u64 dma_ring_addr; + u64 dma_headwb_addr; /* deprecated with AVF 1.0 */ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_txq_info); + +/* VIRTCHNL_OP_CONFIG_RX_QUEUE + * VF sends this message to set up parameters for one RX queue. + * External data buffer contains one instance of virtchnl_rxq_info. + * PF configures requested queue and returns a status code. + */ + +/* Rx queue config info */ +struct virtchnl_rxq_info { + u16 vsi_id; + u16 queue_id; + u32 ring_len; /* number of descriptors, multiple of 32 */ + u16 hdr_size; + u16 splithdr_enabled; /* deprecated with AVF 1.0 */ + u32 databuffer_size; + u32 max_pkt_size; + u32 pad1; + u64 dma_ring_addr; + enum virtchnl_rx_hsplit rx_split_pos; /* deprecated with AVF 1.0 */ + u32 pad2; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_rxq_info); + +/* VIRTCHNL_OP_CONFIG_VSI_QUEUES + * VF sends this message to set parameters for all active TX and RX queues + * associated with the specified VSI. + * PF configures queues and returns status. + * If the number of queues specified is greater than the number of queues + * associated with the VSI, an error is returned and no queues are configured. + */ +struct virtchnl_queue_pair_info { + /* NOTE: vsi_id and queue_id should be identical for both queues. */ + struct virtchnl_txq_info txq; + struct virtchnl_rxq_info rxq; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(64, virtchnl_queue_pair_info); + +struct virtchnl_vsi_queue_config_info { + u16 vsi_id; + u16 num_queue_pairs; + u32 pad; + struct virtchnl_queue_pair_info qpair[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(72, virtchnl_vsi_queue_config_info); + +/* VIRTCHNL_OP_REQUEST_QUEUES + * VF sends this message to request the PF to allocate additional queues to + * this VF. Each VF gets a guaranteed number of queues on init but asking for + * additional queues must be negotiated. This is a best effort request as it + * is possible the PF does not have enough queues left to support the request. + * If the PF cannot support the number requested it will respond with the + * maximum number it is able to support; otherwise it will respond with the + * number requested. + */ + +/* VF resource request */ +struct virtchnl_vf_res_request { + u16 num_queue_pairs; +}; + +/* VIRTCHNL_OP_CONFIG_IRQ_MAP + * VF uses this message to map vectors to queues. + * The rxq_map and txq_map fields are bitmaps used to indicate which queues + * are to be associated with the specified vector. + * The "other" causes are always mapped to vector 0. + * PF configures interrupt mapping and returns status. + */ +struct virtchnl_vector_map { + u16 vsi_id; + u16 vector_id; + u16 rxq_map; + u16 txq_map; + u16 rxitr_idx; + u16 txitr_idx; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_vector_map); + +struct virtchnl_irq_map_info { + u16 num_vectors; + struct virtchnl_vector_map vecmap[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(14, virtchnl_irq_map_info); + +/* VIRTCHNL_OP_ENABLE_QUEUES + * VIRTCHNL_OP_DISABLE_QUEUES + * VF sends these message to enable or disable TX/RX queue pairs. + * The queues fields are bitmaps indicating which queues to act upon. + * (Currently, we only support 16 queues per VF, but we make the field + * u32 to allow for expansion.) + * PF performs requested action and returns status. + */ +struct virtchnl_queue_select { + u16 vsi_id; + u16 pad; + u32 rx_queues; + u32 tx_queues; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_queue_select); + +/* VIRTCHNL_OP_ADD_ETH_ADDR + * VF sends this message in order to add one or more unicast or multicast + * address filters for the specified VSI. + * PF adds the filters and returns status. + */ + +/* VIRTCHNL_OP_DEL_ETH_ADDR + * VF sends this message in order to remove one or more unicast or multicast + * filters for the specified VSI. + * PF removes the filters and returns status. + */ + +struct virtchnl_ether_addr { + u8 addr[VIRTCHNL_ETH_LENGTH_OF_ADDRESS]; + u8 pad[2]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_ether_addr); + +struct virtchnl_ether_addr_list { + u16 vsi_id; + u16 num_elements; + struct virtchnl_ether_addr list[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_ether_addr_list); + +#ifdef VIRTCHNL_SOL_VF_SUPPORT +/* VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG + * VF sends this message to get the default MTU and list of additional ethernet + * addresses it is allowed to use. + * PF responds with an indirect message containing + * virtchnl_addnl_solaris_config with zero or more + * virtchnl_ether_addr structures. + * + * It is expected that this operation will only ever be needed for Solaris VFs + * running under a Solaris PF. + */ +struct virtchnl_addnl_solaris_config { + u16 default_mtu; + struct virtchnl_ether_addr_list al; +}; + +#endif +/* VIRTCHNL_OP_ADD_VLAN + * VF sends this message to add one or more VLAN tag filters for receives. + * PF adds the filters and returns status. + * If a port VLAN is configured by the PF, this operation will return an + * error to the VF. + */ + +/* VIRTCHNL_OP_DEL_VLAN + * VF sends this message to remove one or more VLAN tag filters for receives. + * PF removes the filters and returns status. + * If a port VLAN is configured by the PF, this operation will return an + * error to the VF. + */ + +struct virtchnl_vlan_filter_list { + u16 vsi_id; + u16 num_elements; + u16 vlan_id[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_vlan_filter_list); + +/* VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE + * VF sends VSI id and flags. + * PF returns status code in retval. + * Note: we assume that broadcast accept mode is always enabled. + */ +struct virtchnl_promisc_info { + u16 vsi_id; + u16 flags; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_promisc_info); + +#define FLAG_VF_UNICAST_PROMISC 0x00000001 +#define FLAG_VF_MULTICAST_PROMISC 0x00000002 + +/* VIRTCHNL_OP_GET_STATS + * VF sends this message to request stats for the selected VSI. VF uses + * the virtchnl_queue_select struct to specify the VSI. The queue_id + * field is ignored by the PF. + * + * PF replies with struct eth_stats in an external buffer. + */ + +/* VIRTCHNL_OP_CONFIG_RSS_KEY + * VIRTCHNL_OP_CONFIG_RSS_LUT + * VF sends these messages to configure RSS. Only supported if both PF + * and VF drivers set the VIRTCHNL_VF_OFFLOAD_RSS_PF bit during + * configuration negotiation. If this is the case, then the RSS fields in + * the VF resource struct are valid. + * Both the key and LUT are initialized to 0 by the PF, meaning that + * RSS is effectively disabled until set up by the VF. + */ +struct virtchnl_rss_key { + u16 vsi_id; + u16 key_len; + u8 key[1]; /* RSS hash key, packed bytes */ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_rss_key); + +struct virtchnl_rss_lut { + u16 vsi_id; + u16 lut_entries; + u8 lut[1]; /* RSS lookup table*/ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_rss_lut); + +/* VIRTCHNL_OP_GET_RSS_HENA_CAPS + * VIRTCHNL_OP_SET_RSS_HENA + * VF sends these messages to get and set the hash filter enable bits for RSS. + * By default, the PF sets these to all possible traffic types that the + * hardware supports. The VF can query this value if it wants to change the + * traffic types that are hashed by the hardware. + */ +struct virtchnl_rss_hena { + u64 hena; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_rss_hena); + +/* VIRTCHNL_OP_EVENT + * PF sends this message to inform the VF driver of events that may affect it. + * No direct response is expected from the VF, though it may generate other + * messages in response to this one. + */ +enum virtchnl_event_codes { + VIRTCHNL_EVENT_UNKNOWN = 0, + VIRTCHNL_EVENT_LINK_CHANGE, + VIRTCHNL_EVENT_RESET_IMPENDING, + VIRTCHNL_EVENT_PF_DRIVER_CLOSE, +}; + +#define PF_EVENT_SEVERITY_INFO 0 +#define PF_EVENT_SEVERITY_ATTENTION 1 +#define PF_EVENT_SEVERITY_ACTION_REQUIRED 2 +#define PF_EVENT_SEVERITY_CERTAIN_DOOM 255 + +struct virtchnl_pf_event { + enum virtchnl_event_codes event; + union { + struct { + enum virtchnl_link_speed link_speed; + bool link_status; + } link_event; + } event_data; + + int severity; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_pf_event); + +#ifdef VIRTCHNL_IWARP + +/* VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP + * VF uses this message to request PF to map IWARP vectors to IWARP queues. + * The request for this originates from the VF IWARP driver through + * a client interface between VF LAN and VF IWARP driver. + * A vector could have an AEQ and CEQ attached to it although + * there is a single AEQ per VF IWARP instance in which case + * most vectors will have an INVALID_IDX for aeq and valid idx for ceq. + * There will never be a case where there will be multiple CEQs attached + * to a single vector. + * PF configures interrupt mapping and returns status. + */ + +/* HW does not define a type value for AEQ; only for RX/TX and CEQ. + * In order for us to keep the interface simple, SW will define a + * unique type value for AEQ. + */ +#define QUEUE_TYPE_PE_AEQ 0x80 +#define QUEUE_INVALID_IDX 0xFFFF + +struct virtchnl_iwarp_qv_info { + u32 v_idx; /* msix_vector */ + u16 ceq_idx; + u16 aeq_idx; + u8 itr_idx; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_iwarp_qv_info); + +struct virtchnl_iwarp_qvlist_info { + u32 num_vectors; + struct virtchnl_iwarp_qv_info qv_info[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_iwarp_qvlist_info); + +#endif + +/* VF reset states - these are written into the RSTAT register: + * VFGEN_RSTAT on the VF + * When the PF initiates a reset, it writes 0 + * When the reset is complete, it writes 1 + * When the PF detects that the VF has recovered, it writes 2 + * VF checks this register periodically to determine if a reset has occurred, + * then polls it to know when the reset is complete. + * If either the PF or VF reads the register while the hardware + * is in a reset state, it will return DEADBEEF, which, when masked + * will result in 3. + */ +enum virtchnl_vfr_states { + VIRTCHNL_VFR_INPROGRESS = 0, + VIRTCHNL_VFR_COMPLETED, + VIRTCHNL_VFR_VFACTIVE, +}; + +/** + * virtchnl_vc_validate_vf_msg + * @ver: Virtchnl version info + * @v_opcode: Opcode for the message + * @msg: pointer to the msg buffer + * @msglen: msg length + * + * validate msg format against struct for each opcode + */ +static inline int +virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, + u8 *msg, u16 msglen) +{ + bool err_msg_format = false; + int valid_len = 0; + + /* Validate message length. */ + switch (v_opcode) { + case VIRTCHNL_OP_VERSION: + valid_len = sizeof(struct virtchnl_version_info); + break; + case VIRTCHNL_OP_RESET_VF: + break; + case VIRTCHNL_OP_GET_VF_RESOURCES: + if (VF_IS_V11(ver)) + valid_len = sizeof(u32); + break; + case VIRTCHNL_OP_CONFIG_TX_QUEUE: + valid_len = sizeof(struct virtchnl_txq_info); + break; + case VIRTCHNL_OP_CONFIG_RX_QUEUE: + valid_len = sizeof(struct virtchnl_rxq_info); + break; + case VIRTCHNL_OP_CONFIG_VSI_QUEUES: + valid_len = sizeof(struct virtchnl_vsi_queue_config_info); + if (msglen >= valid_len) { + struct virtchnl_vsi_queue_config_info *vqc = + (struct virtchnl_vsi_queue_config_info *)msg; + valid_len += (vqc->num_queue_pairs * + sizeof(struct + virtchnl_queue_pair_info)); + if (vqc->num_queue_pairs == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_CONFIG_IRQ_MAP: + valid_len = sizeof(struct virtchnl_irq_map_info); + if (msglen >= valid_len) { + struct virtchnl_irq_map_info *vimi = + (struct virtchnl_irq_map_info *)msg; + valid_len += (vimi->num_vectors * + sizeof(struct virtchnl_vector_map)); + if (vimi->num_vectors == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_ENABLE_QUEUES: + case VIRTCHNL_OP_DISABLE_QUEUES: + valid_len = sizeof(struct virtchnl_queue_select); + break; + case VIRTCHNL_OP_ADD_ETH_ADDR: + case VIRTCHNL_OP_DEL_ETH_ADDR: + valid_len = sizeof(struct virtchnl_ether_addr_list); + if (msglen >= valid_len) { + struct virtchnl_ether_addr_list *veal = + (struct virtchnl_ether_addr_list *)msg; + valid_len += veal->num_elements * + sizeof(struct virtchnl_ether_addr); + if (veal->num_elements == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_ADD_VLAN: + case VIRTCHNL_OP_DEL_VLAN: + valid_len = sizeof(struct virtchnl_vlan_filter_list); + if (msglen >= valid_len) { + struct virtchnl_vlan_filter_list *vfl = + (struct virtchnl_vlan_filter_list *)msg; + valid_len += vfl->num_elements * sizeof(u16); + if (vfl->num_elements == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + valid_len = sizeof(struct virtchnl_promisc_info); + break; + case VIRTCHNL_OP_GET_STATS: + valid_len = sizeof(struct virtchnl_queue_select); + break; +#ifdef VIRTCHNL_IWARP + case VIRTCHNL_OP_IWARP: + /* These messages are opaque to us and will be validated in + * the RDMA client code. We just need to check for nonzero + * length. The firmware will enforce max length restrictions. + */ + if (msglen) + valid_len = msglen; + else + err_msg_format = true; + break; + case VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP: + break; + case VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP: + valid_len = sizeof(struct virtchnl_iwarp_qvlist_info); + if (msglen >= valid_len) { + struct virtchnl_iwarp_qvlist_info *qv = + (struct virtchnl_iwarp_qvlist_info *)msg; + if (qv->num_vectors == 0) { + err_msg_format = true; + break; + } + valid_len += ((qv->num_vectors - 1) * + sizeof(struct virtchnl_iwarp_qv_info)); + } + break; +#endif + case VIRTCHNL_OP_CONFIG_RSS_KEY: + valid_len = sizeof(struct virtchnl_rss_key); + if (msglen >= valid_len) { + struct virtchnl_rss_key *vrk = + (struct virtchnl_rss_key *)msg; + valid_len += vrk->key_len - 1; + } + break; + case VIRTCHNL_OP_CONFIG_RSS_LUT: + valid_len = sizeof(struct virtchnl_rss_lut); + if (msglen >= valid_len) { + struct virtchnl_rss_lut *vrl = + (struct virtchnl_rss_lut *)msg; + valid_len += vrl->lut_entries - 1; + } + break; + case VIRTCHNL_OP_GET_RSS_HENA_CAPS: + break; + case VIRTCHNL_OP_SET_RSS_HENA: + valid_len = sizeof(struct virtchnl_rss_hena); + break; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING: + break; + case VIRTCHNL_OP_REQUEST_QUEUES: + valid_len = sizeof(struct virtchnl_vf_res_request); + break; + /* These are always errors coming from the VF. */ + case VIRTCHNL_OP_EVENT: + case VIRTCHNL_OP_UNKNOWN: + default: + return VIRTCHNL_ERR_PARAM; + } + /* few more checks */ + if ((valid_len != msglen) || (err_msg_format)) + return VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH; + + return 0; +} +#endif /* _VIRTCHNL_H_ */ diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c index fd7d3475..5f26e24a 100644 --- a/drivers/net/i40e/i40e_ethdev.c +++ b/drivers/net/i40e/i40e_ethdev.c @@ -74,7 +74,7 @@ /* Maximun number of capability elements */ #define I40E_MAX_CAP_ELE_NUM 128 -/* Wait count and inteval */ +/* Wait count and interval */ #define I40E_CHK_Q_ENA_COUNT 1000 #define I40E_CHK_Q_ENA_INTERVAL_US 1000 @@ -515,6 +515,7 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .get_eeprom = i40e_get_eeprom, .mac_addr_set = i40e_set_default_mac_addr, .mtu_set = i40e_dev_mtu_set, + .tm_ops_get = i40e_tm_ops_get, }; /* store statistics names and its offset in stats structure */ @@ -879,7 +880,7 @@ is_floating_veb_supported(struct rte_devargs *devargs) static void config_floating_veb(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -919,7 +920,7 @@ i40e_init_ethtype_filter_list(struct rte_eth_dev *dev) /* Initialize ethertype filter rule list and hash */ TAILQ_INIT(ðertype_rule->ethertype_list); snprintf(ethertype_hash_name, RTE_HASH_NAMESIZE, - "ethertype_%s", dev->data->name); + "ethertype_%s", dev->device->name); ethertype_rule->hash_table = rte_hash_create(ðertype_hash_params); if (!ethertype_rule->hash_table) { PMD_INIT_LOG(ERR, "Failed to create ethertype hash table!"); @@ -964,7 +965,7 @@ i40e_init_tunnel_filter_list(struct rte_eth_dev *dev) /* Initialize tunnel filter rule list and hash */ TAILQ_INIT(&tunnel_rule->tunnel_list); snprintf(tunnel_hash_name, RTE_HASH_NAMESIZE, - "tunnel_%s", dev->data->name); + "tunnel_%s", dev->device->name); tunnel_rule->hash_table = rte_hash_create(&tunnel_hash_params); if (!tunnel_rule->hash_table) { PMD_INIT_LOG(ERR, "Failed to create tunnel hash table!"); @@ -1009,7 +1010,7 @@ i40e_init_fdir_filter_list(struct rte_eth_dev *dev) /* Initialize flow director filter rule list and hash */ TAILQ_INIT(&fdir_info->fdir_list); snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, - "fdir_%s", dev->data->name); + "fdir_%s", dev->device->name); fdir_info->hash_table = rte_hash_create(&fdir_hash_params); if (!fdir_info->hash_table) { PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); @@ -1061,7 +1062,7 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) return 0; } i40e_set_default_ptype_table(dev); - pci_dev = I40E_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); intr_handle = &pci_dev->intr_handle; rte_eth_copy_pci_info(dev, pci_dev); @@ -1142,11 +1143,8 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) config_floating_veb(dev); /* Clear PXE mode */ i40e_clear_pxe_mode(hw); - ret = i40e_dev_sync_phy_type(hw); - if (ret) { - PMD_INIT_LOG(ERR, "Failed to sync phy type: %d", ret); - goto err_sync_phy_type; - } + i40e_dev_sync_phy_type(hw); + /* * On X710, performance number is far from the expectation on recent * firmware versions. The fix for this issue may not be integrated in @@ -1298,6 +1296,9 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) /* initialize mirror rule list */ TAILQ_INIT(&pf->mirror_list); + /* initialize Traffic Manager configuration */ + i40e_tm_conf_init(dev); + ret = i40e_init_ethtype_filter_list(dev); if (ret < 0) goto err_init_ethtype_filter_list; @@ -1331,7 +1332,6 @@ err_msix_pool_init: err_qp_pool_init: err_parameter_init: err_get_capabilities: -err_sync_phy_type: (void)i40e_shutdown_adminq(hw); return ret; @@ -1414,7 +1414,7 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev) pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - pci_dev = I40E_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); intr_handle = &pci_dev->intr_handle; if (hw->adapter_stopped == 0) @@ -1461,6 +1461,9 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev) rte_free(p_flow); } + /* Remove all Traffic Manager configuration */ + i40e_tm_conf_uninit(dev); + return 0; } @@ -1470,9 +1473,14 @@ i40e_dev_configure(struct rte_eth_dev *dev) struct i40e_adapter *ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode; int i, ret; + ret = i40e_dev_sync_phy_type(hw); + if (ret) + return ret; + /* Initialize to TRUE. If any of Rx queues doesn't meet the * bulk allocation or vector Rx preconditions we will reset it. */ @@ -1547,7 +1555,7 @@ void i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t msix_vect = vsi->msix_intr; @@ -1661,7 +1669,7 @@ void i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t msix_vect = vsi->msix_intr; @@ -1733,7 +1741,7 @@ static void i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t interval = i40e_calc_itr_interval(\ @@ -1765,7 +1773,7 @@ static void i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t msix_intr, i; @@ -1806,11 +1814,15 @@ i40e_parse_link_speeds(uint16_t link_speeds) static int i40e_phy_conf_link(struct i40e_hw *hw, uint8_t abilities, - uint8_t force_speed) + uint8_t force_speed, + bool is_up) { enum i40e_status_code status; struct i40e_aq_get_phy_abilities_resp phy_ab; struct i40e_aq_set_phy_config phy_conf; + enum i40e_aq_phy_type cnt; + uint32_t phy_type_mask = 0; + const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX | I40E_AQ_PHY_FLAG_PAUSE_RX | I40E_AQ_PHY_FLAG_PAUSE_RX | @@ -1828,6 +1840,10 @@ i40e_phy_conf_link(struct i40e_hw *hw, if (status) return ret; + /* If link already up, no need to set up again */ + if (is_up && phy_ab.phy_type != 0) + return I40E_SUCCESS; + memset(&phy_conf, 0, sizeof(phy_conf)); /* bits 0-2 use the values from get_phy_abilities_resp */ @@ -1838,13 +1854,21 @@ i40e_phy_conf_link(struct i40e_hw *hw, if (abilities & I40E_AQ_PHY_AN_ENABLED) phy_conf.link_speed = advt; else - phy_conf.link_speed = force_speed; + phy_conf.link_speed = is_up ? force_speed : phy_ab.link_speed; phy_conf.abilities = abilities; + + + /* To enable link, phy_type mask needs to include each type */ + for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_MAX; cnt++) + phy_type_mask |= 1 << cnt; + /* use get_phy_abilities_resp value for the rest */ - phy_conf.phy_type = phy_ab.phy_type; - phy_conf.phy_type_ext = phy_ab.phy_type_ext; + phy_conf.phy_type = is_up ? cpu_to_le32(phy_type_mask) : 0; + phy_conf.phy_type_ext = is_up ? (I40E_AQ_PHY_TYPE_EXT_25G_KR | + I40E_AQ_PHY_TYPE_EXT_25G_CR | I40E_AQ_PHY_TYPE_EXT_25G_SR | + I40E_AQ_PHY_TYPE_EXT_25G_LR) : 0; phy_conf.fec_config = phy_ab.fec_cfg_curr_mod_ext_info; phy_conf.eee_capability = phy_ab.eee_capability; phy_conf.eeer = phy_ab.eeer_val; @@ -1876,13 +1900,7 @@ i40e_apply_link_speed(struct rte_eth_dev *dev) abilities |= I40E_AQ_PHY_AN_ENABLED; abilities |= I40E_AQ_PHY_LINK_ENABLED; - /* Skip changing speed on 40G interfaces, FW does not support */ - if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types)) { - speed = I40E_LINK_SPEED_UNKNOWN; - abilities |= I40E_AQ_PHY_AN_ENABLED; - } - - return i40e_phy_conf_link(hw, abilities, speed); + return i40e_phy_conf_link(hw, abilities, speed, true); } static int @@ -1892,7 +1910,7 @@ i40e_dev_start(struct rte_eth_dev *dev) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *main_vsi = pf->main_vsi; int ret, i; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t intr_vector = 0; struct i40e_vsi *vsi; @@ -2008,7 +2026,7 @@ i40e_dev_start(struct rte_eth_dev *dev) if (dev->data->dev_conf.intr_conf.lsc != 0) PMD_INIT_LOG(INFO, "lsc won't enable because of no intr multiplex"); - } else if (dev->data->dev_conf.intr_conf.lsc != 0) { + } else { ret = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN | I40E_AQ_EVENT_MODULE_QUAL_FAIL | @@ -2016,7 +2034,7 @@ i40e_dev_start(struct rte_eth_dev *dev) if (ret != I40E_SUCCESS) PMD_DRV_LOG(WARNING, "Fail to set phy mask"); - /* Call get_link_info aq commond to enable LSE */ + /* Call get_link_info aq commond to enable/disable LSE */ i40e_dev_link_update(dev, 0); } @@ -2025,6 +2043,11 @@ i40e_dev_start(struct rte_eth_dev *dev) i40e_filter_restore(pf); + if (pf->tm_conf.root && !pf->tm_conf.committed) + PMD_DRV_LOG(WARNING, + "please call hierarchy_commit() " + "before starting the port"); + return I40E_SUCCESS; err_up: @@ -2038,12 +2061,15 @@ static void i40e_dev_stop(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *main_vsi = pf->main_vsi; struct i40e_mirror_rule *p_mirror; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int i; + if (hw->adapter_stopped == 1) + return; /* Disable all queues */ i40e_dev_switch_queues(pf, FALSE); @@ -2085,6 +2111,11 @@ i40e_dev_stop(struct rte_eth_dev *dev) rte_free(intr_handle->intr_vec); intr_handle->intr_vec = NULL; } + + /* reset hierarchy commit */ + pf->tm_conf.committed = false; + + hw->adapter_stopped = 1; } static void @@ -2092,7 +2123,7 @@ i40e_dev_close(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t reg; int i; @@ -2100,7 +2131,6 @@ i40e_dev_close(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); i40e_dev_stop(dev); - hw->adapter_stopped = 1; i40e_dev_free_queues(dev); /* Disable interrupt */ @@ -2225,7 +2255,7 @@ i40e_dev_set_link_down(struct rte_eth_dev *dev) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK; - return i40e_phy_conf_link(hw, abilities, speed); + return i40e_phy_conf_link(hw, abilities, speed, false); } int @@ -2352,9 +2382,6 @@ i40e_update_vsi_stats(struct i40e_vsi *vsi) i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx), vsi->offset_loaded, &oes->tx_broadcast, &nes->tx_broadcast); - /* exclude CRC bytes */ - nes->tx_bytes -= (nes->tx_unicast + nes->tx_multicast + - nes->tx_broadcast) * ETHER_CRC_LEN; /* GLV_TDPC not supported */ i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded, &oes->tx_errors, &nes->tx_errors); @@ -2390,14 +2417,35 @@ i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) i40e_stat_update_48(hw, I40E_GLV_GORCH(hw->port), I40E_GLV_GORCL(hw->port), pf->offset_loaded, - &pf->internal_rx_bytes_offset, - &pf->internal_rx_bytes); + &pf->internal_stats_offset.rx_bytes, + &pf->internal_stats.rx_bytes); i40e_stat_update_48(hw, I40E_GLV_GOTCH(hw->port), I40E_GLV_GOTCL(hw->port), pf->offset_loaded, - &pf->internal_tx_bytes_offset, - &pf->internal_tx_bytes); + &pf->internal_stats_offset.tx_bytes, + &pf->internal_stats.tx_bytes); + /* Get total internal rx packet count */ + i40e_stat_update_48(hw, I40E_GLV_UPRCH(hw->port), + I40E_GLV_UPRCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.rx_unicast, + &pf->internal_stats.rx_unicast); + i40e_stat_update_48(hw, I40E_GLV_MPRCH(hw->port), + I40E_GLV_MPRCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.rx_multicast, + &pf->internal_stats.rx_multicast); + i40e_stat_update_48(hw, I40E_GLV_BPRCH(hw->port), + I40E_GLV_BPRCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.rx_broadcast, + &pf->internal_stats.rx_broadcast); + + /* exclude CRC size */ + pf->internal_stats.rx_bytes -= (pf->internal_stats.rx_unicast + + pf->internal_stats.rx_multicast + + pf->internal_stats.rx_broadcast) * ETHER_CRC_LEN; /* Get statistics of struct i40e_eth_stats */ i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port), @@ -2420,7 +2468,17 @@ i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) * so subtract ETHER_CRC_LEN from the byte counter for each rx packet. */ ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast + - ns->eth.rx_broadcast) * ETHER_CRC_LEN + pf->internal_rx_bytes; + ns->eth.rx_broadcast) * ETHER_CRC_LEN; + + /* Workaround: it is possible I40E_GLV_GORCH[H/L] is updated before + * I40E_GLPRT_GORCH[H/L], so there is a small window that cause negtive + * value. + */ + if (ns->eth.rx_bytes < pf->internal_stats.rx_bytes) + ns->eth.rx_bytes = 0; + /* exlude internal rx bytes */ + else + ns->eth.rx_bytes -= pf->internal_stats.rx_bytes; i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port), pf->offset_loaded, &os->eth.rx_discards, @@ -2448,7 +2506,14 @@ i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) pf->offset_loaded, &os->eth.tx_broadcast, &ns->eth.tx_broadcast); ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast + - ns->eth.tx_broadcast) * ETHER_CRC_LEN + pf->internal_tx_bytes; + ns->eth.tx_broadcast) * ETHER_CRC_LEN; + + /* exclude internal tx bytes */ + if (ns->eth.tx_bytes < pf->internal_stats.tx_bytes) + ns->eth.tx_bytes = 0; + else + ns->eth.tx_bytes -= pf->internal_stats.tx_bytes; + /* GLPRT_TEPC not supported */ /* additional port specific stats */ @@ -2869,7 +2934,7 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *vsi = pf->main_vsi; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->pci_dev = pci_dev; dev_info->max_rx_queues = vsi->nb_qps; @@ -2973,71 +3038,93 @@ i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) } static int -i40e_vlan_tpid_set(struct rte_eth_dev *dev, - enum rte_vlan_type vlan_type, - uint16_t tpid) +i40e_vlan_tpid_set_by_registers(struct rte_eth_dev *dev, + enum rte_vlan_type vlan_type, + uint16_t tpid, int qinq) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint64_t reg_r = 0, reg_w = 0; - uint16_t reg_id = 0; - int ret = 0; - int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend; + uint64_t reg_r = 0; + uint64_t reg_w = 0; + uint16_t reg_id = 3; + int ret; - switch (vlan_type) { - case ETH_VLAN_TYPE_OUTER: - if (qinq) + if (qinq) { + if (vlan_type == ETH_VLAN_TYPE_OUTER) reg_id = 2; - else - reg_id = 3; - break; - case ETH_VLAN_TYPE_INNER: - if (qinq) - reg_id = 3; - else { - ret = -EINVAL; - PMD_DRV_LOG(ERR, - "Unsupported vlan type in single vlan."); - return ret; - } - break; - default: - ret = -EINVAL; - PMD_DRV_LOG(ERR, "Unsupported vlan type %d", vlan_type); - return ret; } + ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id), ®_r, NULL); if (ret != I40E_SUCCESS) { PMD_DRV_LOG(ERR, "Fail to debug read from I40E_GL_SWT_L2TAGCTRL[%d]", reg_id); - ret = -EIO; - return ret; + return -EIO; } PMD_DRV_LOG(DEBUG, - "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: 0x%08"PRIx64, - reg_id, reg_r); + "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: 0x%08"PRIx64, + reg_id, reg_r); reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK)); reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT); if (reg_r == reg_w) { - ret = 0; PMD_DRV_LOG(DEBUG, "No need to write"); - return ret; + return 0; } ret = i40e_aq_debug_write_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id), reg_w, NULL); if (ret != I40E_SUCCESS) { - ret = -EIO; PMD_DRV_LOG(ERR, - "Fail to debug write to I40E_GL_SWT_L2TAGCTRL[%d]", - reg_id); - return ret; + "Fail to debug write to I40E_GL_SWT_L2TAGCTRL[%d]", + reg_id); + return -EIO; } PMD_DRV_LOG(DEBUG, - "Debug write 0x%08"PRIx64" to I40E_GL_SWT_L2TAGCTRL[%d]", - reg_w, reg_id); + "Debug write 0x%08"PRIx64" to I40E_GL_SWT_L2TAGCTRL[%d]", + reg_w, reg_id); + + return 0; +} + +static int +i40e_vlan_tpid_set(struct rte_eth_dev *dev, + enum rte_vlan_type vlan_type, + uint16_t tpid) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend; + int ret = 0; + + if ((vlan_type != ETH_VLAN_TYPE_INNER && + vlan_type != ETH_VLAN_TYPE_OUTER) || + (!qinq && vlan_type == ETH_VLAN_TYPE_INNER)) { + PMD_DRV_LOG(ERR, + "Unsupported vlan type."); + return -EINVAL; + } + /* 802.1ad frames ability is added in NVM API 1.7*/ + if (hw->flags & I40E_HW_FLAG_802_1AD_CAPABLE) { + if (qinq) { + if (vlan_type == ETH_VLAN_TYPE_OUTER) + hw->first_tag = rte_cpu_to_le_16(tpid); + else if (vlan_type == ETH_VLAN_TYPE_INNER) + hw->second_tag = rte_cpu_to_le_16(tpid); + } else { + if (vlan_type == ETH_VLAN_TYPE_OUTER) + hw->second_tag = rte_cpu_to_le_16(tpid); + } + ret = i40e_aq_set_switch_config(hw, 0, 0, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, + "Set switch config failed aq_err: %d", + hw->aq.asq_last_status); + ret = -EIO; + } + } else + /* If NVM API < 1.7, keep the register setting */ + ret = i40e_vlan_tpid_set_by_registers(dev, vlan_type, + tpid, qinq); return ret; } @@ -3066,7 +3153,7 @@ i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask) if (mask & ETH_VLAN_EXTEND_MASK) { if (dev->data->dev_conf.rxmode.hw_vlan_extend) { i40e_vsi_config_double_vlan(vsi, TRUE); - /* Set global registers with default ether type value */ + /* Set global registers with default ethertype. */ i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, ETHER_TYPE_VLAN); i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER, @@ -3788,7 +3875,7 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_PF_TO_HW(pf); - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); uint16_t qp_count = 0, vsi_count = 0; if (pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) { @@ -4284,6 +4371,8 @@ i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi, for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) if (enabled_tcmap & (1 << i)) total_tc++; + if (total_tc == 0) + total_tc = 1; vsi->enabled_tc = enabled_tcmap; /* Number of queues per enabled TC */ @@ -5216,10 +5305,8 @@ i40e_pf_setup(struct i40e_pf *pf) pf->offset_loaded = FALSE; memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats)); memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats)); - pf->internal_rx_bytes = 0; - pf->internal_tx_bytes = 0; - pf->internal_rx_bytes_offset = 0; - pf->internal_tx_bytes_offset = 0; + memset(&pf->internal_stats, 0, sizeof(struct i40e_eth_stats)); + memset(&pf->internal_stats_offset, 0, sizeof(struct i40e_eth_stats)); ret = i40e_pf_get_switch_config(pf); if (ret != I40E_SUCCESS) { @@ -5734,16 +5821,16 @@ i40e_dev_handle_vfr_event(struct rte_eth_dev *dev) index = abs_vf_id / I40E_UINT32_BIT_SIZE; offset = abs_vf_id % I40E_UINT32_BIT_SIZE; val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index)); - /* VFR event occured */ + /* VFR event occurred */ if (val & (0x1 << offset)) { int ret; /* Clear the event first */ I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index), (0x1 << offset)); - PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id); + PMD_DRV_LOG(INFO, "VF %u reset occurred", abs_vf_id); /** - * Only notify a VF reset event occured, + * Only notify a VF reset event occurred, * don't trigger another SW reset */ ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0); @@ -5804,7 +5891,7 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) ret = i40e_dev_link_update(dev, 0); if (!ret) _rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_INTR_LSC, NULL); + RTE_ETH_EVENT_INTR_LSC, NULL, NULL); break; default: PMD_DRV_LOG(DEBUG, "Request %u is not supported yet", @@ -7460,7 +7547,7 @@ i40e_pf_config_rss(struct i40e_pf *pf) /* * If both VMDQ and RSS enabled, not all of PF queues are configured. - * It's necessary to calulate the actual PF queues that are configured. + * It's necessary to calculate the actual PF queues that are configured. */ if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) num = i40e_pf_calc_configured_queues_num(pf); @@ -8136,7 +8223,7 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, /** * Validate if the input set is allowed for a specific PCTYPE */ -static int +int i40e_validate_input_set(enum i40e_filter_pctype pctype, enum rte_filter_type filter, uint64_t inset) { @@ -8311,7 +8398,7 @@ i40e_parse_input_set(uint64_t *inset, * Translate the input set from bit masks to register aware bit masks * and vice versa */ -static uint64_t +uint64_t i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input) { uint64_t val = 0; @@ -8396,7 +8483,7 @@ i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input) return val; } -static int +int i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem) { uint8_t i, idx = 0; @@ -8444,7 +8531,7 @@ i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem) return idx; } -static void +void i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val) { uint32_t reg = i40e_read_rx_ctl(hw, addr); @@ -9012,7 +9099,7 @@ i40e_dev_filter_ctrl(struct rte_eth_dev *dev, static void i40e_enable_extended_tag(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); uint32_t buf = 0; int ret; @@ -9150,8 +9237,9 @@ i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype) */ /* For both X710 and XL710 */ -#define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x10000200 -#define I40E_GL_SWR_PRI_JOIN_MAP_0 0x26CE00 +#define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1 0x10000200 +#define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2 0x20000200 +#define I40E_GL_SWR_PRI_JOIN_MAP_0 0x26CE00 #define I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x011f0200 #define I40E_GL_SWR_PRI_JOIN_MAP_2 0x26CE08 @@ -9172,13 +9260,22 @@ i40e_dev_sync_phy_type(struct i40e_hw *hw) enum i40e_status_code status; struct i40e_aq_get_phy_abilities_resp phy_ab; int ret = -ENOTSUP; + int retries = 0; status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab, NULL); - if (status) - return ret; - + while (status) { + PMD_INIT_LOG(WARNING, "Failed to sync phy type: status=%d", + status); + retries++; + rte_delay_us(100000); + if (retries < 5) + status = i40e_aq_get_phy_capabilities(hw, false, + true, &phy_ab, NULL); + else + return ret; + } return 0; } @@ -9203,8 +9300,12 @@ i40e_configure_registers(struct i40e_hw *hw) reg_table[i].val = I40E_X722_GL_SWR_PRI_JOIN_MAP_0_VALUE; else /* For X710/XL710/XXV710 */ - reg_table[i].val = - I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE; + if (hw->aq.fw_maj_ver < 6) + reg_table[i].val = + I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1; + else + reg_table[i].val = + I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2; } if (reg_table[i].addr == I40E_GL_SWR_PRI_JOIN_MAP_2) { @@ -10460,7 +10561,7 @@ i40e_dev_get_dcb_info(struct rte_eth_dev *dev, static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t interval = @@ -10494,7 +10595,7 @@ i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t msix_intr; @@ -10738,8 +10839,7 @@ i40e_filter_restore(struct i40e_pf *pf) static bool is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) { - if (strcmp(dev->data->drv_name, - drv->driver.name)) + if (strcmp(dev->device->driver->name, drv->driver.name)) return false; return true; diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h index b0d963c1..48abc05a 100644 --- a/drivers/net/i40e/i40e_ethdev.h +++ b/drivers/net/i40e/i40e_ethdev.h @@ -39,6 +39,7 @@ #include <rte_kvargs.h> #include <rte_hash.h> #include <rte_flow_driver.h> +#include <rte_tm_driver.h> #define I40E_VLAN_TAG_SIZE 4 @@ -102,7 +103,7 @@ /* Linux PF host with virtchnl version 1.1 */ #define PF_IS_V11(vf) \ - (((vf)->version_major == I40E_VIRTCHNL_VERSION_MAJOR) && \ + (((vf)->version_major == VIRTCHNL_VERSION_MAJOR) && \ ((vf)->version_minor == 1)) /* index flex payload per layer */ @@ -251,6 +252,15 @@ enum i40e_flxpld_layer_idx { I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \ I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8) +/* The max bandwidth of i40e is 40Gbps. */ +#define I40E_QOS_BW_MAX 40000 +/* The bandwidth should be the multiple of 50Mbps. */ +#define I40E_QOS_BW_GRANULARITY 50 +/* The min bandwidth weight is 1. */ +#define I40E_QOS_BW_WEIGHT_MIN 1 +/* The max bandwidth weight is 127. */ +#define I40E_QOS_BW_WEIGHT_MAX 127 + /** * The overhead from MTU to max frame size. * Considering QinQ packet, the VLAN tag needs to be counted twice. @@ -405,7 +415,7 @@ enum I40E_VF_STATE { struct i40e_pf_vf { struct i40e_pf *pf; struct i40e_vsi *vsi; - enum I40E_VF_STATE state; /* The number of queue pairs availiable */ + enum I40E_VF_STATE state; /* The number of queue pairs available */ uint16_t vf_idx; /* VF index in pf->vfs */ uint16_t lan_nb_qps; /* Actual queues allocated */ uint16_t reset_cnt; /* Total vf reset times */ @@ -431,6 +441,25 @@ struct i40e_vmdq_info { struct i40e_vsi *vsi; }; +#define I40E_FDIR_MAX_FLEXLEN 16 /**< Max length of flexbytes. */ +#define I40E_MAX_FLX_SOURCE_OFF 480 +#define NONUSE_FLX_PIT_DEST_OFF 63 +#define NONUSE_FLX_PIT_FSIZE 1 +#define I40E_FLX_OFFSET_IN_FIELD_VECTOR 50 +#define MK_FLX_PIT(src_offset, fsize, dst_offset) ( \ + (((src_offset) << I40E_PRTQF_FLX_PIT_SOURCE_OFF_SHIFT) & \ + I40E_PRTQF_FLX_PIT_SOURCE_OFF_MASK) | \ + (((fsize) << I40E_PRTQF_FLX_PIT_FSIZE_SHIFT) & \ + I40E_PRTQF_FLX_PIT_FSIZE_MASK) | \ + ((((dst_offset) == NONUSE_FLX_PIT_DEST_OFF ? \ + NONUSE_FLX_PIT_DEST_OFF : \ + ((dst_offset) + I40E_FLX_OFFSET_IN_FIELD_VECTOR)) << \ + I40E_PRTQF_FLX_PIT_DEST_OFF_SHIFT) & \ + I40E_PRTQF_FLX_PIT_DEST_OFF_MASK)) +#define I40E_WORD(hi, lo) (uint16_t)((((hi) << 8) & 0xFF00) | ((lo) & 0xFF)) +#define I40E_FLEX_WORD_MASK(off) (0x80 >> (off)) +#define I40E_FDIR_IPv6_TC_OFFSET 20 + /* * Structure to store flex pit for flow diretor. */ @@ -442,6 +471,7 @@ struct i40e_fdir_flex_pit { struct i40e_fdir_flex_mask { uint8_t word_mask; /**< Bit i enables word i of flexible payload */ + uint8_t nb_bitmask; struct { uint8_t offset; uint16_t mask; @@ -479,6 +509,12 @@ struct i40e_fdir_info { struct i40e_fdir_filter_list fdir_list; struct i40e_fdir_filter **hash_map; struct rte_hash *hash_table; + + /* Mark if flex pit and mask is set */ + bool flex_pit_flag[I40E_MAX_FLXPLD_LAYER]; + bool flex_mask_flag[I40E_FILTER_PCTYPE_MAX]; + + bool inset_flag[I40E_FILTER_PCTYPE_MAX]; /* Mark if input set is set */ }; /* Ethertype filter number HW supports */ @@ -625,6 +661,67 @@ struct rte_flow { TAILQ_HEAD(i40e_flow_list, rte_flow); +/* Struct to store Traffic Manager shaper profile. */ +struct i40e_tm_shaper_profile { + TAILQ_ENTRY(i40e_tm_shaper_profile) node; + uint32_t shaper_profile_id; + uint32_t reference_count; + struct rte_tm_shaper_params profile; +}; + +TAILQ_HEAD(i40e_shaper_profile_list, i40e_tm_shaper_profile); + +/* node type of Traffic Manager */ +enum i40e_tm_node_type { + I40E_TM_NODE_TYPE_PORT, + I40E_TM_NODE_TYPE_TC, + I40E_TM_NODE_TYPE_QUEUE, + I40E_TM_NODE_TYPE_MAX, +}; + +/* Struct to store Traffic Manager node configuration. */ +struct i40e_tm_node { + TAILQ_ENTRY(i40e_tm_node) node; + uint32_t id; + uint32_t priority; + uint32_t weight; + uint32_t reference_count; + struct i40e_tm_node *parent; + struct i40e_tm_shaper_profile *shaper_profile; + struct rte_tm_node_params params; +}; + +TAILQ_HEAD(i40e_tm_node_list, i40e_tm_node); + +/* Struct to store all the Traffic Manager configuration. */ +struct i40e_tm_conf { + struct i40e_shaper_profile_list shaper_profile_list; + struct i40e_tm_node *root; /* root node - port */ + struct i40e_tm_node_list tc_list; /* node list for all the TCs */ + struct i40e_tm_node_list queue_list; /* node list for all the queues */ + /** + * The number of added TC nodes. + * It should be no more than the TC number of this port. + */ + uint32_t nb_tc_node; + /** + * The number of added queue nodes. + * It should be no more than the queue number of this port. + */ + uint32_t nb_queue_node; + /** + * This flag is used to check if APP can change the TM node + * configuration. + * When it's true, means the configuration is applied to HW, + * APP should not change the configuration. + * As we don't support on-the-fly configuration, when starting + * the port, APP should call the hierarchy_commit API to set this + * flag to true. When stopping the port, this flag should be set + * to false. + */ + bool committed; +}; + /* * Structure to store private data specific for PF instance. */ @@ -639,11 +736,9 @@ struct i40e_pf { struct i40e_hw_port_stats stats_offset; struct i40e_hw_port_stats stats; - /* internal packet byte count, it should be excluded from the total */ - uint64_t internal_rx_bytes; - uint64_t internal_tx_bytes; - uint64_t internal_rx_bytes_offset; - uint64_t internal_tx_bytes_offset; + /* internal packet statistics, it should be excluded from the total */ + struct i40e_eth_stats internal_stats_offset; + struct i40e_eth_stats internal_stats; bool offset_loaded; struct rte_eth_dev_data *dev_data; /* Pointer to the device data */ @@ -690,6 +785,7 @@ struct i40e_pf { struct i40e_flow_list flow_list; bool mpls_replace_flag; /* 1 - MPLS filter replace is done */ bool qinq_replace_flag; /* QINQ filter replace is done */ + struct i40e_tm_conf tm_conf; }; enum pending_msg { @@ -742,7 +838,7 @@ struct i40e_vf { /* Event from pf */ bool dev_closed; bool link_up; - enum i40e_aq_link_speed link_speed; + enum virtchnl_link_speed link_speed; bool vf_reset; volatile uint32_t pend_cmd; /* pending command not finished yet */ int32_t cmd_retval; /* return value of the cmd response from PF */ @@ -750,8 +846,8 @@ struct i40e_vf { uint8_t *aq_resp; /* buffer to store the adminq response from PF */ /* VSI info */ - struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */ - struct i40e_virtchnl_vsi_resource *vsi_res; /* LAN VSI */ + struct virtchnl_vf_resource *vf_res; /* All VSIs */ + struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */ struct i40e_vsi vsi; uint64_t flags; }; @@ -822,8 +918,7 @@ int i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr); void i40e_update_vsi_stats(struct i40e_vsi *vsi); void i40e_pf_disable_irq0(struct i40e_hw *hw); void i40e_pf_enable_irq0(struct i40e_hw *hw); -int i40e_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); +int i40e_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete); void i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi); void i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi); int i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi, @@ -898,6 +993,17 @@ int i40e_add_macvlan_filters(struct i40e_vsi *vsi, int total); bool is_i40e_supported(struct rte_eth_dev *dev); +int i40e_validate_input_set(enum i40e_filter_pctype pctype, + enum rte_filter_type filter, uint64_t inset); +int i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, + uint8_t nb_elem); +uint64_t i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input); +void i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val); + +int i40e_tm_ops_get(struct rte_eth_dev *dev, void *ops); +void i40e_tm_conf_init(struct rte_eth_dev *dev); +void i40e_tm_conf_uninit(struct rte_eth_dev *dev); + #define I40E_DEV_TO_PCI(eth_dev) \ RTE_DEV_TO_PCI((eth_dev)->device) diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c index 6e5839dd..f6d82934 100644 --- a/drivers/net/i40e/i40e_ethdev_vf.c +++ b/drivers/net/i40e/i40e_ethdev_vf.c @@ -56,7 +56,6 @@ #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_ethdev_pci.h> -#include <rte_atomic.h> #include <rte_malloc.h> #include <rte_dev.h> @@ -77,7 +76,7 @@ #define MAX_RESET_WAIT_CNT 20 struct i40evf_arq_msg_info { - enum i40e_virtchnl_ops ops; + enum virtchnl_ops ops; enum i40e_status_code result; uint16_t buf_len; uint16_t msg_len; @@ -85,7 +84,7 @@ struct i40evf_arq_msg_info { }; struct vf_cmd_info { - enum i40e_virtchnl_ops ops; + enum virtchnl_ops ops; uint8_t *in_args; uint32_t in_args_size; uint8_t *out_buffer; @@ -108,7 +107,7 @@ static void i40evf_dev_stop(struct rte_eth_dev *dev); static void i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int i40evf_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); + int wait_to_complete); static void i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static int i40evf_dev_xstats_get(struct rte_eth_dev *dev, @@ -159,7 +158,7 @@ static int i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id); static int i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id); -static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev, +static void i40evf_handle_pf_event(struct rte_eth_dev *dev, uint8_t *msg, uint16_t msglen); @@ -244,7 +243,7 @@ i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_arq_event_info event; - enum i40e_virtchnl_ops opcode; + enum virtchnl_ops opcode; enum i40e_status_code retval; int ret; enum i40evf_aq_result result = I40EVF_MSG_NON; @@ -259,16 +258,16 @@ i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data) return result; } - opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high); + opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high); retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low); /* pf sys event */ - if (opcode == I40E_VIRTCHNL_OP_EVENT) { - struct i40e_virtchnl_pf_event *vpe = - (struct i40e_virtchnl_pf_event *)event.msg_buf; + if (opcode == VIRTCHNL_OP_EVENT) { + struct virtchnl_pf_event *vpe = + (struct virtchnl_pf_event *)event.msg_buf; result = I40EVF_MSG_SYS; switch (vpe->event) { - case I40E_VIRTCHNL_EVENT_LINK_CHANGE: + case VIRTCHNL_EVENT_LINK_CHANGE: vf->link_up = vpe->event_data.link_event.link_status; vf->link_speed = @@ -277,12 +276,12 @@ i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data) PMD_DRV_LOG(INFO, "Link status update:%s", vf->link_up ? "up" : "down"); break; - case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: + case VIRTCHNL_EVENT_RESET_IMPENDING: vf->vf_reset = true; vf->pend_msg |= PFMSG_RESET_IMPENDING; PMD_DRV_LOG(INFO, "vf is reseting"); break; - case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE: + case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: vf->dev_closed = true; vf->pend_msg |= PFMSG_DRIVER_CLOSE; PMD_DRV_LOG(INFO, "PF driver closed"); @@ -312,17 +311,17 @@ static inline void _clear_cmd(struct i40e_vf *vf) { rte_wmb(); - vf->pend_cmd = I40E_VIRTCHNL_OP_UNKNOWN; + vf->pend_cmd = VIRTCHNL_OP_UNKNOWN; } /* * Check there is pending cmd in execution. If none, set new command. */ static inline int -_atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops) +_atomic_set_cmd(struct i40e_vf *vf, enum virtchnl_ops ops) { int ret = rte_atomic32_cmpset(&vf->pend_cmd, - I40E_VIRTCHNL_OP_UNKNOWN, ops); + VIRTCHNL_OP_UNKNOWN, ops); if (!ret) PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd); @@ -347,7 +346,7 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) info.msg = args->out_buffer; info.buf_len = args->out_size; - info.ops = I40E_VIRTCHNL_OP_UNKNOWN; + info.ops = VIRTCHNL_OP_UNKNOWN; info.result = I40E_SUCCESS; err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS, @@ -359,12 +358,12 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) } switch (args->ops) { - case I40E_VIRTCHNL_OP_RESET_VF: + case VIRTCHNL_OP_RESET_VF: /*no need to process in this function */ err = 0; break; - case I40E_VIRTCHNL_OP_VERSION: - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + case VIRTCHNL_OP_VERSION: + case VIRTCHNL_OP_GET_VF_RESOURCES: /* for init adminq commands, need to poll the response */ err = -1; do { @@ -385,13 +384,18 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) /* for other adminq in running time, waiting the cmd done flag */ err = -1; do { - if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) { + if (vf->pend_cmd == VIRTCHNL_OP_UNKNOWN) { err = 0; break; } rte_delay_ms(ASQ_DELAY_MS); /* If don't read msg or read sys event, continue */ } while (i++ < MAX_TRY_TIMES); + /* If there's no response is received, clear command */ + if (i >= MAX_TRY_TIMES) { + PMD_DRV_LOG(WARNING, "No response for %d", args->ops); + _clear_cmd(vf); + } break; } @@ -404,15 +408,15 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) static int i40evf_check_api_version(struct rte_eth_dev *dev) { - struct i40e_virtchnl_version_info version, *pver; + struct virtchnl_version_info version, *pver; int err; struct vf_cmd_info args; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - version.major = I40E_VIRTCHNL_VERSION_MAJOR; - version.minor = I40E_VIRTCHNL_VERSION_MINOR; + version.major = VIRTCHNL_VERSION_MAJOR; + version.minor = VIRTCHNL_VERSION_MINOR; - args.ops = I40E_VIRTCHNL_OP_VERSION; + args.ops = VIRTCHNL_OP_VERSION; args.in_args = (uint8_t *)&version; args.in_args_size = sizeof(version); args.out_buffer = vf->aq_resp; @@ -424,19 +428,19 @@ i40evf_check_api_version(struct rte_eth_dev *dev) return err; } - pver = (struct i40e_virtchnl_version_info *)args.out_buffer; + pver = (struct virtchnl_version_info *)args.out_buffer; vf->version_major = pver->major; vf->version_minor = pver->minor; if (vf->version_major == I40E_DPDK_VERSION_MAJOR) PMD_DRV_LOG(INFO, "Peer is DPDK PF host"); - else if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) && - (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR)) + else if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) && + (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) PMD_DRV_LOG(INFO, "Peer is Linux PF host"); else { PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)", vf->version_major, vf->version_minor, - I40E_VIRTCHNL_VERSION_MAJOR, - I40E_VIRTCHNL_VERSION_MINOR); + VIRTCHNL_VERSION_MAJOR, + VIRTCHNL_VERSION_MINOR); return -1; } @@ -452,15 +456,15 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev) struct vf_cmd_info args; uint32_t caps, len; - args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES; + args.ops = VIRTCHNL_OP_GET_VF_RESOURCES; args.out_buffer = vf->aq_resp; args.out_size = I40E_AQ_BUF_SZ; if (PF_IS_V11(vf)) { - caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | - I40E_VIRTCHNL_VF_OFFLOAD_VLAN | - I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING; + caps = VIRTCHNL_VF_OFFLOAD_L2 | + VIRTCHNL_VF_OFFLOAD_RSS_AQ | + VIRTCHNL_VF_OFFLOAD_RSS_REG | + VIRTCHNL_VF_OFFLOAD_VLAN | + VIRTCHNL_VF_OFFLOAD_RX_POLLING; args.in_args = (uint8_t *)∩︀ args.in_args_size = sizeof(caps); } else { @@ -474,8 +478,8 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev) return err; } - len = sizeof(struct i40e_virtchnl_vf_resource) + - I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource); + len = sizeof(struct virtchnl_vf_resource) + + I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource); (void)rte_memcpy(vf->vf_res, args.out_buffer, RTE_MIN(args.out_size, len)); @@ -492,18 +496,18 @@ i40evf_config_promisc(struct rte_eth_dev *dev, struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); int err; struct vf_cmd_info args; - struct i40e_virtchnl_promisc_info promisc; + struct virtchnl_promisc_info promisc; promisc.flags = 0; promisc.vsi_id = vf->vsi_res->vsi_id; if (enable_unicast) - promisc.flags |= I40E_FLAG_VF_UNICAST_PROMISC; + promisc.flags |= FLAG_VF_UNICAST_PROMISC; if (enable_multicast) - promisc.flags |= I40E_FLAG_VF_MULTICAST_PROMISC; + promisc.flags |= FLAG_VF_MULTICAST_PROMISC; - args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; + args.ops = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; args.in_args = (uint8_t *)&promisc; args.in_args_size = sizeof(promisc); args.out_buffer = vf->aq_resp; @@ -517,30 +521,46 @@ i40evf_config_promisc(struct rte_eth_dev *dev, return err; } -/* Configure vlan and double vlan offload. Use flag to specify which part to configure */ static int -i40evf_config_vlan_offload(struct rte_eth_dev *dev, - bool enable_vlan_strip) +i40evf_enable_vlan_strip(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - int err; struct vf_cmd_info args; - struct i40e_virtchnl_vlan_offload_info offload; - - offload.vsi_id = vf->vsi_res->vsi_id; - offload.enable_vlan_strip = enable_vlan_strip; + int ret; - args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD; - args.in_args = (uint8_t *)&offload; - args.in_args_size = sizeof(offload); + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL_OP_ENABLE_VLAN_STRIPPING; + args.in_args = NULL; + args.in_args_size = 0; args.out_buffer = vf->aq_resp; args.out_size = I40E_AQ_BUF_SZ; + ret = i40evf_execute_vf_cmd(dev, &args); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of " + "VIRTCHNL_OP_ENABLE_VLAN_STRIPPING"); - err = i40evf_execute_vf_cmd(dev, &args); - if (err) - PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_OFFLOAD"); + return ret; +} - return err; +static int +i40evf_disable_vlan_strip(struct rte_eth_dev *dev) +{ + struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct vf_cmd_info args; + int ret; + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL_OP_DISABLE_VLAN_STRIPPING; + args.in_args = NULL; + args.in_args_size = 0; + args.out_buffer = vf->aq_resp; + args.out_size = I40E_AQ_BUF_SZ; + ret = i40evf_execute_vf_cmd(dev, &args); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of " + "VIRTCHNL_OP_DISABLE_VLAN_STRIPPING"); + + return ret; } static int @@ -550,7 +570,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev, struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); int err; struct vf_cmd_info args; - struct i40e_virtchnl_pvid_info tpid_info; + struct virtchnl_pvid_info tpid_info; if (info == NULL) { PMD_DRV_LOG(ERR, "invalid parameters"); @@ -561,7 +581,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev, tpid_info.vsi_id = vf->vsi_res->vsi_id; (void)rte_memcpy(&tpid_info.info, info, sizeof(*info)); - args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID; + args.ops = (enum virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID; args.in_args = (uint8_t *)&tpid_info; args.in_args_size = sizeof(tpid_info); args.out_buffer = vf->aq_resp; @@ -575,7 +595,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev, } static void -i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info, +i40evf_fill_virtchnl_vsi_txq_info(struct virtchnl_txq_info *txq_info, uint16_t vsi_id, uint16_t queue_id, uint16_t nb_txq, @@ -590,7 +610,7 @@ i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info, } static void -i40evf_fill_virtchnl_vsi_rxq_info(struct i40e_virtchnl_rxq_info *rxq_info, +i40evf_fill_virtchnl_vsi_rxq_info(struct virtchnl_rxq_info *rxq_info, uint16_t vsi_id, uint16_t queue_id, uint16_t nb_rxq, @@ -618,8 +638,8 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) (struct i40e_rx_queue **)dev->data->rx_queues; struct i40e_tx_queue **txq = (struct i40e_tx_queue **)dev->data->tx_queues; - struct i40e_virtchnl_vsi_queue_config_info *vc_vqci; - struct i40e_virtchnl_queue_pair_info *vc_qpi; + struct virtchnl_vsi_queue_config_info *vc_vqci; + struct virtchnl_queue_pair_info *vc_qpi; struct vf_cmd_info args; uint16_t i, nb_qp = vf->num_queue_pairs; const uint32_t size = @@ -628,7 +648,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) int ret; memset(buff, 0, sizeof(buff)); - vc_vqci = (struct i40e_virtchnl_vsi_queue_config_info *)buff; + vc_vqci = (struct virtchnl_vsi_queue_config_info *)buff; vc_vqci->vsi_id = vf->vsi_res->vsi_id; vc_vqci->num_queue_pairs = nb_qp; @@ -640,7 +660,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) vf->max_pkt_len, rxq[i]); } memset(&args, 0, sizeof(args)); - args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES; + args.ops = VIRTCHNL_OP_CONFIG_VSI_QUEUES; args.in_args = (uint8_t *)vc_vqci; args.in_args_size = size; args.out_buffer = vf->aq_resp; @@ -648,7 +668,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) ret = i40evf_execute_vf_cmd(dev, &args); if (ret) PMD_DRV_LOG(ERR, "Failed to execute command of " - "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES"); + "VIRTCHNL_OP_CONFIG_VSI_QUEUES"); return ret; } @@ -662,8 +682,8 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev) (struct i40e_rx_queue **)dev->data->rx_queues; struct i40e_tx_queue **txq = (struct i40e_tx_queue **)dev->data->tx_queues; - struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei; - struct i40e_virtchnl_queue_pair_ext_info *vc_qpei; + struct virtchnl_vsi_queue_config_ext_info *vc_vqcei; + struct virtchnl_queue_pair_ext_info *vc_qpei; struct vf_cmd_info args; uint16_t i, nb_qp = vf->num_queue_pairs; const uint32_t size = @@ -672,7 +692,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev) int ret; memset(buff, 0, sizeof(buff)); - vc_vqcei = (struct i40e_virtchnl_vsi_queue_config_ext_info *)buff; + vc_vqcei = (struct virtchnl_vsi_queue_config_ext_info *)buff; vc_vqcei->vsi_id = vf->vsi_res->vsi_id; vc_vqcei->num_queue_pairs = nb_qp; vc_qpei = vc_vqcei->qpair; @@ -693,7 +713,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev) } memset(&args, 0, sizeof(args)); args.ops = - (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT; + (enum virtchnl_ops)VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT; args.in_args = (uint8_t *)vc_vqcei; args.in_args_size = size; args.out_buffer = vf->aq_resp; @@ -701,7 +721,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev) ret = i40evf_execute_vf_cmd(dev, &args); if (ret) PMD_DRV_LOG(ERR, "Failed to execute command of " - "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT"); + "VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT"); return ret; } @@ -724,10 +744,10 @@ i40evf_config_irq_map(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct vf_cmd_info args; - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_irq_map_info) + \ - sizeof(struct i40e_virtchnl_vector_map)]; - struct i40e_virtchnl_irq_map_info *map_info; - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + uint8_t cmd_buffer[sizeof(struct virtchnl_irq_map_info) + \ + sizeof(struct virtchnl_vector_map)]; + struct virtchnl_irq_map_info *map_info; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t vector_id; int i, err; @@ -741,7 +761,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev) vector_id = I40E_MISC_VEC_ID; } - map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer; + map_info = (struct virtchnl_irq_map_info *)cmd_buffer; map_info->num_vectors = 1; map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT; map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id; @@ -756,7 +776,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev) intr_handle->intr_vec[i] = vector_id; } - args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP; + args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP; args.in_args = (u8 *)cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -773,7 +793,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid, bool on) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_queue_select queue_select; + struct virtchnl_queue_select queue_select; int err; struct vf_cmd_info args; memset(&queue_select, 0, sizeof(queue_select)); @@ -785,9 +805,9 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid, queue_select.tx_queues |= 1 << qid; if (on) - args.ops = I40E_VIRTCHNL_OP_ENABLE_QUEUES; + args.ops = VIRTCHNL_OP_ENABLE_QUEUES; else - args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES; + args.ops = VIRTCHNL_OP_DISABLE_QUEUES; args.in_args = (u8 *)&queue_select; args.in_args_size = sizeof(queue_select); args.out_buffer = vf->aq_resp; @@ -861,10 +881,10 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, __rte_unused uint32_t index, __rte_unused uint32_t pool) { - struct i40e_virtchnl_ether_addr_list *list; + struct virtchnl_ether_addr_list *list; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \ - sizeof(struct i40e_virtchnl_ether_addr)]; + uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \ + sizeof(struct virtchnl_ether_addr)]; int err; struct vf_cmd_info args; @@ -876,13 +896,13 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, return I40E_ERR_INVALID_MAC_ADDR; } - list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer; + list = (struct virtchnl_ether_addr_list *)cmd_buffer; list->vsi_id = vf->vsi_res->vsi_id; list->num_elements = 1; (void)rte_memcpy(list->list[0].addr, addr->addr_bytes, sizeof(addr->addr_bytes)); - args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS; + args.ops = VIRTCHNL_OP_ADD_ETH_ADDR; args.in_args = cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -891,6 +911,8 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, if (err) PMD_DRV_LOG(ERR, "fail to execute command " "OP_ADD_ETHER_ADDRESS"); + else + vf->vsi.mac_num++; return err; } @@ -899,10 +921,10 @@ static void i40evf_del_mac_addr_by_addr(struct rte_eth_dev *dev, struct ether_addr *addr) { - struct i40e_virtchnl_ether_addr_list *list; + struct virtchnl_ether_addr_list *list; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \ - sizeof(struct i40e_virtchnl_ether_addr)]; + uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \ + sizeof(struct virtchnl_ether_addr)]; int err; struct vf_cmd_info args; @@ -914,13 +936,13 @@ i40evf_del_mac_addr_by_addr(struct rte_eth_dev *dev, return; } - list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer; + list = (struct virtchnl_ether_addr_list *)cmd_buffer; list->vsi_id = vf->vsi_res->vsi_id; list->num_elements = 1; (void)rte_memcpy(list->list[0].addr, addr->addr_bytes, sizeof(addr->addr_bytes)); - args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS; + args.ops = VIRTCHNL_OP_DEL_ETH_ADDR; args.in_args = cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -929,6 +951,8 @@ i40evf_del_mac_addr_by_addr(struct rte_eth_dev *dev, if (err) PMD_DRV_LOG(ERR, "fail to execute command " "OP_DEL_ETHER_ADDRESS"); + else + vf->vsi.mac_num--; return; } @@ -947,13 +971,13 @@ static int i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_queue_select q_stats; + struct virtchnl_queue_select q_stats; int err; struct vf_cmd_info args; memset(&q_stats, 0, sizeof(q_stats)); q_stats.vsi_id = vf->vsi_res->vsi_id; - args.ops = I40E_VIRTCHNL_OP_GET_STATS; + args.ops = VIRTCHNL_OP_GET_STATS; args.in_args = (u8 *)&q_stats; args.in_args_size = sizeof(q_stats); args.out_buffer = vf->aq_resp; @@ -1050,18 +1074,18 @@ static int i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_vlan_filter_list *vlan_list; - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) + + struct virtchnl_vlan_filter_list *vlan_list; + uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) + sizeof(uint16_t)]; int err; struct vf_cmd_info args; - vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer; + vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer; vlan_list->vsi_id = vf->vsi_res->vsi_id; vlan_list->num_elements = 1; vlan_list->vlan_id[0] = vlanid; - args.ops = I40E_VIRTCHNL_OP_ADD_VLAN; + args.ops = VIRTCHNL_OP_ADD_VLAN; args.in_args = (u8 *)&cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -1077,18 +1101,18 @@ static int i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_vlan_filter_list *vlan_list; - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) + + struct virtchnl_vlan_filter_list *vlan_list; + uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) + sizeof(uint16_t)]; int err; struct vf_cmd_info args; - vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer; + vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer; vlan_list->vsi_id = vf->vsi_res->vsi_id; vlan_list->num_elements = 1; vlan_list->vlan_id[0] = vlanid; - args.ops = I40E_VIRTCHNL_OP_DEL_VLAN; + args.ops = VIRTCHNL_OP_DEL_VLAN; args.in_args = (u8 *)&cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -1178,7 +1202,7 @@ i40evf_reset_vf(struct i40e_hw *hw) reset = rd32(hw, I40E_VFGEN_RSTAT) & I40E_VFGEN_RSTAT_VFR_STATE_MASK; reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT; - if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset) + if (VIRTCHNL_VFR_COMPLETED == reset || VIRTCHNL_VFR_VFACTIVE == reset) break; else rte_delay_ms(50); @@ -1242,8 +1266,8 @@ i40evf_init_vf(struct rte_eth_dev *dev) PMD_INIT_LOG(ERR, "check_api version failed"); goto err_aq; } - bufsz = sizeof(struct i40e_virtchnl_vf_resource) + - (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource)); + bufsz = sizeof(struct virtchnl_vf_resource) + + (I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); if (!vf->vf_res) { PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); @@ -1257,7 +1281,7 @@ i40evf_init_vf(struct rte_eth_dev *dev) /* got VF config message back from PF, now we can parse it */ for (i = 0; i < vf->vf_res->num_vsis; i++) { - if (vf->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV) + if (vf->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) vf->vsi_res = &vf->vf_res->vsi_res[i]; } @@ -1269,7 +1293,7 @@ i40evf_init_vf(struct rte_eth_dev *dev) if (hw->mac.type == I40E_MAC_X722_VF) vf->flags = I40E_FLAG_RSS_AQ_CAPABLE; vf->vsi.vsi_id = vf->vsi_res->vsi_id; - vf->vsi.type = vf->vsi_res->vsi_type; + vf->vsi.type = (enum i40e_vsi_type)vf->vsi_res->vsi_type; vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs; vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); @@ -1318,25 +1342,25 @@ i40evf_uninit_vf(struct rte_eth_dev *dev) } static void -i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev, - uint8_t *msg, - __rte_unused uint16_t msglen) +i40evf_handle_pf_event(struct rte_eth_dev *dev, uint8_t *msg, + __rte_unused uint16_t msglen) { - struct i40e_virtchnl_pf_event *pf_msg = - (struct i40e_virtchnl_pf_event *)msg; + struct virtchnl_pf_event *pf_msg = + (struct virtchnl_pf_event *)msg; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); switch (pf_msg->event) { - case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: + case VIRTCHNL_EVENT_RESET_IMPENDING: PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event"); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, NULL); break; - case I40E_VIRTCHNL_EVENT_LINK_CHANGE: + case VIRTCHNL_EVENT_LINK_CHANGE: PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event"); vf->link_up = pf_msg->event_data.link_event.link_status; vf->link_speed = pf_msg->event_data.link_event.link_speed; break; - case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE: + case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event"); break; default: @@ -1352,7 +1376,7 @@ i40evf_handle_aq_msg(struct rte_eth_dev *dev) struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_arq_event_info info; uint16_t pending, aq_opc; - enum i40e_virtchnl_ops msg_opc; + enum virtchnl_ops msg_opc; enum i40e_status_code msg_ret; int ret; @@ -1377,13 +1401,13 @@ i40evf_handle_aq_msg(struct rte_eth_dev *dev) * cookie_high of struct i40e_aq_desc, while return error code * are stored in cookie_low, Which is done by * i40e_aq_send_msg_to_vf in PF driver.*/ - msg_opc = (enum i40e_virtchnl_ops)rte_le_to_cpu_32( + msg_opc = (enum virtchnl_ops)rte_le_to_cpu_32( info.desc.cookie_high); msg_ret = (enum i40e_status_code)rte_le_to_cpu_32( info.desc.cookie_low); switch (aq_opc) { case i40e_aqc_opc_send_msg_to_vf: - if (msg_opc == I40E_VIRTCHNL_OP_EVENT) + if (msg_opc == VIRTCHNL_OP_EVENT) /* process event*/ i40evf_handle_pf_event(dev, info.msg_buf, info.msg_len); @@ -1460,7 +1484,7 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); PMD_INIT_FUNC_TRACE(); @@ -1592,8 +1616,8 @@ i40evf_dev_configure(struct rte_eth_dev *dev) */ if (!conf->rxmode.hw_strip_crc) { vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) && - (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR)) { + if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) && + (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) { /* Peer is running non-DPDK PF driver. */ PMD_INIT_LOG(ERR, "VF can't disable HW CRC Strip"); return -EINVAL; @@ -1621,22 +1645,15 @@ i40evf_init_vlan(struct rte_eth_dev *dev) static void i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask) { - bool enable_vlan_strip = 0; struct rte_eth_conf *dev_conf = &dev->data->dev_conf; - struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - /* Linux pf host doesn't support vlan offload yet */ - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) { - /* Vlan stripping setting */ - if (mask & ETH_VLAN_STRIP_MASK) { - /* Enable or disable VLAN stripping */ - if (dev_conf->rxmode.hw_vlan_strip) - enable_vlan_strip = 1; - else - enable_vlan_strip = 0; - - i40evf_config_vlan_offload(dev, enable_vlan_strip); - } + /* Vlan stripping setting */ + if (mask & ETH_VLAN_STRIP_MASK) { + /* Enable or disable VLAN stripping */ + if (dev_conf->rxmode.hw_vlan_strip) + i40evf_enable_vlan_strip(dev); + else + i40evf_disable_vlan_strip(dev); } } @@ -1884,7 +1901,7 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; if (!rte_intr_allow_others(intr_handle)) { @@ -1917,7 +1934,7 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; if (!rte_intr_allow_others(intr_handle)) { @@ -1944,7 +1961,7 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev) static int i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t interval = @@ -1979,7 +1996,7 @@ i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) static int i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t msix_intr; @@ -2001,7 +2018,7 @@ i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) static void i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) { - struct i40e_virtchnl_ether_addr_list *list; + struct virtchnl_ether_addr_list *list; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); int err, i, j; int next_begin = 0; @@ -2012,11 +2029,11 @@ i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) do { j = 0; - len = sizeof(struct i40e_virtchnl_ether_addr_list); + len = sizeof(struct virtchnl_ether_addr_list); for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) { if (is_zero_ether_addr(&dev->data->mac_addrs[i])) continue; - len += sizeof(struct i40e_virtchnl_ether_addr); + len += sizeof(struct virtchnl_ether_addr); if (len >= I40E_AQ_BUF_SZ) { next_begin = i + 1; break; @@ -2043,17 +2060,23 @@ i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) } list->vsi_id = vf->vsi_res->vsi_id; list->num_elements = j; - args.ops = add ? I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS : - I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS; + args.ops = add ? VIRTCHNL_OP_ADD_ETH_ADDR : + VIRTCHNL_OP_DEL_ETH_ADDR; args.in_args = (uint8_t *)list; args.in_args_size = len; args.out_buffer = vf->aq_resp; args.out_size = I40E_AQ_BUF_SZ; err = i40evf_execute_vf_cmd(dev, &args); - if (err) + if (err) { PMD_DRV_LOG(ERR, "fail to execute command %s", add ? "OP_ADD_ETHER_ADDRESS" : "OP_DEL_ETHER_ADDRESS"); + } else { + if (add) + vf->vsi.mac_num++; + else + vf->vsi.mac_num--; + } rte_free(list); begin = next_begin; } while (begin < I40E_NUM_MACADDR_MAX); @@ -2064,7 +2087,7 @@ i40evf_dev_start(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t intr_vector = 0; @@ -2130,11 +2153,14 @@ err_queue: static void i40evf_dev_stop(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev); PMD_INIT_FUNC_TRACE(); + if (hw->adapter_stopped == 1) + return; i40evf_stop_queues(dev); i40evf_disable_queues_intr(dev); i40e_dev_clear_queues(dev); @@ -2147,6 +2173,7 @@ i40evf_dev_stop(struct rte_eth_dev *dev) } /* remove all mac addrs */ i40evf_add_del_all_mac_addr(dev, FALSE); + hw->adapter_stopped = 1; } @@ -2261,7 +2288,7 @@ i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); memset(dev_info, 0, sizeof(*dev_info)); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs; dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs; dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN; @@ -2330,11 +2357,10 @@ static void i40evf_dev_close(struct rte_eth_dev *dev) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = I40E_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; i40evf_dev_stop(dev); - hw->adapter_stopped = 1; i40e_dev_free_queues(dev); i40evf_reset_vf(hw); i40e_shutdown_adminq(hw); diff --git a/drivers/net/i40e/i40e_fdir.c b/drivers/net/i40e/i40e_fdir.c index 28cc554f..8013add4 100644 --- a/drivers/net/i40e/i40e_fdir.c +++ b/drivers/net/i40e/i40e_fdir.c @@ -67,15 +67,13 @@ #define I40E_FDIR_IP_DEFAULT_VERSION_IHL 0x45 #define I40E_FDIR_TCP_DEFAULT_DATAOFF 0x50 #define I40E_FDIR_IPv6_DEFAULT_VTC_FLOW 0x60000000 -#define I40E_FDIR_IPv6_TC_OFFSET 20 #define I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS 0xFF #define I40E_FDIR_IPv6_PAYLOAD_LEN 380 #define I40E_FDIR_UDP_DEFAULT_LEN 400 -/* Wait count and interval for fdir filter programming */ -#define I40E_FDIR_WAIT_COUNT 10 -#define I40E_FDIR_WAIT_INTERVAL_US 1000 +/* Wait time for fdir filter programming */ +#define I40E_FDIR_MAX_WAIT_US 10000 /* Wait count and interval for fdir filter flush */ #define I40E_FDIR_FLUSH_RETRY 50 @@ -84,21 +82,6 @@ #define I40E_COUNTER_PF 2 /* Statistic counter index for one pf */ #define I40E_COUNTER_INDEX_FDIR(pf_id) (0 + (pf_id) * I40E_COUNTER_PF) -#define I40E_MAX_FLX_SOURCE_OFF 480 -#define I40E_FLX_OFFSET_IN_FIELD_VECTOR 50 - -#define NONUSE_FLX_PIT_DEST_OFF 63 -#define NONUSE_FLX_PIT_FSIZE 1 -#define MK_FLX_PIT(src_offset, fsize, dst_offset) ( \ - (((src_offset) << I40E_PRTQF_FLX_PIT_SOURCE_OFF_SHIFT) & \ - I40E_PRTQF_FLX_PIT_SOURCE_OFF_MASK) | \ - (((fsize) << I40E_PRTQF_FLX_PIT_FSIZE_SHIFT) & \ - I40E_PRTQF_FLX_PIT_FSIZE_MASK) | \ - ((((dst_offset) == NONUSE_FLX_PIT_DEST_OFF ? \ - NONUSE_FLX_PIT_DEST_OFF : \ - ((dst_offset) + I40E_FLX_OFFSET_IN_FIELD_VECTOR)) << \ - I40E_PRTQF_FLX_PIT_DEST_OFF_SHIFT) & \ - I40E_PRTQF_FLX_PIT_DEST_OFF_MASK)) #define I40E_FDIR_FLOWS ( \ (1 << RTE_ETH_FLOW_FRAG_IPV4) | \ @@ -113,8 +96,6 @@ (1 << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \ (1 << RTE_ETH_FLOW_L2_PAYLOAD)) -#define I40E_FLEX_WORD_MASK(off) (0x80 >> (off)) - static int i40e_fdir_filter_programming(struct i40e_pf *pf, enum i40e_filter_pctype pctype, const struct rte_eth_fdir_filter *filter, @@ -257,7 +238,7 @@ i40e_fdir_setup(struct i40e_pf *pf) /* reserve memory for the fdir programming packet */ snprintf(z_name, sizeof(z_name), "%s_%s_%d", - eth_dev->data->drv_name, + eth_dev->device->driver->name, I40E_FDIR_MZ_NAME, eth_dev->data->port_id); mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN, SOCKET_ID_ANY); @@ -300,8 +281,12 @@ i40e_fdir_teardown(struct i40e_pf *pf) vsi = pf->fdir.fdir_vsi; if (!vsi) return; - i40e_switch_tx_queue(hw, vsi->base_queue, FALSE); - i40e_switch_rx_queue(hw, vsi->base_queue, FALSE); + int err = i40e_switch_tx_queue(hw, vsi->base_queue, FALSE); + if (err) + PMD_DRV_LOG(DEBUG, "Failed to do FDIR TX switch off"); + err = i40e_switch_rx_queue(hw, vsi->base_queue, FALSE); + if (err) + PMD_DRV_LOG(DEBUG, "Failed to do FDIR RX switch off"); i40e_dev_rx_queue_release(pf->fdir.rxq); pf->fdir.rxq = NULL; i40e_dev_tx_queue_release(pf->fdir.txq); @@ -378,8 +363,6 @@ i40e_init_flx_pld(struct i40e_pf *pf) } } -#define I40E_WORD(hi, lo) (uint16_t)((((hi) << 8) & 0xFF00) | ((lo) & 0xFF)) - #define I40E_VALIDATE_FLEX_PIT(flex_pit1, flex_pit2) do { \ if ((flex_pit2).src_offset < \ (flex_pit1).src_offset + (flex_pit1).size) { \ @@ -1295,28 +1278,27 @@ i40e_fdir_filter_programming(struct i40e_pf *pf, /* Update the tx tail register */ rte_wmb(); I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); - - for (i = 0; i < I40E_FDIR_WAIT_COUNT; i++) { - rte_delay_us(I40E_FDIR_WAIT_INTERVAL_US); + for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) { if ((txdp->cmd_type_offset_bsz & rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) == rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) break; + rte_delay_us(1); } - if (i >= I40E_FDIR_WAIT_COUNT) { + if (i >= I40E_FDIR_MAX_WAIT_US) { PMD_DRV_LOG(ERR, "Failed to program FDIR filter:" " time out to get DD on tx queue."); return -ETIMEDOUT; } /* totally delay 10 ms to check programming status*/ - rte_delay_us((I40E_FDIR_WAIT_COUNT - i) * I40E_FDIR_WAIT_INTERVAL_US); - if (i40e_check_fdir_programming_status(rxq) < 0) { - PMD_DRV_LOG(ERR, "Failed to program FDIR filter:" - " programming status reported."); - return -ENOSYS; + for (; i < I40E_FDIR_MAX_WAIT_US; i++) { + if (i40e_check_fdir_programming_status(rxq) >= 0) + return 0; + rte_delay_us(1); } - - return 0; + PMD_DRV_LOG(ERR, + "Failed to program FDIR filter: programming status reported."); + return -ETIMEDOUT; } /* diff --git a/drivers/net/i40e/i40e_flow.c b/drivers/net/i40e/i40e_flow.c index 24e1c658..b92719a3 100644 --- a/drivers/net/i40e/i40e_flow.c +++ b/drivers/net/i40e/i40e_flow.c @@ -52,8 +52,7 @@ #include "base/i40e_prototype.h" #include "i40e_ethdev.h" -#define I40E_IPV4_TC_SHIFT 4 -#define I40E_IPV6_TC_MASK (0x00FF << I40E_IPV4_TC_SHIFT) +#define I40E_IPV6_TC_MASK (0xFF << I40E_FDIR_IPv6_TC_OFFSET) #define I40E_IPV6_FRAG_HEADER 44 #define I40E_TENANT_ARRAY_NUM 3 #define I40E_TCI_MASK 0xFFFF @@ -114,6 +113,12 @@ static int i40e_flow_parse_vxlan_filter(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error, union i40e_filter_t *filter); +static int i40e_flow_parse_nvgre_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); static int i40e_flow_parse_mpls_filter(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item pattern[], @@ -135,7 +140,7 @@ i40e_flow_parse_qinq_filter(struct rte_eth_dev *dev, struct rte_flow_error *error, union i40e_filter_t *filter); static int -i40e_flow_parse_qinq_pattern(__rte_unused struct rte_eth_dev *dev, +i40e_flow_parse_qinq_pattern(struct rte_eth_dev *dev, const struct rte_flow_item *pattern, struct rte_flow_error *error, struct i40e_tunnel_filter_conf *filter); @@ -158,102 +163,1295 @@ static enum rte_flow_item_type pattern_ethertype[] = { /* Pattern matched flow director filter */ static enum rte_flow_item_type pattern_fdir_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv4_ext[] = { +static enum rte_flow_item_type pattern_fdir_ipv4_udp[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv4_udp[] = { +static enum rte_flow_item_type pattern_fdir_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_UDP, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv4_udp_ext[] = { +static enum rte_flow_item_type pattern_fdir_ipv6_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_1[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv4_tcp[] = { +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv4_tcp_ext[] = { +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_2[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv4_sctp[] = { +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv4_sctp_ext[] = { +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_2[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6[] = { +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6_ext[] = { +static enum rte_flow_item_type pattern_fdir_ipv6_raw_2[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6_udp[] = { +static enum rte_flow_item_type pattern_fdir_ipv6_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6_udp_ext[] = { +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_2[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6_tcp[] = { +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6_tcp_ext[] = { +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_2[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6_sctp[] = { +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_SCTP, RTE_FLOW_ITEM_TYPE_END, }; -static enum rte_flow_item_type pattern_fdir_ipv6_sctp_ext[] = { +static enum rte_flow_item_type pattern_fdir_vlan_ipv6[] = { RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, RTE_FLOW_ITEM_TYPE_END, }; @@ -296,7 +1494,40 @@ static enum rte_flow_item_type pattern_vxlan_4[] = { RTE_FLOW_ITEM_TYPE_END, }; -/* Pattern matched MPLS */ +static enum rte_flow_item_type pattern_nvgre_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_nvgre_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_nvgre_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_nvgre_4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + static enum rte_flow_item_type pattern_mpls_1[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, @@ -329,7 +1560,6 @@ static enum rte_flow_item_type pattern_mpls_4[] = { RTE_FLOW_ITEM_TYPE_END, }; -/* Pattern matched QINQ */ static enum rte_flow_item_type pattern_qinq_1[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_VLAN, @@ -340,28 +1570,163 @@ static enum rte_flow_item_type pattern_qinq_1[] = { static struct i40e_valid_pattern i40e_supported_patterns[] = { /* Ethertype */ { pattern_ethertype, i40e_flow_parse_ethertype_filter }, - /* FDIR */ + /* FDIR - support default flow type without flexible payload*/ + { pattern_ethertype, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv4, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv4_ext, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv4_udp, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv4_udp_ext, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv4_tcp, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv4_tcp_ext, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv4_sctp, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv4_sctp_ext, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv6, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv6_ext, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv6_udp, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv6_udp_ext, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv6_tcp, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv6_tcp_ext, i40e_flow_parse_fdir_filter }, { pattern_fdir_ipv6_sctp, i40e_flow_parse_fdir_filter }, - { pattern_fdir_ipv6_sctp_ext, i40e_flow_parse_fdir_filter }, + /* FDIR - support default flow type with flexible payload */ + { pattern_fdir_ethertype_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_3, i40e_flow_parse_fdir_filter }, + /* FDIR - support single vlan input set */ + { pattern_fdir_ethertype_vlan, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_3, i40e_flow_parse_fdir_filter }, + /* FDIR - support VF item */ + { pattern_fdir_ipv4_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, /* VXLAN */ { pattern_vxlan_1, i40e_flow_parse_vxlan_filter }, { pattern_vxlan_2, i40e_flow_parse_vxlan_filter }, { pattern_vxlan_3, i40e_flow_parse_vxlan_filter }, { pattern_vxlan_4, i40e_flow_parse_vxlan_filter }, + /* NVGRE */ + { pattern_nvgre_1, i40e_flow_parse_nvgre_filter }, + { pattern_nvgre_2, i40e_flow_parse_nvgre_filter }, + { pattern_nvgre_3, i40e_flow_parse_nvgre_filter }, + { pattern_nvgre_4, i40e_flow_parse_nvgre_filter }, /* MPLSoUDP & MPLSoGRE */ { pattern_mpls_1, i40e_flow_parse_mpls_filter }, { pattern_mpls_2, i40e_flow_parse_mpls_filter }, @@ -452,10 +1817,10 @@ i40e_match_pattern(enum rte_flow_item_type *item_array, /* Find if there's parse filter function matched */ static parse_filter_t -i40e_find_parse_filter_func(struct rte_flow_item *pattern) +i40e_find_parse_filter_func(struct rte_flow_item *pattern, uint32_t *idx) { parse_filter_t parse_filter = NULL; - uint8_t i = 0; + uint8_t i = *idx; for (; i < RTE_DIM(i40e_supported_patterns); i++) { if (i40e_match_pattern(i40e_supported_patterns[i].items, @@ -465,6 +1830,8 @@ i40e_find_parse_filter_func(struct rte_flow_item *pattern) } } + *idx = ++i; + return parse_filter; } @@ -704,12 +2071,244 @@ i40e_flow_parse_ethertype_filter(struct rte_eth_dev *dev, return ret; } +static int +i40e_flow_check_raw_item(const struct rte_flow_item *item, + const struct rte_flow_item_raw *raw_spec, + struct rte_flow_error *error) +{ + if (!raw_spec->relative) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Relative should be 1."); + return -rte_errno; + } + + if (raw_spec->offset % sizeof(uint16_t)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Offset should be even."); + return -rte_errno; + } + + if (raw_spec->search || raw_spec->limit) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "search or limit is not supported."); + return -rte_errno; + } + + if (raw_spec->offset < 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Offset should be non-negative."); + return -rte_errno; + } + return 0; +} + +static int +i40e_flow_store_flex_pit(struct i40e_pf *pf, + struct i40e_fdir_flex_pit *flex_pit, + enum i40e_flxpld_layer_idx layer_idx, + uint8_t raw_id) +{ + uint8_t field_idx; + + field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + raw_id; + /* Check if the configuration is conflicted */ + if (pf->fdir.flex_pit_flag[layer_idx] && + (pf->fdir.flex_set[field_idx].src_offset != flex_pit->src_offset || + pf->fdir.flex_set[field_idx].size != flex_pit->size || + pf->fdir.flex_set[field_idx].dst_offset != flex_pit->dst_offset)) + return -1; + + /* Check if the configuration exists. */ + if (pf->fdir.flex_pit_flag[layer_idx] && + (pf->fdir.flex_set[field_idx].src_offset == flex_pit->src_offset && + pf->fdir.flex_set[field_idx].size == flex_pit->size && + pf->fdir.flex_set[field_idx].dst_offset == flex_pit->dst_offset)) + return 1; + + pf->fdir.flex_set[field_idx].src_offset = + flex_pit->src_offset; + pf->fdir.flex_set[field_idx].size = + flex_pit->size; + pf->fdir.flex_set[field_idx].dst_offset = + flex_pit->dst_offset; + + return 0; +} + +static int +i40e_flow_store_flex_mask(struct i40e_pf *pf, + enum i40e_filter_pctype pctype, + uint8_t *mask) +{ + struct i40e_fdir_flex_mask flex_mask; + uint16_t mask_tmp; + uint8_t i, nb_bitmask = 0; + + memset(&flex_mask, 0, sizeof(struct i40e_fdir_flex_mask)); + for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i += sizeof(uint16_t)) { + mask_tmp = I40E_WORD(mask[i], mask[i + 1]); + if (mask_tmp) { + flex_mask.word_mask |= + I40E_FLEX_WORD_MASK(i / sizeof(uint16_t)); + if (mask_tmp != UINT16_MAX) { + flex_mask.bitmask[nb_bitmask].mask = ~mask_tmp; + flex_mask.bitmask[nb_bitmask].offset = + i / sizeof(uint16_t); + nb_bitmask++; + if (nb_bitmask > I40E_FDIR_BITMASK_NUM_WORD) + return -1; + } + } + } + flex_mask.nb_bitmask = nb_bitmask; + + if (pf->fdir.flex_mask_flag[pctype] && + (memcmp(&flex_mask, &pf->fdir.flex_mask[pctype], + sizeof(struct i40e_fdir_flex_mask)))) + return -2; + else if (pf->fdir.flex_mask_flag[pctype] && + !(memcmp(&flex_mask, &pf->fdir.flex_mask[pctype], + sizeof(struct i40e_fdir_flex_mask)))) + return 1; + + memcpy(&pf->fdir.flex_mask[pctype], &flex_mask, + sizeof(struct i40e_fdir_flex_mask)); + return 0; +} + +static void +i40e_flow_set_fdir_flex_pit(struct i40e_pf *pf, + enum i40e_flxpld_layer_idx layer_idx, + uint8_t raw_id) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint32_t flx_pit; + uint8_t field_idx; + uint16_t min_next_off = 0; /* in words */ + uint8_t i; + + /* Set flex pit */ + for (i = 0; i < raw_id; i++) { + field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; + flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset, + pf->fdir.flex_set[field_idx].size, + pf->fdir.flex_set[field_idx].dst_offset); + + I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit); + min_next_off = pf->fdir.flex_set[field_idx].src_offset + + pf->fdir.flex_set[field_idx].size; + } + + for (; i < I40E_MAX_FLXPLD_FIED; i++) { + /* set the non-used register obeying register's constrain */ + field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; + flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE, + NONUSE_FLX_PIT_DEST_OFF); + I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit); + min_next_off++; + } + + pf->fdir.flex_pit_flag[layer_idx] = 1; +} + +static void +i40e_flow_set_fdir_flex_msk(struct i40e_pf *pf, + enum i40e_filter_pctype pctype) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_fdir_flex_mask *flex_mask; + uint32_t flxinset, fd_mask; + uint8_t i; + + /* Set flex mask */ + flex_mask = &pf->fdir.flex_mask[pctype]; + flxinset = (flex_mask->word_mask << + I40E_PRTQF_FD_FLXINSET_INSET_SHIFT) & + I40E_PRTQF_FD_FLXINSET_INSET_MASK; + i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), flxinset); + + for (i = 0; i < flex_mask->nb_bitmask; i++) { + fd_mask = (flex_mask->bitmask[i].mask << + I40E_PRTQF_FD_MSK_MASK_SHIFT) & + I40E_PRTQF_FD_MSK_MASK_MASK; + fd_mask |= ((flex_mask->bitmask[i].offset + + I40E_FLX_OFFSET_IN_FIELD_VECTOR) << + I40E_PRTQF_FD_MSK_OFFSET_SHIFT) & + I40E_PRTQF_FD_MSK_OFFSET_MASK; + i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), fd_mask); + } + + pf->fdir.flex_mask_flag[pctype] = 1; +} + +static int +i40e_flow_set_fdir_inset(struct i40e_pf *pf, + enum i40e_filter_pctype pctype, + uint64_t input_set) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint64_t inset_reg = 0; + uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0}; + int i, num; + + /* Check if the input set is valid */ + if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR, + input_set) != 0) { + PMD_DRV_LOG(ERR, "Invalid input set"); + return -EINVAL; + } + + /* Check if the configuration is conflicted */ + if (pf->fdir.inset_flag[pctype] && + memcmp(&pf->fdir.input_set[pctype], &input_set, sizeof(uint64_t))) + return -1; + + if (pf->fdir.inset_flag[pctype] && + !memcmp(&pf->fdir.input_set[pctype], &input_set, sizeof(uint64_t))) + return 0; + + num = i40e_generate_inset_mask_reg(input_set, mask_reg, + I40E_INSET_MASK_NUM_REG); + if (num < 0) + return -EINVAL; + + inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set); + + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); + + for (i = 0; i < num; i++) + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), + mask_reg[i]); + + /*clear unused mask registers of the pctype */ + for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), 0); + I40E_WRITE_FLUSH(hw); + + pf->fdir.input_set[pctype] = input_set; + pf->fdir.inset_flag[pctype] = 1; + return 0; +} + /* 1. Last in item should be NULL as range is not supported. - * 2. Supported flow type and input set: refer to array - * default_inset_table in i40e_ethdev.c. - * 3. Mask of fields which need to be matched should be + * 2. Supported patterns: refer to array i40e_supported_patterns. + * 3. Supported flow type and input set: refer to array + * valid_fdir_inset_table in i40e_ethdev.c. + * 4. Mask of fields which need to be matched should be * filled with 1. - * 4. Mask of fields which needn't to be matched should be + * 5. Mask of fields which needn't to be matched should be * filled with 0. */ static int @@ -721,20 +2320,44 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); const struct rte_flow_item *item = pattern; const struct rte_flow_item_eth *eth_spec, *eth_mask; + const struct rte_flow_item_vlan *vlan_spec, *vlan_mask; const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask; const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask; const struct rte_flow_item_tcp *tcp_spec, *tcp_mask; const struct rte_flow_item_udp *udp_spec, *udp_mask; const struct rte_flow_item_sctp *sctp_spec, *sctp_mask; + const struct rte_flow_item_raw *raw_spec, *raw_mask; const struct rte_flow_item_vf *vf_spec; + uint32_t flow_type = RTE_ETH_FLOW_UNKNOWN; enum i40e_filter_pctype pctype; uint64_t input_set = I40E_INSET_NONE; - uint16_t flag_offset; + uint16_t frag_off; enum rte_flow_item_type item_type; enum rte_flow_item_type l3 = RTE_FLOW_ITEM_TYPE_END; - uint32_t j; + uint32_t i, j; + uint8_t ipv6_addr_mask[16] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; + enum i40e_flxpld_layer_idx layer_idx = I40E_FLXPLD_L2_IDX; + uint8_t raw_id = 0; + int32_t off_arr[I40E_MAX_FLXPLD_FIED]; + uint16_t len_arr[I40E_MAX_FLXPLD_FIED]; + struct i40e_fdir_flex_pit flex_pit; + uint8_t next_dst_off = 0; + uint8_t flex_mask[I40E_FDIR_MAX_FLEX_LEN]; + uint16_t flex_size; + bool cfg_flex_pit = true; + bool cfg_flex_msk = true; + uint16_t outer_tpid; + uint16_t ether_type; + uint32_t vtc_flow_cpu; + int ret; + memset(off_arr, 0, sizeof(off_arr)); + memset(len_arr, 0, sizeof(len_arr)); + memset(flex_mask, 0, I40E_FDIR_MAX_FLEX_LEN); + outer_tpid = i40e_get_outer_vlan(dev); for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { if (item->last) { rte_flow_error_set(error, EINVAL, @@ -748,13 +2371,58 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, case RTE_FLOW_ITEM_TYPE_ETH: eth_spec = (const struct rte_flow_item_eth *)item->spec; eth_mask = (const struct rte_flow_item_eth *)item->mask; - if (eth_spec || eth_mask) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "Invalid ETH spec/mask"); - return -rte_errno; + + if (eth_spec && eth_mask) { + if (!is_zero_ether_addr(ð_mask->src) || + !is_zero_ether_addr(ð_mask->dst)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid MAC_addr mask."); + return -rte_errno; + } + + if ((eth_mask->type & UINT16_MAX) == + UINT16_MAX) { + input_set |= I40E_INSET_LAST_ETHER_TYPE; + filter->input.flow.l2_flow.ether_type = + eth_spec->type; + } + + ether_type = rte_be_to_cpu_16(eth_spec->type); + if (ether_type == ETHER_TYPE_IPv4 || + ether_type == ETHER_TYPE_IPv6 || + ether_type == ETHER_TYPE_ARP || + ether_type == outer_tpid) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unsupported ether_type."); + return -rte_errno; + } + } + + flow_type = RTE_ETH_FLOW_L2_PAYLOAD; + layer_idx = I40E_FLXPLD_L2_IDX; + + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + vlan_spec = + (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = + (const struct rte_flow_item_vlan *)item->mask; + if (vlan_spec && vlan_mask) { + if (vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) { + input_set |= I40E_INSET_VLAN_INNER; + filter->input.flow_ext.vlan_tci = + vlan_spec->tci; + } } + + flow_type = RTE_ETH_FLOW_L2_PAYLOAD; + layer_idx = I40E_FLXPLD_L2_IDX; + break; case RTE_FLOW_ITEM_TYPE_IPV4: l3 = RTE_FLOW_ITEM_TYPE_IPV4; @@ -762,58 +2430,55 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, (const struct rte_flow_item_ipv4 *)item->spec; ipv4_mask = (const struct rte_flow_item_ipv4 *)item->mask; - if (!ipv4_spec || !ipv4_mask) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "NULL IPv4 spec/mask"); - return -rte_errno; - } - /* Check IPv4 mask and update input set */ - if (ipv4_mask->hdr.version_ihl || - ipv4_mask->hdr.total_length || - ipv4_mask->hdr.packet_id || - ipv4_mask->hdr.fragment_offset || - ipv4_mask->hdr.hdr_checksum) { - rte_flow_error_set(error, EINVAL, + if (ipv4_spec && ipv4_mask) { + /* Check IPv4 mask and update input set */ + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Invalid IPv4 mask."); - return -rte_errno; + return -rte_errno; + } + + if (ipv4_mask->hdr.src_addr == UINT32_MAX) + input_set |= I40E_INSET_IPV4_SRC; + if (ipv4_mask->hdr.dst_addr == UINT32_MAX) + input_set |= I40E_INSET_IPV4_DST; + if (ipv4_mask->hdr.type_of_service == UINT8_MAX) + input_set |= I40E_INSET_IPV4_TOS; + if (ipv4_mask->hdr.time_to_live == UINT8_MAX) + input_set |= I40E_INSET_IPV4_TTL; + if (ipv4_mask->hdr.next_proto_id == UINT8_MAX) + input_set |= I40E_INSET_IPV4_PROTO; + + /* Get filter info */ + flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_OTHER; + /* Check if it is fragment. */ + frag_off = ipv4_spec->hdr.fragment_offset; + frag_off = rte_be_to_cpu_16(frag_off); + if (frag_off & IPV4_HDR_OFFSET_MASK || + frag_off & IPV4_HDR_MF_FLAG) + flow_type = RTE_ETH_FLOW_FRAG_IPV4; + + /* Get the filter info */ + filter->input.flow.ip4_flow.proto = + ipv4_spec->hdr.next_proto_id; + filter->input.flow.ip4_flow.tos = + ipv4_spec->hdr.type_of_service; + filter->input.flow.ip4_flow.ttl = + ipv4_spec->hdr.time_to_live; + filter->input.flow.ip4_flow.src_ip = + ipv4_spec->hdr.src_addr; + filter->input.flow.ip4_flow.dst_ip = + ipv4_spec->hdr.dst_addr; } - if (ipv4_mask->hdr.src_addr == UINT32_MAX) - input_set |= I40E_INSET_IPV4_SRC; - if (ipv4_mask->hdr.dst_addr == UINT32_MAX) - input_set |= I40E_INSET_IPV4_DST; - if (ipv4_mask->hdr.type_of_service == UINT8_MAX) - input_set |= I40E_INSET_IPV4_TOS; - if (ipv4_mask->hdr.time_to_live == UINT8_MAX) - input_set |= I40E_INSET_IPV4_TTL; - if (ipv4_mask->hdr.next_proto_id == UINT8_MAX) - input_set |= I40E_INSET_IPV4_PROTO; - - /* Get filter info */ - flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_OTHER; - /* Check if it is fragment. */ - flag_offset = - rte_be_to_cpu_16(ipv4_spec->hdr.fragment_offset); - if (flag_offset & IPV4_HDR_OFFSET_MASK || - flag_offset & IPV4_HDR_MF_FLAG) - flow_type = RTE_ETH_FLOW_FRAG_IPV4; - - /* Get the filter info */ - filter->input.flow.ip4_flow.proto = - ipv4_spec->hdr.next_proto_id; - filter->input.flow.ip4_flow.tos = - ipv4_spec->hdr.type_of_service; - filter->input.flow.ip4_flow.ttl = - ipv4_spec->hdr.time_to_live; - filter->input.flow.ip4_flow.src_ip = - ipv4_spec->hdr.src_addr; - filter->input.flow.ip4_flow.dst_ip = - ipv4_spec->hdr.dst_addr; + layer_idx = I40E_FLXPLD_L3_IDX; break; case RTE_FLOW_ITEM_TYPE_IPV6: @@ -822,232 +2487,276 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, (const struct rte_flow_item_ipv6 *)item->spec; ipv6_mask = (const struct rte_flow_item_ipv6 *)item->mask; - if (!ipv6_spec || !ipv6_mask) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "NULL IPv6 spec/mask"); - return -rte_errno; - } - - /* Check IPv6 mask and update input set */ - if (ipv6_mask->hdr.payload_len) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "Invalid IPv6 mask"); - return -rte_errno; - } - /* SCR and DST address of IPv6 shouldn't be masked */ - for (j = 0; j < RTE_DIM(ipv6_mask->hdr.src_addr); j++) { - if (ipv6_mask->hdr.src_addr[j] != UINT8_MAX || - ipv6_mask->hdr.dst_addr[j] != UINT8_MAX) { + if (ipv6_spec && ipv6_mask) { + /* Check IPv6 mask and update input set */ + if (ipv6_mask->hdr.payload_len) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Invalid IPv6 mask"); return -rte_errno; } + + if (!memcmp(ipv6_mask->hdr.src_addr, + ipv6_addr_mask, + RTE_DIM(ipv6_mask->hdr.src_addr))) + input_set |= I40E_INSET_IPV6_SRC; + if (!memcmp(ipv6_mask->hdr.dst_addr, + ipv6_addr_mask, + RTE_DIM(ipv6_mask->hdr.dst_addr))) + input_set |= I40E_INSET_IPV6_DST; + + if ((ipv6_mask->hdr.vtc_flow & + rte_cpu_to_be_32(I40E_IPV6_TC_MASK)) + == rte_cpu_to_be_32(I40E_IPV6_TC_MASK)) + input_set |= I40E_INSET_IPV6_TC; + if (ipv6_mask->hdr.proto == UINT8_MAX) + input_set |= I40E_INSET_IPV6_NEXT_HDR; + if (ipv6_mask->hdr.hop_limits == UINT8_MAX) + input_set |= I40E_INSET_IPV6_HOP_LIMIT; + + /* Get filter info */ + vtc_flow_cpu = + rte_be_to_cpu_32(ipv6_spec->hdr.vtc_flow); + filter->input.flow.ipv6_flow.tc = + (uint8_t)(vtc_flow_cpu >> + I40E_FDIR_IPv6_TC_OFFSET); + filter->input.flow.ipv6_flow.proto = + ipv6_spec->hdr.proto; + filter->input.flow.ipv6_flow.hop_limits = + ipv6_spec->hdr.hop_limits; + + rte_memcpy(filter->input.flow.ipv6_flow.src_ip, + ipv6_spec->hdr.src_addr, 16); + rte_memcpy(filter->input.flow.ipv6_flow.dst_ip, + ipv6_spec->hdr.dst_addr, 16); + + /* Check if it is fragment. */ + if (ipv6_spec->hdr.proto == + I40E_IPV6_FRAG_HEADER) + flow_type = + RTE_ETH_FLOW_FRAG_IPV6; + else + flow_type = + RTE_ETH_FLOW_NONFRAG_IPV6_OTHER; } - input_set |= I40E_INSET_IPV6_SRC; - input_set |= I40E_INSET_IPV6_DST; - - if ((ipv6_mask->hdr.vtc_flow & - rte_cpu_to_be_16(I40E_IPV6_TC_MASK)) - == rte_cpu_to_be_16(I40E_IPV6_TC_MASK)) - input_set |= I40E_INSET_IPV6_TC; - if (ipv6_mask->hdr.proto == UINT8_MAX) - input_set |= I40E_INSET_IPV6_NEXT_HDR; - if (ipv6_mask->hdr.hop_limits == UINT8_MAX) - input_set |= I40E_INSET_IPV6_HOP_LIMIT; - - /* Get filter info */ - filter->input.flow.ipv6_flow.tc = - (uint8_t)(ipv6_spec->hdr.vtc_flow << - I40E_IPV4_TC_SHIFT); - filter->input.flow.ipv6_flow.proto = - ipv6_spec->hdr.proto; - filter->input.flow.ipv6_flow.hop_limits = - ipv6_spec->hdr.hop_limits; - - rte_memcpy(filter->input.flow.ipv6_flow.src_ip, - ipv6_spec->hdr.src_addr, 16); - rte_memcpy(filter->input.flow.ipv6_flow.dst_ip, - ipv6_spec->hdr.dst_addr, 16); - - /* Check if it is fragment. */ - if (ipv6_spec->hdr.proto == I40E_IPV6_FRAG_HEADER) - flow_type = RTE_ETH_FLOW_FRAG_IPV6; - else - flow_type = RTE_ETH_FLOW_NONFRAG_IPV6_OTHER; + layer_idx = I40E_FLXPLD_L3_IDX; + break; case RTE_FLOW_ITEM_TYPE_TCP: tcp_spec = (const struct rte_flow_item_tcp *)item->spec; tcp_mask = (const struct rte_flow_item_tcp *)item->mask; - if (!tcp_spec || !tcp_mask) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "NULL TCP spec/mask"); - return -rte_errno; - } - /* Check TCP mask and update input set */ - if (tcp_mask->hdr.sent_seq || - tcp_mask->hdr.recv_ack || - tcp_mask->hdr.data_off || - tcp_mask->hdr.tcp_flags || - tcp_mask->hdr.rx_win || - tcp_mask->hdr.cksum || - tcp_mask->hdr.tcp_urp) { - rte_flow_error_set(error, EINVAL, + if (tcp_spec && tcp_mask) { + /* Check TCP mask and update input set */ + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Invalid TCP mask"); - return -rte_errno; - } + return -rte_errno; + } - if (tcp_mask->hdr.src_port != UINT16_MAX || - tcp_mask->hdr.dst_port != UINT16_MAX) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "Invalid TCP mask"); - return -rte_errno; + if (tcp_mask->hdr.src_port == UINT16_MAX) + input_set |= I40E_INSET_SRC_PORT; + if (tcp_mask->hdr.dst_port == UINT16_MAX) + input_set |= I40E_INSET_DST_PORT; + + /* Get filter info */ + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + flow_type = + RTE_ETH_FLOW_NONFRAG_IPV4_TCP; + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + flow_type = + RTE_ETH_FLOW_NONFRAG_IPV6_TCP; + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { + filter->input.flow.tcp4_flow.src_port = + tcp_spec->hdr.src_port; + filter->input.flow.tcp4_flow.dst_port = + tcp_spec->hdr.dst_port; + } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { + filter->input.flow.tcp6_flow.src_port = + tcp_spec->hdr.src_port; + filter->input.flow.tcp6_flow.dst_port = + tcp_spec->hdr.dst_port; + } } - input_set |= I40E_INSET_SRC_PORT; - input_set |= I40E_INSET_DST_PORT; - - /* Get filter info */ - if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) - flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_TCP; - else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) - flow_type = RTE_ETH_FLOW_NONFRAG_IPV6_TCP; - - if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { - filter->input.flow.tcp4_flow.src_port = - tcp_spec->hdr.src_port; - filter->input.flow.tcp4_flow.dst_port = - tcp_spec->hdr.dst_port; - } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { - filter->input.flow.tcp6_flow.src_port = - tcp_spec->hdr.src_port; - filter->input.flow.tcp6_flow.dst_port = - tcp_spec->hdr.dst_port; - } + layer_idx = I40E_FLXPLD_L4_IDX; + break; case RTE_FLOW_ITEM_TYPE_UDP: udp_spec = (const struct rte_flow_item_udp *)item->spec; udp_mask = (const struct rte_flow_item_udp *)item->mask; - if (!udp_spec || !udp_mask) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "NULL UDP spec/mask"); - return -rte_errno; - } - /* Check UDP mask and update input set*/ - if (udp_mask->hdr.dgram_len || - udp_mask->hdr.dgram_cksum) { - rte_flow_error_set(error, EINVAL, + if (udp_spec && udp_mask) { + /* Check UDP mask and update input set*/ + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Invalid UDP mask"); - return -rte_errno; - } + return -rte_errno; + } - if (udp_mask->hdr.src_port != UINT16_MAX || - udp_mask->hdr.dst_port != UINT16_MAX) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "Invalid UDP mask"); - return -rte_errno; + if (udp_mask->hdr.src_port == UINT16_MAX) + input_set |= I40E_INSET_SRC_PORT; + if (udp_mask->hdr.dst_port == UINT16_MAX) + input_set |= I40E_INSET_DST_PORT; + + /* Get filter info */ + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + flow_type = + RTE_ETH_FLOW_NONFRAG_IPV4_UDP; + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + flow_type = + RTE_ETH_FLOW_NONFRAG_IPV6_UDP; + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { + filter->input.flow.udp4_flow.src_port = + udp_spec->hdr.src_port; + filter->input.flow.udp4_flow.dst_port = + udp_spec->hdr.dst_port; + } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { + filter->input.flow.udp6_flow.src_port = + udp_spec->hdr.src_port; + filter->input.flow.udp6_flow.dst_port = + udp_spec->hdr.dst_port; + } } - input_set |= I40E_INSET_SRC_PORT; - input_set |= I40E_INSET_DST_PORT; - - /* Get filter info */ - if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) - flow_type = - RTE_ETH_FLOW_NONFRAG_IPV4_UDP; - else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) - flow_type = - RTE_ETH_FLOW_NONFRAG_IPV6_UDP; - - if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { - filter->input.flow.udp4_flow.src_port = - udp_spec->hdr.src_port; - filter->input.flow.udp4_flow.dst_port = - udp_spec->hdr.dst_port; - } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { - filter->input.flow.udp6_flow.src_port = - udp_spec->hdr.src_port; - filter->input.flow.udp6_flow.dst_port = - udp_spec->hdr.dst_port; - } + layer_idx = I40E_FLXPLD_L4_IDX; + break; case RTE_FLOW_ITEM_TYPE_SCTP: sctp_spec = (const struct rte_flow_item_sctp *)item->spec; sctp_mask = (const struct rte_flow_item_sctp *)item->mask; - if (!sctp_spec || !sctp_mask) { - rte_flow_error_set(error, EINVAL, + + if (sctp_spec && sctp_mask) { + /* Check SCTP mask and update input set */ + if (sctp_mask->hdr.cksum) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, - "NULL SCTP spec/mask"); - return -rte_errno; + "Invalid UDP mask"); + return -rte_errno; + } + + if (sctp_mask->hdr.src_port == UINT16_MAX) + input_set |= I40E_INSET_SRC_PORT; + if (sctp_mask->hdr.dst_port == UINT16_MAX) + input_set |= I40E_INSET_DST_PORT; + if (sctp_mask->hdr.tag == UINT32_MAX) + input_set |= I40E_INSET_SCTP_VT; + + /* Get filter info */ + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + flow_type = + RTE_ETH_FLOW_NONFRAG_IPV4_SCTP; + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + flow_type = + RTE_ETH_FLOW_NONFRAG_IPV6_SCTP; + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { + filter->input.flow.sctp4_flow.src_port = + sctp_spec->hdr.src_port; + filter->input.flow.sctp4_flow.dst_port = + sctp_spec->hdr.dst_port; + filter->input.flow.sctp4_flow.verify_tag + = sctp_spec->hdr.tag; + } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { + filter->input.flow.sctp6_flow.src_port = + sctp_spec->hdr.src_port; + filter->input.flow.sctp6_flow.dst_port = + sctp_spec->hdr.dst_port; + filter->input.flow.sctp6_flow.verify_tag + = sctp_spec->hdr.tag; + } } - /* Check SCTP mask and update input set */ - if (sctp_mask->hdr.cksum) { + layer_idx = I40E_FLXPLD_L4_IDX; + + break; + case RTE_FLOW_ITEM_TYPE_RAW: + raw_spec = (const struct rte_flow_item_raw *)item->spec; + raw_mask = (const struct rte_flow_item_raw *)item->mask; + + if (!raw_spec || !raw_mask) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, - "Invalid UDP mask"); + "NULL RAW spec/mask"); return -rte_errno; } - if (sctp_mask->hdr.src_port != UINT16_MAX || - sctp_mask->hdr.dst_port != UINT16_MAX || - sctp_mask->hdr.tag != UINT32_MAX) { + ret = i40e_flow_check_raw_item(item, raw_spec, error); + if (ret < 0) + return ret; + + off_arr[raw_id] = raw_spec->offset; + len_arr[raw_id] = raw_spec->length; + + flex_size = 0; + memset(&flex_pit, 0, sizeof(struct i40e_fdir_flex_pit)); + flex_pit.size = + raw_spec->length / sizeof(uint16_t); + flex_pit.dst_offset = + next_dst_off / sizeof(uint16_t); + + for (i = 0; i <= raw_id; i++) { + if (i == raw_id) + flex_pit.src_offset += + raw_spec->offset / + sizeof(uint16_t); + else + flex_pit.src_offset += + (off_arr[i] + len_arr[i]) / + sizeof(uint16_t); + flex_size += len_arr[i]; + } + if (((flex_pit.src_offset + flex_pit.size) >= + I40E_MAX_FLX_SOURCE_OFF / sizeof(uint16_t)) || + flex_size > I40E_FDIR_MAX_FLEXLEN) { rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "Invalid UDP mask"); + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Exceeds maxmial payload limit."); return -rte_errno; } - input_set |= I40E_INSET_SRC_PORT; - input_set |= I40E_INSET_DST_PORT; - input_set |= I40E_INSET_SCTP_VT; - - /* Get filter info */ - if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) - flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_SCTP; - else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) - flow_type = RTE_ETH_FLOW_NONFRAG_IPV6_SCTP; - - if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { - filter->input.flow.sctp4_flow.src_port = - sctp_spec->hdr.src_port; - filter->input.flow.sctp4_flow.dst_port = - sctp_spec->hdr.dst_port; - filter->input.flow.sctp4_flow.verify_tag = - sctp_spec->hdr.tag; - } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { - filter->input.flow.sctp6_flow.src_port = - sctp_spec->hdr.src_port; - filter->input.flow.sctp6_flow.dst_port = - sctp_spec->hdr.dst_port; - filter->input.flow.sctp6_flow.verify_tag = - sctp_spec->hdr.tag; + + /* Store flex pit to SW */ + ret = i40e_flow_store_flex_pit(pf, &flex_pit, + layer_idx, raw_id); + if (ret < 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Conflict with the first flexible rule."); + return -rte_errno; + } else if (ret > 0) + cfg_flex_pit = false; + + for (i = 0; i < raw_spec->length; i++) { + j = i + next_dst_off; + filter->input.flow_ext.flexbytes[j] = + raw_spec->pattern[i]; + flex_mask[j] = raw_mask->pattern[i]; } + + next_dst_off += raw_spec->length; + raw_id++; break; case RTE_FLOW_ITEM_TYPE_VF: vf_spec = (const struct rte_flow_item_vf *)item->spec; @@ -1075,14 +2784,44 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, return -rte_errno; } - if (input_set != i40e_get_default_input_set(pctype)) { + ret = i40e_flow_set_fdir_inset(pf, pctype, input_set); + if (ret == -1) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Conflict with the first rule's input set."); + return -rte_errno; + } else if (ret == -EINVAL) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, - "Invalid input set."); + "Invalid pattern mask."); return -rte_errno; } + filter->input.flow_type = flow_type; + /* Store flex mask to SW */ + ret = i40e_flow_store_flex_mask(pf, pctype, flex_mask); + if (ret == -1) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Exceed maximal number of bitmasks"); + return -rte_errno; + } else if (ret == -2) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Conflict with the first flexible rule"); + return -rte_errno; + } else if (ret > 0) + cfg_flex_msk = false; + + if (cfg_flex_pit) + i40e_flow_set_fdir_flex_pit(pf, layer_idx, raw_id); + + if (cfg_flex_msk) + i40e_flow_set_fdir_flex_msk(pf, pctype); + return 0; } @@ -1101,55 +2840,64 @@ i40e_flow_parse_fdir_action(struct rte_eth_dev *dev, const struct rte_flow_action_mark *mark_spec; uint32_t index = 0; - /* Check if the first non-void action is QUEUE or DROP. */ + /* Check if the first non-void action is QUEUE or DROP or PASSTHRU. */ NEXT_ITEM_OF_ACTION(act, actions, index); - if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && - act->type != RTE_FLOW_ACTION_TYPE_DROP) { - rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, - act, "Invalid action."); - return -rte_errno; - } - - act_q = (const struct rte_flow_action_queue *)act->conf; - filter->action.flex_off = 0; - if (act->type == RTE_FLOW_ACTION_TYPE_QUEUE) + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->action.rx_queue = act_q->index; + if ((!filter->input.flow_ext.is_vf && + filter->action.rx_queue >= pf->dev_data->nb_rx_queues) || + (filter->input.flow_ext.is_vf && + filter->action.rx_queue >= pf->vf_nb_qps)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Invalid queue ID for FDIR."); + return -rte_errno; + } filter->action.behavior = RTE_ETH_FDIR_ACCEPT; - else + break; + case RTE_FLOW_ACTION_TYPE_DROP: filter->action.behavior = RTE_ETH_FDIR_REJECT; - - filter->action.report_status = RTE_ETH_FDIR_REPORT_ID; - filter->action.rx_queue = act_q->index; - - if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) { + break; + case RTE_FLOW_ACTION_TYPE_PASSTHRU: + filter->action.behavior = RTE_ETH_FDIR_PASSTHRU; + break; + default: rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, act, - "Invalid queue ID for FDIR."); + "Invalid action."); return -rte_errno; } - /* Check if the next non-void item is MARK or END. */ + /* Check if the next non-void item is MARK or FLAG or END. */ index++; NEXT_ITEM_OF_ACTION(act, actions, index); - if (act->type != RTE_FLOW_ACTION_TYPE_MARK && - act->type != RTE_FLOW_ACTION_TYPE_END) { + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_MARK: + mark_spec = (const struct rte_flow_action_mark *)act->conf; + filter->action.report_status = RTE_ETH_FDIR_REPORT_ID; + filter->soft_id = mark_spec->id; + break; + case RTE_FLOW_ACTION_TYPE_FLAG: + filter->action.report_status = RTE_ETH_FDIR_NO_REPORT_STATUS; + break; + case RTE_FLOW_ACTION_TYPE_END: + return 0; + default: rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, act, "Invalid action."); return -rte_errno; } - if (act->type == RTE_FLOW_ACTION_TYPE_MARK) { - mark_spec = (const struct rte_flow_action_mark *)act->conf; - filter->soft_id = mark_spec->id; - - /* Check if the next non-void item is END */ - index++; - NEXT_ITEM_OF_ACTION(act, actions, index); - if (act->type != RTE_FLOW_ACTION_TYPE_END) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - act, "Invalid action."); - return -rte_errno; - } + /* Check if the next non-void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid action."); + return -rte_errno; } return 0; @@ -1262,27 +3010,27 @@ i40e_flow_parse_tunnel_action(struct rte_eth_dev *dev, return 0; } +static uint16_t i40e_supported_tunnel_filter_types[] = { + ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_TENID | + ETH_TUNNEL_FILTER_IVLAN, + ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_IVLAN, + ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_TENID, + ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_TENID | + ETH_TUNNEL_FILTER_IMAC, + ETH_TUNNEL_FILTER_IMAC, +}; + static int -i40e_check_tenant_id_mask(const uint8_t *mask) +i40e_check_tunnel_filter_type(uint8_t filter_type) { - uint32_t j; - int is_masked = 0; - - for (j = 0; j < I40E_TENANT_ARRAY_NUM; j++) { - if (*(mask + j) == UINT8_MAX) { - if (j > 0 && (*(mask + j) != *(mask + j - 1))) - return -EINVAL; - is_masked = 0; - } else if (*(mask + j) == 0) { - if (j > 0 && (*(mask + j) != *(mask + j - 1))) - return -EINVAL; - is_masked = 1; - } else { - return -EINVAL; - } + uint8_t i; + + for (i = 0; i < RTE_DIM(i40e_supported_tunnel_filter_types); i++) { + if (filter_type == i40e_supported_tunnel_filter_types[i]) + return 0; } - return is_masked; + return -1; } /* 1. Last in item should be NULL as range is not supported. @@ -1302,18 +3050,17 @@ i40e_flow_parse_vxlan_pattern(__rte_unused struct rte_eth_dev *dev, const struct rte_flow_item *item = pattern; const struct rte_flow_item_eth *eth_spec; const struct rte_flow_item_eth *eth_mask; - const struct rte_flow_item_eth *o_eth_spec = NULL; - const struct rte_flow_item_eth *o_eth_mask = NULL; - const struct rte_flow_item_vxlan *vxlan_spec = NULL; - const struct rte_flow_item_vxlan *vxlan_mask = NULL; - const struct rte_flow_item_eth *i_eth_spec = NULL; - const struct rte_flow_item_eth *i_eth_mask = NULL; - const struct rte_flow_item_vlan *vlan_spec = NULL; - const struct rte_flow_item_vlan *vlan_mask = NULL; + const struct rte_flow_item_vxlan *vxlan_spec; + const struct rte_flow_item_vxlan *vxlan_mask; + const struct rte_flow_item_vlan *vlan_spec; + const struct rte_flow_item_vlan *vlan_mask; + uint8_t filter_type = 0; bool is_vni_masked = 0; + uint8_t vni_mask[] = {0xFF, 0xFF, 0xFF}; enum rte_flow_item_type item_type; bool vxlan_flag = 0; uint32_t tenant_id_be = 0; + int ret; for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { if (item->last) { @@ -1328,6 +3075,11 @@ i40e_flow_parse_vxlan_pattern(__rte_unused struct rte_eth_dev *dev, case RTE_FLOW_ITEM_TYPE_ETH: eth_spec = (const struct rte_flow_item_eth *)item->spec; eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Check if ETH item is used for place holder. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ if ((!eth_spec && eth_mask) || (eth_spec && !eth_mask)) { rte_flow_error_set(error, EINVAL, @@ -1351,50 +3103,40 @@ i40e_flow_parse_vxlan_pattern(__rte_unused struct rte_eth_dev *dev, return -rte_errno; } - if (!vxlan_flag) + if (!vxlan_flag) { rte_memcpy(&filter->outer_mac, ð_spec->dst, ETHER_ADDR_LEN); - else + filter_type |= ETH_TUNNEL_FILTER_OMAC; + } else { rte_memcpy(&filter->inner_mac, ð_spec->dst, ETHER_ADDR_LEN); + filter_type |= ETH_TUNNEL_FILTER_IMAC; + } } - - if (!vxlan_flag) { - o_eth_spec = eth_spec; - o_eth_mask = eth_mask; - } else { - i_eth_spec = eth_spec; - i_eth_mask = eth_mask; - } - break; case RTE_FLOW_ITEM_TYPE_VLAN: vlan_spec = (const struct rte_flow_item_vlan *)item->spec; vlan_mask = (const struct rte_flow_item_vlan *)item->mask; - if (vxlan_flag) { - vlan_spec = - (const struct rte_flow_item_vlan *)item->spec; - vlan_mask = - (const struct rte_flow_item_vlan *)item->mask; - if (!(vlan_spec && vlan_mask)) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, - "Invalid vlan item"); - return -rte_errno; - } - } else { - if (vlan_spec || vlan_mask) - rte_flow_error_set(error, EINVAL, + if (!(vlan_spec && vlan_mask)) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Invalid vlan item"); return -rte_errno; } + + if (vlan_spec && vlan_mask) { + if (vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) + filter->inner_vlan = + rte_be_to_cpu_16(vlan_spec->tci) & + I40E_TCI_MASK; + filter_type |= ETH_TUNNEL_FILTER_IVLAN; + } break; case RTE_FLOW_ITEM_TYPE_IPV4: filter->ip_type = I40E_TUNNEL_IPTYPE_IPV4; @@ -1441,7 +3183,7 @@ i40e_flow_parse_vxlan_pattern(__rte_unused struct rte_eth_dev *dev, (const struct rte_flow_item_vxlan *)item->mask; /* Check if VXLAN item is used to describe protocol. * If yes, both spec and mask should be NULL. - * If no, either spec or mask shouldn't be NULL. + * If no, both spec and mask shouldn't be NULL. */ if ((!vxlan_spec && vxlan_mask) || (vxlan_spec && !vxlan_mask)) { @@ -1453,17 +3195,25 @@ i40e_flow_parse_vxlan_pattern(__rte_unused struct rte_eth_dev *dev, } /* Check if VNI is masked. */ - if (vxlan_mask) { + if (vxlan_spec && vxlan_mask) { is_vni_masked = - i40e_check_tenant_id_mask(vxlan_mask->vni); - if (is_vni_masked < 0) { + !!memcmp(vxlan_mask->vni, vni_mask, + RTE_DIM(vni_mask)); + if (is_vni_masked) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Invalid VNI mask"); return -rte_errno; } + + rte_memcpy(((uint8_t *)&tenant_id_be + 1), + vxlan_spec->vni, 3); + filter->tenant_id = + rte_be_to_cpu_32(tenant_id_be); + filter_type |= ETH_TUNNEL_FILTER_TENID; } + vxlan_flag = 1; break; default: @@ -1471,95 +3221,243 @@ i40e_flow_parse_vxlan_pattern(__rte_unused struct rte_eth_dev *dev, } } - /* Check specification and mask to get the filter type */ - if (vlan_spec && vlan_mask && - (vlan_mask->tci == rte_cpu_to_be_16(I40E_TCI_MASK))) { - /* If there's inner vlan */ - filter->inner_vlan = rte_be_to_cpu_16(vlan_spec->tci) - & I40E_TCI_MASK; - if (vxlan_spec && vxlan_mask && !is_vni_masked) { - /* If there's vxlan */ - rte_memcpy(((uint8_t *)&tenant_id_be + 1), - vxlan_spec->vni, 3); - filter->tenant_id = rte_be_to_cpu_32(tenant_id_be); - if (!o_eth_spec && !o_eth_mask && - i_eth_spec && i_eth_mask) - filter->filter_type = - RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID; - else { + ret = i40e_check_tunnel_filter_type(filter_type); + if (ret < 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, + "Invalid filter type"); + return -rte_errno; + } + filter->filter_type = filter_type; + + filter->tunnel_type = I40E_TUNNEL_TYPE_VXLAN; + + return 0; +} + +static int +i40e_flow_parse_vxlan_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct i40e_tunnel_filter_conf *tunnel_filter = + &filter->consistent_tunnel_filter; + int ret; + + ret = i40e_flow_parse_vxlan_pattern(dev, pattern, + error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_tunnel_action(dev, actions, error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_TUNNEL; + + return ret; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported filter types: IMAC_IVLAN_TENID, IMAC_IVLAN, + * IMAC_TENID, OMAC_TENID_IMAC and IMAC. + * 3. Mask of fields which need to be matched should be + * filled with 1. + * 4. Mask of fields which needn't to be matched should be + * filled with 0. + */ +static int +i40e_flow_parse_nvgre_pattern(__rte_unused struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter) +{ + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_item_nvgre *nvgre_spec; + const struct rte_flow_item_nvgre *nvgre_mask; + const struct rte_flow_item_vlan *vlan_spec; + const struct rte_flow_item_vlan *vlan_mask; + enum rte_flow_item_type item_type; + uint8_t filter_type = 0; + bool is_tni_masked = 0; + uint8_t tni_mask[] = {0xFF, 0xFF, 0xFF}; + bool nvgre_flag = 0; + uint32_t tenant_id_be = 0; + int ret; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Check if ETH item is used for place holder. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!eth_spec && eth_mask) || + (eth_spec && !eth_mask)) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, - NULL, - "Invalid filter type"); + item, + "Invalid ether spec/mask"); return -rte_errno; } - } else if (!vxlan_spec && !vxlan_mask) { - /* If there's no vxlan */ - if (!o_eth_spec && !o_eth_mask && - i_eth_spec && i_eth_mask) - filter->filter_type = - RTE_TUNNEL_FILTER_IMAC_IVLAN; - else { + + if (eth_spec && eth_mask) { + /* DST address of inner MAC shouldn't be masked. + * SRC address of Inner MAC should be masked. + */ + if (!is_broadcast_ether_addr(ð_mask->dst) || + !is_zero_ether_addr(ð_mask->src) || + eth_mask->type) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ether spec/mask"); + return -rte_errno; + } + + if (!nvgre_flag) { + rte_memcpy(&filter->outer_mac, + ð_spec->dst, + ETHER_ADDR_LEN); + filter_type |= ETH_TUNNEL_FILTER_OMAC; + } else { + rte_memcpy(&filter->inner_mac, + ð_spec->dst, + ETHER_ADDR_LEN); + filter_type |= ETH_TUNNEL_FILTER_IMAC; + } + } + + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + vlan_spec = + (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = + (const struct rte_flow_item_vlan *)item->mask; + if (!(vlan_spec && vlan_mask)) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, - NULL, - "Invalid filter type"); + item, + "Invalid vlan item"); return -rte_errno; } - } else { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - NULL, - "Invalid filter type"); - return -rte_errno; - } - } else if ((!vlan_spec && !vlan_mask) || - (vlan_spec && vlan_mask && vlan_mask->tci == 0x0)) { - /* If there's no inner vlan */ - if (vxlan_spec && vxlan_mask && !is_vni_masked) { - /* If there's vxlan */ - rte_memcpy(((uint8_t *)&tenant_id_be + 1), - vxlan_spec->vni, 3); - filter->tenant_id = rte_be_to_cpu_32(tenant_id_be); - if (!o_eth_spec && !o_eth_mask && - i_eth_spec && i_eth_mask) - filter->filter_type = - RTE_TUNNEL_FILTER_IMAC_TENID; - else if (o_eth_spec && o_eth_mask && - i_eth_spec && i_eth_mask) - filter->filter_type = - RTE_TUNNEL_FILTER_OMAC_TENID_IMAC; - } else if (!vxlan_spec && !vxlan_mask) { - /* If there's no vxlan */ - if (!o_eth_spec && !o_eth_mask && - i_eth_spec && i_eth_mask) { - filter->filter_type = ETH_TUNNEL_FILTER_IMAC; - } else { + + if (vlan_spec && vlan_mask) { + if (vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) + filter->inner_vlan = + rte_be_to_cpu_16(vlan_spec->tci) & + I40E_TCI_MASK; + filter_type |= ETH_TUNNEL_FILTER_IVLAN; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV4; + /* IPv4 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, NULL, - "Invalid filter type"); + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv4 item"); return -rte_errno; } - } else { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, NULL, - "Invalid filter type"); - return -rte_errno; + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV6; + /* IPv6 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv6 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_NVGRE: + nvgre_spec = + (const struct rte_flow_item_nvgre *)item->spec; + nvgre_mask = + (const struct rte_flow_item_nvgre *)item->mask; + /* Check if NVGRE item is used to describe protocol. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!nvgre_spec && nvgre_mask) || + (nvgre_spec && !nvgre_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + + if (nvgre_spec && nvgre_mask) { + is_tni_masked = + !!memcmp(nvgre_mask->tni, tni_mask, + RTE_DIM(tni_mask)); + if (is_tni_masked) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid TNI mask"); + return -rte_errno; + } + rte_memcpy(((uint8_t *)&tenant_id_be + 1), + nvgre_spec->tni, 3); + filter->tenant_id = + rte_be_to_cpu_32(tenant_id_be); + filter_type |= ETH_TUNNEL_FILTER_TENID; + } + + nvgre_flag = 1; + break; + default: + break; } - } else { + } + + ret = i40e_check_tunnel_filter_type(filter_type); + if (ret < 0) { rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, NULL, - "Not supported by tunnel filter."); + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, + "Invalid filter type"); return -rte_errno; } + filter->filter_type = filter_type; - filter->tunnel_type = I40E_TUNNEL_TYPE_VXLAN; + filter->tunnel_type = I40E_TUNNEL_TYPE_NVGRE; return 0; } static int -i40e_flow_parse_vxlan_filter(struct rte_eth_dev *dev, +i40e_flow_parse_nvgre_filter(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item pattern[], const struct rte_flow_action actions[], @@ -1570,7 +3468,7 @@ i40e_flow_parse_vxlan_filter(struct rte_eth_dev *dev, &filter->consistent_tunnel_filter; int ret; - ret = i40e_flow_parse_vxlan_pattern(dev, pattern, + ret = i40e_flow_parse_nvgre_pattern(dev, pattern, error, tunnel_filter); if (ret) return ret; @@ -1821,8 +3719,10 @@ i40e_flow_parse_qinq_pattern(__rte_unused struct rte_eth_dev *dev, } /* Get filter specification */ - if ((o_vlan_mask->tci == rte_cpu_to_be_16(I40E_TCI_MASK)) && - (i_vlan_mask->tci == rte_cpu_to_be_16(I40E_TCI_MASK))) { + if ((o_vlan_mask != NULL) && (o_vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) && + (i_vlan_mask != NULL) && + (i_vlan_mask->tci == rte_cpu_to_be_16(I40E_TCI_MASK))) { filter->outer_vlan = rte_be_to_cpu_16(o_vlan_spec->tci) & I40E_TCI_MASK; filter->inner_vlan = rte_be_to_cpu_16(i_vlan_spec->tci) @@ -1880,7 +3780,8 @@ i40e_flow_validate(struct rte_eth_dev *dev, parse_filter_t parse_filter; uint32_t item_num = 0; /* non-void item number of pattern*/ uint32_t i = 0; - int ret; + bool flag = false; + int ret = I40E_NOT_SUPPORTED; if (!pattern) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, @@ -1922,16 +3823,21 @@ i40e_flow_validate(struct rte_eth_dev *dev, i40e_pattern_skip_void_item(items, pattern); - /* Find if there's matched parse filter function */ - parse_filter = i40e_find_parse_filter_func(items); - if (!parse_filter) { - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - pattern, "Unsupported pattern"); - return -rte_errno; - } - - ret = parse_filter(dev, attr, items, actions, error, &cons_filter); + i = 0; + do { + parse_filter = i40e_find_parse_filter_func(items, &i); + if (!parse_filter && !flag) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + pattern, "Unsupported pattern"); + rte_free(items); + return -rte_errno; + } + if (parse_filter) + ret = parse_filter(dev, attr, items, actions, + error, &cons_filter); + flag = true; + } while ((ret < 0) && (i < RTE_DIM(i40e_supported_patterns))); rte_free(items); diff --git a/drivers/net/i40e/i40e_pf.c b/drivers/net/i40e/i40e_pf.c index 0758503e..100f8dc2 100644 --- a/drivers/net/i40e/i40e_pf.c +++ b/drivers/net/i40e/i40e_pf.c @@ -61,7 +61,7 @@ static int i40e_pf_host_switch_queues(struct i40e_pf_vf *vf, - struct i40e_virtchnl_queue_select *qsel, + struct virtchnl_queue_select *qsel, bool on); /** @@ -128,7 +128,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) struct i40e_pf *pf; uint16_t vf_id, abs_vf_id, vf_msix_num; int ret; - struct i40e_virtchnl_queue_select qsel; + struct virtchnl_queue_select qsel; if (vf == NULL) return -EINVAL; @@ -139,7 +139,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) abs_vf_id = vf_id + hw->func_caps.vf_base_id; /* Notify VF that we are in VFR progress */ - I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), I40E_VFR_INPROGRESS); + I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), VIRTCHNL_VFR_INPROGRESS); /* * If require a SW VF reset, a VFLR interrupt will be generated, @@ -167,7 +167,6 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) PMD_DRV_LOG(ERR, "VF reset timeout"); return -ETIMEDOUT; } - /* This is not first time to do reset, do cleanup job first */ if (vf->vsi) { /* Disable queues */ @@ -220,7 +219,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) } /* Reset done, Set COMPLETE flag and clear reset bit */ - I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), I40E_VFR_COMPLETED); + I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), VIRTCHNL_VFR_COMPLETED); val = I40E_READ_REG(hw, I40E_VPGEN_VFRTRIG(vf_id)); val &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; I40E_WRITE_REG(hw, I40E_VPGEN_VFRTRIG(vf_id), val); @@ -248,7 +247,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) return -EFAULT; } - I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), I40E_VFR_VFACTIVE); + I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), VIRTCHNL_VFR_VFACTIVE); return ret; } @@ -277,7 +276,7 @@ i40e_pf_host_send_msg_to_vf(struct i40e_pf_vf *vf, static void i40e_pf_host_process_cmd_version(struct i40e_pf_vf *vf, bool b_op) { - struct i40e_virtchnl_version_info info; + struct virtchnl_version_info info; /* Respond like a Linux PF host in order to support both DPDK VF and * Linux VF driver. The expense is original DPDK host specific feature @@ -286,16 +285,16 @@ i40e_pf_host_process_cmd_version(struct i40e_pf_vf *vf, bool b_op) * DPDK VF also can't identify host driver by version number returned. * It always assume talking with Linux PF. */ - info.major = I40E_VIRTCHNL_VERSION_MAJOR; - info.minor = I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS; + info.major = VIRTCHNL_VERSION_MAJOR; + info.minor = VIRTCHNL_VERSION_MINOR_NO_VF_CAPS; if (b_op) - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_VERSION, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_VERSION, I40E_SUCCESS, (uint8_t *)&info, sizeof(info)); else - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_VERSION, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_VERSION, I40E_NOT_SUPPORTED, (uint8_t *)&info, sizeof(info)); @@ -313,22 +312,22 @@ i40e_pf_host_process_cmd_reset_vf(struct i40e_pf_vf *vf) static int i40e_pf_host_process_cmd_get_vf_resource(struct i40e_pf_vf *vf, bool b_op) { - struct i40e_virtchnl_vf_resource *vf_res = NULL; + struct virtchnl_vf_resource *vf_res = NULL; struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); uint32_t len = 0; int ret = I40E_SUCCESS; if (!b_op) { i40e_pf_host_send_msg_to_vf(vf, - I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + VIRTCHNL_OP_GET_VF_RESOURCES, I40E_NOT_SUPPORTED, NULL, 0); return ret; } /* only have 1 VSI by default */ - len = sizeof(struct i40e_virtchnl_vf_resource) + + len = sizeof(struct virtchnl_vf_resource) + I40E_DEFAULT_VF_VSI_NUM * - sizeof(struct i40e_virtchnl_vsi_resource); + sizeof(struct virtchnl_vsi_resource); vf_res = rte_zmalloc("i40e_vf_res", len, 0); if (vf_res == NULL) { @@ -339,21 +338,21 @@ i40e_pf_host_process_cmd_get_vf_resource(struct i40e_pf_vf *vf, bool b_op) goto send_msg; } - vf_res->vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 | - I40E_VIRTCHNL_VF_OFFLOAD_VLAN; + vf_res->vf_offload_flags = VIRTCHNL_VF_OFFLOAD_L2 | + VIRTCHNL_VF_OFFLOAD_VLAN; vf_res->max_vectors = hw->func_caps.num_msix_vectors_vf; vf_res->num_queue_pairs = vf->vsi->nb_qps; vf_res->num_vsis = I40E_DEFAULT_VF_VSI_NUM; /* Change below setting if PF host can support more VSIs for VF */ - vf_res->vsi_res[0].vsi_type = I40E_VSI_SRIOV; + vf_res->vsi_res[0].vsi_type = VIRTCHNL_VSI_SRIOV; vf_res->vsi_res[0].vsi_id = vf->vsi->vsi_id; vf_res->vsi_res[0].num_queue_pairs = vf->vsi->nb_qps; ether_addr_copy(&vf->mac_addr, (struct ether_addr *)vf_res->vsi_res[0].default_mac_addr); send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_GET_VF_RESOURCES, ret, (uint8_t *)vf_res, len); rte_free(vf_res); @@ -363,7 +362,7 @@ send_msg: static int i40e_pf_host_hmc_config_rxq(struct i40e_hw *hw, struct i40e_pf_vf *vf, - struct i40e_virtchnl_rxq_info *rxq, + struct virtchnl_rxq_info *rxq, uint8_t crcstrip) { int err = I40E_SUCCESS; @@ -431,7 +430,7 @@ i40e_vsi_get_tc_of_queue(struct i40e_vsi *vsi, static int i40e_pf_host_hmc_config_txq(struct i40e_hw *hw, struct i40e_pf_vf *vf, - struct i40e_virtchnl_txq_info *txq) + struct virtchnl_txq_info *txq) { int err = I40E_SUCCESS; struct i40e_hmc_obj_txq tx_ctx; @@ -480,14 +479,14 @@ i40e_pf_host_process_cmd_config_vsi_queues(struct i40e_pf_vf *vf, { struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); struct i40e_vsi *vsi = vf->vsi; - struct i40e_virtchnl_vsi_queue_config_info *vc_vqci = - (struct i40e_virtchnl_vsi_queue_config_info *)msg; - struct i40e_virtchnl_queue_pair_info *vc_qpi; + struct virtchnl_vsi_queue_config_info *vc_vqci = + (struct virtchnl_vsi_queue_config_info *)msg; + struct virtchnl_queue_pair_info *vc_qpi; int i, ret = I40E_SUCCESS; if (!b_op) { i40e_pf_host_send_msg_to_vf(vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -511,9 +510,9 @@ i40e_pf_host_process_cmd_config_vsi_queues(struct i40e_pf_vf *vf, /* * Apply VF RX queue setting to HMC. - * If the opcode is I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, + * If the opcode is VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, * then the extra information of - * 'struct i40e_virtchnl_queue_pair_extra_info' is needed, + * 'struct virtchnl_queue_pair_extra_info' is needed, * otherwise set the last parameter to NULL. */ if (i40e_pf_host_hmc_config_rxq(hw, vf, &vc_qpi[i].rxq, @@ -533,7 +532,7 @@ i40e_pf_host_process_cmd_config_vsi_queues(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES, ret, NULL, 0); return ret; @@ -547,15 +546,15 @@ i40e_pf_host_process_cmd_config_vsi_queues_ext(struct i40e_pf_vf *vf, { struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); struct i40e_vsi *vsi = vf->vsi; - struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei = - (struct i40e_virtchnl_vsi_queue_config_ext_info *)msg; - struct i40e_virtchnl_queue_pair_ext_info *vc_qpei; + struct virtchnl_vsi_queue_config_ext_info *vc_vqcei = + (struct virtchnl_vsi_queue_config_ext_info *)msg; + struct virtchnl_queue_pair_ext_info *vc_qpei; int i, ret = I40E_SUCCESS; if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, + VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -578,9 +577,9 @@ i40e_pf_host_process_cmd_config_vsi_queues_ext(struct i40e_pf_vf *vf, } /* * Apply VF RX queue setting to HMC. - * If the opcode is I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, + * If the opcode is VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, * then the extra information of - * 'struct i40e_virtchnl_queue_pair_ext_info' is needed, + * 'struct virtchnl_queue_pair_ext_info' is needed, * otherwise set the last parameter to NULL. */ if (i40e_pf_host_hmc_config_rxq(hw, vf, &vc_qpei[i].rxq, @@ -600,7 +599,7 @@ i40e_pf_host_process_cmd_config_vsi_queues_ext(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, ret, NULL, 0); return ret; @@ -608,7 +607,7 @@ send_msg: static void i40e_pf_config_irq_link_list(struct i40e_pf_vf *vf, - struct i40e_virtchnl_vector_map *vvm) + struct virtchnl_vector_map *vvm) { #define BITS_PER_CHAR 8 uint64_t linklistmap = 0, tempmap; @@ -711,9 +710,9 @@ i40e_pf_host_process_cmd_config_irq_map(struct i40e_pf_vf *vf, int ret = I40E_SUCCESS; struct i40e_pf *pf = vf->pf; struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); - struct i40e_virtchnl_irq_map_info *irqmap = - (struct i40e_virtchnl_irq_map_info *)msg; - struct i40e_virtchnl_vector_map *map; + struct virtchnl_irq_map_info *irqmap = + (struct virtchnl_irq_map_info *)msg; + struct virtchnl_vector_map *map; int i; uint16_t vector_id; unsigned long qbit_max; @@ -721,12 +720,12 @@ i40e_pf_host_process_cmd_config_irq_map(struct i40e_pf_vf *vf, if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_NOT_SUPPORTED, NULL, 0); return ret; } - if (msg == NULL || msglen < sizeof(struct i40e_virtchnl_irq_map_info)) { + if (msg == NULL || msglen < sizeof(struct virtchnl_irq_map_info)) { PMD_DRV_LOG(ERR, "buffer too short"); ret = I40E_ERR_PARAM; goto send_msg; @@ -773,7 +772,7 @@ i40e_pf_host_process_cmd_config_irq_map(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_IRQ_MAP, ret, NULL, 0); return ret; @@ -781,7 +780,7 @@ send_msg: static int i40e_pf_host_switch_queues(struct i40e_pf_vf *vf, - struct i40e_virtchnl_queue_select *qsel, + struct virtchnl_queue_select *qsel, bool on) { int ret = I40E_SUCCESS; @@ -831,8 +830,8 @@ i40e_pf_host_process_cmd_enable_queues(struct i40e_pf_vf *vf, uint16_t msglen) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_queue_select *q_sel = - (struct i40e_virtchnl_queue_select *)msg; + struct virtchnl_queue_select *q_sel = + (struct virtchnl_queue_select *)msg; if (msg == NULL || msglen != sizeof(*q_sel)) { ret = I40E_ERR_PARAM; @@ -841,7 +840,7 @@ i40e_pf_host_process_cmd_enable_queues(struct i40e_pf_vf *vf, ret = i40e_pf_host_switch_queues(vf, q_sel, true); send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ENABLE_QUEUES, ret, NULL, 0); return ret; @@ -854,13 +853,13 @@ i40e_pf_host_process_cmd_disable_queues(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_queue_select *q_sel = - (struct i40e_virtchnl_queue_select *)msg; + struct virtchnl_queue_select *q_sel = + (struct virtchnl_queue_select *)msg; if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_DISABLE_QUEUES, + VIRTCHNL_OP_DISABLE_QUEUES, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -872,7 +871,7 @@ i40e_pf_host_process_cmd_disable_queues(struct i40e_pf_vf *vf, ret = i40e_pf_host_switch_queues(vf, q_sel, false); send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DISABLE_QUEUES, ret, NULL, 0); return ret; @@ -886,8 +885,8 @@ i40e_pf_host_process_cmd_add_ether_address(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_ether_addr_list *addr_list = - (struct i40e_virtchnl_ether_addr_list *)msg; + struct virtchnl_ether_addr_list *addr_list = + (struct virtchnl_ether_addr_list *)msg; struct i40e_mac_filter_info filter; int i; struct ether_addr *mac; @@ -895,7 +894,7 @@ i40e_pf_host_process_cmd_add_ether_address(struct i40e_pf_vf *vf, if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + VIRTCHNL_OP_ADD_ETH_ADDR, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -920,7 +919,7 @@ i40e_pf_host_process_cmd_add_ether_address(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ADD_ETH_ADDR, ret, NULL, 0); return ret; @@ -933,15 +932,15 @@ i40e_pf_host_process_cmd_del_ether_address(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_ether_addr_list *addr_list = - (struct i40e_virtchnl_ether_addr_list *)msg; + struct virtchnl_ether_addr_list *addr_list = + (struct virtchnl_ether_addr_list *)msg; int i; struct ether_addr *mac; if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS, + VIRTCHNL_OP_DEL_ETH_ADDR, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -962,7 +961,7 @@ i40e_pf_host_process_cmd_del_ether_address(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DEL_ETH_ADDR, ret, NULL, 0); return ret; @@ -974,15 +973,15 @@ i40e_pf_host_process_cmd_add_vlan(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_vlan_filter_list *vlan_filter_list = - (struct i40e_virtchnl_vlan_filter_list *)msg; + struct virtchnl_vlan_filter_list *vlan_filter_list = + (struct virtchnl_vlan_filter_list *)msg; int i; uint16_t *vid; if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_ADD_VLAN, + VIRTCHNL_OP_ADD_VLAN, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -1002,7 +1001,7 @@ i40e_pf_host_process_cmd_add_vlan(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_ADD_VLAN, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ADD_VLAN, ret, NULL, 0); return ret; @@ -1015,15 +1014,15 @@ i40e_pf_host_process_cmd_del_vlan(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_vlan_filter_list *vlan_filter_list = - (struct i40e_virtchnl_vlan_filter_list *)msg; + struct virtchnl_vlan_filter_list *vlan_filter_list = + (struct virtchnl_vlan_filter_list *)msg; int i; uint16_t *vid; if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_DEL_VLAN, + VIRTCHNL_OP_DEL_VLAN, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -1042,7 +1041,7 @@ i40e_pf_host_process_cmd_del_vlan(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_DEL_VLAN, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DEL_VLAN, ret, NULL, 0); return ret; @@ -1056,15 +1055,15 @@ i40e_pf_host_process_cmd_config_promisc_mode( bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_promisc_info *promisc = - (struct i40e_virtchnl_promisc_info *)msg; + struct virtchnl_promisc_info *promisc = + (struct virtchnl_promisc_info *)msg; struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); bool unicast = FALSE, multicast = FALSE; if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, + VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_NOT_SUPPORTED, NULL, 0); return ret; } @@ -1074,21 +1073,21 @@ i40e_pf_host_process_cmd_config_promisc_mode( goto send_msg; } - if (promisc->flags & I40E_FLAG_VF_UNICAST_PROMISC) + if (promisc->flags & FLAG_VF_UNICAST_PROMISC) unicast = TRUE; ret = i40e_aq_set_vsi_unicast_promiscuous(hw, vf->vsi->seid, unicast, NULL, true); if (ret != I40E_SUCCESS) goto send_msg; - if (promisc->flags & I40E_FLAG_VF_MULTICAST_PROMISC) + if (promisc->flags & FLAG_VF_MULTICAST_PROMISC) multicast = TRUE; ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vf->vsi->seid, multicast, NULL); send_msg: i40e_pf_host_send_msg_to_vf(vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, ret, NULL, 0); + VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, ret, NULL, 0); return ret; } @@ -1099,12 +1098,12 @@ i40e_pf_host_process_cmd_get_stats(struct i40e_pf_vf *vf, bool b_op) i40e_update_vsi_stats(vf->vsi); if (b_op) - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_GET_STATS, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_GET_STATS, I40E_SUCCESS, (uint8_t *)&vf->vsi->eth_stats, sizeof(vf->vsi->eth_stats)); else - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_GET_STATS, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_GET_STATS, I40E_NOT_SUPPORTED, (uint8_t *)&vf->vsi->eth_stats, sizeof(vf->vsi->eth_stats)); @@ -1113,37 +1112,47 @@ i40e_pf_host_process_cmd_get_stats(struct i40e_pf_vf *vf, bool b_op) } static int -i40e_pf_host_process_cmd_cfg_vlan_offload( - struct i40e_pf_vf *vf, - uint8_t *msg, - uint16_t msglen, - bool b_op) +i40e_pf_host_process_cmd_enable_vlan_strip(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_vlan_offload_info *offload = - (struct i40e_virtchnl_vlan_offload_info *)msg; if (!b_op) { i40e_pf_host_send_msg_to_vf( vf, - I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD, + VIRTCHNL_OP_ENABLE_VLAN_STRIPPING, I40E_NOT_SUPPORTED, NULL, 0); return ret; } - if (msg == NULL || msglen != sizeof(*offload)) { - ret = I40E_ERR_PARAM; - goto send_msg; + ret = i40e_vsi_config_vlan_stripping(vf->vsi, TRUE); + if (ret != 0) + PMD_DRV_LOG(ERR, "Failed to enable vlan stripping"); + + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ENABLE_VLAN_STRIPPING, + ret, NULL, 0); + + return ret; +} + +static int +i40e_pf_host_process_cmd_disable_vlan_strip(struct i40e_pf_vf *vf, bool b_op) +{ + int ret = I40E_SUCCESS; + + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; } - ret = i40e_vsi_config_vlan_stripping(vf->vsi, - !!offload->enable_vlan_strip); + ret = i40e_vsi_config_vlan_stripping(vf->vsi, FALSE); if (ret != 0) - PMD_DRV_LOG(ERR, "Failed to configure vlan stripping"); + PMD_DRV_LOG(ERR, "Failed to disable vlan stripping"); -send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD, - ret, NULL, 0); + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, + ret, NULL, 0); return ret; } @@ -1155,8 +1164,8 @@ i40e_pf_host_process_cmd_cfg_pvid(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_pvid_info *tpid_info = - (struct i40e_virtchnl_pvid_info *)msg; + struct virtchnl_pvid_info *tpid_info = + (struct virtchnl_pvid_info *)msg; if (!b_op) { i40e_pf_host_send_msg_to_vf( @@ -1183,39 +1192,39 @@ send_msg: void i40e_notify_vf_link_status(struct rte_eth_dev *dev, struct i40e_pf_vf *vf) { - struct i40e_virtchnl_pf_event event; + struct virtchnl_pf_event event; - event.event = I40E_VIRTCHNL_EVENT_LINK_CHANGE; + event.event = VIRTCHNL_EVENT_LINK_CHANGE; event.event_data.link_event.link_status = dev->data->dev_link.link_status; - /* need to convert the ETH_SPEED_xxx into I40E_LINK_SPEED_xxx */ + /* need to convert the ETH_SPEED_xxx into VIRTCHNL_LINK_SPEED_xxx */ switch (dev->data->dev_link.link_speed) { case ETH_SPEED_NUM_100M: - event.event_data.link_event.link_speed = I40E_LINK_SPEED_100MB; + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_100MB; break; case ETH_SPEED_NUM_1G: - event.event_data.link_event.link_speed = I40E_LINK_SPEED_1GB; + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_1GB; break; case ETH_SPEED_NUM_10G: - event.event_data.link_event.link_speed = I40E_LINK_SPEED_10GB; + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_10GB; break; case ETH_SPEED_NUM_20G: - event.event_data.link_event.link_speed = I40E_LINK_SPEED_20GB; + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_20GB; break; case ETH_SPEED_NUM_25G: - event.event_data.link_event.link_speed = I40E_LINK_SPEED_25GB; + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_25GB; break; case ETH_SPEED_NUM_40G: - event.event_data.link_event.link_speed = I40E_LINK_SPEED_40GB; + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_40GB; break; default: event.event_data.link_event.link_speed = - I40E_LINK_SPEED_UNKNOWN; + VIRTCHNL_LINK_SPEED_UNKNOWN; break; } - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_EVENT, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_EVENT, I40E_SUCCESS, (uint8_t *)&event, sizeof(event)); } @@ -1231,7 +1240,7 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, struct i40e_pf_vf *vf; /* AdminQ will pass absolute VF id, transfer to internal vf id */ uint16_t vf_id = abs_vf_id - hw->func_caps.vf_base_id; - struct rte_pmd_i40e_mb_event_param cb_param; + struct rte_pmd_i40e_mb_event_param ret_param; bool b_op = TRUE; if (vf_id > pf->vf_num - 1 || !pf->vfs) { @@ -1251,100 +1260,104 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, * initialise structure to send to user application * will return response from user in retval field */ - cb_param.retval = RTE_PMD_I40E_MB_EVENT_PROCEED; - cb_param.vfid = vf_id; - cb_param.msg_type = opcode; - cb_param.msg = (void *)msg; - cb_param.msglen = msglen; + ret_param.retval = RTE_PMD_I40E_MB_EVENT_PROCEED; + ret_param.vfid = vf_id; + ret_param.msg_type = opcode; + ret_param.msg = (void *)msg; + ret_param.msglen = msglen; /** * Ask user application if we're allowed to perform those functions. - * If we get cb_param.retval == RTE_PMD_I40E_MB_EVENT_PROCEED, + * If we get ret_param.retval == RTE_PMD_I40E_MB_EVENT_PROCEED, * then business as usual. * If RTE_PMD_I40E_MB_EVENT_NOOP_ACK or RTE_PMD_I40E_MB_EVENT_NOOP_NACK, * do nothing and send not_supported to VF. As PF must send a response * to VF and ACK/NACK is not defined. */ - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, &cb_param); - if (cb_param.retval != RTE_PMD_I40E_MB_EVENT_PROCEED) { + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); + if (ret_param.retval != RTE_PMD_I40E_MB_EVENT_PROCEED) { PMD_DRV_LOG(WARNING, "VF to PF message(%d) is not permitted!", opcode); b_op = FALSE; } switch (opcode) { - case I40E_VIRTCHNL_OP_VERSION : + case VIRTCHNL_OP_VERSION: PMD_DRV_LOG(INFO, "OP_VERSION received"); i40e_pf_host_process_cmd_version(vf, b_op); break; - case I40E_VIRTCHNL_OP_RESET_VF : + case VIRTCHNL_OP_RESET_VF: PMD_DRV_LOG(INFO, "OP_RESET_VF received"); i40e_pf_host_process_cmd_reset_vf(vf); break; - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + case VIRTCHNL_OP_GET_VF_RESOURCES: PMD_DRV_LOG(INFO, "OP_GET_VF_RESOURCES received"); i40e_pf_host_process_cmd_get_vf_resource(vf, b_op); break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + case VIRTCHNL_OP_CONFIG_VSI_QUEUES: PMD_DRV_LOG(INFO, "OP_CONFIG_VSI_QUEUES received"); i40e_pf_host_process_cmd_config_vsi_queues(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT: + case VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT: PMD_DRV_LOG(INFO, "OP_CONFIG_VSI_QUEUES_EXT received"); i40e_pf_host_process_cmd_config_vsi_queues_ext(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + case VIRTCHNL_OP_CONFIG_IRQ_MAP: PMD_DRV_LOG(INFO, "OP_CONFIG_IRQ_MAP received"); i40e_pf_host_process_cmd_config_irq_map(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + case VIRTCHNL_OP_ENABLE_QUEUES: PMD_DRV_LOG(INFO, "OP_ENABLE_QUEUES received"); if (b_op) { i40e_pf_host_process_cmd_enable_queues(vf, msg, msglen); i40e_notify_vf_link_status(dev, vf); } else { i40e_pf_host_send_msg_to_vf( - vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + vf, VIRTCHNL_OP_ENABLE_QUEUES, I40E_NOT_SUPPORTED, NULL, 0); } break; - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + case VIRTCHNL_OP_DISABLE_QUEUES: PMD_DRV_LOG(INFO, "OP_DISABLE_QUEUE received"); i40e_pf_host_process_cmd_disable_queues(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + case VIRTCHNL_OP_ADD_ETH_ADDR: PMD_DRV_LOG(INFO, "OP_ADD_ETHER_ADDRESS received"); i40e_pf_host_process_cmd_add_ether_address(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + case VIRTCHNL_OP_DEL_ETH_ADDR: PMD_DRV_LOG(INFO, "OP_DEL_ETHER_ADDRESS received"); i40e_pf_host_process_cmd_del_ether_address(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_ADD_VLAN: + case VIRTCHNL_OP_ADD_VLAN: PMD_DRV_LOG(INFO, "OP_ADD_VLAN received"); i40e_pf_host_process_cmd_add_vlan(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_DEL_VLAN: + case VIRTCHNL_OP_DEL_VLAN: PMD_DRV_LOG(INFO, "OP_DEL_VLAN received"); i40e_pf_host_process_cmd_del_vlan(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: PMD_DRV_LOG(INFO, "OP_CONFIG_PROMISCUOUS_MODE received"); i40e_pf_host_process_cmd_config_promisc_mode(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_GET_STATS: + case VIRTCHNL_OP_GET_STATS: PMD_DRV_LOG(INFO, "OP_GET_STATS received"); i40e_pf_host_process_cmd_get_stats(vf, b_op); break; - case I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD: - PMD_DRV_LOG(INFO, "OP_CFG_VLAN_OFFLOAD received"); - i40e_pf_host_process_cmd_cfg_vlan_offload(vf, msg, - msglen, b_op); + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: + PMD_DRV_LOG(INFO, "OP_ENABLE_VLAN_STRIPPING received"); + i40e_pf_host_process_cmd_enable_vlan_strip(vf, b_op); + break; + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING: + PMD_DRV_LOG(INFO, "OP_DISABLE_VLAN_STRIPPING received"); + i40e_pf_host_process_cmd_disable_vlan_strip(vf, b_op); break; case I40E_VIRTCHNL_OP_CFG_VLAN_PVID: PMD_DRV_LOG(INFO, "OP_CFG_VLAN_PVID received"); diff --git a/drivers/net/i40e/i40e_pf.h b/drivers/net/i40e/i40e_pf.h index b4c22876..7afb7eae 100644 --- a/drivers/net/i40e/i40e_pf.h +++ b/drivers/net/i40e/i40e_pf.h @@ -35,7 +35,7 @@ #define _I40E_PF_H_ /* VERSION info to exchange between VF and PF host. In case VF works with - * ND kernel driver, it reads I40E_VIRTCHNL_VERSION_MAJOR/MINOR. In + * ND kernel driver, it reads VIRTCHNL_VERSION_MAJOR/MINOR. In * case works with DPDK host, it reads version below. Then VF realize who it * is talking to and use proper language to communicate. * */ @@ -49,45 +49,44 @@ #define I40E_DPDK_OFFSET 0x100 /* DPDK pf driver specific command to VF */ -enum i40e_virtchnl_ops_dpdk { +enum virtchnl_ops_dpdk { /* * Keep some gap between Linux PF commands and * DPDK PF extended commands. */ - I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD = I40E_VIRTCHNL_OP_VERSION + - I40E_DPDK_OFFSET, - I40E_VIRTCHNL_OP_CFG_VLAN_PVID, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, + I40E_VIRTCHNL_OP_CFG_VLAN_PVID = VIRTCHNL_OP_VERSION + + I40E_DPDK_OFFSET, + VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, }; /* A structure to support extended info of a receive queue. */ -struct i40e_virtchnl_rxq_ext_info { +struct virtchnl_rxq_ext_info { uint8_t crcstrip; }; /* * A structure to support extended info of queue pairs, an additional field - * is added, comparing to original 'struct i40e_virtchnl_queue_pair_info'. + * is added, comparing to original 'struct virtchnl_queue_pair_info'. */ -struct i40e_virtchnl_queue_pair_ext_info { +struct virtchnl_queue_pair_ext_info { /* vsi_id and queue_id should be identical for both rx and tx queues.*/ - struct i40e_virtchnl_txq_info txq; - struct i40e_virtchnl_rxq_info rxq; - struct i40e_virtchnl_rxq_ext_info rxq_ext; + struct virtchnl_txq_info txq; + struct virtchnl_rxq_info rxq; + struct virtchnl_rxq_ext_info rxq_ext; }; /* * A structure to support extended info of VSI queue pairs, - * 'struct i40e_virtchnl_queue_pair_ext_info' is used, see its original - * of 'struct i40e_virtchnl_queue_pair_info'. + * 'struct virtchnl_queue_pair_ext_info' is used, see its original + * of 'struct virtchnl_queue_pair_info'. */ -struct i40e_virtchnl_vsi_queue_config_ext_info { +struct virtchnl_vsi_queue_config_ext_info { uint16_t vsi_id; uint16_t num_queue_pairs; - struct i40e_virtchnl_queue_pair_ext_info qpair[0]; + struct virtchnl_queue_pair_ext_info qpair[0]; }; -struct i40e_virtchnl_vlan_offload_info { +struct virtchnl_vlan_offload_info { uint16_t vsi_id; uint8_t enable_vlan_strip; uint8_t reserved; @@ -106,7 +105,7 @@ struct i40e_virtchnl_vlan_offload_info { * enable op, needs to specify the pvid. PF returns status * code in retval. */ -struct i40e_virtchnl_pvid_info { +struct virtchnl_pvid_info { uint16_t vsi_id; struct i40e_vsi_vlan_pvid_info info; }; @@ -114,7 +113,7 @@ struct i40e_virtchnl_pvid_info { int i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset); void i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, uint16_t abs_vf_id, uint32_t opcode, - __rte_unused uint32_t retval, + uint32_t retval, uint8_t *msg, uint16_t msglen); int i40e_pf_host_init(struct rte_eth_dev *dev); int i40e_pf_host_uninit(struct rte_eth_dev *dev); diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c index 351cb94d..d42c23c0 100644 --- a/drivers/net/i40e/i40e_rxtx.c +++ b/drivers/net/i40e/i40e_rxtx.c @@ -1257,7 +1257,7 @@ end_of_tx: return nb_tx; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int i40e_tx_free_bufs(struct i40e_tx_queue *txq) { struct i40e_tx_entry *txep; @@ -1608,7 +1608,7 @@ i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) rxq = dev->data->rx_queues[rx_queue_id]; /* - * rx_queue_id is queue id aplication refers to, while + * rx_queue_id is queue id application refers to, while * rxq->reg_idx is the real queue index. */ err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE); @@ -1639,7 +1639,7 @@ i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) txq = dev->data->tx_queues[tx_queue_id]; /* - * tx_queue_id is queue id aplication refers to, while + * tx_queue_id is queue id application refers to, while * rxq->reg_idx is the real queue index. */ err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE); @@ -1664,7 +1664,7 @@ i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) txq = dev->data->tx_queues[tx_queue_id]; /* - * tx_queue_id is queue id aplication refers to, while + * tx_queue_id is queue id application refers to, while * txq->reg_idx is the real queue index. */ err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE); @@ -2474,7 +2474,7 @@ i40e_rx_queue_config(struct i40e_rx_queue *rxq) case I40E_FLAG_HEADER_SPLIT_DISABLED: default: rxq->rx_hdr_len = 0; - rxq->rx_buf_len = RTE_ALIGN(buf_size, + rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); rxq->hs_mode = i40e_header_split_none; break; diff --git a/drivers/net/i40e/i40e_rxtx_vec_common.h b/drivers/net/i40e/i40e_rxtx_vec_common.h index 69209668..39a6da06 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_common.h +++ b/drivers/net/i40e/i40e_rxtx_vec_common.h @@ -102,7 +102,7 @@ reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs, return pkt_idx; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int i40e_tx_free_bufs(struct i40e_tx_queue *txq) { struct i40e_tx_entry *txep; @@ -159,7 +159,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq) return txq->tx_rs_thresh; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void tx_backlog_entry(struct i40e_tx_entry *txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { diff --git a/drivers/net/i40e/i40e_rxtx_vec_sse.c b/drivers/net/i40e/i40e_rxtx_vec_sse.c index 3b4a352e..779f14e5 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_sse.c +++ b/drivers/net/i40e/i40e_rxtx_vec_sse.c @@ -87,6 +87,8 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) mb1 = rxep[1].mbuf; /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) != + offsetof(struct rte_mbuf, buf_addr) + 8); vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); @@ -117,7 +119,7 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) } static inline void -desc_to_olflags_v(struct i40e_rx_queue *rxq, __m128i descs[4] __rte_unused, +desc_to_olflags_v(struct i40e_rx_queue *rxq, __m128i descs[4], struct rte_mbuf **rx_pkts) { const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); @@ -203,6 +205,12 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, __m128i descs[4] __rte_unused, rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 4), 0x10); rearm2 = _mm_blend_epi16(mbuf_init, vlan0, 0x10); rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(vlan0, 4), 0x10); + + /* write the rearm data and the olflags in one write */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); @@ -252,6 +260,15 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, -rxq->crc_len, /* sub crc on pkt_len */ 0, 0 /* ignore pkt_type field */ ); + /* + * compile-time check the above crc_adjust layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi16 + * call above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); __m128i dd_check, eop_check; /* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */ @@ -296,6 +313,19 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* pkt_type set as unknown */ 0xFF, 0xFF /*pkt_type set as unknown */ ); + /* + * Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); /* Cache is empty -> need to scan the buffer rings, but first move * the next 'n' mbufs into the cache @@ -621,11 +651,5 @@ i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) int __attribute__((cold)) i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) { -#ifndef RTE_LIBRTE_IEEE1588 - /* need SSE4.1 support */ - if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - return -1; -#endif - return i40e_rx_vec_dev_conf_condition_check_default(dev); } diff --git a/drivers/net/i40e/i40e_tm.c b/drivers/net/i40e/i40e_tm.c new file mode 100644 index 00000000..d90313af --- /dev/null +++ b/drivers/net/i40e/i40e_tm.c @@ -0,0 +1,976 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_malloc.h> + +#include "base/i40e_prototype.h" +#include "i40e_ethdev.h" + +static int i40e_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error); +static int i40e_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error); +static int i40e_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error); +static int i40e_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error); +static int i40e_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error); +static int i40e_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error); +static int i40e_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error); +static int i40e_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error); +static int i40e_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error); + +const struct rte_tm_ops i40e_tm_ops = { + .capabilities_get = i40e_tm_capabilities_get, + .shaper_profile_add = i40e_shaper_profile_add, + .shaper_profile_delete = i40e_shaper_profile_del, + .node_add = i40e_node_add, + .node_delete = i40e_node_delete, + .node_type_get = i40e_node_type_get, + .level_capabilities_get = i40e_level_capabilities_get, + .node_capabilities_get = i40e_node_capabilities_get, + .hierarchy_commit = i40e_hierarchy_commit, +}; + +int +i40e_tm_ops_get(struct rte_eth_dev *dev __rte_unused, + void *arg) +{ + if (!arg) + return -EINVAL; + + *(const void **)arg = &i40e_tm_ops; + + return 0; +} + +void +i40e_tm_conf_init(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + /* initialize shaper profile list */ + TAILQ_INIT(&pf->tm_conf.shaper_profile_list); + + /* initialize node configuration */ + pf->tm_conf.root = NULL; + TAILQ_INIT(&pf->tm_conf.tc_list); + TAILQ_INIT(&pf->tm_conf.queue_list); + pf->tm_conf.nb_tc_node = 0; + pf->tm_conf.nb_queue_node = 0; + pf->tm_conf.committed = false; +} + +void +i40e_tm_conf_uninit(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_shaper_profile *shaper_profile; + struct i40e_tm_node *tm_node; + + /* clear node configuration */ + while ((tm_node = TAILQ_FIRST(&pf->tm_conf.queue_list))) { + TAILQ_REMOVE(&pf->tm_conf.queue_list, tm_node, node); + rte_free(tm_node); + } + pf->tm_conf.nb_queue_node = 0; + while ((tm_node = TAILQ_FIRST(&pf->tm_conf.tc_list))) { + TAILQ_REMOVE(&pf->tm_conf.tc_list, tm_node, node); + rte_free(tm_node); + } + pf->tm_conf.nb_tc_node = 0; + if (pf->tm_conf.root) { + rte_free(pf->tm_conf.root); + pf->tm_conf.root = NULL; + } + + /* Remove all shaper profiles */ + while ((shaper_profile = + TAILQ_FIRST(&pf->tm_conf.shaper_profile_list))) { + TAILQ_REMOVE(&pf->tm_conf.shaper_profile_list, + shaper_profile, node); + rte_free(shaper_profile); + } +} + +static inline uint16_t +i40e_tc_nb_get(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_vsi *main_vsi = pf->main_vsi; + uint16_t sum = 0; + int i; + + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (main_vsi->enabled_tc & BIT_ULL(i)) + sum++; + } + + return sum; +} + +static int +i40e_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint16_t tc_nb = i40e_tc_nb_get(dev); + + if (!cap || !error) + return -EINVAL; + + if (tc_nb > hw->func_caps.num_tx_qp) + return -EINVAL; + + error->type = RTE_TM_ERROR_TYPE_NONE; + + /* set all the parameters to 0 first. */ + memset(cap, 0, sizeof(struct rte_tm_capabilities)); + + /** + * support port + TCs + queues + * here shows the max capability not the current configuration. + */ + cap->n_nodes_max = 1 + I40E_MAX_TRAFFIC_CLASS + hw->func_caps.num_tx_qp; + cap->n_levels_max = 3; /* port, TC, queue */ + cap->non_leaf_nodes_identical = 1; + cap->leaf_nodes_identical = 1; + cap->shaper_n_max = cap->n_nodes_max; + cap->shaper_private_n_max = cap->n_nodes_max; + cap->shaper_private_dual_rate_n_max = 0; + cap->shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->shaper_private_rate_max = 5000000000ull; + cap->shaper_shared_n_max = 0; + cap->shaper_shared_n_nodes_per_shaper_max = 0; + cap->shaper_shared_n_shapers_per_node_max = 0; + cap->shaper_shared_dual_rate_n_max = 0; + cap->shaper_shared_rate_min = 0; + cap->shaper_shared_rate_max = 0; + cap->sched_n_children_max = hw->func_caps.num_tx_qp; + /** + * HW supports SP. But no plan to support it now. + * So, all the nodes should have the same priority. + */ + cap->sched_sp_n_priorities_max = 1; + cap->sched_wfq_n_children_per_group_max = 0; + cap->sched_wfq_n_groups_max = 0; + /** + * SW only supports fair round robin now. + * So, all the nodes should have the same weight. + */ + cap->sched_wfq_weight_max = 1; + cap->cman_head_drop_supported = 0; + cap->dynamic_update_mask = 0; + cap->shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD; + cap->shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + cap->cman_wred_context_n_max = 0; + cap->cman_wred_context_private_n_max = 0; + cap->cman_wred_context_shared_n_max = 0; + cap->cman_wred_context_shared_n_nodes_per_context_max = 0; + cap->cman_wred_context_shared_n_contexts_per_node_max = 0; + cap->stats_mask = 0; + + return 0; +} + +static inline struct i40e_tm_shaper_profile * +i40e_shaper_profile_search(struct rte_eth_dev *dev, + uint32_t shaper_profile_id) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_shaper_profile_list *shaper_profile_list = + &pf->tm_conf.shaper_profile_list; + struct i40e_tm_shaper_profile *shaper_profile; + + TAILQ_FOREACH(shaper_profile, shaper_profile_list, node) { + if (shaper_profile_id == shaper_profile->shaper_profile_id) + return shaper_profile; + } + + return NULL; +} + +static int +i40e_shaper_profile_param_check(struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + /* min rate not supported */ + if (profile->committed.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE; + error->message = "committed rate not supported"; + return -EINVAL; + } + /* min bucket size not supported */ + if (profile->committed.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE; + error->message = "committed bucket size not supported"; + return -EINVAL; + } + /* max bucket size not supported */ + if (profile->peak.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE; + error->message = "peak bucket size not supported"; + return -EINVAL; + } + /* length adjustment not supported */ + if (profile->pkt_length_adjust) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN; + error->message = "packet length adjustment not supported"; + return -EINVAL; + } + + return 0; +} + +static int +i40e_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_shaper_profile *shaper_profile; + int ret; + + if (!profile || !error) + return -EINVAL; + + ret = i40e_shaper_profile_param_check(profile, error); + if (ret) + return ret; + + shaper_profile = i40e_shaper_profile_search(dev, shaper_profile_id); + + if (shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID exist"; + return -EINVAL; + } + + shaper_profile = rte_zmalloc("i40e_tm_shaper_profile", + sizeof(struct i40e_tm_shaper_profile), + 0); + if (!shaper_profile) + return -ENOMEM; + shaper_profile->shaper_profile_id = shaper_profile_id; + (void)rte_memcpy(&shaper_profile->profile, profile, + sizeof(struct rte_tm_shaper_params)); + TAILQ_INSERT_TAIL(&pf->tm_conf.shaper_profile_list, + shaper_profile, node); + + return 0; +} + +static int +i40e_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_shaper_profile *shaper_profile; + + if (!error) + return -EINVAL; + + shaper_profile = i40e_shaper_profile_search(dev, shaper_profile_id); + + if (!shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID not exist"; + return -EINVAL; + } + + /* don't delete a profile if it's used by one or several nodes */ + if (shaper_profile->reference_count) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "profile in use"; + return -EINVAL; + } + + TAILQ_REMOVE(&pf->tm_conf.shaper_profile_list, shaper_profile, node); + rte_free(shaper_profile); + + return 0; +} + +static inline struct i40e_tm_node * +i40e_tm_node_search(struct rte_eth_dev *dev, + uint32_t node_id, enum i40e_tm_node_type *node_type) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_node_list *queue_list = &pf->tm_conf.queue_list; + struct i40e_tm_node_list *tc_list = &pf->tm_conf.tc_list; + struct i40e_tm_node *tm_node; + + if (pf->tm_conf.root && pf->tm_conf.root->id == node_id) { + *node_type = I40E_TM_NODE_TYPE_PORT; + return pf->tm_conf.root; + } + + TAILQ_FOREACH(tm_node, tc_list, node) { + if (tm_node->id == node_id) { + *node_type = I40E_TM_NODE_TYPE_TC; + return tm_node; + } + } + + TAILQ_FOREACH(tm_node, queue_list, node) { + if (tm_node->id == node_id) { + *node_type = I40E_TM_NODE_TYPE_QUEUE; + return tm_node; + } + } + + return NULL; +} + +static int +i40e_node_param_check(uint32_t node_id, uint32_t parent_node_id, + uint32_t priority, uint32_t weight, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + if (priority) { + error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY; + error->message = "priority should be 0"; + return -EINVAL; + } + + if (weight != 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT; + error->message = "weight must be 1"; + return -EINVAL; + } + + /* not support shared shaper */ + if (params->shared_shaper_id) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID; + error->message = "shared shaper not supported"; + return -EINVAL; + } + if (params->n_shared_shapers) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS; + error->message = "shared shaper not supported"; + return -EINVAL; + } + + /* for root node */ + if (parent_node_id == RTE_TM_NODE_ID_NULL) { + if (params->nonleaf.wfq_weight_mode) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFQ not supported"; + return -EINVAL; + } + if (params->nonleaf.n_sp_priorities != 1) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES; + error->message = "SP priority not supported"; + return -EINVAL; + } else if (params->nonleaf.wfq_weight_mode && + !(*params->nonleaf.wfq_weight_mode)) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFP should be byte mode"; + return -EINVAL; + } + + return 0; + } + + /* for TC or queue node */ + if (params->leaf.cman) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN; + error->message = "Congestion management not supported"; + return -EINVAL; + } + if (params->leaf.wred.wred_profile_id != + RTE_TM_WRED_PROFILE_ID_NONE) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.shared_wred_context_id) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.n_shared_wred_contexts) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS; + error->message = "WRED not supported"; + return -EINVAL; + } + + return 0; +} + +/** + * Now the TC and queue configuration is controlled by DCB. + * We need check if the node configuration follows the DCB configuration. + * In the future, we may use TM to cover DCB. + */ +static int +i40e_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + enum i40e_tm_node_type node_type = I40E_TM_NODE_TYPE_MAX; + enum i40e_tm_node_type parent_node_type = I40E_TM_NODE_TYPE_MAX; + struct i40e_tm_shaper_profile *shaper_profile; + struct i40e_tm_node *tm_node; + struct i40e_tm_node *parent_node; + uint16_t tc_nb = 0; + int ret; + + if (!params || !error) + return -EINVAL; + + /* if already committed */ + if (pf->tm_conf.committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + ret = i40e_node_param_check(node_id, parent_node_id, priority, weight, + params, error); + if (ret) + return ret; + + /* check if the node ID is already used */ + if (i40e_tm_node_search(dev, node_id, &node_type)) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "node id already used"; + return -EINVAL; + } + + /* check the shaper profile id */ + shaper_profile = i40e_shaper_profile_search(dev, + params->shaper_profile_id); + if (!shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID; + error->message = "shaper profile not exist"; + return -EINVAL; + } + + /* root node if not have a parent */ + if (parent_node_id == RTE_TM_NODE_ID_NULL) { + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id > I40E_TM_NODE_TYPE_PORT) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* obviously no more than one root */ + if (pf->tm_conf.root) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "already have a root"; + return -EINVAL; + } + + /* add the root node */ + tm_node = rte_zmalloc("i40e_tm_node", + sizeof(struct i40e_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->parent = NULL; + tm_node->shaper_profile = shaper_profile; + (void)rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + pf->tm_conf.root = tm_node; + + /* increase the reference counter of the shaper profile */ + shaper_profile->reference_count++; + + return 0; + } + + /* TC or queue node */ + /* check the parent node */ + parent_node = i40e_tm_node_search(dev, parent_node_id, + &parent_node_type); + if (!parent_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent not exist"; + return -EINVAL; + } + if (parent_node_type != I40E_TM_NODE_TYPE_PORT && + parent_node_type != I40E_TM_NODE_TYPE_TC) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent is not port or TC"; + return -EINVAL; + } + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id != parent_node_type + 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* check the node number */ + if (parent_node_type == I40E_TM_NODE_TYPE_PORT) { + /* check the TC number */ + tc_nb = i40e_tc_nb_get(dev); + if (pf->tm_conf.nb_tc_node >= tc_nb) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many TCs"; + return -EINVAL; + } + } else { + /* check the queue number */ + if (pf->tm_conf.nb_queue_node >= hw->func_caps.num_tx_qp) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many queues"; + return -EINVAL; + } + + /** + * check the node id. + * For queue, the node id means queue id. + */ + if (node_id >= hw->func_caps.num_tx_qp) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too large queue id"; + return -EINVAL; + } + } + + /* add the TC or queue node */ + tm_node = rte_zmalloc("i40e_tm_node", + sizeof(struct i40e_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->parent = pf->tm_conf.root; + tm_node->shaper_profile = shaper_profile; + (void)rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + if (parent_node_type == I40E_TM_NODE_TYPE_PORT) { + TAILQ_INSERT_TAIL(&pf->tm_conf.tc_list, + tm_node, node); + pf->tm_conf.nb_tc_node++; + } else { + TAILQ_INSERT_TAIL(&pf->tm_conf.queue_list, + tm_node, node); + pf->tm_conf.nb_queue_node++; + } + tm_node->parent->reference_count++; + + /* increase the reference counter of the shaper profile */ + shaper_profile->reference_count++; + + return 0; +} + +static int +i40e_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + enum i40e_tm_node_type node_type = I40E_TM_NODE_TYPE_MAX; + struct i40e_tm_node *tm_node; + + if (!error) + return -EINVAL; + + /* if already committed */ + if (pf->tm_conf.committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = i40e_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + /* the node should have no child */ + if (tm_node->reference_count) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = + "cannot delete a node which has children"; + return -EINVAL; + } + + /* root node */ + if (node_type == I40E_TM_NODE_TYPE_PORT) { + tm_node->shaper_profile->reference_count--; + rte_free(tm_node); + pf->tm_conf.root = NULL; + return 0; + } + + /* TC or queue node */ + tm_node->shaper_profile->reference_count--; + tm_node->parent->reference_count--; + if (node_type == I40E_TM_NODE_TYPE_TC) { + TAILQ_REMOVE(&pf->tm_conf.tc_list, tm_node, node); + pf->tm_conf.nb_tc_node--; + } else { + TAILQ_REMOVE(&pf->tm_conf.queue_list, tm_node, node); + pf->tm_conf.nb_queue_node--; + } + rte_free(tm_node); + + return 0; +} + +static int +i40e_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error) +{ + enum i40e_tm_node_type node_type = I40E_TM_NODE_TYPE_MAX; + struct i40e_tm_node *tm_node; + + if (!is_leaf || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = i40e_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + if (node_type == I40E_TM_NODE_TYPE_QUEUE) + *is_leaf = true; + else + *is_leaf = false; + + return 0; +} + +static int +i40e_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!cap || !error) + return -EINVAL; + + if (level_id >= I40E_TM_NODE_TYPE_MAX) { + error->type = RTE_TM_ERROR_TYPE_LEVEL_ID; + error->message = "too deep level"; + return -EINVAL; + } + + /* root node */ + if (level_id == I40E_TM_NODE_TYPE_PORT) { + cap->n_nodes_max = 1; + cap->n_nodes_nonleaf_max = 1; + cap->n_nodes_leaf_max = 0; + cap->non_leaf_nodes_identical = true; + cap->leaf_nodes_identical = true; + cap->nonleaf.shaper_private_supported = true; + cap->nonleaf.shaper_private_dual_rate_supported = false; + cap->nonleaf.shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->nonleaf.shaper_private_rate_max = 5000000000ull; + cap->nonleaf.shaper_shared_n_max = 0; + cap->nonleaf.sched_n_children_max = I40E_MAX_TRAFFIC_CLASS; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + cap->nonleaf.stats_mask = 0; + + return 0; + } + + /* TC or queue node */ + if (level_id == I40E_TM_NODE_TYPE_TC) { + /* TC */ + cap->n_nodes_max = I40E_MAX_TRAFFIC_CLASS; + cap->n_nodes_nonleaf_max = I40E_MAX_TRAFFIC_CLASS; + cap->n_nodes_leaf_max = 0; + cap->non_leaf_nodes_identical = true; + } else { + /* queue */ + cap->n_nodes_max = hw->func_caps.num_tx_qp; + cap->n_nodes_nonleaf_max = 0; + cap->n_nodes_leaf_max = hw->func_caps.num_tx_qp; + cap->non_leaf_nodes_identical = true; + } + cap->leaf_nodes_identical = true; + cap->leaf.shaper_private_supported = true; + cap->leaf.shaper_private_dual_rate_supported = false; + cap->leaf.shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->leaf.shaper_private_rate_max = 5000000000ull; + cap->leaf.shaper_shared_n_max = 0; + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + cap->leaf.stats_mask = 0; + + return 0; +} + +static int +i40e_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + enum i40e_tm_node_type node_type; + struct i40e_tm_node *tm_node; + + if (!cap || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = i40e_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + cap->shaper_private_supported = true; + cap->shaper_private_dual_rate_supported = false; + cap->shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->shaper_private_rate_max = 5000000000ull; + cap->shaper_shared_n_max = 0; + + if (node_type == I40E_TM_NODE_TYPE_QUEUE) { + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + } else { + if (node_type == I40E_TM_NODE_TYPE_PORT) + cap->nonleaf.sched_n_children_max = + I40E_MAX_TRAFFIC_CLASS; + else + cap->nonleaf.sched_n_children_max = + hw->func_caps.num_tx_qp; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + } + + cap->stats_mask = 0; + + return 0; +} + +static int +i40e_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_node_list *tc_list = &pf->tm_conf.tc_list; + struct i40e_tm_node_list *queue_list = &pf->tm_conf.queue_list; + struct i40e_tm_node *tm_node; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + struct i40e_aqc_configure_vsi_ets_sla_bw_data tc_bw; + uint64_t bw; + uint8_t tc_map; + int ret; + int i; + + if (!error) + return -EINVAL; + + /* check the setting */ + if (!pf->tm_conf.root) + goto done; + + vsi = pf->main_vsi; + hw = I40E_VSI_TO_HW(vsi); + + /** + * Don't support bandwidth control for port and TCs in parallel. + * If the port has a max bandwidth, the TCs should have none. + */ + /* port */ + bw = pf->tm_conf.root->shaper_profile->profile.peak.rate; + if (bw) { + /* check if any TC has a max bandwidth */ + TAILQ_FOREACH(tm_node, tc_list, node) { + if (tm_node->shaper_profile->profile.peak.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "no port and TC max bandwidth" + " in parallel"; + goto fail_clear; + } + } + + /* interpret Bps to 50Mbps */ + bw = bw * 8 / 1000 / 1000 / I40E_QOS_BW_GRANULARITY; + + /* set the max bandwidth */ + ret = i40e_aq_config_vsi_bw_limit(hw, vsi->seid, + (uint16_t)bw, 0, NULL); + if (ret) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "fail to set port max bandwidth"; + goto fail_clear; + } + + goto done; + } + + /* TC */ + memset(&tc_bw, 0, sizeof(tc_bw)); + tc_bw.tc_valid_bits = vsi->enabled_tc; + tc_map = vsi->enabled_tc; + TAILQ_FOREACH(tm_node, tc_list, node) { + if (!tm_node->reference_count) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "TC without queue assigned"; + goto fail_clear; + } + + i = 0; + while (i < I40E_MAX_TRAFFIC_CLASS && !(tc_map & BIT_ULL(i))) + i++; + if (i >= I40E_MAX_TRAFFIC_CLASS) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "cannot find the TC"; + goto fail_clear; + } + tc_map &= ~BIT_ULL(i); + + bw = tm_node->shaper_profile->profile.peak.rate; + if (!bw) + continue; + + /* interpret Bps to 50Mbps */ + bw = bw * 8 / 1000 / 1000 / I40E_QOS_BW_GRANULARITY; + + tc_bw.tc_bw_credits[i] = rte_cpu_to_le_16((uint16_t)bw); + } + + TAILQ_FOREACH(tm_node, queue_list, node) { + bw = tm_node->shaper_profile->profile.peak.rate; + if (bw) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "not support queue QoS"; + goto fail_clear; + } + } + + ret = i40e_aq_config_vsi_ets_sla_bw_limit(hw, vsi->seid, &tc_bw, NULL); + if (ret) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "fail to set TC max bandwidth"; + goto fail_clear; + } + +done: + pf->tm_conf.committed = true; + return 0; + +fail_clear: + /* clear all the traffic manager configuration */ + if (clear_on_fail) { + i40e_tm_conf_uninit(dev); + i40e_tm_conf_init(dev); + } + return -EINVAL; +} diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c index f7ce62bb..f12b7f4a 100644 --- a/drivers/net/i40e/rte_pmd_i40e.c +++ b/drivers/net/i40e/rte_pmd_i40e.c @@ -40,15 +40,6 @@ #include "i40e_rxtx.h" #include "rte_pmd_i40e.h" -/* The max bandwidth of i40e is 40Gbps. */ -#define I40E_QOS_BW_MAX 40000 -/* The bandwidth should be the multiple of 50Mbps. */ -#define I40E_QOS_BW_GRANULARITY 50 -/* The min bandwidth weight is 1. */ -#define I40E_QOS_BW_WEIGHT_MIN 1 -/* The max bandwidth weight is 127. */ -#define I40E_QOS_BW_WEIGHT_MAX 127 - int rte_pmd_i40e_ping_vfs(uint8_t port, uint16_t vf) { @@ -1468,7 +1459,7 @@ rte_pmd_i40e_set_tc_strict_prio(uint8_t port, uint8_t tc_map) return ret; } -#define I40E_PROFILE_INFO_SIZE 48 +#define I40E_PROFILE_INFO_SIZE sizeof(struct rte_pmd_i40e_profile_info) #define I40E_MAX_PROFILE_NUM 16 static void @@ -1520,9 +1511,6 @@ i40e_add_rm_profile_info(struct i40e_hw *hw, uint8_t *profile_info_sec) return status; } -#define I40E_PROFILE_INFO_SIZE 48 -#define I40E_MAX_PROFILE_NUM 16 - /* Check if the profile info exists */ static int i40e_check_profile_info(uint8_t port, uint8_t *profile_info_sec) @@ -1557,11 +1545,7 @@ i40e_check_profile_info(uint8_t port, uint8_t *profile_info_sec) sizeof(struct i40e_profile_section_header)); for (i = 0; i < p_list->p_count; i++) { p = &p_list->p_info[i]; - if ((pinfo->track_id == p->track_id) && - !memcmp(&pinfo->version, &p->version, - sizeof(struct i40e_ddp_version)) && - !memcmp(&pinfo->name, &p->name, - I40E_DDP_NAME_SIZE)) { + if (pinfo->track_id == p->track_id) { PMD_DRV_LOG(INFO, "Profile exists."); rte_free(buff); return 1; @@ -1587,6 +1571,13 @@ rte_pmd_i40e_process_ddp_package(uint8_t port, uint8_t *buff, int is_exist; enum i40e_status_code status = I40E_SUCCESS; + if (op != RTE_PMD_I40E_PKG_OP_WR_ADD && + op != RTE_PMD_I40E_PKG_OP_WR_ONLY && + op != RTE_PMD_I40E_PKG_OP_WR_DEL) { + PMD_DRV_LOG(ERR, "Operation not supported."); + return -ENOTSUP; + } + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; @@ -1623,6 +1614,10 @@ rte_pmd_i40e_process_ddp_package(uint8_t port, uint8_t *buff, return -EINVAL; } track_id = ((struct i40e_metadata_segment *)metadata_seg_hdr)->track_id; + if (track_id == I40E_DDP_TRACKID_INVALID) { + PMD_DRV_LOG(ERR, "Invalid track_id"); + return -EINVAL; + } /* Find profile segment */ profile_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_I40E, @@ -1642,46 +1637,231 @@ rte_pmd_i40e_process_ddp_package(uint8_t port, uint8_t *buff, return -EINVAL; } + /* Check if the profile already loaded */ + i40e_generate_profile_info_sec( + ((struct i40e_profile_segment *)profile_seg_hdr)->name, + &((struct i40e_profile_segment *)profile_seg_hdr)->version, + track_id, profile_info_sec, + op == RTE_PMD_I40E_PKG_OP_WR_ADD); + is_exist = i40e_check_profile_info(port, profile_info_sec); + if (is_exist < 0) { + PMD_DRV_LOG(ERR, "Failed to check profile."); + rte_free(profile_info_sec); + return -EINVAL; + } + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) { - /* Check if the profile exists */ - i40e_generate_profile_info_sec( - ((struct i40e_profile_segment *)profile_seg_hdr)->name, - &((struct i40e_profile_segment *)profile_seg_hdr)->version, - track_id, profile_info_sec, 1); - is_exist = i40e_check_profile_info(port, profile_info_sec); - if (is_exist > 0) { + if (is_exist) { PMD_DRV_LOG(ERR, "Profile already exists."); rte_free(profile_info_sec); - return 1; - } else if (is_exist < 0) { - PMD_DRV_LOG(ERR, "Failed to check profile."); + return -EEXIST; + } + } else if (op == RTE_PMD_I40E_PKG_OP_WR_DEL) { + if (!is_exist) { + PMD_DRV_LOG(ERR, "Profile does not exist."); rte_free(profile_info_sec); - return -EINVAL; + return -EACCES; } + } - /* Write profile to HW */ + if (op == RTE_PMD_I40E_PKG_OP_WR_DEL) { + status = i40e_rollback_profile( + hw, + (struct i40e_profile_segment *)profile_seg_hdr, + track_id); + if (status) { + PMD_DRV_LOG(ERR, "Failed to write profile for delete."); + rte_free(profile_info_sec); + return status; + } + } else { status = i40e_write_profile( - hw, - (struct i40e_profile_segment *)profile_seg_hdr, - track_id); + hw, + (struct i40e_profile_segment *)profile_seg_hdr, + track_id); if (status) { - PMD_DRV_LOG(ERR, "Failed to write profile."); + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) + PMD_DRV_LOG(ERR, "Failed to write profile for add."); + else + PMD_DRV_LOG(ERR, "Failed to write profile."); rte_free(profile_info_sec); return status; } + } - /* Add profile info to info list */ + if (track_id && (op != RTE_PMD_I40E_PKG_OP_WR_ONLY)) { + /* Modify loaded profiles info list */ status = i40e_add_rm_profile_info(hw, profile_info_sec); - if (status) - PMD_DRV_LOG(ERR, "Failed to add profile info."); - } else { - PMD_DRV_LOG(ERR, "Operation not supported."); + if (status) { + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) + PMD_DRV_LOG(ERR, "Failed to add profile to info list."); + else + PMD_DRV_LOG(ERR, "Failed to delete profile from info list."); + } } rte_free(profile_info_sec); return status; } +int rte_pmd_i40e_get_ddp_info(uint8_t *pkg_buff, uint32_t pkg_size, + uint8_t *info_buff, uint32_t info_size, + enum rte_pmd_i40e_package_info type) +{ + uint32_t ret_size; + struct i40e_package_header *pkg_hdr; + struct i40e_generic_seg_header *i40e_seg_hdr; + struct i40e_generic_seg_header *note_seg_hdr; + struct i40e_generic_seg_header *metadata_seg_hdr; + + if (!info_buff) { + PMD_DRV_LOG(ERR, "Output info buff is invalid."); + return -EINVAL; + } + + if (!pkg_buff || pkg_size < (sizeof(struct i40e_package_header) + + sizeof(struct i40e_metadata_segment) + + sizeof(uint32_t) * 2)) { + PMD_DRV_LOG(ERR, "Package buff is invalid."); + return -EINVAL; + } + + pkg_hdr = (struct i40e_package_header *)pkg_buff; + if (pkg_hdr->segment_count < 2) { + PMD_DRV_LOG(ERR, "Segment_count should be 2 at least."); + return -EINVAL; + } + + /* Find metadata segment */ + metadata_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_METADATA, + pkg_hdr); + + /* Find global notes segment */ + note_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_NOTES, + pkg_hdr); + + /* Find i40e profile segment */ + i40e_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_I40E, pkg_hdr); + + /* get global header info */ + if (type == RTE_PMD_I40E_PKG_INFO_GLOBAL_HEADER) { + struct rte_pmd_i40e_profile_info *info = + (struct rte_pmd_i40e_profile_info *)info_buff; + + if (info_size < sizeof(struct rte_pmd_i40e_profile_info)) { + PMD_DRV_LOG(ERR, "Output info buff size is invalid."); + return -EINVAL; + } + + if (!metadata_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find metadata segment header"); + return -EINVAL; + } + + memset(info, 0, sizeof(struct rte_pmd_i40e_profile_info)); + info->owner = RTE_PMD_I40E_DDP_OWNER_UNKNOWN; + info->track_id = + ((struct i40e_metadata_segment *)metadata_seg_hdr)->track_id; + + memcpy(info->name, + ((struct i40e_metadata_segment *)metadata_seg_hdr)->name, + I40E_DDP_NAME_SIZE); + memcpy(&info->version, + &((struct i40e_metadata_segment *)metadata_seg_hdr)->version, + sizeof(struct i40e_ddp_version)); + return I40E_SUCCESS; + } + + /* get global note size */ + if (type == RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES_SIZE) { + if (info_size < sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + if (note_seg_hdr == NULL) + ret_size = 0; + else + ret_size = note_seg_hdr->size; + *(uint32_t *)info_buff = ret_size; + return I40E_SUCCESS; + } + + /* get global note */ + if (type == RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES) { + if (note_seg_hdr == NULL) + return -ENOTSUP; + if (info_size < note_seg_hdr->size) { + PMD_DRV_LOG(ERR, "Information buffer size is too small"); + return -EINVAL; + } + memcpy(info_buff, ¬e_seg_hdr[1], note_seg_hdr->size); + return I40E_SUCCESS; + } + + /* get i40e segment header info */ + if (type == RTE_PMD_I40E_PKG_INFO_HEADER) { + struct rte_pmd_i40e_profile_info *info = + (struct rte_pmd_i40e_profile_info *)info_buff; + + if (info_size < sizeof(struct rte_pmd_i40e_profile_info)) { + PMD_DRV_LOG(ERR, "Output info buff size is invalid."); + return -EINVAL; + } + + if (!metadata_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find metadata segment header"); + return -EINVAL; + } + + if (!i40e_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find i40e segment header"); + return -EINVAL; + } + + memset(info, 0, sizeof(struct rte_pmd_i40e_profile_info)); + info->owner = RTE_PMD_I40E_DDP_OWNER_UNKNOWN; + info->track_id = + ((struct i40e_metadata_segment *)metadata_seg_hdr)->track_id; + + memcpy(info->name, + ((struct i40e_profile_segment *)i40e_seg_hdr)->name, + I40E_DDP_NAME_SIZE); + memcpy(&info->version, + &((struct i40e_profile_segment *)i40e_seg_hdr)->version, + sizeof(struct i40e_ddp_version)); + return I40E_SUCCESS; + } + + /* get number of devices */ + if (type == RTE_PMD_I40E_PKG_INFO_DEVID_NUM) { + if (info_size < sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + *(uint32_t *)info_buff = + ((struct i40e_profile_segment *)i40e_seg_hdr)->device_table_count; + return I40E_SUCCESS; + } + + /* get list of devices */ + if (type == RTE_PMD_I40E_PKG_INFO_DEVID_LIST) { + uint32_t dev_num; + dev_num = + ((struct i40e_profile_segment *)i40e_seg_hdr)->device_table_count; + if (info_size < sizeof(struct rte_pmd_i40e_ddp_device_id) * dev_num) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + memcpy(info_buff, + ((struct i40e_profile_segment *)i40e_seg_hdr)->device_table, + sizeof(struct rte_pmd_i40e_ddp_device_id) * dev_num); + return I40E_SUCCESS; + } + + PMD_DRV_LOG(ERR, "Info type %u is invalid.", type); + return -EINVAL; +} + int rte_pmd_i40e_get_ddp_list(uint8_t port, uint8_t *buff, uint32_t size) { diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h index 1efb2c4b..356fa89d 100644 --- a/drivers/net/i40e/rte_pmd_i40e.h +++ b/drivers/net/i40e/rte_pmd_i40e.h @@ -59,7 +59,7 @@ enum rte_pmd_i40e_mb_event_rsp { */ struct rte_pmd_i40e_mb_event_param { uint16_t vfid; /**< Virtual Function number */ - uint16_t msg_type; /**< VF to PF message type, see i40e_virtchnl_ops */ + uint16_t msg_type; /**< VF to PF message type, see virtchnl_ops */ uint16_t retval; /**< return value */ void *msg; /**< pointer to message */ uint16_t msglen; /**< length of the message */ @@ -71,9 +71,26 @@ struct rte_pmd_i40e_mb_event_param { enum rte_pmd_i40e_package_op { RTE_PMD_I40E_PKG_OP_UNDEFINED = 0, RTE_PMD_I40E_PKG_OP_WR_ADD, /**< load package and add to info list */ + RTE_PMD_I40E_PKG_OP_WR_DEL, /**< load package and delete from info list */ + RTE_PMD_I40E_PKG_OP_WR_ONLY, /**< load package without modifying info list */ RTE_PMD_I40E_PKG_OP_MAX = 32 }; +/** + * Types of package information. + */ +enum rte_pmd_i40e_package_info { + RTE_PMD_I40E_PKG_INFO_UNDEFINED = 0, + RTE_PMD_I40E_PKG_INFO_GLOBAL_HEADER, + RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES_SIZE, + RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES, + RTE_PMD_I40E_PKG_INFO_GLOBAL_MAX = 1024, + RTE_PMD_I40E_PKG_INFO_HEADER, + RTE_PMD_I40E_PKG_INFO_DEVID_NUM, + RTE_PMD_I40E_PKG_INFO_DEVID_LIST, + RTE_PMD_I40E_PKG_INFO_MAX = 0xFFFFFFFF +}; + #define RTE_PMD_I40E_DDP_NAME_SIZE 32 /** @@ -88,6 +105,14 @@ struct rte_pmd_i40e_ddp_version { }; /** + * Device ID for dynamic device personalization. + */ +struct rte_pmd_i40e_ddp_device_id { + uint32_t vendor_dev_id; + uint32_t sub_vendor_dev_id; +}; + +/** * Profile information in profile info list. */ struct rte_pmd_i40e_profile_info { @@ -98,6 +123,8 @@ struct rte_pmd_i40e_profile_info { uint8_t name[RTE_PMD_I40E_DDP_NAME_SIZE]; }; +#define RTE_PMD_I40E_DDP_OWNER_UNKNOWN 0xFF + /** * Profile information list returned from HW. */ @@ -492,13 +519,36 @@ int rte_pmd_i40e_set_tc_strict_prio(uint8_t port, uint8_t tc_map); * - (0) if successful. * - (-ENODEV) if *port* invalid. * - (-EINVAL) if bad parameter. - * - (1) if profile exists. + * - (-EEXIST) if profile exists. + * - (-EACCES) if profile does not exist. + * - (-ENOTSUP) if operation not supported. */ int rte_pmd_i40e_process_ddp_package(uint8_t port, uint8_t *buff, uint32_t size, enum rte_pmd_i40e_package_op op); /** + * rte_pmd_i40e_get_ddp_info - Get profile's info + * @param pkg + * buffer of package. + * @param pkg_size + * package buffer size + * @param info + * buffer for response + * @param size + * response buffer size + * @param type + * type of information requested + * @return + * - (0) if successful. + * - (-ENOTSUP) if information type not supported by the profile. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_get_ddp_info(uint8_t *pkg, uint32_t pkg_size, + uint8_t *info, uint32_t size, + enum rte_pmd_i40e_package_info type); + +/** * rte_pmd_i40e_get_ddp_list - Get loaded profile list * @param port * port id diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map index 3b0e805d..20cc9801 100644 --- a/drivers/net/i40e/rte_pmd_i40e_version.map +++ b/drivers/net/i40e/rte_pmd_i40e_version.map @@ -38,3 +38,10 @@ DPDK_17.05 { rte_pmd_i40e_get_ddp_list; } DPDK_17.02; + +DPDK_17.08 { + global: + + rte_pmd_i40e_get_ddp_info; + +} DPDK_17.05; diff --git a/drivers/net/ixgbe/Makefile b/drivers/net/ixgbe/Makefile index 5529d81c..5e57cb35 100644 --- a/drivers/net/ixgbe/Makefile +++ b/drivers/net/ixgbe/Makefile @@ -119,11 +119,12 @@ else SRCS-$(CONFIG_RTE_IXGBE_INC_VECTOR) += ixgbe_rxtx_vec_sse.c endif -ifeq ($(CONFIG_RTE_NIC_BYPASS),y) +ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_BYPASS),y) SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_bypass.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_82599_bypass.c endif SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += rte_pmd_ixgbe.c +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_tm.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_IXGBE_PMD)-include := rte_pmd_ixgbe.h diff --git a/drivers/net/ixgbe/base/README b/drivers/net/ixgbe/base/README index a61617be..8c833b44 100644 --- a/drivers/net/ixgbe/base/README +++ b/drivers/net/ixgbe/base/README @@ -34,7 +34,7 @@ Intel® IXGBE driver =================== This directory contains source code of FreeBSD ixgbe driver of version -cid-10g-shared-code.2017.03.29 released by the team which develop +cid-10g-shared-code.2017.05.16 released by the team which develop basic drivers for any ixgbe NIC. The sub-directory of base/ contains the original source package. This driver is valid for the product(s) listed below diff --git a/drivers/net/ixgbe/base/ixgbe_common.c b/drivers/net/ixgbe/base/ixgbe_common.c index 4dabb434..7f85713e 100644 --- a/drivers/net/ixgbe/base/ixgbe_common.c +++ b/drivers/net/ixgbe/base/ixgbe_common.c @@ -504,7 +504,8 @@ s32 ixgbe_init_hw_generic(struct ixgbe_hw *hw) } /* Initialize the LED link active for LED blink support */ - hw->mac.ops.init_led_link_act(hw); + if (hw->mac.ops.init_led_link_act) + hw->mac.ops.init_led_link_act(hw); if (status != IXGBE_SUCCESS) DEBUGOUT1("Failed to initialize HW, STATUS = %d\n", status); diff --git a/drivers/net/ixgbe/base/ixgbe_x550.c b/drivers/net/ixgbe/base/ixgbe_x550.c index 674dc144..9862391b 100644 --- a/drivers/net/ixgbe/base/ixgbe_x550.c +++ b/drivers/net/ixgbe/base/ixgbe_x550.c @@ -86,6 +86,10 @@ s32 ixgbe_init_ops_X550(struct ixgbe_hw *hw) /* Manageability interface */ mac->ops.set_fw_drv_ver = ixgbe_set_fw_drv_ver_x550; switch (hw->device_id) { + case IXGBE_DEV_ID_X550EM_X_1G_T: + hw->mac.ops.led_on = NULL; + hw->mac.ops.led_off = NULL; + break; case IXGBE_DEV_ID_X550EM_X_10G_T: case IXGBE_DEV_ID_X550EM_A_10G_T: hw->mac.ops.led_on = ixgbe_led_on_t_X550em; @@ -459,9 +463,13 @@ STATIC s32 ixgbe_identify_phy_x550em(struct ixgbe_hw *hw) hw->phy.type = ixgbe_phy_x550em_kr; break; case IXGBE_DEV_ID_X550EM_A_10G_T: - case IXGBE_DEV_ID_X550EM_X_1G_T: case IXGBE_DEV_ID_X550EM_X_10G_T: return ixgbe_identify_phy_generic(hw); + case IXGBE_DEV_ID_X550EM_X_1G_T: + hw->phy.type = ixgbe_phy_ext_1g_t; + hw->phy.ops.read_reg = NULL; + hw->phy.ops.write_reg = NULL; + break; case IXGBE_DEV_ID_X550EM_A_1G_T: case IXGBE_DEV_ID_X550EM_A_1G_T_L: hw->phy.type = ixgbe_phy_fw; @@ -751,6 +759,11 @@ s32 ixgbe_init_ops_X550EM(struct ixgbe_hw *hw) phy->ops.set_phy_power = NULL; phy->ops.get_firmware_version = NULL; break; + case IXGBE_DEV_ID_X550EM_X_1G_T: + mac->ops.setup_fc = NULL; + phy->ops.identify = ixgbe_identify_phy_x550em; + phy->ops.set_phy_power = NULL; + break; default: phy->ops.identify = ixgbe_identify_phy_x550em; } @@ -945,6 +958,11 @@ s32 ixgbe_init_ops_X550EM_x(struct ixgbe_hw *hw) ixgbe_write_i2c_combined_generic_unlocked; link->addr = IXGBE_CS4227; + if (hw->device_id == IXGBE_DEV_ID_X550EM_X_1G_T) { + mac->ops.setup_fc = NULL; + mac->ops.setup_eee = NULL; + mac->ops.init_led_link_act = NULL; + } return ret_val; } @@ -1915,6 +1933,8 @@ void ixgbe_init_mac_link_ops_X550em(struct ixgbe_hw *hw) ixgbe_setup_mac_link_sfp_x550em; break; case ixgbe_media_type_copper: + if (hw->device_id == IXGBE_DEV_ID_X550EM_X_1G_T) + break; if (hw->mac.type == ixgbe_mac_X550EM_a) { if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) { @@ -2380,10 +2400,6 @@ s32 ixgbe_init_phy_ops_X550em(struct ixgbe_hw *hw) /* set up for CS4227 usage */ hw->phy.phy_semaphore_mask = IXGBE_GSSR_SHARED_I2C_SM; break; - case IXGBE_DEV_ID_X550EM_X_1G_T: - phy->ops.read_reg_mdi = ixgbe_read_phy_reg_mdi_22; - phy->ops.write_reg_mdi = ixgbe_write_phy_reg_mdi_22; - break; default: break; } @@ -2414,6 +2430,7 @@ s32 ixgbe_init_phy_ops_X550em(struct ixgbe_hw *hw) case ixgbe_phy_ext_1g_t: /* link is managed by FW */ phy->ops.setup_link = NULL; + phy->ops.reset = NULL; break; case ixgbe_phy_x550em_xfi: /* link is managed by HW */ @@ -2565,10 +2582,9 @@ mac_reset_top: status = hw->mac.ops.acquire_swfw_sync(hw, swfw_mask); if (status != IXGBE_SUCCESS) { ERROR_REPORT2(IXGBE_ERROR_CAUTION, - "semaphore failed with %d", status); + "semaphore failed with %d", status); return IXGBE_ERR_SWFW_SYNC; } - ctrl |= IXGBE_READ_REG(hw, IXGBE_CTRL); IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); IXGBE_WRITE_FLUSH(hw); diff --git a/drivers/net/ixgbe/ixgbe_bypass.c b/drivers/net/ixgbe/ixgbe_bypass.c index 70069284..38a44936 100644 --- a/drivers/net/ixgbe/ixgbe_bypass.c +++ b/drivers/net/ixgbe/ixgbe_bypass.c @@ -36,6 +36,7 @@ #include <rte_ethdev.h> #include "ixgbe_ethdev.h" #include "ixgbe_bypass_api.h" +#include "rte_pmd_ixgbe.h" #define BYPASS_STATUS_OFF_MASK 3 @@ -284,7 +285,7 @@ ixgbe_bypass_wd_timeout_store(struct rte_eth_dev *dev, u32 timeout) FUNC_PTR_OR_ERR_RET(adapter->bps.ops.bypass_set, -ENOTSUP); /* disable the timer with timeout of zero */ - if (timeout == RTE_BYPASS_TMT_OFF) { + if (timeout == RTE_PMD_IXGBE_BYPASS_TMT_OFF) { status = 0x0; /* WDG enable off */ mask = BYPASS_WDT_ENABLE_M; } else { @@ -355,7 +356,7 @@ ixgbe_bypass_wd_timeout_show(struct rte_eth_dev *dev, u32 *wd_timeout) wdg = by_ctl & BYPASS_WDT_ENABLE_M; if (!wdg) - *wd_timeout = RTE_BYPASS_TMT_OFF; + *wd_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; else *wd_timeout = (by_ctl >> BYPASS_WDT_TIME_SHIFT) & BYPASS_WDT_MASK; diff --git a/drivers/net/ixgbe/ixgbe_bypass.h b/drivers/net/ixgbe/ixgbe_bypass.h index 5f5c63e3..09155bb3 100644 --- a/drivers/net/ixgbe/ixgbe_bypass.h +++ b/drivers/net/ixgbe/ixgbe_bypass.h @@ -34,7 +34,7 @@ #ifndef _IXGBE_BYPASS_H_ #define _IXGBE_BYPASS_H_ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS struct ixgbe_bypass_mac_ops { s32 (*bypass_rw)(struct ixgbe_hw *hw, u32 cmd, u32 *status); @@ -63,6 +63,6 @@ s32 ixgbe_bypass_wd_reset(struct rte_eth_dev *dev); s32 ixgbe_bypass_init_shared_code(struct ixgbe_hw *hw); s32 ixgbe_bypass_init_hw(struct ixgbe_hw *hw); -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ #endif /* _IXGBE_BYPASS_H_ */ diff --git a/drivers/net/ixgbe/ixgbe_bypass_api.h b/drivers/net/ixgbe/ixgbe_bypass_api.h index aec8f1ec..d52fde04 100644 --- a/drivers/net/ixgbe/ixgbe_bypass_api.h +++ b/drivers/net/ixgbe/ixgbe_bypass_api.h @@ -34,7 +34,7 @@ #ifndef _IXGBE_BYPASS_API_H_ #define _IXGBE_BYPASS_API_H_ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS #include "ixgbe_bypass_defines.h" /** @@ -295,6 +295,6 @@ static s32 ixgbe_bypass_rd_eep_generic(struct ixgbe_hw *hw, u32 addr, u8 *value) return 0; } -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ #endif /* _IXGBE_BYPASS_API_H_ */ diff --git a/drivers/net/ixgbe/ixgbe_bypass_defines.h b/drivers/net/ixgbe/ixgbe_bypass_defines.h index cafcb278..d12c2714 100644 --- a/drivers/net/ixgbe/ixgbe_bypass_defines.h +++ b/drivers/net/ixgbe/ixgbe_bypass_defines.h @@ -34,7 +34,7 @@ #ifndef _IXGBE_BYPASS_DEFINES_H_ #define _IXGBE_BYPASS_DEFINES_H_ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS #define msleep(x) rte_delay_us(x*1000) #define usleep_range(min, max) rte_delay_us(min) @@ -155,6 +155,6 @@ enum ixgbe_state_t { #define IXGBE_BYPASS_FW_WRITE_FAILURE -35 -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ #endif /* _IXGBE_BYPASS_DEFINES_H_ */ diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c index aeaa432c..22171d86 100644 --- a/drivers/net/ixgbe/ixgbe_ethdev.c +++ b/drivers/net/ixgbe/ixgbe_ethdev.c @@ -57,7 +57,6 @@ #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_ethdev_pci.h> -#include <rte_atomic.h> #include <rte_malloc.h> #include <rte_random.h> #include <rte_dev.h> @@ -187,13 +186,13 @@ ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, uint64_t *values, unsigned int n); static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev); static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev); -static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, +static int ixgbe_dev_xstats_get_names(struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, - __rte_unused unsigned int size); -static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, - struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit); + unsigned int size); +static int ixgbevf_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, unsigned limit); static int ixgbe_dev_xstats_get_names_by_id( - __rte_unused struct rte_eth_dev *dev, + struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, unsigned int limit); @@ -240,7 +239,7 @@ static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size); static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev); -static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev); +static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev); static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev); @@ -262,6 +261,8 @@ static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev); static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev); static int ixgbevf_dev_configure(struct rte_eth_dev *dev); static int ixgbevf_dev_start(struct rte_eth_dev *dev); +static int ixgbevf_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete); static void ixgbevf_dev_stop(struct rte_eth_dev *dev); static void ixgbevf_dev_close(struct rte_eth_dev *dev); static void ixgbevf_intr_disable(struct ixgbe_hw *hw); @@ -302,9 +303,6 @@ static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, uint8_t queue, uint8_t msix_vector); static void ixgbe_configure_msix(struct rte_eth_dev *dev); -static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, - uint16_t queue_idx, uint16_t tx_rate); - static int ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, uint32_t pool); @@ -444,13 +442,8 @@ static const struct rte_pci_id pci_id_ixgbe_map[] = { { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE) }, - { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP) }, - { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_SFP) }, - { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_RNDC) }, - { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_560FLR) }, - { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_ECNA_DP) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM) }, @@ -481,7 +474,7 @@ static const struct rte_pci_id pci_id_ixgbe_map[] = { { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4) }, { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR) }, -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS) }, #endif { .vendor_id = 0, /* sentinel */ }, @@ -575,17 +568,6 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .set_queue_rate_limit = ixgbe_set_queue_rate_limit, .reta_update = ixgbe_dev_rss_reta_update, .reta_query = ixgbe_dev_rss_reta_query, -#ifdef RTE_NIC_BYPASS - .bypass_init = ixgbe_bypass_init, - .bypass_state_set = ixgbe_bypass_state_store, - .bypass_state_show = ixgbe_bypass_state_show, - .bypass_event_set = ixgbe_bypass_event_store, - .bypass_event_show = ixgbe_bypass_event_show, - .bypass_wd_timeout_set = ixgbe_bypass_wd_timeout_store, - .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show, - .bypass_ver_show = ixgbe_bypass_ver_show, - .bypass_wd_reset = ixgbe_bypass_wd_reset, -#endif /* RTE_NIC_BYPASS */ .rss_hash_update = ixgbe_dev_rss_hash_update, .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, .filter_ctrl = ixgbe_dev_filter_ctrl, @@ -608,6 +590,7 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .l2_tunnel_offload_set = ixgbe_dev_l2_tunnel_offload_set, .udp_tunnel_port_add = ixgbe_dev_udp_tunnel_port_add, .udp_tunnel_port_del = ixgbe_dev_udp_tunnel_port_del, + .tm_ops_get = ixgbe_tm_ops_get, }; /* @@ -618,7 +601,7 @@ static const struct eth_dev_ops ixgbevf_eth_dev_ops = { .dev_configure = ixgbevf_dev_configure, .dev_start = ixgbevf_dev_start, .dev_stop = ixgbevf_dev_stop, - .link_update = ixgbe_dev_link_update, + .link_update = ixgbevf_dev_link_update, .stats_get = ixgbevf_dev_stats_get, .xstats_get = ixgbevf_dev_xstats_get, .stats_reset = ixgbevf_dev_stats_reset, @@ -1131,7 +1114,7 @@ ixgbe_swfw_lock_reset(struct ixgbe_hw *hw) static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); @@ -1190,11 +1173,11 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) hw->allow_unsupported_sfp = 1; /* Initialize the shared code (base driver) */ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS diag = ixgbe_bypass_init_shared_code(hw); #else diag = ixgbe_init_shared_code(hw); -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ if (diag != IXGBE_SUCCESS) { PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); @@ -1227,11 +1210,11 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) return -EIO; } -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS diag = ixgbe_bypass_init_hw(hw); #else diag = ixgbe_init_hw(hw); -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ /* * Devices with copper phys will fail to initialise if ixgbe_init_hw() @@ -1359,13 +1342,16 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) /* initialize bandwidth configuration info */ memset(bw_conf, 0, sizeof(struct ixgbe_bw_conf)); + /* initialize Traffic Manager configuration */ + ixgbe_tm_conf_init(eth_dev); + return 0; } static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw; @@ -1412,6 +1398,9 @@ eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) /* clear all the filters list */ ixgbe_filterlist_flush(); + /* Remove all Traffic Manager configuration */ + ixgbe_tm_conf_uninit(eth_dev); + return 0; } @@ -1491,7 +1480,7 @@ static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev) TAILQ_INIT(&fdir_info->fdir_list); snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, - "fdir_%s", eth_dev->data->name); + "fdir_%s", eth_dev->device->name); fdir_info->hash_handle = rte_hash_create(&fdir_hash_params); if (!fdir_info->hash_handle) { PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); @@ -1527,7 +1516,7 @@ static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev) TAILQ_INIT(&l2_tn_info->l2_tn_list); snprintf(l2_tn_hash_name, RTE_HASH_NAMESIZE, - "l2_tn_%s", eth_dev->data->name); + "l2_tn_%s", eth_dev->device->name); l2_tn_info->hash_handle = rte_hash_create(&l2_tn_hash_params); if (!l2_tn_info->hash_handle) { PMD_INIT_LOG(ERR, "Failed to create L2 TN hash table!"); @@ -1598,7 +1587,7 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) { int diag; uint32_t tc, tcs; - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); @@ -1747,7 +1736,7 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw; @@ -2176,7 +2165,7 @@ ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) static int ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); switch (nb_rx_q) { case 1: @@ -2425,7 +2414,7 @@ ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, uint16_t total_rate = 0; struct rte_pci_device *pci_dev; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); rte_eth_link_get_nowait(dev->data->port_id, &link); if (vf >= pci_dev->max_vfs) @@ -2496,7 +2485,7 @@ ixgbe_dev_start(struct rte_eth_dev *dev) IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t intr_vector = 0; int err, link_up = 0, negotiate = 0; @@ -2505,6 +2494,8 @@ ixgbe_dev_start(struct rte_eth_dev *dev) int status; uint16_t vf, idx; uint32_t *link_speeds; + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); @@ -2651,9 +2642,22 @@ ixgbe_dev_start(struct rte_eth_dev *dev) speed = 0x0; if (*link_speeds == ETH_LINK_SPEED_AUTONEG) { - speed = (hw->mac.type != ixgbe_mac_82598EB) ? - IXGBE_LINK_SPEED_82599_AUTONEG : - IXGBE_LINK_SPEED_82598_AUTONEG; + switch (hw->mac.type) { + case ixgbe_mac_82598EB: + speed = IXGBE_LINK_SPEED_82598_AUTONEG; + break; + case ixgbe_mac_82599EB: + case ixgbe_mac_X540: + speed = IXGBE_LINK_SPEED_82599_AUTONEG; + break; + case ixgbe_mac_X550: + case ixgbe_mac_X550EM_x: + case ixgbe_mac_X550EM_a: + speed = IXGBE_LINK_SPEED_X550_AUTONEG; + break; + default: + speed = IXGBE_LINK_SPEED_82599_AUTONEG; + } } else { if (*link_speeds & ETH_LINK_SPEED_10G) speed |= IXGBE_LINK_SPEED_10GB_FULL; @@ -2672,7 +2676,9 @@ skip_link_setup: if (rte_intr_allow_others(intr_handle)) { /* check if lsc interrupt is enabled */ if (dev->data->dev_conf.intr_conf.lsc != 0) - ixgbe_dev_lsc_interrupt_setup(dev); + ixgbe_dev_lsc_interrupt_setup(dev, TRUE); + else + ixgbe_dev_lsc_interrupt_setup(dev, FALSE); ixgbe_dev_macsec_interrupt_setup(dev); } else { rte_intr_callback_unregister(intr_handle, @@ -2695,6 +2701,11 @@ skip_link_setup: ixgbe_l2_tunnel_conf(dev); ixgbe_filter_restore(dev); + if (tm_conf->root && !tm_conf->committed) + PMD_DRV_LOG(WARNING, + "please call hierarchy_commit() " + "before starting the port"); + return 0; error: @@ -2714,9 +2725,11 @@ ixgbe_dev_stop(struct rte_eth_dev *dev) IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int vf; + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); @@ -2763,6 +2776,9 @@ ixgbe_dev_stop(struct rte_eth_dev *dev) rte_free(intr_handle->intr_vec); intr_handle->intr_vec = NULL; } + + /* reset hierarchy commit */ + tm_conf->committed = false; } /* @@ -2774,7 +2790,7 @@ ixgbe_dev_set_link_up(struct rte_eth_dev *dev) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); if (hw->mac.type == ixgbe_mac_82599EB) { -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { /* Not suported in bypass mode */ PMD_INIT_LOG(ERR, "Set link up is not supported " @@ -2804,7 +2820,7 @@ ixgbe_dev_set_link_down(struct rte_eth_dev *dev) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); if (hw->mac.type == ixgbe_mac_82599EB) { -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { /* Not suported in bypass mode */ PMD_INIT_LOG(ERR, "Set link down is not supported " @@ -3194,7 +3210,7 @@ static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, } static int ixgbe_dev_xstats_get_names_by_id( - __rte_unused struct rte_eth_dev *dev, + struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, unsigned int limit) @@ -3582,7 +3598,7 @@ ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) static void ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_conf *dev_conf = &dev->data->dev_conf; @@ -3685,6 +3701,10 @@ ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) hw->mac.type == ixgbe_mac_X550_vf) { dev_info->speed_capa |= ETH_LINK_SPEED_100M; } + if (hw->mac.type == ixgbe_mac_X550) { + dev_info->speed_capa |= ETH_LINK_SPEED_2_5G; + dev_info->speed_capa |= ETH_LINK_SPEED_5G; + } } static const uint32_t * @@ -3717,6 +3737,12 @@ ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc || dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc) return ptypes; + +#if defined(RTE_ARCH_X86) + if (dev->rx_pkt_burst == ixgbe_recv_pkts_vec || + dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec) + return ptypes; +#endif return NULL; } @@ -3724,7 +3750,7 @@ static void ixgbevf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); dev_info->pci_dev = pci_dev; @@ -3776,9 +3802,116 @@ ixgbevf_dev_info_get(struct rte_eth_dev *dev, dev_info->tx_desc_lim = tx_desc_lim; } +static int +ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + int *link_up, int wait_to_complete) +{ + /** + * for a quick link status checking, wait_to_compelet == 0, + * skip PF link status checking + */ + bool no_pflink_check = wait_to_complete == 0; + struct ixgbe_mbx_info *mbx = &hw->mbx; + struct ixgbe_mac_info *mac = &hw->mac; + uint32_t links_reg, in_msg; + int ret_val = 0; + + /* If we were hit with a reset drop the link */ + if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) + mac->get_link_status = true; + + if (!mac->get_link_status) + goto out; + + /* if link status is down no point in checking to see if pf is up */ + links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + if (!(links_reg & IXGBE_LINKS_UP)) + goto out; + + /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs + * before the link status is correct + */ + if (mac->type == ixgbe_mac_82599_vf) { + int i; + + for (i = 0; i < 5; i++) { + rte_delay_us(100); + links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + + if (!(links_reg & IXGBE_LINKS_UP)) + goto out; + } + } + + switch (links_reg & IXGBE_LINKS_SPEED_82599) { + case IXGBE_LINKS_SPEED_10G_82599: + *speed = IXGBE_LINK_SPEED_10GB_FULL; + if (hw->mac.type >= ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_2_5GB_FULL; + } + break; + case IXGBE_LINKS_SPEED_1G_82599: + *speed = IXGBE_LINK_SPEED_1GB_FULL; + break; + case IXGBE_LINKS_SPEED_100_82599: + *speed = IXGBE_LINK_SPEED_100_FULL; + if (hw->mac.type == ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_5GB_FULL; + } + break; + case IXGBE_LINKS_SPEED_10_X550EM_A: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + /* Since Reserved in older MAC's */ + if (hw->mac.type >= ixgbe_mac_X550) + *speed = IXGBE_LINK_SPEED_10_FULL; + break; + default: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + } + + if (no_pflink_check) { + if (*speed == IXGBE_LINK_SPEED_UNKNOWN) + mac->get_link_status = true; + else + mac->get_link_status = false; + + goto out; + } + /* if the read failed it could just be a mailbox collision, best wait + * until we are called again and don't report an error + */ + if (mbx->ops.read(hw, &in_msg, 1, 0)) + goto out; + + if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { + /* msg is not CTS and is NACK we must have lost CTS status */ + if (in_msg & IXGBE_VT_MSGTYPE_NACK) + ret_val = -1; + goto out; + } + + /* the pf is talking, if we timed out in the past we reinit */ + if (!mbx->timeout) { + ret_val = -1; + goto out; + } + + /* if we passed all the tests above then the link is up and we no + * longer need to check for link + */ + mac->get_link_status = false; + +out: + *link_up = !mac->get_link_status; + return ret_val; +} + /* return 0 means link status changed, -1 means not changed */ static int -ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +ixgbe_dev_link_update_share(struct rte_eth_dev *dev, + int wait_to_complete, int vf) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_link link, old; @@ -3788,6 +3921,7 @@ ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) int link_up; int diag; u32 speed = 0; + int wait = 1; bool autoneg = false; link.link_status = ETH_LINK_DOWN; @@ -3808,9 +3942,12 @@ ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) /* check if it needs to wait to complete, if lsc interrupt is enabled */ if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0) - diag = ixgbe_check_link(hw, &link_speed, &link_up, 0); + wait = 0; + + if (vf) + diag = ixgbevf_check_link(hw, &link_speed, &link_up, wait); else - diag = ixgbe_check_link(hw, &link_speed, &link_up, 1); + diag = ixgbe_check_link(hw, &link_speed, &link_up, wait); if (diag != 0) { link.link_speed = ETH_SPEED_NUM_100M; @@ -3847,6 +3984,14 @@ ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) link.link_speed = ETH_SPEED_NUM_1G; break; + case IXGBE_LINK_SPEED_2_5GB_FULL: + link.link_speed = ETH_SPEED_NUM_2_5G; + break; + + case IXGBE_LINK_SPEED_5GB_FULL: + link.link_speed = ETH_SPEED_NUM_5G; + break; + case IXGBE_LINK_SPEED_10GB_FULL: link.link_speed = ETH_SPEED_NUM_10G; break; @@ -3859,6 +4004,18 @@ ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) return 0; } +static int +ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + return ixgbe_dev_link_update_share(dev, wait_to_complete, 0); +} + +static int +ixgbevf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + return ixgbe_dev_link_update_share(dev, wait_to_complete, 1); +} + static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev) { @@ -3916,19 +4073,24 @@ ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev) * * @param dev * Pointer to struct rte_eth_dev. + * @param on + * Enable or Disable. * * @return * - On success, zero. * - On failure, a negative value. */ static int -ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev) +ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) { struct ixgbe_interrupt *intr = IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); ixgbe_dev_link_status_print(dev); - intr->mask |= IXGBE_EICR_LSC; + if (on) + intr->mask |= IXGBE_EICR_LSC; + else + intr->mask &= ~IXGBE_EICR_LSC; return 0; } @@ -4035,7 +4197,7 @@ ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev) static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_eth_link link; memset(&link, 0, sizeof(link)); @@ -4143,7 +4305,7 @@ static void ixgbe_dev_interrupt_delayed_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_interrupt *intr = IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); @@ -4166,12 +4328,13 @@ ixgbe_dev_interrupt_delayed_handler(void *param) ixgbe_dev_link_update(dev, 0); intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; ixgbe_dev_link_status_print(dev); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } if (intr->flags & IXGBE_FLAG_MACSEC) { _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC, - NULL); + NULL, NULL); intr->flags &= ~IXGBE_FLAG_MACSEC; } @@ -4660,7 +4823,7 @@ ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index) static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); ixgbe_remove_rar(dev, 0); @@ -4670,7 +4833,7 @@ ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr) static bool is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) { - if (strcmp(dev->data->drv_name, drv->driver.name)) + if (strcmp(dev->device->driver->name, drv->driver.name)) return false; return true; @@ -4690,7 +4853,7 @@ ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) struct ixgbe_hw *hw; struct rte_eth_dev_info dev_info; uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; - struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; + struct rte_eth_dev_data *dev_data = dev->data; ixgbe_dev_info_get(dev, &dev_info); @@ -4698,13 +4861,15 @@ ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen)) return -EINVAL; - /* refuse mtu that requires the support of scattered packets when this - * feature has not been enabled before. + /* If device is started, refuse mtu that requires the support of + * scattered packets when this feature has not been enabled before. */ - if (!rx_conf->enable_scatter && + if (dev_data->dev_started && !dev_data->scattered_rx && (frame_size + 2 * IXGBE_VLAN_TAG_SIZE > - dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) + dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { + PMD_INIT_LOG(ERR, "Stop port first."); return -EINVAL; + } hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); @@ -4799,7 +4964,7 @@ ixgbevf_dev_start(struct rte_eth_dev *dev) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t intr_vector = 0; - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int err, mask = 0; @@ -4863,7 +5028,7 @@ static void ixgbevf_dev_stop(struct rte_eth_dev *dev) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; PMD_INIT_FUNC_TRACE(); @@ -5316,6 +5481,9 @@ ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) if (ixgbe_vt_check(hw) < 0) return -ENOTSUP; + if (rule_id >= IXGBE_MAX_MIRROR_RULES) + return -EINVAL; + memset(&mr_info->mr_conf[rule_id], 0, sizeof(struct rte_eth_mirror_conf)); @@ -5336,7 +5504,7 @@ ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t mask; struct ixgbe_hw *hw = @@ -5370,7 +5538,7 @@ ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t mask; struct ixgbe_hw *hw = @@ -5473,7 +5641,8 @@ ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, tmp |= (msix_vector << (8 * (queue & 0x3))); IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp); } else if ((hw->mac.type == ixgbe_mac_82599EB) || - (hw->mac.type == ixgbe_mac_X540)) { + (hw->mac.type == ixgbe_mac_X540) || + (hw->mac.type == ixgbe_mac_X550)) { if (direction == -1) { /* other causes */ idx = ((queue & 1) * 8); @@ -5495,7 +5664,7 @@ ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, static void ixgbevf_configure_msix(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -5529,7 +5698,7 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev) static void ixgbe_configure_msix(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -5581,6 +5750,7 @@ ixgbe_configure_msix(struct rte_eth_dev *dev) break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: + case ixgbe_mac_X550: ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); break; default: @@ -5598,8 +5768,9 @@ ixgbe_configure_msix(struct rte_eth_dev *dev) IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); } -static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, - uint16_t queue_idx, uint16_t tx_rate) +int +ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, + uint16_t queue_idx, uint16_t tx_rate) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t rf_dec, rf_int; @@ -7531,7 +7702,7 @@ ixgbe_e_tag_insertion_en_dis(struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel, bool en) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); int ret = 0; uint32_t vmtir, vmvir; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -7879,7 +8050,8 @@ static void ixgbevf_mbx_process(struct rte_eth_dev *dev) /* PF reset VF event */ if (in_msg == IXGBE_PF_CONTROL_MSG) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, NULL); } static int diff --git a/drivers/net/ixgbe/ixgbe_ethdev.h b/drivers/net/ixgbe/ixgbe_ethdev.h index b576a6f4..caa50c8b 100644 --- a/drivers/net/ixgbe/ixgbe_ethdev.h +++ b/drivers/net/ixgbe/ixgbe_ethdev.h @@ -33,12 +33,15 @@ #ifndef _IXGBE_ETHDEV_H_ #define _IXGBE_ETHDEV_H_ +#include "base/ixgbe_type.h" #include "base/ixgbe_dcb.h" #include "base/ixgbe_dcb_82599.h" #include "base/ixgbe_dcb_82598.h" #include "ixgbe_bypass.h" #include <rte_time.h> #include <rte_hash.h> +#include <rte_pci.h> +#include <rte_tm_driver.h> /* need update link, bit flag */ #define IXGBE_FLAG_NEED_LINK_UPDATE (uint32_t)(1 << 0) @@ -72,7 +75,7 @@ #endif #define IXGBE_HWSTRIP_BITMAP_SIZE (IXGBE_MAX_RX_QUEUE_NUM / (sizeof(uint32_t) * NBBY)) -/* EITR Inteval is in 2048ns uinits for 1G and 10G link */ +/* EITR Interval is in 2048ns uinits for 1G and 10G link */ #define IXGBE_EITR_INTERVAL_UNIT_NS 2048 #define IXGBE_EITR_ITR_INT_SHIFT 3 #define IXGBE_EITR_INTERVAL_US(us) \ @@ -152,6 +155,13 @@ return -ENOTSUP;\ } while (0) +/* Link speed for X550 auto negotiation */ +#define IXGBE_LINK_SPEED_X550_AUTONEG (IXGBE_LINK_SPEED_100_FULL | \ + IXGBE_LINK_SPEED_1GB_FULL | \ + IXGBE_LINK_SPEED_2_5GB_FULL | \ + IXGBE_LINK_SPEED_5GB_FULL | \ + IXGBE_LINK_SPEED_10GB_FULL) + /* * Information about the fdir mode. */ @@ -189,6 +199,7 @@ struct ixgbe_fdir_rule { uint32_t fdirflags; /* drop or forward */ uint32_t soft_id; /* an unique value for this rule */ uint8_t queue; /* assigned rx queue */ + uint8_t flex_bytes_offset; }; struct ixgbe_hw_fdir_info { @@ -434,6 +445,68 @@ struct ixgbe_bw_conf { uint8_t tc_num; /* Number of TCs. */ }; +/* Struct to store Traffic Manager shaper profile. */ +struct ixgbe_tm_shaper_profile { + TAILQ_ENTRY(ixgbe_tm_shaper_profile) node; + uint32_t shaper_profile_id; + uint32_t reference_count; + struct rte_tm_shaper_params profile; +}; + +TAILQ_HEAD(ixgbe_shaper_profile_list, ixgbe_tm_shaper_profile); + +/* node type of Traffic Manager */ +enum ixgbe_tm_node_type { + IXGBE_TM_NODE_TYPE_PORT, + IXGBE_TM_NODE_TYPE_TC, + IXGBE_TM_NODE_TYPE_QUEUE, + IXGBE_TM_NODE_TYPE_MAX, +}; + +/* Struct to store Traffic Manager node configuration. */ +struct ixgbe_tm_node { + TAILQ_ENTRY(ixgbe_tm_node) node; + uint32_t id; + uint32_t priority; + uint32_t weight; + uint32_t reference_count; + uint16_t no; + struct ixgbe_tm_node *parent; + struct ixgbe_tm_shaper_profile *shaper_profile; + struct rte_tm_node_params params; +}; + +TAILQ_HEAD(ixgbe_tm_node_list, ixgbe_tm_node); + +/* The configuration of Traffic Manager */ +struct ixgbe_tm_conf { + struct ixgbe_shaper_profile_list shaper_profile_list; + struct ixgbe_tm_node *root; /* root node - port */ + struct ixgbe_tm_node_list tc_list; /* node list for all the TCs */ + struct ixgbe_tm_node_list queue_list; /* node list for all the queues */ + /** + * The number of added TC nodes. + * It should be no more than the TC number of this port. + */ + uint32_t nb_tc_node; + /** + * The number of added queue nodes. + * It should be no more than the queue number of this port. + */ + uint32_t nb_queue_node; + /** + * This flag is used to check if APP can change the TM node + * configuration. + * When it's true, means the configuration is applied to HW, + * APP should not change the configuration. + * As we don't support on-the-fly configuration, when starting + * the port, APP should call the hierarchy_commit API to set this + * flag to true. When stopping the port, this flag should be set + * to false. + */ + bool committed; +}; + /* * Structure to store private data for each driver instance (for each port). */ @@ -450,9 +523,9 @@ struct ixgbe_adapter { struct ixgbe_mirror_info mr_data; struct ixgbe_vf_info *vfdata; struct ixgbe_uta_info uta_info; -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS struct ixgbe_bypass_info bps; -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ struct ixgbe_filter_info filter; struct ixgbe_l2_tn_info l2_tn; struct ixgbe_bw_conf bw_conf; @@ -462,11 +535,9 @@ struct ixgbe_adapter { struct rte_timecounter systime_tc; struct rte_timecounter rx_tstamp_tc; struct rte_timecounter tx_tstamp_tc; + struct ixgbe_tm_conf tm_conf; }; -#define IXGBE_DEV_TO_PCI(eth_dev) \ - RTE_DEV_TO_PCI((eth_dev)->device) - #define IXGBE_DEV_PRIVATE_TO_HW(adapter)\ (&((struct ixgbe_adapter *)adapter)->hw) @@ -512,6 +583,9 @@ struct ixgbe_adapter { #define IXGBE_DEV_PRIVATE_TO_BW_CONF(adapter) \ (&((struct ixgbe_adapter *)adapter)->bw_conf) +#define IXGBE_DEV_PRIVATE_TO_TM_CONF(adapter) \ + (&((struct ixgbe_adapter *)adapter)->tm_conf) + /* * RX/TX function prototypes */ @@ -624,6 +698,8 @@ void ixgbe_filterlist_flush(void); */ int ixgbe_fdir_configure(struct rte_eth_dev *dev); int ixgbe_fdir_set_input_mask(struct rte_eth_dev *dev); +int ixgbe_fdir_set_flexbytes_offset(struct rte_eth_dev *dev, + uint16_t offset); int ixgbe_fdir_filter_program(struct rte_eth_dev *dev, struct ixgbe_fdir_rule *rule, bool del, bool update); @@ -671,6 +747,11 @@ int ixgbe_vt_check(struct ixgbe_hw *hw); int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, uint16_t tx_rate, uint64_t q_msk); bool is_ixgbe_supported(struct rte_eth_dev *dev); +int ixgbe_tm_ops_get(struct rte_eth_dev *dev, void *ops); +void ixgbe_tm_conf_init(struct rte_eth_dev *dev); +void ixgbe_tm_conf_uninit(struct rte_eth_dev *dev); +int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, uint16_t queue_idx, + uint16_t tx_rate); static inline int ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info, diff --git a/drivers/net/ixgbe/ixgbe_fdir.c b/drivers/net/ixgbe/ixgbe_fdir.c index 7f6c7b58..eb2d5581 100644 --- a/drivers/net/ixgbe/ixgbe_fdir.c +++ b/drivers/net/ixgbe/ixgbe_fdir.c @@ -302,7 +302,7 @@ fdir_set_input_mask_82599(struct rte_eth_dev *dev) * mask VM pool and DIPv6 since there are currently not supported * mask FLEX byte, it will be set in flex_conf */ - uint32_t fdirm = IXGBE_FDIRM_POOL | IXGBE_FDIRM_DIPv6 | IXGBE_FDIRM_FLEX; + uint32_t fdirm = IXGBE_FDIRM_POOL | IXGBE_FDIRM_DIPv6; uint32_t fdirtcpm; /* TCP source and destination port masks. */ uint32_t fdiripv6m; /* IPv6 source and destination masks. */ volatile uint32_t *reg; @@ -333,6 +333,10 @@ fdir_set_input_mask_82599(struct rte_eth_dev *dev) return -EINVAL; } + /* flex byte mask */ + if (info->mask.flex_bytes_mask == 0) + fdirm |= IXGBE_FDIRM_FLEX; + IXGBE_WRITE_REG(hw, IXGBE_FDIRM, fdirm); /* store the TCP/UDP port masks, bit reversed from port layout */ @@ -533,6 +537,31 @@ ixgbe_fdir_set_input_mask(struct rte_eth_dev *dev) return -ENOTSUP; } +int +ixgbe_fdir_set_flexbytes_offset(struct rte_eth_dev *dev, + uint16_t offset) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t fdirctrl; + int i; + + fdirctrl = IXGBE_READ_REG(hw, IXGBE_FDIRCTRL); + + fdirctrl &= ~IXGBE_FDIRCTRL_FLEX_MASK; + fdirctrl |= ((offset >> 1) /* convert to word offset */ + << IXGBE_FDIRCTRL_FLEX_SHIFT); + + IXGBE_WRITE_REG(hw, IXGBE_FDIRCTRL, fdirctrl); + IXGBE_WRITE_FLUSH(hw); + for (i = 0; i < IXGBE_FDIR_INIT_DONE_POLL; i++) { + if (IXGBE_READ_REG(hw, IXGBE_FDIRCTRL) & + IXGBE_FDIRCTRL_INIT_DONE) + break; + msec_delay(1); + } + return 0; +} + static int fdir_set_input_mask(struct rte_eth_dev *dev, const struct rte_eth_fdir_masks *input_mask) @@ -654,7 +683,7 @@ ixgbe_fdir_configure(struct rte_eth_dev *dev) /* * The defaults in the HW for RX PB 1-7 are not zero and so should be - * intialized to zero for non DCB mode otherwise actual total RX PB + * initialized to zero for non DCB mode otherwise actual total RX PB * would be bigger than programmed and filter space would run into * the PB 0 region. */ @@ -1243,7 +1272,9 @@ ixgbe_fdir_filter_program(struct rte_eth_dev *dev, hw->mac.type == ixgbe_mac_X550EM_x || hw->mac.type == ixgbe_mac_X550EM_a) && (rule->ixgbe_fdir.formatted.flow_type == - IXGBE_ATR_FLOW_TYPE_IPV4) && + IXGBE_ATR_FLOW_TYPE_IPV4 || + rule->ixgbe_fdir.formatted.flow_type == + IXGBE_ATR_FLOW_TYPE_IPV6) && (info->mask.src_port_mask != 0 || info->mask.dst_port_mask != 0)) { PMD_DRV_LOG(ERR, "By this device," diff --git a/drivers/net/ixgbe/ixgbe_flow.c b/drivers/net/ixgbe/ixgbe_flow.c index 9aeb71e4..d6796088 100644 --- a/drivers/net/ixgbe/ixgbe_flow.c +++ b/drivers/net/ixgbe/ixgbe_flow.c @@ -56,7 +56,6 @@ #include <rte_alarm.h> #include <rte_ether.h> #include <rte_ethdev.h> -#include <rte_atomic.h> #include <rte_malloc.h> #include <rte_random.h> #include <rte_dev.h> @@ -78,23 +77,40 @@ #define IXGBE_MIN_N_TUPLE_PRIO 1 #define IXGBE_MAX_N_TUPLE_PRIO 7 -#define NEXT_ITEM_OF_PATTERN(item, pattern, index)\ - do { \ - item = pattern + index;\ - while (item->type == RTE_FLOW_ITEM_TYPE_VOID) {\ - index++; \ - item = pattern + index; \ - } \ - } while (0) - -#define NEXT_ITEM_OF_ACTION(act, actions, index)\ - do { \ - act = actions + index; \ - while (act->type == RTE_FLOW_ACTION_TYPE_VOID) {\ - index++; \ - act = actions + index; \ - } \ - } while (0) +#define IXGBE_MAX_FLX_SOURCE_OFF 62 + +/** + * Endless loop will never happen with below assumption + * 1. there is at least one no-void item(END) + * 2. cur is before END. + */ +static inline +const struct rte_flow_item *next_no_void_pattern( + const struct rte_flow_item pattern[], + const struct rte_flow_item *cur) +{ + const struct rte_flow_item *next = + cur ? cur + 1 : &pattern[0]; + while (1) { + if (next->type != RTE_FLOW_ITEM_TYPE_VOID) + return next; + next++; + } +} + +static inline +const struct rte_flow_action *next_no_void_action( + const struct rte_flow_action actions[], + const struct rte_flow_action *cur) +{ + const struct rte_flow_action *next = + cur ? cur + 1 : &actions[0]; + while (1) { + if (next->type != RTE_FLOW_ACTION_TYPE_VOID) + return next; + next++; + } +} /** * Please aware there's an asumption for all the parsers. @@ -144,7 +160,6 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, const struct rte_flow_item_udp *udp_mask; const struct rte_flow_item_sctp *sctp_spec; const struct rte_flow_item_sctp *sctp_mask; - uint32_t index; if (!pattern) { rte_flow_error_set(error, @@ -166,11 +181,8 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, return -rte_errno; } - /* parse pattern */ - index = 0; - /* the first not void item can be MAC or IPv4 */ - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, NULL); if (item->type != RTE_FLOW_ITEM_TYPE_ETH && item->type != RTE_FLOW_ITEM_TYPE_IPV4) { @@ -198,8 +210,7 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, return -rte_errno; } /* check if the next not void item is IPv4 */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -252,11 +263,11 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, filter->proto = ipv4_spec->hdr.next_proto_id; /* check if the next not void item is TCP or UDP */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_TCP && item->type != RTE_FLOW_ITEM_TYPE_UDP && - item->type != RTE_FLOW_ITEM_TYPE_SCTP) { + item->type != RTE_FLOW_ITEM_TYPE_SCTP && + item->type != RTE_FLOW_ITEM_TYPE_END) { memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -265,7 +276,8 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, } /* get the TCP/UDP info */ - if (!item->spec || !item->mask) { + if ((item->type != RTE_FLOW_ITEM_TYPE_END) && + (!item->spec || !item->mask)) { memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -345,7 +357,7 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, udp_spec = (const struct rte_flow_item_udp *)item->spec; filter->dst_port = udp_spec->hdr.dst_port; filter->src_port = udp_spec->hdr.src_port; - } else { + } else if (item->type == RTE_FLOW_ITEM_TYPE_SCTP) { sctp_mask = (const struct rte_flow_item_sctp *)item->mask; /** @@ -368,11 +380,12 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, sctp_spec = (const struct rte_flow_item_sctp *)item->spec; filter->dst_port = sctp_spec->hdr.dst_port; filter->src_port = sctp_spec->hdr.src_port; + } else { + goto action; } /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_END) { memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); rte_flow_error_set(error, EINVAL, @@ -381,14 +394,13 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, return -rte_errno; } - /* parse action */ - index = 0; +action: /** * n-tuple only supports forwarding, * check if the first not void action is QUEUE. */ - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, NULL); if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); rte_flow_error_set(error, EINVAL, @@ -400,8 +412,7 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr, ((const struct rte_flow_action_queue *)act->conf)->index; /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, act); if (act->type != RTE_FLOW_ACTION_TYPE_END) { memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); rte_flow_error_set(error, EINVAL, @@ -482,9 +493,7 @@ ixgbe_parse_ntuple_filter(struct rte_eth_dev *dev, return -rte_errno; } - if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM || - filter->priority > IXGBE_5TUPLE_MAX_PRI || - filter->priority < IXGBE_5TUPLE_MIN_PRI) + if (filter->queue >= dev->data->nb_rx_queues) return -rte_errno; /* fixed value for ixgbe */ @@ -520,7 +529,6 @@ cons_parse_ethertype_filter(const struct rte_flow_attr *attr, const struct rte_flow_item_eth *eth_spec; const struct rte_flow_item_eth *eth_mask; const struct rte_flow_action_queue *act_q; - uint32_t index; if (!pattern) { rte_flow_error_set(error, EINVAL, @@ -543,15 +551,8 @@ cons_parse_ethertype_filter(const struct rte_flow_attr *attr, return -rte_errno; } - /* Parse pattern */ - index = 0; - + item = next_no_void_pattern(pattern, NULL); /* The first non-void item should be MAC. */ - item = pattern + index; - while (item->type == RTE_FLOW_ITEM_TYPE_VOID) { - index++; - item = pattern + index; - } if (item->type != RTE_FLOW_ITEM_TYPE_ETH) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -610,12 +611,7 @@ cons_parse_ethertype_filter(const struct rte_flow_attr *attr, filter->ether_type = rte_be_to_cpu_16(eth_spec->type); /* Check if the next non-void item is END. */ - index++; - item = pattern + index; - while (item->type == RTE_FLOW_ITEM_TYPE_VOID) { - index++; - item = pattern + index; - } + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_END) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -625,13 +621,7 @@ cons_parse_ethertype_filter(const struct rte_flow_attr *attr, /* Parse action */ - index = 0; - /* Check if the first non-void action is QUEUE or DROP. */ - act = actions + index; - while (act->type == RTE_FLOW_ACTION_TYPE_VOID) { - index++; - act = actions + index; - } + act = next_no_void_action(actions, NULL); if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && act->type != RTE_FLOW_ACTION_TYPE_DROP) { rte_flow_error_set(error, EINVAL, @@ -648,12 +638,7 @@ cons_parse_ethertype_filter(const struct rte_flow_attr *attr, } /* Check if the next non-void item is END */ - index++; - act = actions + index; - while (act->type == RTE_FLOW_ACTION_TYPE_VOID) { - index++; - act = actions + index; - } + act = next_no_void_action(actions, act); if (act->type != RTE_FLOW_ACTION_TYPE_END) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -725,7 +710,7 @@ ixgbe_parse_ethertype_filter(struct rte_eth_dev *dev, return -rte_errno; } - if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) { + if (filter->queue >= dev->data->nb_rx_queues) { memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -793,7 +778,6 @@ cons_parse_syn_filter(const struct rte_flow_attr *attr, const struct rte_flow_item_tcp *tcp_spec; const struct rte_flow_item_tcp *tcp_mask; const struct rte_flow_action_queue *act_q; - uint32_t index; if (!pattern) { rte_flow_error_set(error, EINVAL, @@ -816,11 +800,9 @@ cons_parse_syn_filter(const struct rte_flow_attr *attr, return -rte_errno; } - /* parse pattern */ - index = 0; /* the first not void item should be MAC or IPv4 or IPv6 or TCP */ - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, NULL); if (item->type != RTE_FLOW_ITEM_TYPE_ETH && item->type != RTE_FLOW_ITEM_TYPE_IPV4 && item->type != RTE_FLOW_ITEM_TYPE_IPV6 && @@ -849,8 +831,7 @@ cons_parse_syn_filter(const struct rte_flow_attr *attr, } /* check if the next not void item is IPv4 or IPv6 */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && item->type != RTE_FLOW_ITEM_TYPE_IPV6) { rte_flow_error_set(error, EINVAL, @@ -872,8 +853,7 @@ cons_parse_syn_filter(const struct rte_flow_attr *attr, } /* check if the next not void item is TCP */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_TCP) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -917,8 +897,7 @@ cons_parse_syn_filter(const struct rte_flow_attr *attr, } /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_END) { memset(filter, 0, sizeof(struct rte_eth_syn_filter)); rte_flow_error_set(error, EINVAL, @@ -927,11 +906,8 @@ cons_parse_syn_filter(const struct rte_flow_attr *attr, return -rte_errno; } - /* parse action */ - index = 0; - /* check if the first not void action is QUEUE. */ - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, NULL); if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { memset(filter, 0, sizeof(struct rte_eth_syn_filter)); rte_flow_error_set(error, EINVAL, @@ -951,8 +927,7 @@ cons_parse_syn_filter(const struct rte_flow_attr *attr, } /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, act); if (act->type != RTE_FLOW_ACTION_TYPE_END) { memset(filter, 0, sizeof(struct rte_eth_syn_filter)); rte_flow_error_set(error, EINVAL, @@ -1012,6 +987,9 @@ ixgbe_parse_syn_filter(struct rte_eth_dev *dev, ret = cons_parse_syn_filter(attr, pattern, actions, filter, error); + if (filter->queue >= dev->data->nb_rx_queues) + return -rte_errno; + if (ret) return ret; @@ -1048,7 +1026,6 @@ cons_parse_l2_tn_filter(const struct rte_flow_attr *attr, const struct rte_flow_item_e_tag *e_tag_mask; const struct rte_flow_action *act; const struct rte_flow_action_queue *act_q; - uint32_t index; if (!pattern) { rte_flow_error_set(error, EINVAL, @@ -1070,11 +1047,9 @@ cons_parse_l2_tn_filter(const struct rte_flow_attr *attr, NULL, "NULL attribute."); return -rte_errno; } - /* parse pattern */ - index = 0; /* The first not void item should be e-tag. */ - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, NULL); if (item->type != RTE_FLOW_ITEM_TYPE_E_TAG) { memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); rte_flow_error_set(error, EINVAL, @@ -1121,8 +1096,7 @@ cons_parse_l2_tn_filter(const struct rte_flow_attr *attr, filter->tunnel_id = rte_be_to_cpu_16(e_tag_spec->rsvd_grp_ecid_b); /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_END) { memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); rte_flow_error_set(error, EINVAL, @@ -1159,11 +1133,8 @@ cons_parse_l2_tn_filter(const struct rte_flow_attr *attr, return -rte_errno; } - /* parse action */ - index = 0; - /* check if the first not void action is QUEUE. */ - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, NULL); if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); rte_flow_error_set(error, EINVAL, @@ -1176,8 +1147,7 @@ cons_parse_l2_tn_filter(const struct rte_flow_attr *attr, filter->pool = act_q->index; /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, act); if (act->type != RTE_FLOW_ACTION_TYPE_END) { memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); rte_flow_error_set(error, EINVAL, @@ -1213,6 +1183,9 @@ ixgbe_parse_l2_tn_filter(struct rte_eth_dev *dev, return -rte_errno; } + if (l2_tn_filter->pool >= dev->data->nb_rx_queues) + return -rte_errno; + return ret; } @@ -1226,7 +1199,6 @@ ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, const struct rte_flow_action *act; const struct rte_flow_action_queue *act_q; const struct rte_flow_action_mark *mark; - uint32_t index; /* parse attr */ /* must be input direction */ @@ -1256,11 +1228,8 @@ ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, return -rte_errno; } - /* parse action */ - index = 0; - /* check if the first not void action is QUEUE or DROP. */ - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, NULL); if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && act->type != RTE_FLOW_ACTION_TYPE_DROP) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); @@ -1274,12 +1243,19 @@ ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, act_q = (const struct rte_flow_action_queue *)act->conf; rule->queue = act_q->index; } else { /* drop */ + /* signature mode does not support drop action. */ + if (rule->mode == RTE_FDIR_MODE_SIGNATURE) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } rule->fdirflags = IXGBE_FDIRCMD_DROP; } /* check if the next not void item is MARK */ - index++; - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, act); if ((act->type != RTE_FLOW_ACTION_TYPE_MARK) && (act->type != RTE_FLOW_ACTION_TYPE_END)) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); @@ -1294,8 +1270,7 @@ ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, if (act->type == RTE_FLOW_ACTION_TYPE_MARK) { mark = (const struct rte_flow_action_mark *)act->conf; rule->soft_id = mark->id; - index++; - NEXT_ITEM_OF_ACTION(act, actions, index); + act = next_no_void_action(actions, act); } /* check if the next not void item is END */ @@ -1310,14 +1285,78 @@ ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, return 0; } +/* search next no void pattern and skip fuzzy */ +static inline +const struct rte_flow_item *next_no_fuzzy_pattern( + const struct rte_flow_item pattern[], + const struct rte_flow_item *cur) +{ + const struct rte_flow_item *next = + next_no_void_pattern(pattern, cur); + while (1) { + if (next->type != RTE_FLOW_ITEM_TYPE_FUZZY) + return next; + next = next_no_void_pattern(pattern, next); + } +} + +static inline uint8_t signature_match(const struct rte_flow_item pattern[]) +{ + const struct rte_flow_item_fuzzy *spec, *last, *mask; + const struct rte_flow_item *item; + uint32_t sh, lh, mh; + int i = 0; + + while (1) { + item = pattern + i; + if (item->type == RTE_FLOW_ITEM_TYPE_END) + break; + + if (item->type == RTE_FLOW_ITEM_TYPE_FUZZY) { + spec = + (const struct rte_flow_item_fuzzy *)item->spec; + last = + (const struct rte_flow_item_fuzzy *)item->last; + mask = + (const struct rte_flow_item_fuzzy *)item->mask; + + if (!spec || !mask) + return 0; + + sh = spec->thresh; + + if (!last) + lh = sh; + else + lh = last->thresh; + + mh = mask->thresh; + sh = sh & mh; + lh = lh & mh; + + if (!sh || sh > lh) + return 0; + + return 1; + } + + i++; + } + + return 0; +} + /** * Parse the rule to see if it is a IP or MAC VLAN flow director rule. * And get the flow director filter info BTW. * UDP/TCP/SCTP PATTERN: - * The first not void item can be ETH or IPV4. - * The second not void item must be IPV4 if the first one is ETH. - * The third not void item must be UDP or TCP or SCTP. + * The first not void item can be ETH or IPV4 or IPV6 + * The second not void item must be IPV4 or IPV6 if the first one is ETH. + * The next not void item could be UDP or TCP or SCTP (optional) + * The next not void item could be RAW (for flexbyte, optional) * The next not void item must be END. + * A Fuzzy Match pattern can appear at any place before END. + * Fuzzy Match is optional for IPV4 but is required for IPV6 * MAC VLAN PATTERN: * The first not void item must be ETH. * The second not void item must be MAC VLAN. @@ -1334,6 +1373,14 @@ ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, * dst_addr 192.167.3.50 0xFFFFFFFF * UDP/TCP/SCTP src_port 80 0xFFFF * dst_port 80 0xFFFF + * FLEX relative 0 0x1 + * search 0 0x1 + * reserved 0 0 + * offset 12 0xFFFFFFFF + * limit 0 0xFFFF + * length 2 0xFFFF + * pattern[0] 0x86 0xFF + * pattern[1] 0xDD 0xFF * END * MAC VLAN pattern example: * ITEM Spec Mask @@ -1346,7 +1393,8 @@ ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, * Item->last should be NULL. */ static int -ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, +ixgbe_parse_fdir_filter_normal(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, const struct rte_flow_item pattern[], const struct rte_flow_action actions[], struct ixgbe_fdir_rule *rule, @@ -1357,6 +1405,8 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, const struct rte_flow_item_eth *eth_mask; const struct rte_flow_item_ipv4 *ipv4_spec; const struct rte_flow_item_ipv4 *ipv4_mask; + const struct rte_flow_item_ipv6 *ipv6_spec; + const struct rte_flow_item_ipv6 *ipv6_mask; const struct rte_flow_item_tcp *tcp_spec; const struct rte_flow_item_tcp *tcp_mask; const struct rte_flow_item_udp *udp_spec; @@ -1365,8 +1415,11 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, const struct rte_flow_item_sctp *sctp_mask; const struct rte_flow_item_vlan *vlan_spec; const struct rte_flow_item_vlan *vlan_mask; + const struct rte_flow_item_raw *raw_mask; + const struct rte_flow_item_raw *raw_spec; + uint8_t j; - uint32_t index, j; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); if (!pattern) { rte_flow_error_set(error, EINVAL, @@ -1396,17 +1449,16 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); memset(&rule->mask, 0xFF, sizeof(struct ixgbe_hw_fdir_mask)); rule->mask.vlan_tci_mask = 0; - - /* parse pattern */ - index = 0; + rule->mask.flex_bytes_mask = 0; /** * The first not void item should be * MAC or IPv4 or TCP or UDP or SCTP. */ - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_fuzzy_pattern(pattern, NULL); if (item->type != RTE_FLOW_ITEM_TYPE_ETH && item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6 && item->type != RTE_FLOW_ITEM_TYPE_TCP && item->type != RTE_FLOW_ITEM_TYPE_UDP && item->type != RTE_FLOW_ITEM_TYPE_SCTP) { @@ -1417,7 +1469,10 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, return -rte_errno; } - rule->mode = RTE_FDIR_MODE_PERFECT; + if (signature_match(pattern)) + rule->mode = RTE_FDIR_MODE_SIGNATURE; + else + rule->mode = RTE_FDIR_MODE_PERFECT; /*Not supported last point for range*/ if (item->last) { @@ -1454,14 +1509,13 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, if (item->mask) { - /* If ethernet has meaning, it means MAC VLAN mode. */ - rule->mode = RTE_FDIR_MODE_PERFECT_MAC_VLAN; rule->b_mask = TRUE; eth_mask = (const struct rte_flow_item_eth *)item->mask; /* Ether type should be masked. */ - if (eth_mask->type) { + if (eth_mask->type || + rule->mode == RTE_FDIR_MODE_SIGNATURE) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -1469,6 +1523,9 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, return -rte_errno; } + /* If ethernet has meaning, it means MAC VLAN mode. */ + rule->mode = RTE_FDIR_MODE_PERFECT_MAC_VLAN; + /** * src MAC address must be masked, * and don't support dst MAC address mask. @@ -1497,8 +1554,7 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, * Check if the next not void item is vlan or ipv4. * IPv6 is not supported. */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_fuzzy_pattern(pattern, item); if (rule->mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { if (item->type != RTE_FLOW_ITEM_TYPE_VLAN) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); @@ -1544,18 +1600,9 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, rule->mask.vlan_tci_mask &= rte_cpu_to_be_16(0xEFFF); /* More than one tags are not supported. */ - /** - * Check if the next not void item is not vlan. - */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); - if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { - memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - item, "Not supported by fdir filter"); - return -rte_errno; - } else if (item->type != RTE_FLOW_ITEM_TYPE_END) { + /* Next not void item must be END */ + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -1564,7 +1611,7 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, } } - /* Get the IP info. */ + /* Get the IPV4 info. */ if (item->type == RTE_FLOW_ITEM_TYPE_IPV4) { /** * Set the flow type even if there's no content @@ -1624,12 +1671,104 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, * Check if the next not void item is * TCP or UDP or SCTP or END. */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_fuzzy_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_TCP && item->type != RTE_FLOW_ITEM_TYPE_UDP && item->type != RTE_FLOW_ITEM_TYPE_SCTP && - item->type != RTE_FLOW_ITEM_TYPE_END) { + item->type != RTE_FLOW_ITEM_TYPE_END && + item->type != RTE_FLOW_ITEM_TYPE_RAW) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Get the IPV6 info. */ + if (item->type == RTE_FLOW_ITEM_TYPE_IPV6) { + /** + * Set the flow type even if there's no content + * as we must have a flow type. + */ + rule->ixgbe_fdir.formatted.flow_type = + IXGBE_ATR_FLOW_TYPE_IPV6; + + /** + * 1. must signature match + * 2. not support last + * 3. mask must not null + */ + if (rule->mode != RTE_FDIR_MODE_SIGNATURE || + item->last || + !item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + rule->b_mask = TRUE; + ipv6_mask = + (const struct rte_flow_item_ipv6 *)item->mask; + if (ipv6_mask->hdr.vtc_flow || + ipv6_mask->hdr.payload_len || + ipv6_mask->hdr.proto || + ipv6_mask->hdr.hop_limits) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /* check src addr mask */ + for (j = 0; j < 16; j++) { + if (ipv6_mask->hdr.src_addr[j] == UINT8_MAX) { + rule->mask.src_ipv6_mask |= 1 << j; + } else if (ipv6_mask->hdr.src_addr[j] != 0) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* check dst addr mask */ + for (j = 0; j < 16; j++) { + if (ipv6_mask->hdr.dst_addr[j] == UINT8_MAX) { + rule->mask.dst_ipv6_mask |= 1 << j; + } else if (ipv6_mask->hdr.dst_addr[j] != 0) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + if (item->spec) { + rule->b_spec = TRUE; + ipv6_spec = + (const struct rte_flow_item_ipv6 *)item->spec; + rte_memcpy(rule->ixgbe_fdir.formatted.src_ip, + ipv6_spec->hdr.src_addr, 16); + rte_memcpy(rule->ixgbe_fdir.formatted.dst_ip, + ipv6_spec->hdr.dst_addr, 16); + } + + /** + * Check if the next not void item is + * TCP or UDP or SCTP or END. + */ + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP && + item->type != RTE_FLOW_ITEM_TYPE_END && + item->type != RTE_FLOW_ITEM_TYPE_RAW) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -1644,8 +1783,8 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, * Set the flow type even if there's no content * as we must have a flow type. */ - rule->ixgbe_fdir.formatted.flow_type = - IXGBE_ATR_FLOW_TYPE_TCPV4; + rule->ixgbe_fdir.formatted.flow_type |= + IXGBE_ATR_L4TYPE_TCP; /*Not supported last point for range*/ if (item->last) { rte_flow_error_set(error, EINVAL, @@ -1690,6 +1829,17 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, rule->ixgbe_fdir.formatted.dst_port = tcp_spec->hdr.dst_port; } + + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } /* Get the UDP info */ @@ -1698,8 +1848,8 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, * Set the flow type even if there's no content * as we must have a flow type. */ - rule->ixgbe_fdir.formatted.flow_type = - IXGBE_ATR_FLOW_TYPE_UDPV4; + rule->ixgbe_fdir.formatted.flow_type |= + IXGBE_ATR_L4TYPE_UDP; /*Not supported last point for range*/ if (item->last) { rte_flow_error_set(error, EINVAL, @@ -1739,6 +1889,17 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, rule->ixgbe_fdir.formatted.dst_port = udp_spec->hdr.dst_port; } + + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } /* Get the SCTP info */ @@ -1747,8 +1908,8 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, * Set the flow type even if there's no content * as we must have a flow type. */ - rule->ixgbe_fdir.formatted.flow_type = - IXGBE_ATR_FLOW_TYPE_SCTPV4; + rule->ixgbe_fdir.formatted.flow_type |= + IXGBE_ATR_L4TYPE_SCTP; /*Not supported last point for range*/ if (item->last) { rte_flow_error_set(error, EINVAL, @@ -1756,46 +1917,147 @@ ixgbe_parse_fdir_filter_normal(const struct rte_flow_attr *attr, item, "Not supported last point for range"); return -rte_errno; } - /** - * Only care about src & dst ports, - * others should be masked. - */ - if (!item->mask) { + + /* only x550 family only support sctp port */ + if (hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_X550EM_x || + hw->mac.type == ixgbe_mac_X550EM_a) { + /** + * Only care about src & dst ports, + * others should be masked. + */ + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->b_mask = TRUE; + sctp_mask = + (const struct rte_flow_item_sctp *)item->mask; + if (sctp_mask->hdr.tag || + sctp_mask->hdr.cksum) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->mask.src_port_mask = sctp_mask->hdr.src_port; + rule->mask.dst_port_mask = sctp_mask->hdr.dst_port; + + if (item->spec) { + rule->b_spec = TRUE; + sctp_spec = + (const struct rte_flow_item_sctp *)item->spec; + rule->ixgbe_fdir.formatted.src_port = + sctp_spec->hdr.src_port; + rule->ixgbe_fdir.formatted.dst_port = + sctp_spec->hdr.dst_port; + } + /* others even sctp port is not supported */ + } else { + sctp_mask = + (const struct rte_flow_item_sctp *)item->mask; + if (sctp_mask && + (sctp_mask->hdr.src_port || + sctp_mask->hdr.dst_port || + sctp_mask->hdr.tag || + sctp_mask->hdr.cksum)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Not supported by fdir filter"); return -rte_errno; } - rule->b_mask = TRUE; - sctp_mask = - (const struct rte_flow_item_sctp *)item->mask; - if (sctp_mask->hdr.tag || - sctp_mask->hdr.cksum) { + } + + /* Get the flex byte info */ + if (item->type == RTE_FLOW_ITEM_TYPE_RAW) { + /* Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + /* mask should not be null */ + if (!item->mask || !item->spec) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "Not supported by fdir filter"); return -rte_errno; } - rule->mask.src_port_mask = sctp_mask->hdr.src_port; - rule->mask.dst_port_mask = sctp_mask->hdr.dst_port; - if (item->spec) { - rule->b_spec = TRUE; - sctp_spec = - (const struct rte_flow_item_sctp *)item->spec; - rule->ixgbe_fdir.formatted.src_port = - sctp_spec->hdr.src_port; - rule->ixgbe_fdir.formatted.dst_port = - sctp_spec->hdr.dst_port; + raw_mask = (const struct rte_flow_item_raw *)item->mask; + + /* check mask */ + if (raw_mask->relative != 0x1 || + raw_mask->search != 0x1 || + raw_mask->reserved != 0x0 || + (uint32_t)raw_mask->offset != 0xffffffff || + raw_mask->limit != 0xffff || + raw_mask->length != 0xffff) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; } + + raw_spec = (const struct rte_flow_item_raw *)item->spec; + + /* check spec */ + if (raw_spec->relative != 0 || + raw_spec->search != 0 || + raw_spec->reserved != 0 || + raw_spec->offset > IXGBE_MAX_FLX_SOURCE_OFF || + raw_spec->offset % 2 || + raw_spec->limit != 0 || + raw_spec->length != 2 || + /* pattern can't be 0xffff */ + (raw_spec->pattern[0] == 0xff && + raw_spec->pattern[1] == 0xff)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /* check pattern mask */ + if (raw_mask->pattern[0] != 0xff || + raw_mask->pattern[1] != 0xff) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + rule->mask.flex_bytes_mask = 0xffff; + rule->ixgbe_fdir.formatted.flex_bytes = + (((uint16_t)raw_spec->pattern[1]) << 8) | + raw_spec->pattern[0]; + rule->flex_bytes_offset = raw_spec->offset; } if (item->type != RTE_FLOW_ITEM_TYPE_END) { /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_fuzzy_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_END) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, @@ -1863,7 +2125,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, const struct rte_flow_item_eth *eth_mask; const struct rte_flow_item_vlan *vlan_spec; const struct rte_flow_item_vlan *vlan_mask; - uint32_t index, j; + uint32_t j; if (!pattern) { rte_flow_error_set(error, EINVAL, @@ -1894,14 +2156,11 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, memset(&rule->mask, 0xFF, sizeof(struct ixgbe_hw_fdir_mask)); rule->mask.vlan_tci_mask = 0; - /* parse pattern */ - index = 0; - /** * The first not void item should be * MAC or IPv4 or IPv6 or UDP or VxLAN. */ - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, NULL); if (item->type != RTE_FLOW_ITEM_TYPE_ETH && item->type != RTE_FLOW_ITEM_TYPE_IPV4 && item->type != RTE_FLOW_ITEM_TYPE_IPV6 && @@ -1927,7 +2186,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, item, "Not supported by fdir filter"); return -rte_errno; } - /*Not supported last point for range*/ + /* Not supported last point for range*/ if (item->last) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -1936,8 +2195,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, } /* Check if the next not void item is IPv4 or IPv6. */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && item->type != RTE_FLOW_ITEM_TYPE_IPV6) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); @@ -1968,8 +2226,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, } /* Check if the next not void item is UDP or NVGRE. */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_UDP && item->type != RTE_FLOW_ITEM_TYPE_NVGRE) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); @@ -1999,8 +2256,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, } /* Check if the next not void item is VxLAN. */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, @@ -2156,8 +2412,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, } /* check if the next not void item is MAC */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_ETH) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, @@ -2240,8 +2495,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, * Check if the next not void item is vlan or ipv4. * IPv6 is not supported. */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if ((item->type != RTE_FLOW_ITEM_TYPE_VLAN) && (item->type != RTE_FLOW_ITEM_TYPE_IPV4)) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); @@ -2277,8 +2531,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, /* More than one tags are not supported. */ /* check if the next not void item is END */ - index++; - NEXT_ITEM_OF_PATTERN(item, pattern, index); + item = next_no_void_pattern(pattern, item); if (item->type != RTE_FLOW_ITEM_TYPE_END) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); @@ -2316,7 +2569,7 @@ ixgbe_parse_fdir_filter(struct rte_eth_dev *dev, hw->mac.type != ixgbe_mac_X550EM_a) return -ENOTSUP; - ret = ixgbe_parse_fdir_filter_normal(attr, pattern, + ret = ixgbe_parse_fdir_filter_normal(dev, attr, pattern, actions, rule, error); if (!ret) @@ -2325,10 +2578,24 @@ ixgbe_parse_fdir_filter(struct rte_eth_dev *dev, ret = ixgbe_parse_fdir_filter_tunnel(attr, pattern, actions, rule, error); + if (ret) + return ret; + step_next: + + if (hw->mac.type == ixgbe_mac_82599EB && + rule->fdirflags == IXGBE_FDIRCMD_DROP && + (rule->ixgbe_fdir.formatted.src_port != 0 || + rule->ixgbe_fdir.formatted.dst_port != 0)) + return -ENOTSUP; + if (fdir_mode == RTE_FDIR_MODE_NONE || fdir_mode != rule->mode) return -ENOTSUP; + + if (rule->queue >= dev->data->nb_rx_queues) + return -ENOTSUP; + return ret; } @@ -2414,6 +2681,7 @@ ixgbe_flow_create(struct rte_eth_dev *dev, struct ixgbe_eth_l2_tunnel_conf_ele *l2_tn_filter_ptr; struct ixgbe_fdir_rule_ele *fdir_rule_ptr; struct ixgbe_flow_mem *ixgbe_flow_mem_ptr; + uint8_t first_mask = FALSE; flow = rte_zmalloc("ixgbe_rte_flow", sizeof(struct rte_flow), 0); if (!flow) { @@ -2505,11 +2773,19 @@ ixgbe_flow_create(struct rte_eth_dev *dev, rte_memcpy(&fdir_info->mask, &fdir_rule.mask, sizeof(struct ixgbe_hw_fdir_mask)); + fdir_info->flex_bytes_offset = + fdir_rule.flex_bytes_offset; + + if (fdir_rule.mask.flex_bytes_mask) + ixgbe_fdir_set_flexbytes_offset(dev, + fdir_rule.flex_bytes_offset); + ret = ixgbe_fdir_set_input_mask(dev); if (ret) goto out; fdir_info->mask_added = TRUE; + first_mask = TRUE; } else { /** * Only support one global mask, @@ -2520,6 +2796,10 @@ ixgbe_flow_create(struct rte_eth_dev *dev, sizeof(struct ixgbe_hw_fdir_mask)); if (ret) goto out; + + if (fdir_info->flex_bytes_offset != + fdir_rule.flex_bytes_offset) + goto out; } } @@ -2540,8 +2820,15 @@ ixgbe_flow_create(struct rte_eth_dev *dev, return flow; } - if (ret) + if (ret) { + /** + * clean the mask_added flag if fail to + * program + **/ + if (first_mask) + fdir_info->mask_added = FALSE; goto out; + } } goto out; @@ -2583,7 +2870,7 @@ out: * the HW. Because there can be no enough room for the rule. */ static int -ixgbe_flow_validate(__rte_unused struct rte_eth_dev *dev, +ixgbe_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item pattern[], const struct rte_flow_action actions[], @@ -2774,9 +3061,8 @@ ixgbe_flow_flush(struct rte_eth_dev *dev, } const struct rte_flow_ops ixgbe_flow_ops = { - ixgbe_flow_validate, - ixgbe_flow_create, - ixgbe_flow_destroy, - ixgbe_flow_flush, - NULL, + .validate = ixgbe_flow_validate, + .create = ixgbe_flow_create, + .destroy = ixgbe_flow_destroy, + .flush = ixgbe_flow_flush, }; diff --git a/drivers/net/ixgbe/ixgbe_pf.c b/drivers/net/ixgbe/ixgbe_pf.c index d88832e5..c0d86c76 100644 --- a/drivers/net/ixgbe/ixgbe_pf.c +++ b/drivers/net/ixgbe/ixgbe_pf.c @@ -61,7 +61,7 @@ static inline uint16_t dev_num_vf(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); return pci_dev->max_vfs; } @@ -511,7 +511,7 @@ ixgbe_vf_set_mac_addr(struct rte_eth_dev *dev, uint32_t vf, uint32_t *msgbuf) } static int -ixgbe_vf_set_multicast(struct rte_eth_dev *dev, __rte_unused uint32_t vf, uint32_t *msgbuf) +ixgbe_vf_set_multicast(struct rte_eth_dev *dev, uint32_t vf, uint32_t *msgbuf) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = @@ -683,7 +683,7 @@ ixgbe_rcv_msg_from_vf(struct rte_eth_dev *dev, uint16_t vf) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); - struct rte_pmd_ixgbe_mb_event_param cb_param; + struct rte_pmd_ixgbe_mb_event_param ret_param; retval = ixgbe_read_mbx(hw, msgbuf, mbx_size, vf); if (retval) { @@ -702,10 +702,10 @@ ixgbe_rcv_msg_from_vf(struct rte_eth_dev *dev, uint16_t vf) * initialise structure to send to user application * will return response from user in retval field */ - cb_param.retval = RTE_PMD_IXGBE_MB_EVENT_PROCEED; - cb_param.vfid = vf; - cb_param.msg_type = msgbuf[0] & 0xFFFF; - cb_param.msg = (void *)msgbuf; + ret_param.retval = RTE_PMD_IXGBE_MB_EVENT_PROCEED; + ret_param.vfid = vf; + ret_param.msg_type = msgbuf[0] & 0xFFFF; + ret_param.msg = (void *)msgbuf; /* perform VF reset */ if (msgbuf[0] == IXGBE_VF_RESET) { @@ -714,20 +714,22 @@ ixgbe_rcv_msg_from_vf(struct rte_eth_dev *dev, uint16_t vf) vfinfo[vf].clear_to_send = true; /* notify application about VF reset */ - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, &cb_param); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); return ret; } /** * ask user application if we allowed to perform those functions - * if we get cb_param.retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED + * if we get ret_param.retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED * then business as usual, * if 0, do nothing and send ACK to VF - * if cb_param.retval > 1, do nothing and send NAK to VF + * if ret_param.retval > 1, do nothing and send NAK to VF */ - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, &cb_param); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); - retval = cb_param.retval; + retval = ret_param.retval; /* check & process VF to PF mailbox message */ switch ((msgbuf[0] & 0xFFFF)) { diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c index 1e078959..64bff258 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.c +++ b/drivers/net/ixgbe/ixgbe_rxtx.c @@ -126,7 +126,7 @@ uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, * Check for descriptors with their DD bit set and free mbufs. * Return the total number of buffers freed. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) { struct ixgbe_tx_entry *txep; @@ -1084,282 +1084,279 @@ ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD -#define IXGBE_PACKET_TYPE_MAX 0X80 -#define IXGBE_PACKET_TYPE_TN_MAX 0X100 -#define IXGBE_PACKET_TYPE_SHIFT 0X04 +/** + * Use 2 different table for normal packet and tunnel packet + * to save the space. + */ +const uint32_t + ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = { + [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER, + [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4, + [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT, + [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6, + [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] = + RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, +}; + +const uint32_t + ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = { + [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER, + [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT, + [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | + RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_UDP, + + [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER, + [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT, + [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, +}; /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */ static inline uint32_t ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask) { - /** - * Use 2 different table for normal packet and tunnel packet - * to save the space. - */ - static const uint32_t - ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = { - [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER, - [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4, - [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT, - [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6, - [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] = - RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, - }; - - static const uint32_t - ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = { - [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER, - [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT, - [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | - RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | - RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | - RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | - RTE_PTYPE_INNER_L4_UDP, - - [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER, - [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT, - [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, - }; if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF)) return RTE_PTYPE_UNKNOWN; @@ -4111,10 +4108,7 @@ ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq) return -ENOMEM; } - rte_mbuf_refcnt_set(mbuf, 1); - mbuf->next = NULL; mbuf->data_off = RTE_PKTMBUF_HEADROOM; - mbuf->nb_segs = 1; mbuf->port = rxq->port_id; dma_addr = diff --git a/drivers/net/ixgbe/ixgbe_rxtx.h b/drivers/net/ixgbe/ixgbe_rxtx.h index 1ffab4cc..85feb0bd 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.h +++ b/drivers/net/ixgbe/ixgbe_rxtx.h @@ -87,6 +87,10 @@ #define IXGBE_PACKET_TYPE_MASK_TUNNEL 0XFF #define IXGBE_PACKET_TYPE_TUNNEL_BIT 0X1000 +#define IXGBE_PACKET_TYPE_MAX 0X80 +#define IXGBE_PACKET_TYPE_TN_MAX 0X100 +#define IXGBE_PACKET_TYPE_SHIFT 0X04 + /** * Structure associated with each descriptor of the RX ring of a RX queue. */ @@ -309,6 +313,9 @@ int ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev); int ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq); void ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq); +extern const uint32_t ptype_table[IXGBE_PACKET_TYPE_MAX]; +extern const uint32_t ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX]; + #ifdef RTE_IXGBE_INC_VECTOR uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h index 1c34bb5f..9fc112b1 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h @@ -101,7 +101,7 @@ reassemble_packets(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_bufs, return pkt_idx; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) { struct ixgbe_tx_entry_v *txep; @@ -158,7 +158,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) return txq->tx_rs_thresh; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void tx_backlog_entry(struct ixgbe_tx_entry_v *txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c index a7bc199f..e704a7f3 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c @@ -87,6 +87,8 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) mb1 = rxep[1].mbuf; /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) != + offsetof(struct rte_mbuf, buf_addr) + 8); vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr)); vaddr1 = _mm_loadu_si128((__m128i *)&(mb1->buf_addr)); @@ -214,32 +216,84 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, * appropriate flags means that we have to do a shift and blend for * each mbuf before we do the write. */ -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 - rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 8), 0x10); rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 6), 0x10); rearm2 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 4), 0x10); rearm3 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 2), 0x10); -#else - rearm0 = _mm_slli_si128(vtag1, 14); - rearm1 = _mm_slli_si128(vtag1, 12); - rearm2 = _mm_slli_si128(vtag1, 10); - rearm3 = _mm_slli_si128(vtag1, 8); - - rearm0 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm0, 48)); - rearm1 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm1, 48)); - rearm2 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm2, 48)); - rearm3 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm3, 48)); - -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ - + /* write the rearm data and the olflags in one write */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); } +static inline uint32_t get_packet_type(int index, + uint32_t pkt_info, + uint32_t etqf_check, + uint32_t tunnel_check) +{ + if (etqf_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) + return RTE_PTYPE_UNKNOWN; + + if (tunnel_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) { + pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL; + return ptype_table_tn[pkt_info]; + } + + pkt_info &= IXGBE_PACKET_TYPE_MASK_82599; + return ptype_table[pkt_info]; +} + +static inline void +desc_to_ptype_v(__m128i descs[4], uint16_t pkt_type_mask, + struct rte_mbuf **rx_pkts) +{ + __m128i etqf_mask = _mm_set_epi64x(0x800000008000LL, 0x800000008000LL); + __m128i ptype_mask = _mm_set_epi32( + pkt_type_mask, pkt_type_mask, pkt_type_mask, pkt_type_mask); + __m128i tunnel_mask = + _mm_set_epi64x(0x100000001000LL, 0x100000001000LL); + + uint32_t etqf_check, tunnel_check, pkt_info; + + __m128i ptype0 = _mm_unpacklo_epi32(descs[0], descs[2]); + __m128i ptype1 = _mm_unpacklo_epi32(descs[1], descs[3]); + + /* interleave low 32 bits, + * now we have 4 ptypes in a XMM register + */ + ptype0 = _mm_unpacklo_epi32(ptype0, ptype1); + + /* create a etqf bitmask based on the etqf bit. */ + etqf_check = _mm_movemask_epi8(_mm_and_si128(ptype0, etqf_mask)); + + /* shift left by IXGBE_PACKET_TYPE_SHIFT, and apply ptype mask */ + ptype0 = _mm_and_si128(_mm_srli_epi32(ptype0, IXGBE_PACKET_TYPE_SHIFT), + ptype_mask); + + /* create a tunnel bitmask based on the tunnel bit */ + tunnel_check = _mm_movemask_epi8( + _mm_slli_epi32(_mm_and_si128(ptype0, tunnel_mask), 0x3)); + + pkt_info = _mm_extract_epi32(ptype0, 0); + rx_pkts[0]->packet_type = + get_packet_type(0, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 1); + rx_pkts[1]->packet_type = + get_packet_type(1, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 2); + rx_pkts[2]->packet_type = + get_packet_type(2, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 3); + rx_pkts[3]->packet_type = + get_packet_type(3, pkt_info, etqf_check, tunnel_check); +} + /* * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * @@ -266,6 +320,15 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, -rxq->crc_len, /* sub crc on pkt_len */ 0, 0 /* ignore pkt_type field */ ); + /* + * compile-time check the above crc_adjust layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi16 + * call above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); __m128i dd_check, eop_check; __m128i mbuf_init; uint8_t vlan_flags; @@ -312,6 +375,19 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* skip 32 bit pkt_type */ 0xFF, 0xFF ); + /* + * Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); @@ -447,6 +523,8 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, pkt_mb1); + desc_to_ptype_v(descs, rxq->pkt_type_mask, &rx_pkts[pos]); + /* C.4 calc avaialbe number of desc */ var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); nb_pkts_recd += var; diff --git a/drivers/net/ixgbe/ixgbe_tm.c b/drivers/net/ixgbe/ixgbe_tm.c new file mode 100644 index 00000000..cdcf45cb --- /dev/null +++ b/drivers/net/ixgbe/ixgbe_tm.c @@ -0,0 +1,1043 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_malloc.h> + +#include "ixgbe_ethdev.h" + +static int ixgbe_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error); +static int ixgbe_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error); +static int ixgbe_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error); +static int ixgbe_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error); +static int ixgbe_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error); +static int ixgbe_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error); +static int ixgbe_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error); +static int ixgbe_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error); +static int ixgbe_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error); + +const struct rte_tm_ops ixgbe_tm_ops = { + .capabilities_get = ixgbe_tm_capabilities_get, + .shaper_profile_add = ixgbe_shaper_profile_add, + .shaper_profile_delete = ixgbe_shaper_profile_del, + .node_add = ixgbe_node_add, + .node_delete = ixgbe_node_delete, + .node_type_get = ixgbe_node_type_get, + .level_capabilities_get = ixgbe_level_capabilities_get, + .node_capabilities_get = ixgbe_node_capabilities_get, + .hierarchy_commit = ixgbe_hierarchy_commit, +}; + +int +ixgbe_tm_ops_get(struct rte_eth_dev *dev __rte_unused, + void *arg) +{ + if (!arg) + return -EINVAL; + + *(const void **)arg = &ixgbe_tm_ops; + + return 0; +} + +void +ixgbe_tm_conf_init(struct rte_eth_dev *dev) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + + /* initialize shaper profile list */ + TAILQ_INIT(&tm_conf->shaper_profile_list); + + /* initialize node configuration */ + tm_conf->root = NULL; + TAILQ_INIT(&tm_conf->queue_list); + TAILQ_INIT(&tm_conf->tc_list); + tm_conf->nb_tc_node = 0; + tm_conf->nb_queue_node = 0; + tm_conf->committed = false; +} + +void +ixgbe_tm_conf_uninit(struct rte_eth_dev *dev) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_shaper_profile *shaper_profile; + struct ixgbe_tm_node *tm_node; + + /* clear node configuration */ + while ((tm_node = TAILQ_FIRST(&tm_conf->queue_list))) { + TAILQ_REMOVE(&tm_conf->queue_list, tm_node, node); + rte_free(tm_node); + } + tm_conf->nb_queue_node = 0; + while ((tm_node = TAILQ_FIRST(&tm_conf->tc_list))) { + TAILQ_REMOVE(&tm_conf->tc_list, tm_node, node); + rte_free(tm_node); + } + tm_conf->nb_tc_node = 0; + if (tm_conf->root) { + rte_free(tm_conf->root); + tm_conf->root = NULL; + } + + /* Remove all shaper profiles */ + while ((shaper_profile = + TAILQ_FIRST(&tm_conf->shaper_profile_list))) { + TAILQ_REMOVE(&tm_conf->shaper_profile_list, + shaper_profile, node); + rte_free(shaper_profile); + } +} + +static inline uint8_t +ixgbe_tc_nb_get(struct rte_eth_dev *dev) +{ + struct rte_eth_conf *eth_conf; + uint8_t nb_tcs = 0; + + eth_conf = &dev->data->dev_conf; + if (eth_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { + nb_tcs = eth_conf->tx_adv_conf.dcb_tx_conf.nb_tcs; + } else if (eth_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) { + if (eth_conf->tx_adv_conf.vmdq_dcb_tx_conf.nb_queue_pools == + ETH_32_POOLS) + nb_tcs = ETH_4_TCS; + else + nb_tcs = ETH_8_TCS; + } else { + nb_tcs = 1; + } + + return nb_tcs; +} + +static int +ixgbe_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint8_t tc_nb = ixgbe_tc_nb_get(dev); + + if (!cap || !error) + return -EINVAL; + + if (tc_nb > hw->mac.max_tx_queues) + return -EINVAL; + + error->type = RTE_TM_ERROR_TYPE_NONE; + + /* set all the parameters to 0 first. */ + memset(cap, 0, sizeof(struct rte_tm_capabilities)); + + /** + * here is the max capability not the current configuration. + */ + /* port + TCs + queues */ + cap->n_nodes_max = 1 + IXGBE_DCB_MAX_TRAFFIC_CLASS + + hw->mac.max_tx_queues; + cap->n_levels_max = 3; + cap->non_leaf_nodes_identical = 1; + cap->leaf_nodes_identical = 1; + cap->shaper_n_max = cap->n_nodes_max; + cap->shaper_private_n_max = cap->n_nodes_max; + cap->shaper_private_dual_rate_n_max = 0; + cap->shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->shaper_private_rate_max = 1250000000ull; + cap->shaper_shared_n_max = 0; + cap->shaper_shared_n_nodes_per_shaper_max = 0; + cap->shaper_shared_n_shapers_per_node_max = 0; + cap->shaper_shared_dual_rate_n_max = 0; + cap->shaper_shared_rate_min = 0; + cap->shaper_shared_rate_max = 0; + cap->sched_n_children_max = hw->mac.max_tx_queues; + /** + * HW supports SP. But no plan to support it now. + * So, all the nodes should have the same priority. + */ + cap->sched_sp_n_priorities_max = 1; + cap->sched_wfq_n_children_per_group_max = 0; + cap->sched_wfq_n_groups_max = 0; + /** + * SW only supports fair round robin now. + * So, all the nodes should have the same weight. + */ + cap->sched_wfq_weight_max = 1; + cap->cman_head_drop_supported = 0; + cap->dynamic_update_mask = 0; + cap->shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD; + cap->shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + cap->cman_wred_context_n_max = 0; + cap->cman_wred_context_private_n_max = 0; + cap->cman_wred_context_shared_n_max = 0; + cap->cman_wred_context_shared_n_nodes_per_context_max = 0; + cap->cman_wred_context_shared_n_contexts_per_node_max = 0; + cap->stats_mask = 0; + + return 0; +} + +static inline struct ixgbe_tm_shaper_profile * +ixgbe_shaper_profile_search(struct rte_eth_dev *dev, + uint32_t shaper_profile_id) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_shaper_profile_list *shaper_profile_list = + &tm_conf->shaper_profile_list; + struct ixgbe_tm_shaper_profile *shaper_profile; + + TAILQ_FOREACH(shaper_profile, shaper_profile_list, node) { + if (shaper_profile_id == shaper_profile->shaper_profile_id) + return shaper_profile; + } + + return NULL; +} + +static int +ixgbe_shaper_profile_param_check(struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + /* min rate not supported */ + if (profile->committed.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE; + error->message = "committed rate not supported"; + return -EINVAL; + } + /* min bucket size not supported */ + if (profile->committed.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE; + error->message = "committed bucket size not supported"; + return -EINVAL; + } + /* max bucket size not supported */ + if (profile->peak.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE; + error->message = "peak bucket size not supported"; + return -EINVAL; + } + /* length adjustment not supported */ + if (profile->pkt_length_adjust) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN; + error->message = "packet length adjustment not supported"; + return -EINVAL; + } + + return 0; +} + +static int +ixgbe_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_shaper_profile *shaper_profile; + int ret; + + if (!profile || !error) + return -EINVAL; + + ret = ixgbe_shaper_profile_param_check(profile, error); + if (ret) + return ret; + + shaper_profile = ixgbe_shaper_profile_search(dev, shaper_profile_id); + + if (shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID exist"; + return -EINVAL; + } + + shaper_profile = rte_zmalloc("ixgbe_tm_shaper_profile", + sizeof(struct ixgbe_tm_shaper_profile), + 0); + if (!shaper_profile) + return -ENOMEM; + shaper_profile->shaper_profile_id = shaper_profile_id; + (void)rte_memcpy(&shaper_profile->profile, profile, + sizeof(struct rte_tm_shaper_params)); + TAILQ_INSERT_TAIL(&tm_conf->shaper_profile_list, + shaper_profile, node); + + return 0; +} + +static int +ixgbe_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_shaper_profile *shaper_profile; + + if (!error) + return -EINVAL; + + shaper_profile = ixgbe_shaper_profile_search(dev, shaper_profile_id); + + if (!shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID not exist"; + return -EINVAL; + } + + /* don't delete a profile if it's used by one or several nodes */ + if (shaper_profile->reference_count) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "profile in use"; + return -EINVAL; + } + + TAILQ_REMOVE(&tm_conf->shaper_profile_list, shaper_profile, node); + rte_free(shaper_profile); + + return 0; +} + +static inline struct ixgbe_tm_node * +ixgbe_tm_node_search(struct rte_eth_dev *dev, uint32_t node_id, + enum ixgbe_tm_node_type *node_type) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_node *tm_node; + + if (tm_conf->root && tm_conf->root->id == node_id) { + *node_type = IXGBE_TM_NODE_TYPE_PORT; + return tm_conf->root; + } + + TAILQ_FOREACH(tm_node, &tm_conf->tc_list, node) { + if (tm_node->id == node_id) { + *node_type = IXGBE_TM_NODE_TYPE_TC; + return tm_node; + } + } + + TAILQ_FOREACH(tm_node, &tm_conf->queue_list, node) { + if (tm_node->id == node_id) { + *node_type = IXGBE_TM_NODE_TYPE_QUEUE; + return tm_node; + } + } + + return NULL; +} + +static void +ixgbe_queue_base_nb_get(struct rte_eth_dev *dev, uint16_t tc_node_no, + uint16_t *base, uint16_t *nb) +{ + uint8_t nb_tcs = ixgbe_tc_nb_get(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + uint16_t vf_num = pci_dev->max_vfs; + + *base = 0; + *nb = 0; + + /* VT on */ + if (vf_num) { + /* no DCB */ + if (nb_tcs == 1) { + if (vf_num >= ETH_32_POOLS) { + *nb = 2; + *base = vf_num * 2; + } else if (vf_num >= ETH_16_POOLS) { + *nb = 4; + *base = vf_num * 4; + } else { + *nb = 8; + *base = vf_num * 8; + } + } else { + /* DCB */ + *nb = 1; + *base = vf_num * nb_tcs + tc_node_no; + } + } else { + /* VT off */ + if (nb_tcs == ETH_8_TCS) { + switch (tc_node_no) { + case 0: + *base = 0; + *nb = 32; + break; + case 1: + *base = 32; + *nb = 32; + break; + case 2: + *base = 64; + *nb = 16; + break; + case 3: + *base = 80; + *nb = 16; + break; + case 4: + *base = 96; + *nb = 8; + break; + case 5: + *base = 104; + *nb = 8; + break; + case 6: + *base = 112; + *nb = 8; + break; + case 7: + *base = 120; + *nb = 8; + break; + default: + return; + } + } else { + switch (tc_node_no) { + /** + * If no VF and no DCB, only 64 queues can be used. + * This case also be covered by this "case 0". + */ + case 0: + *base = 0; + *nb = 64; + break; + case 1: + *base = 64; + *nb = 32; + break; + case 2: + *base = 96; + *nb = 16; + break; + case 3: + *base = 112; + *nb = 16; + break; + default: + return; + } + } + } +} + +static int +ixgbe_node_param_check(uint32_t node_id, uint32_t parent_node_id, + uint32_t priority, uint32_t weight, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + if (priority) { + error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY; + error->message = "priority should be 0"; + return -EINVAL; + } + + if (weight != 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT; + error->message = "weight must be 1"; + return -EINVAL; + } + + /* not support shared shaper */ + if (params->shared_shaper_id) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID; + error->message = "shared shaper not supported"; + return -EINVAL; + } + if (params->n_shared_shapers) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS; + error->message = "shared shaper not supported"; + return -EINVAL; + } + + /* for root node */ + if (parent_node_id == RTE_TM_NODE_ID_NULL) { + /* check the unsupported parameters */ + if (params->nonleaf.wfq_weight_mode) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFQ not supported"; + return -EINVAL; + } + if (params->nonleaf.n_sp_priorities != 1) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES; + error->message = "SP priority not supported"; + return -EINVAL; + } else if (params->nonleaf.wfq_weight_mode && + !(*params->nonleaf.wfq_weight_mode)) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFP should be byte mode"; + return -EINVAL; + } + + return 0; + } + + /* for TC or queue node */ + /* check the unsupported parameters */ + if (params->leaf.cman) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN; + error->message = "Congestion management not supported"; + return -EINVAL; + } + if (params->leaf.wred.wred_profile_id != + RTE_TM_WRED_PROFILE_ID_NONE) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.shared_wred_context_id) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.n_shared_wred_contexts) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS; + error->message = "WRED not supported"; + return -EINVAL; + } + + return 0; +} + +/** + * Now the TC and queue configuration is controlled by DCB. + * We need check if the node configuration follows the DCB configuration. + * In the future, we may use TM to cover DCB. + */ +static int +ixgbe_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + enum ixgbe_tm_node_type parent_node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_shaper_profile *shaper_profile; + struct ixgbe_tm_node *tm_node; + struct ixgbe_tm_node *parent_node; + uint8_t nb_tcs; + uint16_t q_base = 0; + uint16_t q_nb = 0; + int ret; + + if (!params || !error) + return -EINVAL; + + /* if already committed */ + if (tm_conf->committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + ret = ixgbe_node_param_check(node_id, parent_node_id, priority, weight, + params, error); + if (ret) + return ret; + + /* check if the node ID is already used */ + if (ixgbe_tm_node_search(dev, node_id, &node_type)) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "node id already used"; + return -EINVAL; + } + + /* check the shaper profile id */ + shaper_profile = ixgbe_shaper_profile_search(dev, + params->shaper_profile_id); + if (!shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID; + error->message = "shaper profile not exist"; + return -EINVAL; + } + + /* root node if not have a parent */ + if (parent_node_id == RTE_TM_NODE_ID_NULL) { + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id > IXGBE_TM_NODE_TYPE_PORT) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* obviously no more than one root */ + if (tm_conf->root) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "already have a root"; + return -EINVAL; + } + + /* add the root node */ + tm_node = rte_zmalloc("ixgbe_tm_node", + sizeof(struct ixgbe_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->no = 0; + tm_node->parent = NULL; + tm_node->shaper_profile = shaper_profile; + (void)rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + tm_conf->root = tm_node; + + /* increase the reference counter of the shaper profile */ + shaper_profile->reference_count++; + + return 0; + } + + /* TC or queue node */ + /* check the parent node */ + parent_node = ixgbe_tm_node_search(dev, parent_node_id, + &parent_node_type); + if (!parent_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent not exist"; + return -EINVAL; + } + if (parent_node_type != IXGBE_TM_NODE_TYPE_PORT && + parent_node_type != IXGBE_TM_NODE_TYPE_TC) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent is not port or TC"; + return -EINVAL; + } + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id != parent_node_type + 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* check the node number */ + if (parent_node_type == IXGBE_TM_NODE_TYPE_PORT) { + /* check TC number */ + nb_tcs = ixgbe_tc_nb_get(dev); + if (tm_conf->nb_tc_node >= nb_tcs) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many TCs"; + return -EINVAL; + } + } else { + /* check queue number */ + if (tm_conf->nb_queue_node >= dev->data->nb_tx_queues) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many queues"; + return -EINVAL; + } + + ixgbe_queue_base_nb_get(dev, parent_node->no, &q_base, &q_nb); + if (parent_node->reference_count >= q_nb) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many queues than TC supported"; + return -EINVAL; + } + + /** + * check the node id. + * For queue, the node id means queue id. + */ + if (node_id >= dev->data->nb_tx_queues) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too large queue id"; + return -EINVAL; + } + } + + /* add the TC or queue node */ + tm_node = rte_zmalloc("ixgbe_tm_node", + sizeof(struct ixgbe_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->parent = parent_node; + tm_node->shaper_profile = shaper_profile; + (void)rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + if (parent_node_type == IXGBE_TM_NODE_TYPE_PORT) { + tm_node->no = parent_node->reference_count; + TAILQ_INSERT_TAIL(&tm_conf->tc_list, + tm_node, node); + tm_conf->nb_tc_node++; + } else { + tm_node->no = q_base + parent_node->reference_count; + TAILQ_INSERT_TAIL(&tm_conf->queue_list, + tm_node, node); + tm_conf->nb_queue_node++; + } + tm_node->parent->reference_count++; + + /* increase the reference counter of the shaper profile */ + shaper_profile->reference_count++; + + return 0; +} + +static int +ixgbe_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_node *tm_node; + + if (!error) + return -EINVAL; + + /* if already committed */ + if (tm_conf->committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check the if the node id exists */ + tm_node = ixgbe_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + /* the node should have no child */ + if (tm_node->reference_count) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = + "cannot delete a node which has children"; + return -EINVAL; + } + + /* root node */ + if (node_type == IXGBE_TM_NODE_TYPE_PORT) { + tm_node->shaper_profile->reference_count--; + rte_free(tm_node); + tm_conf->root = NULL; + return 0; + } + + /* TC or queue node */ + tm_node->shaper_profile->reference_count--; + tm_node->parent->reference_count--; + if (node_type == IXGBE_TM_NODE_TYPE_TC) { + TAILQ_REMOVE(&tm_conf->tc_list, tm_node, node); + tm_conf->nb_tc_node--; + } else { + TAILQ_REMOVE(&tm_conf->queue_list, tm_node, node); + tm_conf->nb_queue_node--; + } + rte_free(tm_node); + + return 0; +} + +static int +ixgbe_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error) +{ + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_node *tm_node; + + if (!is_leaf || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = ixgbe_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + if (node_type == IXGBE_TM_NODE_TYPE_QUEUE) + *is_leaf = true; + else + *is_leaf = false; + + return 0; +} + +static int +ixgbe_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!cap || !error) + return -EINVAL; + + if (level_id >= IXGBE_TM_NODE_TYPE_MAX) { + error->type = RTE_TM_ERROR_TYPE_LEVEL_ID; + error->message = "too deep level"; + return -EINVAL; + } + + /* root node */ + if (level_id == IXGBE_TM_NODE_TYPE_PORT) { + cap->n_nodes_max = 1; + cap->n_nodes_nonleaf_max = 1; + cap->n_nodes_leaf_max = 0; + cap->non_leaf_nodes_identical = true; + cap->leaf_nodes_identical = true; + cap->nonleaf.shaper_private_supported = true; + cap->nonleaf.shaper_private_dual_rate_supported = false; + cap->nonleaf.shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->nonleaf.shaper_private_rate_max = 1250000000ull; + cap->nonleaf.shaper_shared_n_max = 0; + cap->nonleaf.sched_n_children_max = IXGBE_DCB_MAX_TRAFFIC_CLASS; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + cap->nonleaf.stats_mask = 0; + + return 0; + } + + /* TC or queue node */ + if (level_id == IXGBE_TM_NODE_TYPE_TC) { + /* TC */ + cap->n_nodes_max = IXGBE_DCB_MAX_TRAFFIC_CLASS; + cap->n_nodes_nonleaf_max = IXGBE_DCB_MAX_TRAFFIC_CLASS; + cap->n_nodes_leaf_max = 0; + cap->non_leaf_nodes_identical = true; + } else { + /* queue */ + cap->n_nodes_max = hw->mac.max_tx_queues; + cap->n_nodes_nonleaf_max = 0; + cap->n_nodes_leaf_max = hw->mac.max_tx_queues; + cap->non_leaf_nodes_identical = true; + } + cap->leaf_nodes_identical = true; + cap->leaf.shaper_private_supported = true; + cap->leaf.shaper_private_dual_rate_supported = false; + cap->leaf.shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->leaf.shaper_private_rate_max = 1250000000ull; + cap->leaf.shaper_shared_n_max = 0; + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + cap->leaf.stats_mask = 0; + + return 0; +} + +static int +ixgbe_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_node *tm_node; + + if (!cap || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = ixgbe_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + cap->shaper_private_supported = true; + cap->shaper_private_dual_rate_supported = false; + cap->shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->shaper_private_rate_max = 1250000000ull; + cap->shaper_shared_n_max = 0; + + if (node_type == IXGBE_TM_NODE_TYPE_QUEUE) { + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + } else { + if (node_type == IXGBE_TM_NODE_TYPE_PORT) + cap->nonleaf.sched_n_children_max = + IXGBE_DCB_MAX_TRAFFIC_CLASS; + else + cap->nonleaf.sched_n_children_max = + hw->mac.max_tx_queues; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + } + + cap->stats_mask = 0; + + return 0; +} + +static int +ixgbe_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_node *tm_node; + uint64_t bw; + int ret; + + if (!error) + return -EINVAL; + + /* check the setting */ + if (!tm_conf->root) + goto done; + + /* not support port max bandwidth yet */ + if (tm_conf->root->shaper_profile->profile.peak.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "no port max bandwidth"; + goto fail_clear; + } + + /* HW not support TC max bandwidth */ + TAILQ_FOREACH(tm_node, &tm_conf->tc_list, node) { + if (tm_node->shaper_profile->profile.peak.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "no TC max bandwidth"; + goto fail_clear; + } + } + + /* queue max bandwidth */ + TAILQ_FOREACH(tm_node, &tm_conf->queue_list, node) { + bw = tm_node->shaper_profile->profile.peak.rate; + if (bw) { + /* interpret Bps to Mbps */ + bw = bw * 8 / 1000 / 1000; + ret = ixgbe_set_queue_rate_limit(dev, tm_node->no, bw); + if (ret) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = + "failed to set queue max bandwidth"; + goto fail_clear; + } + } + } + +done: + tm_conf->committed = true; + return 0; + +fail_clear: + /* clear all the traffic manager configuration */ + if (clear_on_fail) { + ixgbe_tm_conf_uninit(dev); + ixgbe_tm_conf_init(dev); + } + return -EINVAL; +} diff --git a/drivers/net/ixgbe/rte_pmd_ixgbe.c b/drivers/net/ixgbe/rte_pmd_ixgbe.c index e8fc9a64..79897ff6 100644 --- a/drivers/net/ixgbe/rte_pmd_ixgbe.c +++ b/drivers/net/ixgbe/rte_pmd_ixgbe.c @@ -51,7 +51,7 @@ rte_pmd_ixgbe_set_vf_mac_addr(uint8_t port, uint16_t vf, RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -84,7 +84,7 @@ rte_pmd_ixgbe_ping_vf(uint8_t port, uint16_t vf) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -115,7 +115,7 @@ rte_pmd_ixgbe_set_vf_vlan_anti_spoof(uint8_t port, uint16_t vf, uint8_t on) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -145,7 +145,7 @@ rte_pmd_ixgbe_set_vf_mac_anti_spoof(uint8_t port, uint16_t vf, uint8_t on) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -174,7 +174,7 @@ rte_pmd_ixgbe_set_vf_vlan_insert(uint8_t port, uint16_t vf, uint16_t vlan_id) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -270,7 +270,7 @@ rte_pmd_ixgbe_set_vf_split_drop_en(uint8_t port, uint16_t vf, uint8_t on) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -306,7 +306,7 @@ rte_pmd_ixgbe_set_vf_vlan_stripq(uint8_t port, uint16_t vf, uint8_t on) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); if (!is_ixgbe_supported(dev)) @@ -354,7 +354,7 @@ rte_pmd_ixgbe_set_vf_rxmode(uint8_t port, uint16_t vf, RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -401,7 +401,7 @@ rte_pmd_ixgbe_set_vf_rx(uint8_t port, uint16_t vf, uint8_t on) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -452,7 +452,7 @@ rte_pmd_ixgbe_set_vf_tx(uint8_t port, uint16_t vf, uint8_t on) RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); dev = &rte_eth_devices[port]; - pci_dev = IXGBE_DEV_TO_PCI(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (!is_ixgbe_supported(dev)) return -ENOTSUP; @@ -908,3 +908,136 @@ rte_pmd_ixgbe_set_tc_bw_alloc(uint8_t port, return 0; } + +#ifdef RTE_LIBRTE_IXGBE_BYPASS +int +rte_pmd_ixgbe_bypass_init(uint8_t port_id) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + ixgbe_bypass_init(dev); + return 0; +} + +int +rte_pmd_ixgbe_bypass_state_show(uint8_t port_id, uint32_t *state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_state_show(dev, state); +} + +int +rte_pmd_ixgbe_bypass_state_set(uint8_t port_id, uint32_t *new_state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_state_store(dev, new_state); +} + +int +rte_pmd_ixgbe_bypass_event_show(uint8_t port_id, + uint32_t event, + uint32_t *state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_event_show(dev, event, state); +} + +int +rte_pmd_ixgbe_bypass_event_store(uint8_t port_id, + uint32_t event, + uint32_t state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_event_store(dev, event, state); +} + +int +rte_pmd_ixgbe_bypass_wd_timeout_store(uint8_t port_id, uint32_t timeout) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_wd_timeout_store(dev, timeout); +} + +int +rte_pmd_ixgbe_bypass_ver_show(uint8_t port_id, uint32_t *ver) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_ver_show(dev, ver); +} + +int +rte_pmd_ixgbe_bypass_wd_timeout_show(uint8_t port_id, uint32_t *wd_timeout) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_wd_timeout_show(dev, wd_timeout); +} + +int +rte_pmd_ixgbe_bypass_wd_reset(uint8_t port_id) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_wd_reset(dev); +} +#endif diff --git a/drivers/net/ixgbe/rte_pmd_ixgbe.h b/drivers/net/ixgbe/rte_pmd_ixgbe.h index 1f2b1bd7..d33c285d 100644 --- a/drivers/net/ixgbe/rte_pmd_ixgbe.h +++ b/drivers/net/ixgbe/rte_pmd_ixgbe.h @@ -427,6 +427,177 @@ int rte_pmd_ixgbe_set_tc_bw_alloc(uint8_t port, uint8_t tc_num, uint8_t *bw_weight); + +/** + * Initialize bypass logic. This function needs to be called before + * executing any other bypass API. + * + * @param port + * The port identifier of the Ethernet device. + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_init(uint8_t port); + +/** + * Return bypass state. + * + * @param port + * The port identifier of the Ethernet device. + * @param state + * The return bypass state. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_state_show(uint8_t port, uint32_t *state); + +/** + * Set bypass state + * + * @param port + * The port identifier of the Ethernet device. + * @param new_state + * The current bypass state. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_state_set(uint8_t port, uint32_t *new_state); + +/** + * Return bypass state when given event occurs. + * + * @param port + * The port identifier of the Ethernet device. + * @param event + * The bypass event + * - (1) Main power on (power button is pushed) + * - (2) Auxiliary power on (power supply is being plugged) + * - (3) Main power off (system shutdown and power supply is left plugged in) + * - (4) Auxiliary power off (power supply is being unplugged) + * - (5) Display or set the watchdog timer + * @param state + * The bypass state when given event occurred. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_event_show(uint8_t port, + uint32_t event, + uint32_t *state); + +/** + * Set bypass state when given event occurs. + * + * @param port + * The port identifier of the Ethernet device. + * @param event + * The bypass event + * - (1) Main power on (power button is pushed) + * - (2) Auxiliary power on (power supply is being plugged) + * - (3) Main power off (system shutdown and power supply is left plugged in) + * - (4) Auxiliary power off (power supply is being unplugged) + * - (5) Display or set the watchdog timer + * @param state + * The assigned state when given event occurs. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_event_store(uint8_t port, + uint32_t event, + uint32_t state); + +/** + * Set bypass watchdog timeout count. + * + * @param port + * The port identifier of the Ethernet device. + * @param timeout + * The timeout to be set. + * - (0) 0 seconds (timer is off) + * - (1) 1.5 seconds + * - (2) 2 seconds + * - (3) 3 seconds + * - (4) 4 seconds + * - (5) 8 seconds + * - (6) 16 seconds + * - (7) 32 seconds + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_wd_timeout_store(uint8_t port, uint32_t timeout); + +/** + * Get bypass firmware version. + * + * @param port + * The port identifier of the Ethernet device. + * @param ver + * The firmware version + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_ver_show(uint8_t port, uint32_t *ver); + +/** + * Return bypass watchdog timeout in seconds + * + * @param port + * The port identifier of the Ethernet device. + * @param wd_timeout + * The return watchdog timeout. "0" represents timer expired + * - (0) 0 seconds (timer is off) + * - (1) 1.5 seconds + * - (2) 2 seconds + * - (3) 3 seconds + * - (4) 4 seconds + * - (5) 8 seconds + * - (6) 16 seconds + * - (7) 32 seconds + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_wd_timeout_show(uint8_t port, uint32_t *wd_timeout); + +/** + * Reset bypass watchdog timer + * + * @param port + * The port identifier of the Ethernet device. + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_wd_reset(uint8_t port); + + /** * Response sent back to ixgbe driver from user app after callback */ @@ -446,4 +617,48 @@ struct rte_pmd_ixgbe_mb_event_param { uint16_t retval; /**< return value */ void *msg; /**< pointer to message */ }; +enum { + RTE_PMD_IXGBE_BYPASS_MODE_NONE, + RTE_PMD_IXGBE_BYPASS_MODE_NORMAL, + RTE_PMD_IXGBE_BYPASS_MODE_BYPASS, + RTE_PMD_IXGBE_BYPASS_MODE_ISOLATE, + RTE_PMD_IXGBE_BYPASS_MODE_NUM, +}; + +#define RTE_PMD_IXGBE_BYPASS_MODE_VALID(x) \ + ((x) > RTE_PMD_IXGBE_BYPASS_MODE_NONE && \ + (x) < RTE_PMD_IXGBE_BYPASS_MODE_NUM) + +enum { + RTE_PMD_IXGBE_BYPASS_EVENT_NONE, + RTE_PMD_IXGBE_BYPASS_EVENT_START, + RTE_PMD_IXGBE_BYPASS_EVENT_OS_ON = RTE_PMD_IXGBE_BYPASS_EVENT_START, + RTE_PMD_IXGBE_BYPASS_EVENT_POWER_ON, + RTE_PMD_IXGBE_BYPASS_EVENT_OS_OFF, + RTE_PMD_IXGBE_BYPASS_EVENT_POWER_OFF, + RTE_PMD_IXGBE_BYPASS_EVENT_TIMEOUT, + RTE_PMD_IXGBE_BYPASS_EVENT_NUM +}; + +#define RTE_PMD_IXGBE_BYPASS_EVENT_VALID(x) \ + ((x) > RTE_PMD_IXGBE_BYPASS_EVENT_NONE && \ + (x) < RTE_PMD_IXGBE_BYPASS_MODE_NUM) + +enum { + RTE_PMD_IXGBE_BYPASS_TMT_OFF, /* timeout disabled. */ + RTE_PMD_IXGBE_BYPASS_TMT_1_5_SEC, /* timeout for 1.5 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_2_SEC, /* timeout for 2 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_3_SEC, /* timeout for 3 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_4_SEC, /* timeout for 4 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_8_SEC, /* timeout for 8 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_16_SEC, /* timeout for 16 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_32_SEC, /* timeout for 32 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_NUM +}; + +#define RTE_PMD_IXGBE_BYPASS_TMT_VALID(x) \ + ((x) == RTE_PMD_IXGBE_BYPASS_TMT_OFF || \ + ((x) > RTE_PMD_IXGBE_BYPASS_TMT_OFF && \ + (x) < RTE_PMD_IXGBE_BYPASS_TMT_NUM)) + #endif /* _PMD_IXGBE_H_ */ diff --git a/drivers/net/ixgbe/rte_pmd_ixgbe_version.map b/drivers/net/ixgbe/rte_pmd_ixgbe_version.map index 45a57e33..bf776742 100644 --- a/drivers/net/ixgbe/rte_pmd_ixgbe_version.map +++ b/drivers/net/ixgbe/rte_pmd_ixgbe_version.map @@ -38,3 +38,17 @@ DPDK_17.05 { rte_pmd_ixgbe_ping_vf; rte_pmd_ixgbe_set_tc_bw_alloc; } DPDK_17.02; + +DPDK_17.08 { + global: + + rte_pmd_ixgbe_bypass_event_show; + rte_pmd_ixgbe_bypass_event_store; + rte_pmd_ixgbe_bypass_init; + rte_pmd_ixgbe_bypass_state_set; + rte_pmd_ixgbe_bypass_state_show; + rte_pmd_ixgbe_bypass_ver_show; + rte_pmd_ixgbe_bypass_wd_reset; + rte_pmd_ixgbe_bypass_wd_timeout_show; + rte_pmd_ixgbe_bypass_wd_timeout_store; +} DPDK_17.05; diff --git a/drivers/net/kni/rte_eth_kni.c b/drivers/net/kni/rte_eth_kni.c index f688d919..72a2733b 100644 --- a/drivers/net/kni/rte_eth_kni.c +++ b/drivers/net/kni/rte_eth_kni.c @@ -145,7 +145,7 @@ eth_kni_start(struct rte_eth_dev *dev) uint16_t port_id = dev->data->port_id; struct rte_mempool *mb_pool; struct rte_kni_conf conf; - const char *name = dev->data->name + 4; /* remove net_ */ + const char *name = dev->device->name + 4; /* remove net_ */ snprintf(conf.name, RTE_KNI_NAMESIZE, "%s", name); conf.force_bind = 0; diff --git a/drivers/net/liquidio/base/lio_hw_defs.h b/drivers/net/liquidio/base/lio_hw_defs.h index 67eaa452..de58c7cc 100644 --- a/drivers/net/liquidio/base/lio_hw_defs.h +++ b/drivers/net/liquidio/base/lio_hw_defs.h @@ -42,6 +42,12 @@ #define LIO_CN23XX_VF_VID 0x9712 +/* CN23xx subsystem device ids */ +#define PCI_SUBSYS_DEV_ID_CN2350_210 0x0004 +#define PCI_SUBSYS_DEV_ID_CN2360_210 0x0005 +#define PCI_SUBSYS_DEV_ID_CN2360_225 0x0006 +#define PCI_SUBSYS_DEV_ID_CN2350_225 0x0007 + /* --------------------------CONFIG VALUES------------------------ */ /* CN23xx IQ configuration macros */ diff --git a/drivers/net/liquidio/lio_ethdev.c b/drivers/net/liquidio/lio_ethdev.c index c7f1fb64..479936a5 100644 --- a/drivers/net/liquidio/lio_ethdev.c +++ b/drivers/net/liquidio/lio_ethdev.c @@ -395,6 +395,25 @@ lio_dev_info_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *devinfo) { struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + + devinfo->pci_dev = pci_dev; + + switch (pci_dev->id.subsystem_device_id) { + /* CN23xx 10G cards */ + case PCI_SUBSYS_DEV_ID_CN2350_210: + case PCI_SUBSYS_DEV_ID_CN2360_210: + devinfo->speed_capa = ETH_LINK_SPEED_10G; + break; + /* CN23xx 25G cards */ + case PCI_SUBSYS_DEV_ID_CN2350_225: + case PCI_SUBSYS_DEV_ID_CN2360_225: + devinfo->speed_capa = ETH_LINK_SPEED_25G; + break; + default: + lio_dev_err(lio_dev, + "Unknown CN23XX subsystem device id. Not setting speed capability.\n"); + } devinfo->max_rx_queues = lio_dev->max_rx_queues; devinfo->max_tx_queues = lio_dev->max_tx_queues; @@ -1977,7 +1996,7 @@ lio_eth_dev_uninit(struct rte_eth_dev *eth_dev) static int lio_eth_dev_init(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pdev = RTE_DEV_TO_PCI(eth_dev->device); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev); struct lio_device *lio_dev = LIO_DEV(eth_dev); PMD_INIT_FUNC_TRACE(); diff --git a/drivers/net/liquidio/lio_rxtx.c b/drivers/net/liquidio/lio_rxtx.c index 9533015c..2bbb893c 100644 --- a/drivers/net/liquidio/lio_rxtx.c +++ b/drivers/net/liquidio/lio_rxtx.c @@ -81,28 +81,6 @@ lio_droq_destroy_ring_buffers(struct lio_droq *droq) lio_droq_reset_indices(droq); } -static void * -lio_recv_buffer_alloc(struct lio_device *lio_dev, int q_no) -{ - struct lio_droq *droq = lio_dev->droq[q_no]; - struct rte_mempool *mpool = droq->mpool; - struct rte_mbuf *m; - - m = rte_pktmbuf_alloc(mpool); - if (m == NULL) { - lio_dev_err(lio_dev, "Cannot allocate\n"); - return NULL; - } - - rte_mbuf_refcnt_set(m, 1); - m->next = NULL; - m->data_off = RTE_PKTMBUF_HEADROOM; - m->nb_segs = 1; - m->pool = mpool; - - return m; -} - static int lio_droq_setup_ring_buffers(struct lio_device *lio_dev, struct lio_droq *droq) @@ -112,7 +90,7 @@ lio_droq_setup_ring_buffers(struct lio_device *lio_dev, void *buf; for (i = 0; i < droq->max_count; i++) { - buf = lio_recv_buffer_alloc(lio_dev, droq->q_no); + buf = rte_pktmbuf_alloc(droq->mpool); if (buf == NULL) { lio_dev_err(lio_dev, "buffer alloc failed\n"); droq->stats.rx_alloc_failure++; @@ -378,7 +356,6 @@ lio_droq_refill_pullup_descs(struct lio_droq *droq, /* lio_droq_refill * - * @param lio_dev - pointer to the lio device structure * @param droq - droq in which descriptors require new buffers. * * Description: @@ -394,7 +371,7 @@ lio_droq_refill_pullup_descs(struct lio_droq *droq, * This routine is called with droq->lock held. */ static uint32_t -lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq) +lio_droq_refill(struct lio_droq *droq) { struct lio_droq_desc *desc_ring; uint32_t desc_refilled = 0; @@ -407,7 +384,7 @@ lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq) * reuse the buffer, else allocate. */ if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) { - buf = lio_recv_buffer_alloc(lio_dev, droq->q_no); + buf = rte_pktmbuf_alloc(droq->mpool); /* If a buffer could not be allocated, no point in * continuing */ @@ -489,9 +466,6 @@ lio_droq_fast_process_packet(struct lio_device *lio_dev, droq->refill_count++; if (likely(nicbuf != NULL)) { - nicbuf->data_off = RTE_PKTMBUF_HEADROOM; - nicbuf->nb_segs = 1; - nicbuf->next = NULL; /* We don't have a way to pass flags yet */ nicbuf->ol_flags = 0; if (rh->r_dh.has_hash) { @@ -545,9 +519,6 @@ lio_droq_fast_process_packet(struct lio_device *lio_dev, if (!pkt_len) first_buf = nicbuf; - nicbuf->data_off = RTE_PKTMBUF_HEADROOM; - nicbuf->nb_segs = 1; - nicbuf->next = NULL; nicbuf->port = lio_dev->port_id; /* We don't have a way to pass * flags yet @@ -617,7 +588,7 @@ lio_droq_fast_process_packet(struct lio_device *lio_dev, } if (droq->refill_count >= droq->refill_threshold) { - int desc_refilled = lio_droq_refill(lio_dev, droq); + int desc_refilled = lio_droq_refill(droq); /* Flush the droq descriptor data to memory to be sure * that when we update the credits the data in memory is diff --git a/drivers/net/liquidio/lio_struct.h b/drivers/net/liquidio/lio_struct.h index 26f803f9..d9cbf000 100644 --- a/drivers/net/liquidio/lio_struct.h +++ b/drivers/net/liquidio/lio_struct.h @@ -94,7 +94,7 @@ struct lio_droq_stats { /** Num of vxlan packets received; */ uint64_t rx_vxlan; - /** Num of failures of lio_recv_buffer_alloc() */ + /** Num of failures of rte_pktmbuf_alloc() */ uint64_t rx_alloc_failure; }; diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile index e873fb48..c045bd79 100644 --- a/drivers/net/mlx4/Makefile +++ b/drivers/net/mlx4/Makefile @@ -40,7 +40,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_flow.c # Basic CFLAGS. CFLAGS += -O3 -CFLAGS += -std=gnu99 -Wall -Wextra +CFLAGS += -std=c11 -Wall -Wextra CFLAGS += -g CFLAGS += -I. CFLAGS += -D_BSD_SOURCE @@ -84,6 +84,10 @@ ifdef CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS CFLAGS += -DMLX4_PMD_SOFT_COUNTERS=$(CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS) endif +ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS),y) +CFLAGS += -DMLX4_PMD_DEBUG_BROKEN_VERBS +endif + include $(RTE_SDK)/mk/rte.lib.mk # Generate and clean-up mlx4_autoconf.h. diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index ec4419a8..055de49a 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -75,6 +75,7 @@ #include <rte_memory.h> #include <rte_flow.h> #include <rte_kvargs.h> +#include <rte_interrupts.h> /* Generated configuration header. */ #include "mlx4_autoconf.h" @@ -127,6 +128,18 @@ const char *pmd_mlx4_init_params[] = { NULL, }; +static int +mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx); + +static int +mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx); + +static int +priv_rx_intr_vec_enable(struct priv *priv); + +static void +priv_rx_intr_vec_disable(struct priv *priv); + /** * Check if running as a secondary process. * @@ -533,13 +546,93 @@ txq_cleanup(struct txq *txq); static int rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, - unsigned int socket, int inactive, const struct rte_eth_rxconf *conf, - struct rte_mempool *mp); + unsigned int socket, int inactive, + const struct rte_eth_rxconf *conf, + struct rte_mempool *mp, int children_n, + struct rxq *rxq_parent); static void rxq_cleanup(struct rxq *rxq); /** + * Create RSS parent queue. + * + * The new parent is inserted in front of the list in the private structure. + * + * @param priv + * Pointer to private structure. + * @param queues + * Queues indices array, if NULL use all Rx queues. + * @param children_n + * The number of entries in queues[]. + * + * @return + * Pointer to a parent rxq structure, NULL on failure. + */ +struct rxq * +priv_parent_create(struct priv *priv, + uint16_t queues[], + uint16_t children_n) +{ + int ret; + uint16_t i; + struct rxq *parent; + + parent = rte_zmalloc("parent queue", + sizeof(*parent), + RTE_CACHE_LINE_SIZE); + if (!parent) { + ERROR("cannot allocate memory for RSS parent queue"); + return NULL; + } + ret = rxq_setup(priv->dev, parent, 0, 0, 0, + NULL, NULL, children_n, NULL); + if (ret) { + rte_free(parent); + return NULL; + } + parent->rss.queues_n = children_n; + if (queues) { + for (i = 0; i < children_n; ++i) + parent->rss.queues[i] = queues[i]; + } else { + /* the default RSS ring case */ + assert(priv->rxqs_n == children_n); + for (i = 0; i < priv->rxqs_n; ++i) + parent->rss.queues[i] = i; + } + LIST_INSERT_HEAD(&priv->parents, parent, next); + return parent; +} + +/** + * Clean up RX queue parent structure. + * + * @param parent + * RX queue parent structure. + */ +void +rxq_parent_cleanup(struct rxq *parent) +{ + LIST_REMOVE(parent, next); + rxq_cleanup(parent); + rte_free(parent); +} + +/** + * Clean up parent structures from the parent list. + * + * @param priv + * Pointer to private structure. + */ +static void +priv_parent_list_cleanup(struct priv *priv) +{ + while (!LIST_EMPTY(&priv->parents)) + rxq_parent_cleanup(LIST_FIRST(&priv->parents)); +} + +/** * Ethernet device configuration. * * Prepare the driver for a given number of TX and RX queues. @@ -558,7 +651,6 @@ dev_configure(struct rte_eth_dev *dev) unsigned int rxqs_n = dev->data->nb_rx_queues; unsigned int txqs_n = dev->data->nb_tx_queues; unsigned int tmp; - int ret; priv->rxqs = (void *)dev->data->rx_queues; priv->txqs = (void *)dev->data->tx_queues; @@ -569,7 +661,7 @@ dev_configure(struct rte_eth_dev *dev) } if (rxqs_n == priv->rxqs_n) return 0; - if (!rte_is_power_of_2(rxqs_n)) { + if (!rte_is_power_of_2(rxqs_n) && !priv->isolated) { unsigned n_active; n_active = rte_align32pow2(rxqs_n + 1) >> 1; @@ -588,7 +680,7 @@ dev_configure(struct rte_eth_dev *dev) for (i = 0; (i != priv->rxqs_n); ++i) if ((*priv->rxqs)[i] != NULL) return EINVAL; - rxq_cleanup(&priv->rxq_parent); + priv_parent_list_cleanup(priv); priv->rss = 0; priv->rxqs_n = 0; } @@ -613,14 +705,14 @@ dev_configure(struct rte_eth_dev *dev) priv->rss = 1; tmp = priv->rxqs_n; priv->rxqs_n = rxqs_n; - ret = rxq_setup(dev, &priv->rxq_parent, 0, 0, 0, NULL, NULL); - if (!ret) + if (priv->isolated) + return 0; + if (priv_parent_create(priv, NULL, priv->rxqs_n)) return 0; /* Failure, rollback. */ priv->rss = 0; priv->rxqs_n = tmp; - assert(ret > 0); - return ret; + return ENOMEM; } /** @@ -1098,7 +1190,7 @@ static int mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, /* For best performance, this function should not be inlined. */ static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, struct rte_mempool *) - __attribute__((noinline)); + __rte_noinline; /** * Register mempool as a memory region. @@ -2499,11 +2591,12 @@ priv_mac_addr_del(struct priv *priv, unsigned int mac_index) { unsigned int i; + assert(!priv->isolated); assert(mac_index < elemof(priv->mac)); if (!BITFIELD_ISSET(priv->mac_configured, mac_index)) return; if (priv->rss) { - rxq_mac_addr_del(&priv->rxq_parent, mac_index); + rxq_mac_addr_del(LIST_FIRST(&priv->parents), mac_index); goto end; } for (i = 0; (i != priv->dev->data->nb_rx_queues); ++i) @@ -2570,7 +2663,7 @@ priv_mac_addr_add(struct priv *priv, unsigned int mac_index, goto end; } if (priv->rss) { - ret = rxq_mac_addr_add(&priv->rxq_parent, mac_index); + ret = rxq_mac_addr_add(LIST_FIRST(&priv->parents), mac_index); if (ret) return ret; goto end; @@ -2748,14 +2841,17 @@ rxq_cleanup(struct rxq *rxq) rxq->if_cq, ¶ms)); } - if (rxq->qp != NULL) { + if (rxq->qp != NULL && !rxq->priv->isolated) { rxq_promiscuous_disable(rxq); rxq_allmulticast_disable(rxq); rxq_mac_addrs_del(rxq); - claim_zero(ibv_destroy_qp(rxq->qp)); } + if (rxq->qp != NULL) + claim_zero(ibv_destroy_qp(rxq->qp)); if (rxq->cq != NULL) claim_zero(ibv_destroy_cq(rxq->cq)); + if (rxq->channel != NULL) + claim_zero(ibv_destroy_comp_channel(rxq->channel)); if (rxq->rd != NULL) { struct ibv_exp_destroy_res_domain_attr attr = { .comp_mask = 0, @@ -2987,6 +3083,13 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) NB_SEGS(rep) = 0x2a; PORT(rep) = 0x2a; rep->ol_flags = -1; + /* + * Clear special flags in mbuf to avoid + * crashing while freeing. + */ + rep->ol_flags &= + ~(uint64_t)(IND_ATTACHED_MBUF | + CTRL_MBUF_FLAG); #endif assert(rep->buf_len == seg->buf_len); /* Reconfigure sge to use rep instead of seg. */ @@ -3330,15 +3433,18 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc, * Completion queue to associate with QP. * @param desc * Number of descriptors in QP (hint only). - * @param parent - * If nonzero, create a parent QP, otherwise a child. + * @param children_n + * If nonzero, a number of children for parent QP and zero for a child. + * @param rxq_parent + * Pointer for a parent in a child case, NULL otherwise. * * @return * QP pointer or NULL in case of error. */ static struct ibv_qp * rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, - int parent, struct ibv_exp_res_domain *rd) + int children_n, struct ibv_exp_res_domain *rd, + struct rxq *rxq_parent) { struct ibv_exp_qp_init_attr attr = { /* CQ to be associated with the send queue. */ @@ -3368,16 +3474,16 @@ rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, attr.max_inl_recv = priv->inl_recv_size, attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV; #endif - if (parent) { + if (children_n > 0) { attr.qpg.qpg_type = IBV_EXP_QPG_PARENT; /* TSS isn't necessary. */ attr.qpg.parent_attrib.tss_child_count = 0; attr.qpg.parent_attrib.rss_child_count = - rte_align32pow2(priv->rxqs_n + 1) >> 1; + rte_align32pow2(children_n + 1) >> 1; DEBUG("initializing parent RSS queue"); } else { attr.qpg.qpg_type = IBV_EXP_QPG_CHILD_RX; - attr.qpg.qpg_parent = priv->rxq_parent.qp; + attr.qpg.qpg_parent = rxq_parent->qp; DEBUG("initializing child RSS queue"); } return ibv_exp_create_qp(priv->ctx, &attr); @@ -3413,13 +3519,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) struct ibv_recv_wr *bad_wr; unsigned int mb_len; int err; - int parent = (rxq == &priv->rxq_parent); - if (parent) { - ERROR("%p: cannot rehash parent queue %p", - (void *)dev, (void *)rxq); - return EINVAL; - } mb_len = rte_pktmbuf_data_room_size(rxq->mp); DEBUG("%p: rehashing queue %p", (void *)dev, (void *)rxq); /* Number of descriptors and mbufs currently allocated. */ @@ -3451,7 +3551,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) return 0; } /* Remove attached flows if RSS is disabled (no parent queue). */ - if (!priv->rss) { + if (!priv->rss && !priv->isolated) { rxq_allmulticast_disable(&tmpl); rxq_promiscuous_disable(&tmpl); rxq_mac_addrs_del(&tmpl); @@ -3464,6 +3564,8 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) } /* From now on, any failure will render the queue unusable. * Reinitialize QP. */ + if (!tmpl.qp) + goto skip_init; mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RESET }; err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); if (err) { @@ -3471,12 +3573,6 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) assert(err > 0); return err; } - err = ibv_resize_cq(tmpl.cq, desc_n); - if (err) { - ERROR("%p: cannot resize CQ: %s", (void *)dev, strerror(err)); - assert(err > 0); - return err; - } mod = (struct ibv_exp_qp_attr){ /* Move the QP to this state. */ .qp_state = IBV_QPS_INIT, @@ -3485,9 +3581,6 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) }; err = ibv_exp_modify_qp(tmpl.qp, &mod, (IBV_EXP_QP_STATE | -#ifdef RSS_SUPPORT - (parent ? IBV_EXP_QP_GROUP_RSS : 0) | -#endif /* RSS_SUPPORT */ IBV_EXP_QP_PORT)); if (err) { ERROR("%p: QP state to IBV_QPS_INIT failed: %s", @@ -3495,8 +3588,15 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) assert(err > 0); return err; }; +skip_init: + err = ibv_resize_cq(tmpl.cq, desc_n); + if (err) { + ERROR("%p: cannot resize CQ: %s", (void *)dev, strerror(err)); + assert(err > 0); + return err; + } /* Reconfigure flows. Do not care for errors. */ - if (!priv->rss) { + if (!priv->rss && !priv->isolated) { rxq_mac_addrs_add(&tmpl); if (priv->promisc) rxq_promiscuous_enable(&tmpl); @@ -3562,6 +3662,8 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) rxq->elts_n = 0; rte_free(rxq->elts.sp); rxq->elts.sp = NULL; + if (!tmpl.qp) + goto skip_rtr; /* Post WRs. */ err = ibv_post_recv(tmpl.qp, (tmpl.sp ? @@ -3589,6 +3691,116 @@ skip_rtr: } /** + * Create verbs QP resources associated with a rxq. + * + * @param rxq + * Pointer to RX queue structure. + * @param desc + * Number of descriptors to configure in queue. + * @param inactive + * If true, the queue is disabled because its index is higher or + * equal to the real number of queues, which must be a power of 2. + * @param children_n + * The number of children in a parent case, zero for a child. + * @param rxq_parent + * The pointer to a parent RX structure for a child in RSS case, + * NULL for parent. + * + * @return + * 0 on success, errno value on failure. + */ +int +rxq_create_qp(struct rxq *rxq, + uint16_t desc, + int inactive, + int children_n, + struct rxq *rxq_parent) +{ + int ret; + struct ibv_exp_qp_attr mod; + struct ibv_exp_query_intf_params params; + enum ibv_exp_query_intf_status status; + struct ibv_recv_wr *bad_wr; + int parent = (children_n > 0); + struct priv *priv = rxq->priv; + +#ifdef RSS_SUPPORT + if (priv->rss && !inactive && (rxq_parent || parent)) + rxq->qp = rxq_setup_qp_rss(priv, rxq->cq, desc, + children_n, rxq->rd, + rxq_parent); + else +#endif /* RSS_SUPPORT */ + rxq->qp = rxq_setup_qp(priv, rxq->cq, desc, rxq->rd); + if (rxq->qp == NULL) { + ret = (errno ? errno : EINVAL); + ERROR("QP creation failure: %s", + strerror(ret)); + return ret; + } + mod = (struct ibv_exp_qp_attr){ + /* Move the QP to this state. */ + .qp_state = IBV_QPS_INIT, + /* Primary port number. */ + .port_num = priv->port + }; + ret = ibv_exp_modify_qp(rxq->qp, &mod, + (IBV_EXP_QP_STATE | +#ifdef RSS_SUPPORT + (parent ? IBV_EXP_QP_GROUP_RSS : 0) | +#endif /* RSS_SUPPORT */ + IBV_EXP_QP_PORT)); + if (ret) { + ERROR("QP state to IBV_QPS_INIT failed: %s", + strerror(ret)); + return ret; + } + if (!priv->isolated && (parent || !priv->rss)) { + /* Configure MAC and broadcast addresses. */ + ret = rxq_mac_addrs_add(rxq); + if (ret) { + ERROR("QP flow attachment failed: %s", + strerror(ret)); + return ret; + } + } + if (!parent) { + ret = ibv_post_recv(rxq->qp, + (rxq->sp ? + &(*rxq->elts.sp)[0].wr : + &(*rxq->elts.no_sp)[0].wr), + &bad_wr); + if (ret) { + ERROR("ibv_post_recv() failed for WR %p: %s", + (void *)bad_wr, + strerror(ret)); + return ret; + } + } + mod = (struct ibv_exp_qp_attr){ + .qp_state = IBV_QPS_RTR + }; + ret = ibv_exp_modify_qp(rxq->qp, &mod, IBV_EXP_QP_STATE); + if (ret) { + ERROR("QP state to IBV_QPS_RTR failed: %s", + strerror(ret)); + return ret; + } + params = (struct ibv_exp_query_intf_params){ + .intf_scope = IBV_EXP_INTF_GLOBAL, + .intf = IBV_EXP_INTF_QP_BURST, + .obj = rxq->qp, + }; + rxq->if_qp = ibv_exp_query_intf(priv->ctx, ¶ms, &status); + if (rxq->if_qp == NULL) { + ERROR("QP interface family query failed with status %d", + status); + return errno; + } + return 0; +} + +/** * Configure a RX queue. * * @param dev @@ -3606,14 +3818,21 @@ skip_rtr: * Thresholds parameters. * @param mp * Memory pool for buffer allocations. + * @param children_n + * The number of children in a parent case, zero for a child. + * @param rxq_parent + * The pointer to a parent RX structure (or NULL) in a child case, + * NULL for parent. * * @return * 0 on success, errno value on failure. */ static int rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, - unsigned int socket, int inactive, const struct rte_eth_rxconf *conf, - struct rte_mempool *mp) + unsigned int socket, int inactive, + const struct rte_eth_rxconf *conf, + struct rte_mempool *mp, int children_n, + struct rxq *rxq_parent) { struct priv *priv = dev->data->dev_private; struct rxq tmpl = { @@ -3621,17 +3840,15 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, .mp = mp, .socket = socket }; - struct ibv_exp_qp_attr mod; union { struct ibv_exp_query_intf_params params; struct ibv_exp_cq_init_attr cq; struct ibv_exp_res_domain_init_attr rd; } attr; enum ibv_exp_query_intf_status status; - struct ibv_recv_wr *bad_wr; unsigned int mb_len; int ret = 0; - int parent = (rxq == &priv->rxq_parent); + int parent = (children_n > 0); (void)conf; /* Thresholds configuration (ignored). */ /* @@ -3696,11 +3913,22 @@ skip_mr: (void *)dev, strerror(ret)); goto error; } + if (dev->data->dev_conf.intr_conf.rxq) { + tmpl.channel = ibv_create_comp_channel(priv->ctx); + if (tmpl.channel == NULL) { + ret = ENOMEM; + ERROR("%p: Rx interrupt completion channel creation" + " failure: %s", + (void *)dev, strerror(ret)); + goto error; + } + } attr.cq = (struct ibv_exp_cq_init_attr){ .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, .res_domain = tmpl.rd, }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); + tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0, + &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -3711,45 +3939,6 @@ skip_mr: priv->device_attr.max_qp_wr); DEBUG("priv->device_attr.max_sge is %d", priv->device_attr.max_sge); -#ifdef RSS_SUPPORT - if (priv->rss && !inactive) - tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent, - tmpl.rd); - else -#endif /* RSS_SUPPORT */ - tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd); - if (tmpl.qp == NULL) { - ret = (errno ? errno : EINVAL); - ERROR("%p: QP creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - mod = (struct ibv_exp_qp_attr){ - /* Move the QP to this state. */ - .qp_state = IBV_QPS_INIT, - /* Primary port number. */ - .port_num = priv->port - }; - ret = ibv_exp_modify_qp(tmpl.qp, &mod, - (IBV_EXP_QP_STATE | -#ifdef RSS_SUPPORT - (parent ? IBV_EXP_QP_GROUP_RSS : 0) | -#endif /* RSS_SUPPORT */ - IBV_EXP_QP_PORT)); - if (ret) { - ERROR("%p: QP state to IBV_QPS_INIT failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - if ((parent) || (!priv->rss)) { - /* Configure MAC and broadcast addresses. */ - ret = rxq_mac_addrs_add(&tmpl); - if (ret) { - ERROR("%p: QP flow attachment failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - } /* Allocate descriptors for RX queues, except for the RSS parent. */ if (parent) goto skip_alloc; @@ -3760,29 +3949,14 @@ skip_mr: if (ret) { ERROR("%p: RXQ allocation failed: %s", (void *)dev, strerror(ret)); - goto error; - } - ret = ibv_post_recv(tmpl.qp, - (tmpl.sp ? - &(*tmpl.elts.sp)[0].wr : - &(*tmpl.elts.no_sp)[0].wr), - &bad_wr); - if (ret) { - ERROR("%p: ibv_post_recv() failed for WR %p: %s", - (void *)dev, - (void *)bad_wr, - strerror(ret)); - goto error; + return ret; } skip_alloc: - mod = (struct ibv_exp_qp_attr){ - .qp_state = IBV_QPS_RTR - }; - ret = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); - if (ret) { - ERROR("%p: QP state to IBV_QPS_RTR failed: %s", - (void *)dev, strerror(ret)); - goto error; + if (parent || rxq_parent || !priv->rss) { + ret = rxq_create_qp(&tmpl, desc, inactive, + children_n, rxq_parent); + if (ret) + goto error; } /* Save port ID. */ tmpl.port_id = dev->data->port_id; @@ -3794,21 +3968,11 @@ skip_alloc: }; tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); if (tmpl.if_cq == NULL) { + ret = EINVAL; ERROR("%p: CQ interface family query failed with status %d", (void *)dev, status); goto error; } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_QP_BURST, - .obj = tmpl.qp, - }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_qp == NULL) { - ERROR("%p: QP interface family query failed with status %d", - (void *)dev, status); - goto error; - } /* Clean up rxq in case we're reinitializing it. */ DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq); rxq_cleanup(rxq); @@ -3846,6 +4010,7 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, struct rte_mempool *mp) { + struct rxq *parent; struct priv *priv = dev->data->dev_private; struct rxq *rxq = (*priv->rxqs)[idx]; int inactive = 0; @@ -3880,9 +4045,16 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, return -ENOMEM; } } - if (idx >= rte_align32pow2(priv->rxqs_n + 1) >> 1) - inactive = 1; - ret = rxq_setup(dev, rxq, desc, socket, inactive, conf, mp); + if (priv->rss && !priv->isolated) { + /* The list consists of the single default one. */ + parent = LIST_FIRST(&priv->parents); + if (idx >= rte_align32pow2(priv->rxqs_n + 1) >> 1) + inactive = 1; + } else { + parent = NULL; + } + ret = rxq_setup(dev, rxq, desc, socket, + inactive, conf, mp, 0, parent); if (ret) rte_free(rxq); else { @@ -3919,7 +4091,6 @@ mlx4_rx_queue_release(void *dpdk_rxq) return; priv = rxq->priv; priv_lock(priv); - assert(rxq != &priv->rxq_parent); for (i = 0; (i != priv->rxqs_n); ++i) if ((*priv->rxqs)[i] == rxq) { DEBUG("%p: removing RX queue %p from list", @@ -3970,8 +4141,11 @@ mlx4_dev_start(struct rte_eth_dev *dev) } DEBUG("%p: attaching configured flows to all RX queues", (void *)dev); priv->started = 1; - if (priv->rss) { - rxq = &priv->rxq_parent; + if (priv->isolated) { + rxq = NULL; + r = 1; + } else if (priv->rss) { + rxq = LIST_FIRST(&priv->parents); r = 1; } else { rxq = (*priv->rxqs)[0]; @@ -4005,6 +4179,12 @@ mlx4_dev_start(struct rte_eth_dev *dev) (void *)dev); goto err; } + ret = priv_rx_intr_vec_enable(priv); + if (ret) { + ERROR("%p: Rx interrupt vector creation failed", + (void *)dev); + goto err; + } ret = mlx4_priv_flow_start(priv); if (ret) { ERROR("%p: flow start failed: %s", @@ -4053,8 +4233,11 @@ mlx4_dev_stop(struct rte_eth_dev *dev) } DEBUG("%p: detaching flows from all RX queues", (void *)dev); priv->started = 0; - if (priv->rss) { - rxq = &priv->rxq_parent; + if (priv->isolated) { + rxq = NULL; + r = 1; + } else if (priv->rss) { + rxq = LIST_FIRST(&priv->parents); r = 1; } else { rxq = (*priv->rxqs)[0]; @@ -4188,7 +4371,7 @@ mlx4_dev_close(struct rte_eth_dev *dev) priv->txqs = NULL; } if (priv->rss) - rxq_cleanup(&priv->rxq_parent); + priv_parent_list_cleanup(priv); if (priv->pd != NULL) { assert(priv->ctx != NULL); claim_zero(ibv_dealloc_pd(priv->pd)); @@ -4197,6 +4380,7 @@ mlx4_dev_close(struct rte_eth_dev *dev) assert(priv->ctx == NULL); priv_dev_removal_interrupt_handler_uninstall(priv, dev); priv_dev_link_interrupt_handler_uninstall(priv, dev); + priv_rx_intr_vec_disable(priv); priv_unlock(priv); memset(priv, 0, sizeof(*priv)); } @@ -4300,7 +4484,7 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) unsigned int max; char ifname[IF_NAMESIZE]; - info->pci_dev = RTE_DEV_TO_PCI(dev->device); + info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (priv == NULL) return; @@ -4481,6 +4665,8 @@ mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) if (mlx4_is_secondary()) return; priv_lock(priv); + if (priv->isolated) + goto end; DEBUG("%p: removing MAC address from index %" PRIu32, (void *)dev, index); /* Last array entry is reserved for broadcast. */ @@ -4514,6 +4700,12 @@ mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, return -ENOTSUP; (void)vmdq; priv_lock(priv); + if (priv->isolated) { + DEBUG("%p: cannot add MAC address, " + "device is in isolated mode", (void *)dev); + re = EPERM; + goto end; + } DEBUG("%p: adding MAC address at index %" PRIu32, (void *)dev, index); /* Last array entry is reserved for broadcast. */ @@ -4561,6 +4753,12 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev) if (mlx4_is_secondary()) return; priv_lock(priv); + if (priv->isolated) { + DEBUG("%p: cannot enable promiscuous, " + "device is in isolated mode", (void *)dev); + priv_unlock(priv); + return; + } if (priv->promisc) { priv_unlock(priv); return; @@ -4569,7 +4767,7 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev) if (!priv->started) goto end; if (priv->rss) { - ret = rxq_promiscuous_enable(&priv->rxq_parent); + ret = rxq_promiscuous_enable(LIST_FIRST(&priv->parents)); if (ret) { priv_unlock(priv); return; @@ -4609,12 +4807,12 @@ mlx4_promiscuous_disable(struct rte_eth_dev *dev) if (mlx4_is_secondary()) return; priv_lock(priv); - if (!priv->promisc) { + if (!priv->promisc || priv->isolated) { priv_unlock(priv); return; } if (priv->rss) { - rxq_promiscuous_disable(&priv->rxq_parent); + rxq_promiscuous_disable(LIST_FIRST(&priv->parents)); goto end; } for (i = 0; (i != priv->rxqs_n); ++i) @@ -4641,6 +4839,12 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev) if (mlx4_is_secondary()) return; priv_lock(priv); + if (priv->isolated) { + DEBUG("%p: cannot enable allmulticast, " + "device is in isolated mode", (void *)dev); + priv_unlock(priv); + return; + } if (priv->allmulti) { priv_unlock(priv); return; @@ -4649,7 +4853,7 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev) if (!priv->started) goto end; if (priv->rss) { - ret = rxq_allmulticast_enable(&priv->rxq_parent); + ret = rxq_allmulticast_enable(LIST_FIRST(&priv->parents)); if (ret) { priv_unlock(priv); return; @@ -4689,12 +4893,12 @@ mlx4_allmulticast_disable(struct rte_eth_dev *dev) if (mlx4_is_secondary()) return; priv_lock(priv); - if (!priv->allmulti) { + if (!priv->allmulti || priv->isolated) { priv_unlock(priv); return; } if (priv->rss) { - rxq_allmulticast_disable(&priv->rxq_parent); + rxq_allmulticast_disable(LIST_FIRST(&priv->parents)); goto end; } for (i = 0; (i != priv->rxqs_n); ++i) @@ -4832,7 +5036,7 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) } /* Reenable non-RSS queue attributes. No need to check * for errors at this stage. */ - if (!priv->rss) { + if (!priv->rss && !priv->isolated) { rxq_mac_addrs_add(rxq); if (priv->promisc) rxq_promiscuous_enable(rxq); @@ -5003,7 +5207,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) * Rehashing flows in all RX queues is necessary. */ if (priv->rss) - rxq_mac_addrs_del(&priv->rxq_parent); + rxq_mac_addrs_del(LIST_FIRST(&priv->parents)); else for (i = 0; (i != priv->rxqs_n); ++i) if ((*priv->rxqs)[i] != NULL) @@ -5011,7 +5215,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) priv->vlan_filter[j].enabled = 1; if (priv->started) { if (priv->rss) - rxq_mac_addrs_add(&priv->rxq_parent); + rxq_mac_addrs_add(LIST_FIRST(&priv->parents)); else for (i = 0; (i != priv->rxqs_n); ++i) { if ((*priv->rxqs)[i] == NULL) @@ -5025,7 +5229,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) * Rehashing flows in all RX queues is necessary. */ if (priv->rss) - rxq_mac_addrs_del(&priv->rxq_parent); + rxq_mac_addrs_del(LIST_FIRST(&priv->parents)); else for (i = 0; (i != priv->rxqs_n); ++i) if ((*priv->rxqs)[i] != NULL) @@ -5033,7 +5237,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) priv->vlan_filter[j].enabled = 0; if (priv->started) { if (priv->rss) - rxq_mac_addrs_add(&priv->rxq_parent); + rxq_mac_addrs_add(LIST_FIRST(&priv->parents)); else for (i = 0; (i != priv->rxqs_n); ++i) { if ((*priv->rxqs)[i] == NULL) @@ -5067,6 +5271,12 @@ mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) if (mlx4_is_secondary()) return -E_RTE_SECONDARY; priv_lock(priv); + if (priv->isolated) { + DEBUG("%p: cannot set vlan filter, " + "device is in isolated mode", (void *)dev); + priv_unlock(priv); + return -EINVAL; + } ret = vlan_filter_set(dev, vlan_id, on); priv_unlock(priv); assert(ret >= 0); @@ -5079,6 +5289,7 @@ const struct rte_flow_ops mlx4_flow_ops = { .destroy = mlx4_flow_destroy, .flush = mlx4_flow_flush, .query = NULL, + .isolate = mlx4_flow_isolate, }; /** @@ -5157,6 +5368,8 @@ static const struct eth_dev_ops mlx4_dev_ops = { .mac_addr_set = mlx4_mac_addr_set, .mtu_set = mlx4_dev_set_mtu, .filter_ctrl = mlx4_dev_filter_ctrl, + .rx_queue_intr_enable = mlx4_rx_intr_enable, + .rx_queue_intr_disable = mlx4_rx_intr_disable, }; /** @@ -5196,7 +5409,7 @@ mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, /* Extract information. */ if (sscanf(line, "PCI_SLOT_NAME=" - "%" SCNx16 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", + "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", &pci_addr->domain, &pci_addr->bus, &pci_addr->devid, @@ -5308,6 +5521,7 @@ priv_dev_status_handler(struct priv *priv, struct rte_eth_dev *dev, { struct ibv_async_event event; int port_change = 0; + struct rte_eth_link *link = &dev->data->dev_link; int ret = 0; *events = 0; @@ -5329,22 +5543,20 @@ priv_dev_status_handler(struct priv *priv, struct rte_eth_dev *dev, event.event_type, event.element.port_num); ibv_ack_async_event(&event); } - - if (port_change ^ priv->pending_alarm) { - struct rte_eth_link *link = &dev->data->dev_link; - - priv->pending_alarm = 0; - mlx4_link_update(dev, 0); - if (((link->link_speed == 0) && link->link_status) || - ((link->link_speed != 0) && !link->link_status)) { + if (!port_change) + return ret; + mlx4_link_update(dev, 0); + if (((link->link_speed == 0) && link->link_status) || + ((link->link_speed != 0) && !link->link_status)) { + if (!priv->pending_alarm) { /* Inconsistent status, check again later. */ priv->pending_alarm = 1; rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US, mlx4_dev_link_status_handler, dev); - } else { - *events |= (1 << RTE_ETH_EVENT_INTR_LSC); } + } else { + *events |= (1 << RTE_ETH_EVENT_INTR_LSC); } return ret; } @@ -5365,10 +5577,12 @@ mlx4_dev_link_status_handler(void *arg) priv_lock(priv); assert(priv->pending_alarm == 1); + priv->pending_alarm = 0; ret = priv_dev_status_handler(priv, dev, &events); priv_unlock(priv); if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC)) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, + NULL); } /** @@ -5397,7 +5611,8 @@ mlx4_dev_interrupt_handler(void *cb_arg) i++) { if (ev & (1 << i)) { ev &= ~(1 << i); - _rte_eth_dev_callback_process(dev, i, NULL); + _rte_eth_dev_callback_process(dev, i, NULL, + NULL); ret--; } } @@ -5592,6 +5807,154 @@ priv_dev_removal_interrupt_handler_install(struct priv *priv, } /** + * Allocate queue vector and fill epoll fd list for Rx interrupts. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success, negative on failure. + */ +static int +priv_rx_intr_vec_enable(struct priv *priv) +{ + unsigned int i; + unsigned int rxqs_n = priv->rxqs_n; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + unsigned int count = 0; + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + + if (!priv->dev->data->dev_conf.intr_conf.rxq) + return 0; + priv_rx_intr_vec_disable(priv); + intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n])); + if (intr_handle->intr_vec == NULL) { + ERROR("failed to allocate memory for interrupt vector," + " Rx interrupts will not be supported"); + return -ENOMEM; + } + intr_handle->type = RTE_INTR_HANDLE_EXT; + for (i = 0; i != n; ++i) { + struct rxq *rxq = (*priv->rxqs)[i]; + int fd; + int flags; + int rc; + + /* Skip queues that cannot request interrupts. */ + if (!rxq || !rxq->channel) { + /* Use invalid intr_vec[] index to disable entry. */ + intr_handle->intr_vec[i] = + RTE_INTR_VEC_RXTX_OFFSET + + RTE_MAX_RXTX_INTR_VEC_ID; + continue; + } + if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { + ERROR("too many Rx queues for interrupt vector size" + " (%d), Rx interrupts cannot be enabled", + RTE_MAX_RXTX_INTR_VEC_ID); + priv_rx_intr_vec_disable(priv); + return -1; + } + fd = rxq->channel->fd; + flags = fcntl(fd, F_GETFL); + rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK); + if (rc < 0) { + ERROR("failed to make Rx interrupt file descriptor" + " %d non-blocking for queue index %d", fd, i); + priv_rx_intr_vec_disable(priv); + return rc; + } + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; + intr_handle->efds[count] = fd; + count++; + } + if (!count) + priv_rx_intr_vec_disable(priv); + else + intr_handle->nb_efd = count; + return 0; +} + +/** + * Clean up Rx interrupts handler. + * + * @param priv + * Pointer to private structure. + */ +static void +priv_rx_intr_vec_disable(struct priv *priv) +{ + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + + rte_intr_free_epoll_fd(intr_handle); + free(intr_handle->intr_vec); + intr_handle->nb_efd = 0; + intr_handle->intr_vec = NULL; +} + +/** + * DPDK callback for Rx queue interrupt enable. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Rx queue index. + * + * @return + * 0 on success, negative on failure. + */ +static int +mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) +{ + struct priv *priv = dev->data->dev_private; + struct rxq *rxq = (*priv->rxqs)[idx]; + int ret; + + if (!rxq || !rxq->channel) + ret = EINVAL; + else + ret = ibv_req_notify_cq(rxq->cq, 0); + if (ret) + WARN("unable to arm interrupt on rx queue %d", idx); + return -ret; +} + +/** + * DPDK callback for Rx queue interrupt disable. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Rx queue index. + * + * @return + * 0 on success, negative on failure. + */ +static int +mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx) +{ + struct priv *priv = dev->data->dev_private; + struct rxq *rxq = (*priv->rxqs)[idx]; + struct ibv_cq *ev_cq; + void *ev_ctx; + int ret; + + if (!rxq || !rxq->channel) { + ret = EINVAL; + } else { + ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx); + if (ret || ev_cq != rxq->cq) + ret = EINVAL; + } + if (ret) + WARN("unable to disable interrupt on rx queue %d", + idx); + else + ibv_ack_cq_events(rxq->cq, 1); + return -ret; +} + +/** * Verify and store value for device argument. * * @param[in] key @@ -5760,12 +6123,15 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_dev = list[i]; DEBUG("device opened"); - if (ibv_query_device(attr_ctx, &device_attr)) + if (ibv_query_device(attr_ctx, &device_attr)) { + err = ENODEV; goto error; + } INFO("%u port(s) detected", device_attr.phys_port_cnt); if (mlx4_args(pci_dev->device.devargs, &conf)) { ERROR("failed to process device arguments"); + err = EINVAL; goto error; } /* Use all ports when none are defined */ @@ -5799,19 +6165,23 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) DEBUG("using port %u (%08" PRIx32 ")", port, test); ctx = ibv_open_device(ibv_dev); - if (ctx == NULL) + if (ctx == NULL) { + err = ENODEV; goto port_error; + } /* Check port status. */ err = ibv_query_port(ctx, port, &port_attr); if (err) { ERROR("port query failed: %s", strerror(err)); + err = ENODEV; goto port_error; } if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { ERROR("port %d is not configured in Ethernet mode", port); + err = EINVAL; goto port_error; } @@ -5848,6 +6218,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) #ifdef HAVE_EXP_QUERY_DEVICE if (ibv_exp_query_device(ctx, &exp_device_attr)) { ERROR("ibv_exp_query_device() failed"); + err = ENODEV; goto port_error; } #ifdef RSS_SUPPORT @@ -5923,6 +6294,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) if (priv_get_mac(priv, &mac.addr_bytes)) { ERROR("cannot get MAC address, is mlx4_en loaded?" " (errno: %s)", strerror(errno)); + err = ENODEV; goto port_error; } INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", @@ -6002,8 +6374,18 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) eth_dev->device->driver = &mlx4_driver.driver; + /* + * Copy and override interrupt handle to prevent it from + * being shared between all ethdev instances of a given PCI + * device. This is required to properly handle Rx interrupts + * on all ports. + */ + priv->intr_handle_dev = *eth_dev->intr_handle; + eth_dev->intr_handle = &priv->intr_handle_dev; + priv->dev = eth_dev; eth_dev->dev_ops = &mlx4_dev_ops; + eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h index 9a3bae90..c0ade4f1 100644 --- a/drivers/net/mlx4/mlx4.h +++ b/drivers/net/mlx4/mlx4.h @@ -182,7 +182,13 @@ enum { (DEBUG__(__VA_ARGS__), 0) \ })[0]) #define DEBUG(...) DEBUG_(__VA_ARGS__, '\n') +#ifndef MLX4_PMD_DEBUG_BROKEN_VERBS #define claim_zero(...) assert((__VA_ARGS__) == 0) +#else /* MLX4_PMD_DEBUG_BROKEN_VERBS */ +#define claim_zero(...) \ + (void)(((__VA_ARGS__) == 0) || \ + DEBUG("Assertion `(" # __VA_ARGS__ ") == 0' failed (IGNORED).")) +#endif /* MLX4_PMD_DEBUG_BROKEN_VERBS */ #define claim_nonzero(...) assert((__VA_ARGS__) != 0) #define claim_positive(...) assert((__VA_ARGS__) >= 0) #else /* NDEBUG */ @@ -219,6 +225,7 @@ struct rxq_elt { /* RX queue descriptor. */ struct rxq { + LIST_ENTRY(rxq) next; /* Used by parent queue only */ struct priv *priv; /* Back pointer to private data. */ struct rte_mempool *mp; /* Memory Pool for allocations. */ struct ibv_mr *mr; /* Memory Region (for mp). */ @@ -226,6 +233,7 @@ struct rxq { struct ibv_qp *qp; /* Queue Pair. */ struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ struct ibv_exp_cq_family *if_cq; /* CQ interface. */ + struct ibv_comp_channel *channel; /* * Each VLAN ID requires a separate flow steering rule. */ @@ -246,6 +254,10 @@ struct rxq { struct mlx4_rxq_stats stats; /* RX queue counters. */ unsigned int socket; /* CPU socket ID for allocations. */ struct ibv_exp_res_domain *rd; /* Resource Domain. */ + struct { + uint16_t queues_n; + uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; + } rss; }; /* TX element. */ @@ -334,24 +346,41 @@ struct priv { unsigned int rss:1; /* RSS is enabled. */ unsigned int vf:1; /* This is a VF device. */ unsigned int pending_alarm:1; /* An alarm is pending. */ + unsigned int isolated:1; /* Toggle isolated mode. */ #ifdef INLINE_RECV unsigned int inl_recv_size; /* Inline recv size */ #endif unsigned int max_rss_tbl_sz; /* Maximum number of RSS queues. */ /* RX/TX queues. */ - struct rxq rxq_parent; /* Parent queue when RSS is enabled. */ unsigned int rxqs_n; /* RX queues array size. */ unsigned int txqs_n; /* TX queues array size. */ struct rxq *(*rxqs)[]; /* RX queues. */ struct txq *(*txqs)[]; /* TX queues. */ + struct rte_intr_handle intr_handle_dev; /* Device interrupt handler. */ struct rte_intr_handle intr_handle; /* Interrupt handler. */ struct rte_flow_drop *flow_drop_queue; /* Flow drop queue. */ LIST_HEAD(mlx4_flows, rte_flow) flows; struct rte_intr_conf intr_conf; /* Active interrupt configuration. */ + LIST_HEAD(mlx4_parents, rxq) parents; rte_spinlock_t lock; /* Lock for control functions. */ }; void priv_lock(struct priv *priv); void priv_unlock(struct priv *priv); +int +rxq_create_qp(struct rxq *rxq, + uint16_t desc, + int inactive, + int children_n, + struct rxq *rxq_parent); + +void +rxq_parent_cleanup(struct rxq *parent); + +struct rxq * +priv_parent_create(struct priv *priv, + uint16_t queues[], + uint16_t children_n); + #endif /* RTE_PMD_MLX4_H_ */ diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c index edfac038..925c89c5 100644 --- a/drivers/net/mlx4/mlx4_flow.c +++ b/drivers/net/mlx4/mlx4_flow.c @@ -112,6 +112,7 @@ struct rte_flow_drop { static const enum rte_flow_action_type valid_actions[] = { RTE_FLOW_ACTION_TYPE_DROP, RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_RSS, RTE_FLOW_ACTION_TYPE_END, }; @@ -672,6 +673,76 @@ priv_flow_validate(struct priv *priv, if (!queue || (queue->index > (priv->rxqs_n - 1))) goto exit_action_not_supported; action.queue = 1; + action.queues_n = 1; + action.queues[0] = queue->index; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { + int i; + int ierr; + const struct rte_flow_action_rss *rss = + (const struct rte_flow_action_rss *) + actions->conf; + + if (!priv->hw_rss) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "RSS cannot be used with " + "the current configuration"); + return -rte_errno; + } + if (!priv->isolated) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "RSS cannot be used without " + "isolated mode"); + return -rte_errno; + } + if (!rte_is_power_of_2(rss->num)) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "the number of queues " + "should be power of two"); + return -rte_errno; + } + if (priv->max_rss_tbl_sz < rss->num) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "the number of queues " + "is too large"); + return -rte_errno; + } + /* checking indexes array */ + ierr = 0; + for (i = 0; i < rss->num; ++i) { + int j; + if (rss->queue[i] >= priv->rxqs_n) + ierr = 1; + /* + * Prevent the user from specifying + * the same queue twice in the RSS array. + */ + for (j = i + 1; j < rss->num && !ierr; ++j) + if (rss->queue[j] == rss->queue[i]) + ierr = 1; + if (ierr) { + rte_flow_error_set( + error, + ENOTSUP, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "RSS action only supports " + "unique queue indices " + "in a list"); + return -rte_errno; + } + } + action.queue = 1; + action.queues_n = rss->num; + for (i = 0; i < rss->num; ++i) + action.queues[i] = rss->queue[i]; } else { goto exit_action_not_supported; } @@ -797,6 +868,82 @@ err: } /** + * Get RSS parent rxq structure for given queues. + * + * Creates a new or returns an existed one. + * + * @param priv + * Pointer to private structure. + * @param queues + * queues indices array, NULL in default RSS case. + * @param children_n + * the size of queues array. + * + * @return + * Pointer to a parent rxq structure, NULL on failure. + */ +static struct rxq * +priv_parent_get(struct priv *priv, + uint16_t queues[], + uint16_t children_n, + struct rte_flow_error *error) +{ + unsigned int i; + struct rxq *parent; + + for (parent = LIST_FIRST(&priv->parents); + parent; + parent = LIST_NEXT(parent, next)) { + unsigned int same = 0; + unsigned int overlap = 0; + + /* + * Find out whether an appropriate parent queue already exists + * and can be reused, otherwise make sure there are no overlaps. + */ + for (i = 0; i < children_n; ++i) { + unsigned int j; + + for (j = 0; j < parent->rss.queues_n; ++j) { + if (parent->rss.queues[j] != queues[i]) + continue; + ++overlap; + if (i == j) + ++same; + } + } + if (same == children_n && + children_n == parent->rss.queues_n) + return parent; + else if (overlap) + goto error; + } + /* Exclude the cases when some QPs were created without RSS */ + for (i = 0; i < children_n; ++i) { + struct rxq *rxq = (*priv->rxqs)[queues[i]]; + if (rxq->qp) + goto error; + } + parent = priv_parent_create(priv, queues, children_n); + if (!parent) { + rte_flow_error_set(error, + ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "flow rule creation failure"); + return NULL; + } + return parent; + +error: + rte_flow_error_set(error, + EEXIST, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "sharing a queue between several" + " RSS groups is not supported"); + return NULL; +} + +/** * Complete flow rule creation. * * @param priv @@ -819,6 +966,7 @@ priv_flow_create_action_queue(struct priv *priv, { struct ibv_qp *qp; struct rte_flow *rte_flow; + struct rxq *rxq_parent = NULL; assert(priv->pd); assert(priv->ctx); @@ -829,14 +977,46 @@ priv_flow_create_action_queue(struct priv *priv, return NULL; } if (action->drop) { - qp = priv->flow_drop_queue->qp; + qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL; } else { - struct rxq *rxq = (*priv->rxqs)[action->queue_id]; + int ret; + unsigned int i; + struct rxq *rxq = NULL; - qp = rxq->qp; + if (action->queues_n > 1) { + rxq_parent = priv_parent_get(priv, action->queues, + action->queues_n, error); + if (!rxq_parent) + goto error; + } + for (i = 0; i < action->queues_n; ++i) { + rxq = (*priv->rxqs)[action->queues[i]]; + /* + * In case of isolated mode we postpone + * ibv receive queue creation till the first + * rte_flow rule will be applied on that queue. + */ + if (!rxq->qp) { + assert(priv->isolated); + ret = rxq_create_qp(rxq, rxq->elts_n, + 0, 0, rxq_parent); + if (ret) { + rte_flow_error_set( + error, + ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "flow rule creation failure"); + goto error; + } + } + } + qp = action->queues_n > 1 ? rxq_parent->qp : rxq->qp; rte_flow->qp = qp; } rte_flow->ibv_attr = ibv_attr; + if (!priv->started) + return rte_flow; rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr); if (!rte_flow->ibv_flow) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, @@ -846,6 +1026,8 @@ priv_flow_create_action_queue(struct priv *priv, return rte_flow; error: + if (rxq_parent) + rxq_parent_cleanup(rxq_parent); rte_free(rte_flow); return NULL; } @@ -909,11 +1091,22 @@ priv_flow_create(struct priv *priv, continue; } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { action.queue = 1; - action.queue_id = + action.queues_n = 1; + action.queues[0] = ((const struct rte_flow_action_queue *) actions->conf)->index; } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { action.drop = 1; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { + unsigned int i; + const struct rte_flow_action_rss *rss = + (const struct rte_flow_action_rss *) + actions->conf; + + action.queue = 1; + action.queues_n = rss->num; + for (i = 0; i < rss->num; ++i) + action.queues[i] = rss->queue[i]; } else { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -957,6 +1150,43 @@ mlx4_flow_create(struct rte_eth_dev *dev, } /** + * @see rte_flow_isolate() + * + * Must be done before calling dev_configure(). + * + * @param dev + * Pointer to the ethernet device structure. + * @param enable + * Nonzero to enter isolated mode, attempt to leave it otherwise. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative value on error. + */ +int +mlx4_flow_isolate(struct rte_eth_dev *dev, + int enable, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + + priv_lock(priv); + if (priv->rxqs) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "isolated mode must be set" + " before configuring the device"); + priv_unlock(priv); + return -rte_errno; + } + priv->isolated = !!enable; + priv_unlock(priv); + return 0; +} + +/** * Destroy a flow. * * @param priv diff --git a/drivers/net/mlx4/mlx4_flow.h b/drivers/net/mlx4/mlx4_flow.h index 12a293e4..beabcf2d 100644 --- a/drivers/net/mlx4/mlx4_flow.h +++ b/drivers/net/mlx4/mlx4_flow.h @@ -90,10 +90,16 @@ struct mlx4_flow { unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */ }; +int +mlx4_flow_isolate(struct rte_eth_dev *dev, + int enable, + struct rte_flow_error *error); + struct mlx4_flow_action { uint32_t drop:1; /**< Target is a drop queue. */ uint32_t queue:1; /**< Target is a receive queue. */ - uint32_t queue_id; /**< Identifier of the queue. */ + uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queue indices to use. */ + uint16_t queues_n; /**< Number of entries in queue[] */ }; int mlx4_priv_flow_start(struct priv *priv); diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index c0799591..8736de5d 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -39,6 +39,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c +ifeq ($(CONFIG_RTE_ARCH_X86_64),y) +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec_sse.c +endif SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_trigger.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mac.c @@ -52,7 +55,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c # Basic CFLAGS. CFLAGS += -O3 -CFLAGS += -std=gnu99 -Wall -Wextra +CFLAGS += -std=c11 -Wall -Wextra CFLAGS += -g CFLAGS += -I. CFLAGS += -D_BSD_SOURCE @@ -101,6 +104,11 @@ mlx5_autoconf.h.new: FORCE mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh $Q $(RM) -f -- '$@' $Q sh -- '$<' '$@' \ + HAVE_VERBS_IBV_EXP_FLOW_SPEC_ACTION_DROP \ + infiniband/verbs_exp.h \ + enum IBV_EXP_FLOW_SPEC_ACTION_DROP \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ HAVE_VERBS_IBV_EXP_CQ_COMPRESSED_CQE \ infiniband/verbs_exp.h \ enum IBV_EXP_CQ_COMPRESSED_CQE \ diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index bcb2c1b2..b7e50463 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -94,6 +94,12 @@ /* Device parameter to enable hardware TSO offload. */ #define MLX5_TSO "tso" +/* Device parameter to enable hardware Tx vector. */ +#define MLX5_TX_VEC_EN "tx_vec_en" + +/* Device parameter to enable hardware Rx vector. */ +#define MLX5_RX_VEC_EN "rx_vec_en" + /* Default PMD specific parameter value. */ #define MLX5_ARG_UNSET (-1) @@ -105,6 +111,8 @@ struct mlx5_args { int mpw_hdr_dseg; int inline_max_packet_sz; int tso; + int tx_vec_en; + int rx_vec_en; }; /** * Retrieve integer value from environment variable. @@ -246,8 +254,10 @@ static const struct eth_dev_ops mlx5_dev_ops = { .filter_ctrl = mlx5_dev_filter_ctrl, .rx_descriptor_status = mlx5_rx_descriptor_status, .tx_descriptor_status = mlx5_tx_descriptor_status, +#ifdef HAVE_UPDATE_CQ_CI .rx_queue_intr_enable = mlx5_rx_intr_enable, .rx_queue_intr_disable = mlx5_rx_intr_disable, +#endif }; static struct { @@ -322,6 +332,10 @@ mlx5_args_check(const char *key, const char *val, void *opaque) args->inline_max_packet_sz = tmp; } else if (strcmp(MLX5_TSO, key) == 0) { args->tso = !!tmp; + } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { + args->tx_vec_en = !!tmp; + } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { + args->rx_vec_en = !!tmp; } else { WARN("%s: unknown parameter", key); return -EINVAL; @@ -351,6 +365,8 @@ mlx5_args(struct mlx5_args *args, struct rte_devargs *devargs) MLX5_TXQ_MPW_HDR_DSEG_EN, MLX5_TXQ_MAX_INLINE_LEN, MLX5_TSO, + MLX5_TX_VEC_EN, + MLX5_RX_VEC_EN, NULL, }; struct rte_kvargs *kvlist; @@ -406,6 +422,10 @@ mlx5_args_assign(struct priv *priv, struct mlx5_args *args) priv->inline_max_packet_sz = args->inline_max_packet_sz; if (args->tso != MLX5_ARG_UNSET) priv->tso = args->tso; + if (args->tx_vec_en != MLX5_ARG_UNSET) + priv->tx_vec_en = args->tx_vec_en; + if (args->rx_vec_en != MLX5_ARG_UNSET) + priv->rx_vec_en = args->rx_vec_en; } /** @@ -551,6 +571,8 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) .mpw_hdr_dseg = MLX5_ARG_UNSET, .inline_max_packet_sz = MLX5_ARG_UNSET, .tso = MLX5_ARG_UNSET, + .tx_vec_en = MLX5_ARG_UNSET, + .rx_vec_en = MLX5_ARG_UNSET, }; exp_device_attr.comp_mask = @@ -613,6 +635,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv->mps = mps; /* Enable MPW by default if supported. */ priv->cqe_comp = 1; /* Enable compression by default. */ priv->tunnel_en = tunnel_en; + /* Enable vector by default if supported. */ + priv->tx_vec_en = 1; + priv->rx_vec_en = 1; err = mlx5_args(&args, pci_dev->device.devargs); if (err) { ERROR("failed to process device arguments: %s", @@ -786,6 +811,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) eth_dev->device = &pci_dev->device; rte_eth_copy_pci_info(eth_dev, pci_dev); + eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; eth_dev->device->driver = &mlx5_driver.driver; priv->dev = eth_dev; eth_dev->dev_ops = &mlx5_dev_ops; @@ -885,6 +911,8 @@ RTE_INIT(rte_mlx5_pmd_init); static void rte_mlx5_pmd_init(void) { + /* Build the static table for ptype conversion. */ + mlx5_set_ptype_table(); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use * huge pages. Calling ibv_fork_init() during init allows diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 1148dee3..43c53841 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -54,6 +54,7 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#include <rte_pci.h> #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_spinlock.h> @@ -129,6 +130,9 @@ struct priv { unsigned int pending_alarm:1; /* An alarm is pending. */ unsigned int tso:1; /* Whether TSO is supported. */ unsigned int tunnel_en:1; + unsigned int isolated:1; /* Whether isolated mode is enabled. */ + unsigned int tx_vec_en:1; /* Whether Tx vector is enabled. */ + unsigned int rx_vec_en:1; /* Whether Rx vector is enabled. */ /* Whether Tx offloads for tunneled packets are supported. */ unsigned int max_tso_payload_sz; /* Maximum TCP payload for TSO. */ unsigned int txq_inline; /* Maximum packet size for inlining. */ @@ -311,6 +315,7 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *, int mlx5_flow_destroy(struct rte_eth_dev *, struct rte_flow *, struct rte_flow_error *); int mlx5_flow_flush(struct rte_eth_dev *, struct rte_flow_error *); +int mlx5_flow_isolate(struct rte_eth_dev *, int, struct rte_flow_error *); int priv_flow_start(struct priv *); void priv_flow_stop(struct priv *); int priv_flow_rxq_in_use(struct priv *, struct rxq *); diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h index 201bb336..a76bc6f6 100644 --- a/drivers/net/mlx5/mlx5_defs.h +++ b/drivers/net/mlx5/mlx5_defs.h @@ -89,4 +89,22 @@ /* Maximum Packet headers size (L2+L3+L4) for TSO. */ #define MLX5_MAX_TSO_HEADER 128 +/* Default minimum number of Tx queues for vectorized Tx. */ +#define MLX5_VPMD_MIN_TXQS 4 + +/* Threshold of buffer replenishment for vectorized Rx. */ +#define MLX5_VPMD_RXQ_RPLNSH_THRESH 64U + +/* Maximum size of burst for vectorized Rx. */ +#define MLX5_VPMD_RX_MAX_BURST MLX5_VPMD_RXQ_RPLNSH_THRESH + +/* + * Maximum size of burst for vectorized Tx. This is related to the maximum size + * of Enhaned MPW (eMPW) WQE as vectorized Tx is supported with eMPW. + */ +#define MLX5_VPMD_TX_MAX_BURST 32U + +/* Number of packets vectorized Rx can simultaneously process in a loop. */ +#define MLX5_VPMD_DESCS_PER_LOOP 4 + #endif /* RTE_PMD_MLX5_DEFS_H_ */ diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 3fd22cb8..b0eb3cdf 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -119,6 +119,7 @@ struct ethtool_link_settings { #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 #endif +#define ETHTOOL_LINK_MODE_MASK_MAX_KERNEL_NU32 (SCHAR_MAX) /** * Return private structure associated with an Ethernet device. @@ -661,7 +662,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) unsigned int max; char ifname[IF_NAMESIZE]; - info->pci_dev = RTE_DEV_TO_PCI(dev->device); + info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); priv_lock(priv); /* FIXME: we should ask the device for these values. */ @@ -715,15 +716,24 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) { static const uint32_t ptypes[] = { /* refers to rxq_cq_to_pkt_type() */ + RTE_PTYPE_L2_ETHER, RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_NONFRAG, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_INNER_L4_NONFRAG, + RTE_PTYPE_INNER_L4_FRAG, + RTE_PTYPE_INNER_L4_TCP, + RTE_PTYPE_INNER_L4_UDP, RTE_PTYPE_UNKNOWN - }; - if (dev->rx_pkt_burst == mlx5_rx_burst) + if (dev->rx_pkt_burst == mlx5_rx_burst || + dev->rx_pkt_burst == mlx5_rx_burst_vec) return ptypes; return NULL; } @@ -806,9 +816,12 @@ static int mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) { struct priv *priv = mlx5_get_priv(dev); - struct ethtool_link_settings edata = { - .cmd = ETHTOOL_GLINKSETTINGS, - }; + __extension__ struct { + struct ethtool_link_settings edata; + uint32_t link_mode_data[3 * + ETHTOOL_LINK_MODE_MASK_MAX_KERNEL_NU32]; + } ecmd; + struct ifreq ifr; struct rte_eth_link dev_link; uint64_t sc; @@ -821,15 +834,23 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) memset(&dev_link, 0, sizeof(dev_link)); dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); - ifr.ifr_data = (void *)&edata; + memset(&ecmd, 0, sizeof(ecmd)); + ecmd.edata.cmd = ETHTOOL_GLINKSETTINGS; + ifr.ifr_data = (void *)&ecmd; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", + strerror(errno)); + return -1; + } + ecmd.edata.link_mode_masks_nwords = -ecmd.edata.link_mode_masks_nwords; if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", strerror(errno)); return -1; } - dev_link.link_speed = edata.speed; - sc = edata.link_mode_masks[0] | - ((uint64_t)edata.link_mode_masks[1] << 32); + dev_link.link_speed = ecmd.edata.speed; + sc = ecmd.edata.link_mode_masks[0] | + ((uint64_t)ecmd.edata.link_mode_masks[1] << 32); priv->link_speed_capa = 0; if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; @@ -865,7 +886,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) priv->link_speed_capa |= ETH_LINK_SPEED_100G; - dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? + dev_link.link_duplex = ((ecmd.edata.duplex == DUPLEX_HALF) ? ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED); @@ -903,12 +924,6 @@ mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) /** * DPDK callback to change the MTU. * - * Setting the MTU affects hardware MRU (packets larger than the MTU cannot be - * received). Use this as a hint to enable/disable scattered packets support - * and improve performance when not needed. - * Since failure is not an option, reconfiguring queues on the fly is not - * recommended. - * * @param dev * Pointer to Ethernet device structure. * @param in_mtu @@ -921,123 +936,33 @@ int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { struct priv *priv = dev->data->dev_private; + uint16_t kern_mtu; int ret = 0; - unsigned int i; - uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) = - mlx5_rx_burst; - unsigned int max_frame_len; - int rehash; - int restart = priv->started; if (mlx5_is_secondary()) return -E_RTE_SECONDARY; priv_lock(priv); + ret = priv_get_mtu(priv, &kern_mtu); + if (ret) + goto out; /* Set kernel interface MTU first. */ - if (priv_set_mtu(priv, mtu)) { - ret = errno; - WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, - strerror(ret)); + ret = priv_set_mtu(priv, mtu); + if (ret) goto out; - } else + ret = priv_get_mtu(priv, &kern_mtu); + if (ret) + goto out; + if (kern_mtu == mtu) { + priv->mtu = mtu; DEBUG("adapter port %u MTU set to %u", priv->port, mtu); - /* Temporarily replace RX handler with a fake one, assuming it has not - * been copied elsewhere. */ - dev->rx_pkt_burst = removed_rx_burst; - /* Make sure everyone has left mlx5_rx_burst() and uses - * removed_rx_burst() instead. */ - rte_wmb(); - usleep(1000); - /* MTU does not include header and CRC. */ - max_frame_len = ETHER_HDR_LEN + mtu + ETHER_CRC_LEN; - /* Check if at least one queue is going to need a SGE update. */ - for (i = 0; i != priv->rxqs_n; ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; - unsigned int mb_len; - unsigned int size = RTE_PKTMBUF_HEADROOM + max_frame_len; - unsigned int sges_n; - - if (rxq == NULL) - continue; - mb_len = rte_pktmbuf_data_room_size(rxq->mp); - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - /* - * Determine the number of SGEs needed for a full packet - * and round it to the next power of two. - */ - sges_n = log2above((size / mb_len) + !!(size % mb_len)); - if (sges_n != rxq->sges_n) - break; - } - /* - * If all queues have the right number of SGEs, a simple rehash - * of their buffers is enough, otherwise SGE information can only - * be updated in a queue by recreating it. All resources that depend - * on queues (flows, indirection tables) must be recreated as well in - * that case. - */ - rehash = (i == priv->rxqs_n); - if (!rehash) { - /* Clean up everything as with mlx5_dev_stop(). */ - priv_special_flow_disable_all(priv); - priv_mac_addrs_disable(priv); - priv_destroy_hash_rxqs(priv); - priv_fdir_disable(priv); - priv_dev_interrupt_handler_uninstall(priv, dev); } -recover: - /* Reconfigure each RX queue. */ - for (i = 0; (i != priv->rxqs_n); ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; - struct rxq_ctrl *rxq_ctrl = - container_of(rxq, struct rxq_ctrl, rxq); - unsigned int mb_len; - unsigned int tmp; - - if (rxq == NULL) - continue; - mb_len = rte_pktmbuf_data_room_size(rxq->mp); - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - /* Provide new values to rxq_setup(). */ - dev->data->dev_conf.rxmode.jumbo_frame = - (max_frame_len > ETHER_MAX_LEN); - dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame_len; - if (rehash) - ret = rxq_rehash(dev, rxq_ctrl); - else - ret = rxq_ctrl_setup(dev, rxq_ctrl, 1 << rxq->elts_n, - rxq_ctrl->socket, NULL, rxq->mp); - if (!ret) - continue; - /* Attempt to roll back in case of error. */ - tmp = (mb_len << rxq->sges_n) - RTE_PKTMBUF_HEADROOM; - if (max_frame_len != tmp) { - max_frame_len = tmp; - goto recover; - } - /* Double fault, disable RX. */ - break; - } - /* - * Use a safe RX burst function in case of error, otherwise mimic - * mlx5_dev_start(). - */ - if (ret) { - ERROR("unable to reconfigure RX queues, RX disabled"); - rx_func = removed_rx_burst; - } else if (restart && - !rehash && - !priv_create_hash_rxqs(priv) && - !priv_rehash_flows(priv)) { - if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_NONE) - priv_fdir_enable(priv); - priv_dev_interrupt_handler_install(priv, dev); - } - priv->mtu = mtu; - /* Burst functions can now be called again. */ - rte_wmb(); - dev->rx_pkt_burst = rx_func; + priv_unlock(priv); + return 0; out: + ret = errno; + WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, + strerror(ret)); priv_unlock(priv); assert(ret >= 0); return -ret; @@ -1185,7 +1110,7 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, /* Extract information. */ if (sscanf(line, "PCI_SLOT_NAME=" - "%" SCNx16 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", + "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", &pci_addr->domain, &pci_addr->bus, &pci_addr->devid, @@ -1262,7 +1187,8 @@ mlx5_dev_link_status_handler(void *arg) ret = priv_dev_link_status_handler(priv, dev); priv_unlock(priv); if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, + NULL); } /** @@ -1284,7 +1210,8 @@ mlx5_dev_interrupt_handler(void *cb_arg) ret = priv_dev_link_status_handler(priv, dev); priv_unlock(priv); if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, + NULL); } /** @@ -1584,9 +1511,16 @@ priv_select_tx_function(struct priv *priv) priv->dev->tx_pkt_burst = mlx5_tx_burst; /* Select appropriate TX function. */ if (priv->mps == MLX5_MPW_ENHANCED) { - priv->dev->tx_pkt_burst = - mlx5_tx_burst_empw; - DEBUG("selected Enhanced MPW TX function"); + if (priv_check_vec_tx_support(priv) > 0) { + if (priv_check_raw_vec_tx_support(priv) > 0) + priv->dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; + else + priv->dev->tx_pkt_burst = mlx5_tx_burst_vec; + DEBUG("selected Enhanced MPW TX vectorized function"); + } else { + priv->dev->tx_pkt_burst = mlx5_tx_burst_empw; + DEBUG("selected Enhanced MPW TX function"); + } } else if (priv->mps && priv->txq_inline) { priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; DEBUG("selected MPW inline TX function"); @@ -1605,5 +1539,11 @@ priv_select_tx_function(struct priv *priv) void priv_select_rx_function(struct priv *priv) { - priv->dev->rx_pkt_burst = mlx5_rx_burst; + if (priv_check_vec_rx_support(priv) > 0) { + priv_prep_vec_rx_function(priv); + priv->dev->rx_pkt_burst = mlx5_rx_burst_vec; + DEBUG("selected RX vectorized function"); + } else { + priv->dev->rx_pkt_burst = mlx5_rx_burst; + } } diff --git a/drivers/net/mlx5/mlx5_fdir.c b/drivers/net/mlx5/mlx5_fdir.c index c8d47489..34a7e69f 100644 --- a/drivers/net/mlx5/mlx5_fdir.c +++ b/drivers/net/mlx5/mlx5_fdir.c @@ -1053,6 +1053,7 @@ static const struct rte_flow_ops mlx5_flow_ops = { .destroy = mlx5_flow_destroy, .flush = mlx5_flow_flush, .query = NULL, + .isolate = mlx5_flow_isolate, }; /** diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 8b3957ba..86be9291 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -53,7 +53,11 @@ #include "mlx5_prm.h" /* Number of Work Queue necessary for the DROP queue. */ +#ifndef HAVE_VERBS_IBV_EXP_FLOW_SPEC_ACTION_DROP #define MLX5_DROP_WQ_N 4 +#else +#define MLX5_DROP_WQ_N 1 +#endif static int mlx5_flow_create_eth(const struct rte_flow_item *item, @@ -576,6 +580,10 @@ priv_flow_validate(struct priv *priv, } if (action->mark && !flow->ibv_attr && !action->drop) flow->offset += sizeof(struct ibv_exp_flow_spec_action_tag); +#ifdef HAVE_VERBS_IBV_EXP_FLOW_SPEC_ACTION_DROP + if (!flow->ibv_attr && action->drop) + flow->offset += sizeof(struct ibv_exp_flow_spec_action_drop); +#endif if (!action->queue && !action->drop) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "no valid action"); @@ -993,6 +1001,10 @@ priv_flow_create_action_queue_drop(struct priv *priv, struct rte_flow_error *error) { struct rte_flow *rte_flow; +#ifdef HAVE_VERBS_IBV_EXP_FLOW_SPEC_ACTION_DROP + struct ibv_exp_flow_spec_action_drop *drop; + unsigned int size = sizeof(struct ibv_exp_flow_spec_action_drop); +#endif assert(priv->pd); assert(priv->ctx); @@ -1003,10 +1015,19 @@ priv_flow_create_action_queue_drop(struct priv *priv, return NULL; } rte_flow->drop = 1; +#ifdef HAVE_VERBS_IBV_EXP_FLOW_SPEC_ACTION_DROP + drop = (void *)((uintptr_t)flow->ibv_attr + flow->offset); + *drop = (struct ibv_exp_flow_spec_action_drop){ + .type = IBV_EXP_FLOW_SPEC_ACTION_DROP, + .size = size, + }; + ++flow->ibv_attr->num_of_specs; + flow->offset += sizeof(struct ibv_exp_flow_spec_action_drop); +#endif rte_flow->ibv_attr = flow->ibv_attr; - rte_flow->qp = priv->flow_drop_queue->qp; if (!priv->started) return rte_flow; + rte_flow->qp = priv->flow_drop_queue->qp; rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp, rte_flow->ibv_attr); if (!rte_flow->ibv_flow) { @@ -1579,3 +1600,30 @@ priv_flow_rxq_in_use(struct priv *priv, struct rxq *rxq) } return 0; } + +/** + * Isolated mode. + * + * @see rte_flow_isolate() + * @see rte_flow_ops + */ +int +mlx5_flow_isolate(struct rte_eth_dev *dev, + int enable, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + + priv_lock(priv); + if (priv->started) { + rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "port must be stopped first"); + priv_unlock(priv); + return -rte_errno; + } + priv->isolated = !!enable; + priv_unlock(priv); + return 0; +} diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index 79e0c410..8489ea67 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -443,6 +443,8 @@ priv_mac_addrs_enable(struct priv *priv) unsigned int i; int ret; + if (priv->isolated) + return 0; if (!priv_allow_flow_type(priv, HASH_RXQ_FLOW_TYPE_MAC)) return 0; for (i = 0; (i != priv->hash_rxqs_n); ++i) { diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 0a363846..28733517 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -207,7 +207,8 @@ txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx) sizeof(txq_ctrl->txq.mp2mr[0]))); } /* Store the new entry. */ - txq_ctrl->txq.mp2mr[idx].mp = mp; + txq_ctrl->txq.mp2mr[idx].start = (uintptr_t)mr->addr; + txq_ctrl->txq.mp2mr[idx].end = (uintptr_t)mr->addr + mr->length; txq_ctrl->txq.mp2mr[idx].mr = mr; txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey); DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, @@ -265,18 +266,28 @@ txq_mp2mr_iter(struct rte_mempool *mp, void *arg) struct txq_mp2mr_mbuf_check_data data = { .ret = 0, }; + uintptr_t start; + uintptr_t end; unsigned int i; /* Register mempool only if the first element looks like a mbuf. */ if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 || data.ret == -1) return; + if (mlx5_check_mempool(mp, &start, &end) != 0) { + ERROR("mempool %p: not virtually contiguous", + (void *)mp); + return; + } for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) { - if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) { + struct ibv_mr *mr = txq_ctrl->txq.mp2mr[i].mr; + + if (unlikely(mr == NULL)) { /* Unknown MP, add a new MR for it. */ break; } - if (txq_ctrl->txq.mp2mr[i].mp == mp) + if (start >= (uintptr_t)mr->addr && + end <= (uintptr_t)mr->addr + mr->length) return; } txq_mp2mr_reg(&txq_ctrl->txq, mp, i); diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c index 173e6e84..a67e5426 100644 --- a/drivers/net/mlx5/mlx5_rxmode.c +++ b/drivers/net/mlx5/mlx5_rxmode.c @@ -347,6 +347,8 @@ priv_special_flow_enable_all(struct priv *priv) { enum hash_rxq_flow_type flow_type; + if (priv->isolated) + return 0; for (flow_type = HASH_RXQ_FLOW_TYPE_PROMISC; flow_type != HASH_RXQ_FLOW_TYPE_MAC; ++flow_type) { diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 2a268398..74387a79 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -363,6 +363,8 @@ priv_create_hash_rxqs(struct priv *priv) assert(priv->hash_rxqs_n == 0); assert(priv->pd != NULL); assert(priv->ctx != NULL); + if (priv->isolated) + return 0; if (priv->rxqs_n == 0) return EINVAL; assert(priv->rxqs != NULL); @@ -632,6 +634,32 @@ priv_rehash_flows(struct priv *priv) } /** + * Unlike regular Rx function, vPMD Rx doesn't replace mbufs immediately when + * receiving packets. Instead it replaces later in bulk. In rxq->elts[], entries + * from rq_pi to rq_ci are owned by device but the rest is already delivered to + * application. In order not to reuse those mbufs by rxq_alloc_elts(), this + * function must be called to replace used mbufs. + * + * @param rxq + * Pointer to RX queue structure. + */ +static void +rxq_trim_elts(struct rxq *rxq) +{ + const uint16_t q_n = (1 << rxq->elts_n); + const uint16_t q_mask = q_n - 1; + uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi); + uint16_t i; + + if (!rxq->trim_elts) + return; + for (i = 0; i < used; ++i) + (*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL; + rxq->trim_elts = 0; + return; +} + +/** * Allocate RX queue elements. * * @param rxq_ctrl @@ -659,15 +687,13 @@ rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n, volatile struct mlx5_wqe_data_seg *scat = &(*rxq_ctrl->rxq.wqes)[i]; - if (pool != NULL) { - buf = (*pool)[i]; - assert(buf != NULL); + buf = (pool != NULL) ? (*pool)[i] : NULL; + if (buf != NULL) { rte_pktmbuf_reset(buf); rte_pktmbuf_refcnt_update(buf, 1); } else buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp); if (buf == NULL) { - assert(pool == NULL); ERROR("%p: empty mbuf pool", (void *)rxq_ctrl); ret = ENOMEM; goto error; @@ -722,6 +748,7 @@ rxq_free_elts(struct rxq_ctrl *rxq_ctrl) { unsigned int i; + rxq_trim_elts(&rxq_ctrl->rxq); DEBUG("%p: freeing WRs", (void *)rxq_ctrl); if (rxq_ctrl->rxq.elts == NULL) return; @@ -760,72 +787,6 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl) } /** - * Reconfigure RX queue buffers. - * - * rxq_rehash() does not allocate mbufs, which, if not done from the right - * thread (such as a control thread), may corrupt the pool. - * In case of failure, the queue is left untouched. - * - * @param dev - * Pointer to Ethernet device structure. - * @param rxq_ctrl - * RX queue pointer. - * - * @return - * 0 on success, errno value on failure. - */ -int -rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl) -{ - unsigned int elts_n = 1 << rxq_ctrl->rxq.elts_n; - unsigned int i; - struct ibv_exp_wq_attr mod; - int err; - - DEBUG("%p: rehashing queue %p with %u SGE(s) per packet", - (void *)dev, (void *)rxq_ctrl, 1 << rxq_ctrl->rxq.sges_n); - assert(!(elts_n % (1 << rxq_ctrl->rxq.sges_n))); - /* From now on, any failure will render the queue unusable. - * Reinitialize WQ. */ - mod = (struct ibv_exp_wq_attr){ - .attr_mask = IBV_EXP_WQ_ATTR_STATE, - .wq_state = IBV_EXP_WQS_RESET, - }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); - if (err) { - ERROR("%p: cannot reset WQ: %s", (void *)dev, strerror(err)); - assert(err > 0); - return err; - } - /* Snatch mbufs from original queue. */ - claim_zero(rxq_alloc_elts(rxq_ctrl, elts_n, rxq_ctrl->rxq.elts)); - for (i = 0; i != elts_n; ++i) { - struct rte_mbuf *buf = (*rxq_ctrl->rxq.elts)[i]; - - assert(rte_mbuf_refcnt_read(buf) == 2); - rte_pktmbuf_free_seg(buf); - } - /* Change queue state to ready. */ - mod = (struct ibv_exp_wq_attr){ - .attr_mask = IBV_EXP_WQ_ATTR_STATE, - .wq_state = IBV_EXP_WQS_RDY, - }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); - if (err) { - ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s", - (void *)dev, strerror(err)); - goto error; - } - /* Update doorbell counter. */ - rxq_ctrl->rxq.rq_ci = elts_n >> rxq_ctrl->rxq.sges_n; - rte_wmb(); - *rxq_ctrl->rxq.rq_db = htonl(rxq_ctrl->rxq.rq_ci); -error: - assert(err >= 0); - return err; -} - -/** * Initialize RX queue. * * @param tmpl @@ -858,6 +819,7 @@ rxq_setup(struct rxq_ctrl *tmpl) tmpl->rxq.cqe_n = log2above(cq_info.cqe_cnt); tmpl->rxq.cq_ci = 0; tmpl->rxq.rq_ci = 0; + tmpl->rxq.rq_pi = 0; tmpl->rxq.cq_db = cq_info.dbrec; tmpl->rxq.wqes = (volatile struct mlx5_wqe_data_seg (*)[]) @@ -888,7 +850,7 @@ rxq_setup(struct rxq_ctrl *tmpl) * @return * 0 on success, errno value on failure. */ -int +static int rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, struct rte_mempool *mp) @@ -978,10 +940,10 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, if (dev->data->dev_conf.intr_conf.rxq) { tmpl.channel = ibv_create_comp_channel(priv->ctx); if (tmpl.channel == NULL) { - dev->data->dev_conf.intr_conf.rxq = 0; ret = ENOMEM; - ERROR("%p: Comp Channel creation failure: %s", - (void *)dev, strerror(ret)); + ERROR("%p: Rx interrupt completion channel creation" + " failure: %s", + (void *)dev, strerror(ret)); goto error; } } @@ -991,7 +953,12 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, if (priv->cqe_comp) { attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS; attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE; - cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */ + /* + * For vectorized Rx, it must not be doubled in order to + * make cq_ci and rq_ci aligned. + */ + if (rxq_check_vec_support(&tmpl.rxq) < 0) + cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */ } tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0, &attr.cq); @@ -1101,6 +1068,7 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, if (rxq_ctrl->rxq.elts_n) { assert(1 << rxq_ctrl->rxq.elts_n == desc); assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts); + rxq_trim_elts(&rxq_ctrl->rxq); ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts); } else ret = rxq_alloc_elts(&tmpl, desc, NULL); @@ -1163,6 +1131,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct priv *priv = dev->data->dev_private; struct rxq *rxq = (*priv->rxqs)[idx]; struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); + const uint16_t desc_pad = MLX5_VPMD_DESCS_PER_LOOP; /* For vPMD. */ int ret; if (mlx5_is_secondary()) @@ -1196,7 +1165,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, if (rxq_ctrl->rxq.elts_n != log2above(desc)) { rxq_ctrl = rte_realloc(rxq_ctrl, sizeof(*rxq_ctrl) + - desc * sizeof(struct rte_mbuf *), + (desc + desc_pad) * + sizeof(struct rte_mbuf *), RTE_CACHE_LINE_SIZE); if (!rxq_ctrl) { ERROR("%p: unable to reallocate queue index %u", @@ -1207,7 +1177,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, } } else { rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) + - desc * sizeof(struct rte_mbuf *), + (desc + desc_pad) * + sizeof(struct rte_mbuf *), 0, socket); if (rxq_ctrl == NULL) { ERROR("%p: unable to allocate queue index %u", @@ -1224,8 +1195,6 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, DEBUG("%p: adding RX queue %p to list", (void *)dev, (void *)rxq_ctrl); (*priv->rxqs)[idx] = &rxq_ctrl->rxq; - /* Update receive callback. */ - priv_select_rx_function(priv); } priv_unlock(priv); return -ret; @@ -1310,111 +1279,159 @@ mlx5_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, } /** - * Fill epoll fd list for rxq interrupts. + * Allocate queue vector and fill epoll fd list for Rx interrupts. * * @param priv - * Private structure. + * Pointer to private structure. * * @return * 0 on success, negative on failure. */ int -priv_intr_efd_enable(struct priv *priv) +priv_rx_intr_vec_enable(struct priv *priv) { unsigned int i; unsigned int rxqs_n = priv->rxqs_n; unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + unsigned int count = 0; struct rte_intr_handle *intr_handle = priv->dev->intr_handle; - if (n == 0) + if (!priv->dev->data->dev_conf.intr_conf.rxq) return 0; - if (n < rxqs_n) { - WARN("rxqs num is larger than EAL max interrupt vector " - "%u > %u unable to supprt rxq interrupts", - rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); - return -EINVAL; + priv_rx_intr_vec_disable(priv); + intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n])); + if (intr_handle->intr_vec == NULL) { + ERROR("failed to allocate memory for interrupt vector," + " Rx interrupts will not be supported"); + return -ENOMEM; } intr_handle->type = RTE_INTR_HANDLE_EXT; for (i = 0; i != n; ++i) { struct rxq *rxq = (*priv->rxqs)[i]; struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - int fd = rxq_ctrl->channel->fd; + int fd; int flags; int rc; + /* Skip queues that cannot request interrupts. */ + if (!rxq || !rxq_ctrl->channel) { + /* Use invalid intr_vec[] index to disable entry. */ + intr_handle->intr_vec[i] = + RTE_INTR_VEC_RXTX_OFFSET + + RTE_MAX_RXTX_INTR_VEC_ID; + continue; + } + if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { + ERROR("too many Rx queues for interrupt vector size" + " (%d), Rx interrupts cannot be enabled", + RTE_MAX_RXTX_INTR_VEC_ID); + priv_rx_intr_vec_disable(priv); + return -1; + } + fd = rxq_ctrl->channel->fd; flags = fcntl(fd, F_GETFL); rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK); if (rc < 0) { - WARN("failed to change rxq interrupt file " - "descriptor %d for queue index %d", fd, i); + ERROR("failed to make Rx interrupt file descriptor" + " %d non-blocking for queue index %d", fd, i); + priv_rx_intr_vec_disable(priv); return -1; } - intr_handle->efds[i] = fd; + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; + intr_handle->efds[count] = fd; + count++; } - intr_handle->nb_efd = n; + if (!count) + priv_rx_intr_vec_disable(priv); + else + intr_handle->nb_efd = count; return 0; } /** - * Clean epoll fd list for rxq interrupts. + * Clean up Rx interrupts handler. * * @param priv - * Private structure. + * Pointer to private structure. */ void -priv_intr_efd_disable(struct priv *priv) +priv_rx_intr_vec_disable(struct priv *priv) { struct rte_intr_handle *intr_handle = priv->dev->intr_handle; rte_intr_free_epoll_fd(intr_handle); + free(intr_handle->intr_vec); + intr_handle->nb_efd = 0; + intr_handle->intr_vec = NULL; } +#ifdef HAVE_UPDATE_CQ_CI + /** - * Create and init interrupt vector array. + * DPDK callback for Rx queue interrupt enable. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device structure. + * @param rx_queue_id + * Rx queue number. * * @return * 0 on success, negative on failure. */ int -priv_create_intr_vec(struct priv *priv) +mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - unsigned int rxqs_n = priv->rxqs_n; - unsigned int i; - struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + struct priv *priv = mlx5_get_priv(dev); + struct rxq *rxq = (*priv->rxqs)[rx_queue_id]; + struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); + int ret; - if (rxqs_n == 0) - return 0; - intr_handle->intr_vec = (int *) - rte_malloc("intr_vec", rxqs_n * sizeof(int), 0); - if (intr_handle->intr_vec == NULL) { - WARN("Failed to allocate memory for intr_vec " - "rxq interrupt will not be supported"); - return -ENOMEM; - } - for (i = 0; i != rxqs_n; ++i) { - /* 1:1 mapping between rxq and interrupt. */ - intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i; + if (!rxq || !rxq_ctrl->channel) { + ret = EINVAL; + } else { + ibv_mlx5_exp_update_cq_ci(rxq_ctrl->cq, rxq->cq_ci); + ret = ibv_req_notify_cq(rxq_ctrl->cq, 0); } - return 0; + if (ret) + WARN("unable to arm interrupt on rx queue %d", rx_queue_id); + return -ret; } /** - * Destroy init interrupt vector array. + * DPDK callback for Rx queue interrupt disable. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device structure. + * @param rx_queue_id + * Rx queue number. * * @return * 0 on success, negative on failure. */ -void -priv_destroy_intr_vec(struct priv *priv) +int +mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + struct priv *priv = mlx5_get_priv(dev); + struct rxq *rxq = (*priv->rxqs)[rx_queue_id]; + struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); + struct ibv_cq *ev_cq; + void *ev_ctx; + int ret; - rte_free(intr_handle->intr_vec); + if (!rxq || !rxq_ctrl->channel) { + ret = EINVAL; + } else { + ret = ibv_get_cq_event(rxq_ctrl->cq->channel, &ev_cq, &ev_ctx); + if (ret || ev_cq != rxq_ctrl->cq) + ret = EINVAL; + } + if (ret) + WARN("unable to disable interrupt on rx queue %d", + rx_queue_id); + else + ibv_ack_cq_events(rxq_ctrl->cq, 1); + return -ret; } + +#endif /* HAVE_UPDATE_CQ_CI */ diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index de6e0fa4..b07bcd11 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -69,129 +69,131 @@ #include "mlx5_defs.h" #include "mlx5_prm.h" -static inline int -check_cqe(volatile struct mlx5_cqe *cqe, - unsigned int cqes_n, const uint16_t ci) - __attribute__((always_inline)); - -static inline void -txq_complete(struct txq *txq) __attribute__((always_inline)); +static __rte_always_inline uint32_t +rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe); -static inline uint32_t -txq_mp2mr(struct txq *txq, struct rte_mempool *mp) - __attribute__((always_inline)); - -static inline void -mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe) - __attribute__((always_inline)); - -static inline uint32_t -rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe) - __attribute__((always_inline)); - -static inline int +static __rte_always_inline int mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe, - uint16_t cqe_cnt, uint32_t *rss_hash) - __attribute__((always_inline)); + uint16_t cqe_cnt, uint32_t *rss_hash); -static inline uint32_t -rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe) - __attribute__((always_inline)); +static __rte_always_inline uint32_t +rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe); -#ifndef NDEBUG +uint32_t mlx5_ptype_table[] __rte_cache_aligned = { + [0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */ +}; /** - * Verify or set magic value in CQE. - * - * @param cqe - * Pointer to CQE. + * Build a table to translate Rx completion flags to packet type. * - * @return - * 0 the first time. + * @note: fix mlx5_dev_supported_ptypes_get() if any change here. */ -static inline int -check_cqe_seen(volatile struct mlx5_cqe *cqe) +void +mlx5_set_ptype_table(void) { - static const uint8_t magic[] = "seen"; - volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0; - int ret = 1; unsigned int i; + uint32_t (*p)[RTE_DIM(mlx5_ptype_table)] = &mlx5_ptype_table; - for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i) - if (!ret || (*buf)[i] != magic[i]) { - ret = 0; - (*buf)[i] = magic[i]; - } - return ret; -} - -#endif /* NDEBUG */ - -/** - * Check whether CQE is valid. - * - * @param cqe - * Pointer to CQE. - * @param cqes_n - * Size of completion queue. - * @param ci - * Consumer index. - * - * @return - * 0 on success, 1 on failure. - */ -static inline int -check_cqe(volatile struct mlx5_cqe *cqe, - unsigned int cqes_n, const uint16_t ci) -{ - uint16_t idx = ci & cqes_n; - uint8_t op_own = cqe->op_own; - uint8_t op_owner = MLX5_CQE_OWNER(op_own); - uint8_t op_code = MLX5_CQE_OPCODE(op_own); - - if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID))) - return 1; /* No CQE. */ -#ifndef NDEBUG - if ((op_code == MLX5_CQE_RESP_ERR) || - (op_code == MLX5_CQE_REQ_ERR)) { - volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe; - uint8_t syndrome = err_cqe->syndrome; - - if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) || - (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR)) - return 0; - if (!check_cqe_seen(cqe)) - ERROR("unexpected CQE error %u (0x%02x)" - " syndrome 0x%02x", - op_code, op_code, syndrome); - return 1; - } else if ((op_code != MLX5_CQE_RESP_SEND) && - (op_code != MLX5_CQE_REQ)) { - if (!check_cqe_seen(cqe)) - ERROR("unexpected CQE opcode %u (0x%02x)", - op_code, op_code); - return 1; - } -#endif /* NDEBUG */ - return 0; -} - -/** - * Return the address of the WQE. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe_ci - * WQE consumer index. - * - * @return - * WQE address. - */ -static inline uintptr_t * -tx_mlx5_wqe(struct txq *txq, uint16_t ci) -{ - ci &= ((1 << txq->wqe_n) - 1); - return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE); + /* Last entry must not be overwritten, reserved for errored packet. */ + for (i = 0; i < RTE_DIM(mlx5_ptype_table) - 1; ++i) + (*p)[i] = RTE_PTYPE_UNKNOWN; + /* + * The index to the array should have: + * bit[1:0] = l3_hdr_type + * bit[4:2] = l4_hdr_type + * bit[5] = ip_frag + * bit[6] = tunneled + * bit[7] = outer_l3_type + */ + /* L3 */ + (*p)[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + (*p)[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + /* Fragmented */ + (*p)[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + (*p)[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + /* TCP */ + (*p)[0x05] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + /* UDP */ + (*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + (*p)[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + /* Repeat with outer_l3_type being set. Just in case. */ + (*p)[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + (*p)[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + (*p)[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + (*p)[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + (*p)[0x85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + (*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + /* Tunneled - L3 */ + (*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + (*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + (*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + (*p)[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + /* Tunneled - Fragmented */ + (*p)[0x61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + (*p)[0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + (*p)[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + (*p)[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + /* Tunneled - TCP */ + (*p)[0x45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + /* Tunneled - UDP */ + (*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + (*p)[0x4a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + (*p)[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + (*p)[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; } /** @@ -251,156 +253,6 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n, } /** - * Manage TX completions. - * - * When sending a burst, mlx5_tx_burst() posts several WRs. - * - * @param txq - * Pointer to TX queue structure. - */ -static inline void -txq_complete(struct txq *txq) -{ - const unsigned int elts_n = 1 << txq->elts_n; - const unsigned int cqe_n = 1 << txq->cqe_n; - const unsigned int cqe_cnt = cqe_n - 1; - uint16_t elts_free = txq->elts_tail; - uint16_t elts_tail; - uint16_t cq_ci = txq->cq_ci; - volatile struct mlx5_cqe *cqe = NULL; - volatile struct mlx5_wqe_ctrl *ctrl; - - do { - volatile struct mlx5_cqe *tmp; - - tmp = &(*txq->cqes)[cq_ci & cqe_cnt]; - if (check_cqe(tmp, cqe_n, cq_ci)) - break; - cqe = tmp; -#ifndef NDEBUG - if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) { - if (!check_cqe_seen(cqe)) - ERROR("unexpected compressed CQE, TX stopped"); - return; - } - if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) || - (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) { - if (!check_cqe_seen(cqe)) - ERROR("unexpected error CQE, TX stopped"); - return; - } -#endif /* NDEBUG */ - ++cq_ci; - } while (1); - if (unlikely(cqe == NULL)) - return; - txq->wqe_pi = ntohs(cqe->wqe_counter); - ctrl = (volatile struct mlx5_wqe_ctrl *) - tx_mlx5_wqe(txq, txq->wqe_pi); - elts_tail = ctrl->ctrl3; - assert(elts_tail < (1 << txq->wqe_n)); - /* Free buffers. */ - while (elts_free != elts_tail) { - struct rte_mbuf *elt = (*txq->elts)[elts_free]; - unsigned int elts_free_next = - (elts_free + 1) & (elts_n - 1); - struct rte_mbuf *elt_next = (*txq->elts)[elts_free_next]; - -#ifndef NDEBUG - /* Poisoning. */ - memset(&(*txq->elts)[elts_free], - 0x66, - sizeof((*txq->elts)[elts_free])); -#endif - RTE_MBUF_PREFETCH_TO_FREE(elt_next); - /* Only one segment needs to be freed. */ - rte_pktmbuf_free_seg(elt); - elts_free = elts_free_next; - } - txq->cq_ci = cq_ci; - txq->elts_tail = elts_tail; - /* Update the consumer index. */ - rte_wmb(); - *txq->cq_db = htonl(cq_ci); -} - -/** - * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which - * the cloned mbuf is allocated is returned instead. - * - * @param buf - * Pointer to mbuf. - * - * @return - * Memory pool where data is located for given mbuf. - */ -static struct rte_mempool * -txq_mb2mp(struct rte_mbuf *buf) -{ - if (unlikely(RTE_MBUF_INDIRECT(buf))) - return rte_mbuf_from_indirect(buf)->pool; - return buf->pool; -} - -/** - * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[]. - * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full, - * remove an entry first. - * - * @param txq - * Pointer to TX queue structure. - * @param[in] mp - * Memory Pool for which a Memory Region lkey must be returned. - * - * @return - * mr->lkey on success, (uint32_t)-1 on failure. - */ -static inline uint32_t -txq_mp2mr(struct txq *txq, struct rte_mempool *mp) -{ - unsigned int i; - uint32_t lkey = (uint32_t)-1; - - for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) { - if (unlikely(txq->mp2mr[i].mp == NULL)) { - /* Unknown MP, add a new MR for it. */ - break; - } - if (txq->mp2mr[i].mp == mp) { - assert(txq->mp2mr[i].lkey != (uint32_t)-1); - assert(htonl(txq->mp2mr[i].mr->lkey) == - txq->mp2mr[i].lkey); - lkey = txq->mp2mr[i].lkey; - break; - } - } - if (unlikely(lkey == (uint32_t)-1)) - lkey = txq_mp2mr_reg(txq, mp, i); - return lkey; -} - -/** - * Ring TX queue doorbell. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the last WQE posted in the NIC. - */ -static inline void -mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe) -{ - uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg); - volatile uint64_t *src = ((volatile uint64_t *)wqe); - - rte_wmb(); - *txq->qp_db = htonl(txq->wqe_ci); - /* Ensure ordering between DB record and BF copy. */ - rte_wmb(); - *dst = *src; -} - -/** * DPDK callback to check the status of a tx descriptor. * * @param tx_queue @@ -415,12 +267,10 @@ int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset) { struct txq *txq = tx_queue; - const unsigned int elts_n = 1 << txq->elts_n; - const unsigned int elts_cnt = elts_n - 1; - unsigned int used; + uint16_t used; - txq_complete(txq); - used = (txq->elts_head - txq->elts_tail) & elts_cnt; + mlx5_tx_complete(txq); + used = txq->elts_head - txq->elts_tail; if (offset < used) return RTE_ETH_TX_DESC_FULL; return RTE_ETH_TX_DESC_DONE; @@ -494,11 +344,12 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct txq *txq = (struct txq *)dpdk_txq; uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = 1 << txq->elts_n; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; unsigned int i = 0; unsigned int j = 0; unsigned int k = 0; - unsigned int max; + uint16_t max_elts; unsigned int max_inline = txq->max_inline; const unsigned int inline_en = !!max_inline && txq->inline_en; uint16_t max_wqe; @@ -514,10 +365,8 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) /* Prefetch first packet cacheline. */ rte_prefetch0(*pkts); /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); if (unlikely(!max_wqe)) return 0; @@ -533,6 +382,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) uint16_t ehdr; uint8_t cs_flags = 0; uint64_t tso = 0; + uint16_t tso_segsz = 0; #ifdef MLX5_PMD_SOFT_COUNTERS uint32_t total_length = 0; #endif @@ -545,9 +395,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) * that one ring entry remains unused. */ assert(segs_n); - if (max < segs_n + 1) + if (max_elts < segs_n) break; - max -= segs_n; + max_elts -= segs_n; --segs_n; if (unlikely(--max_wqe == 0)) break; @@ -566,7 +416,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if (length < (MLX5_WQE_DWORD_SIZE + 2)) break; /* Update element. */ - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head & elts_m] = buf; /* Prefetch next buffer data. */ if (pkts_n - i > 1) rte_prefetch0( @@ -628,6 +478,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) tso_header_sz = buf->l2_len + vlan_sz + buf->l3_len + buf->l4_len; + tso_segsz = buf->tso_segsz; if (is_tunneled && txq->tunnel_en) { tso_header_sz += buf->outer_l2_len + @@ -762,7 +613,7 @@ use_dseg: naddr = htonll(addr); *dseg = (rte_v128u32_t){ htonl(length), - txq_mp2mr(txq, txq_mb2mp(buf)), + mlx5_tx_mb2mr(txq, buf), naddr, naddr >> 32, }; @@ -801,12 +652,11 @@ next_seg: naddr = htonll(rte_pktmbuf_mtod(buf, uintptr_t)); *dseg = (rte_v128u32_t){ htonl(length), - txq_mp2mr(txq, txq_mb2mp(buf)), + mlx5_tx_mb2mr(txq, buf), naddr, naddr >> 32, }; - elts_head = (elts_head + 1) & (elts_n - 1); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[++elts_head & elts_m] = buf; ++sg; /* Advance counter only if all segs are successfully posted. */ if (sg < segs_n) @@ -814,7 +664,7 @@ next_seg: else j += sg; next_pkt: - elts_head = (elts_head + 1) & (elts_n - 1); + ++elts_head; ++pkts; ++i; /* Initialize known and common part of the WQE structure. */ @@ -827,7 +677,7 @@ next_pkt: }; wqe->eseg = (rte_v128u32_t){ 0, - cs_flags | (htons(buf->tso_segsz) << 16), + cs_flags | (htons(tso_segsz) << 16), 0, (ehdr << 16) | htons(tso_header_sz), }; @@ -857,7 +707,7 @@ next_wqe: /* Take a shortcut if nothing must be sent. */ if (unlikely((i + k) == 0)) return 0; - txq->elts_head = (txq->elts_head + i + j) & (elts_n - 1); + txq->elts_head += (i + j); /* Check whether completion threshold has been reached. */ comp = txq->elts_comp + i + j + k; if (comp >= MLX5_TX_COMP_THRESH) { @@ -964,10 +814,11 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct txq *txq = (struct txq *)dpdk_txq; uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = 1 << txq->elts_n; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; unsigned int i = 0; unsigned int j = 0; - unsigned int max; + uint16_t max_elts; uint16_t max_wqe; unsigned int comp; struct mlx5_mpw mpw = { @@ -980,16 +831,13 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci)); rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1)); /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); if (unlikely(!max_wqe)) return 0; do { struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; uint32_t length; unsigned int segs_n = buf->nb_segs; uint32_t cs_flags = 0; @@ -999,12 +847,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) * that one ring entry remains unused. */ assert(segs_n); - if (max < segs_n + 1) + if (max_elts < segs_n) break; /* Do not bother with large packets MPW cannot handle. */ if (segs_n > MLX5_MPW_DSEG_MAX) break; - max -= segs_n; + max_elts -= segs_n; --pkts_n; /* Should we enable HW CKSUM offload */ if (buf->ol_flags & @@ -1040,17 +888,15 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) volatile struct mlx5_wqe_data_seg *dseg; uintptr_t addr; - elts_head_next = (elts_head + 1) & (elts_n - 1); assert(buf); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; dseg = mpw.data.dseg[mpw.pkts_n]; addr = rte_pktmbuf_mtod(buf, uintptr_t); *dseg = (struct mlx5_wqe_data_seg){ .byte_count = htonl(DATA_LEN(buf)), - .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), + .lkey = mlx5_tx_mb2mr(txq, buf), .addr = htonll(addr), }; - elts_head = elts_head_next; #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) length += DATA_LEN(buf); #endif @@ -1061,7 +907,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) assert(length == mpw.len); if (mpw.pkts_n == MLX5_MPW_DSEG_MAX) mlx5_mpw_close(txq, &mpw); - elts_head = elts_head_next; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += length; @@ -1179,10 +1024,11 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, { struct txq *txq = (struct txq *)dpdk_txq; uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = 1 << txq->elts_n; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; unsigned int i = 0; unsigned int j = 0; - unsigned int max; + uint16_t max_elts; uint16_t max_wqe; unsigned int comp; unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE; @@ -1208,13 +1054,10 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci)); rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1)); /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); do { struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; uintptr_t addr; uint32_t length; unsigned int segs_n = buf->nb_segs; @@ -1225,12 +1068,12 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, * that one ring entry remains unused. */ assert(segs_n); - if (max < segs_n + 1) + if (max_elts < segs_n) break; /* Do not bother with large packets MPW cannot handle. */ if (segs_n > MLX5_MPW_DSEG_MAX) break; - max -= segs_n; + max_elts -= segs_n; --pkts_n; /* * Compute max_wqe in case less WQE were consumed in previous @@ -1291,18 +1134,15 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, do { volatile struct mlx5_wqe_data_seg *dseg; - elts_head_next = - (elts_head + 1) & (elts_n - 1); assert(buf); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; dseg = mpw.data.dseg[mpw.pkts_n]; addr = rte_pktmbuf_mtod(buf, uintptr_t); *dseg = (struct mlx5_wqe_data_seg){ .byte_count = htonl(DATA_LEN(buf)), - .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), + .lkey = mlx5_tx_mb2mr(txq, buf), .addr = htonll(addr), }; - elts_head = elts_head_next; #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) length += DATA_LEN(buf); #endif @@ -1319,9 +1159,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, assert(mpw.state == MLX5_MPW_INL_STATE_OPENED); assert(length <= inline_room); assert(length == DATA_LEN(buf)); - elts_head_next = (elts_head + 1) & (elts_n - 1); addr = rte_pktmbuf_mtod(buf, uintptr_t); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; /* Maximum number of bytes before wrapping. */ max = ((((uintptr_t)(txq->wqes)) + (1 << txq->wqe_n) * @@ -1358,7 +1197,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, inline_room -= length; } } - elts_head = elts_head_next; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += length; @@ -1480,10 +1318,11 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct txq *txq = (struct txq *)dpdk_txq; uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = 1 << txq->elts_n; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; unsigned int i = 0; unsigned int j = 0; - unsigned int max_elts; + uint16_t max_elts; uint16_t max_wqe; unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE; unsigned int mpw_room = 0; @@ -1496,10 +1335,8 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if (unlikely(!pkts_n)) return 0; /* Start processing. */ - txq_complete(txq); + mlx5_tx_complete(txq); max_elts = (elts_n - (elts_head - txq->elts_tail)); - if (max_elts > elts_n) - max_elts -= elts_n; /* A CQE slot must always be available. */ assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci)); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); @@ -1507,7 +1344,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) return 0; do { struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; uintptr_t addr; uint64_t naddr; unsigned int n; @@ -1521,7 +1357,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) * that one ring entry remains unused. */ assert(segs_n); - if (max_elts - j < segs_n + 1) + if (max_elts - j < segs_n) break; /* Do not bother with large packets MPW cannot handle. */ if (segs_n > MLX5_MPW_DSEG_MAX) @@ -1605,18 +1441,15 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) do { volatile struct mlx5_wqe_data_seg *dseg; - elts_head_next = - (elts_head + 1) & (elts_n - 1); assert(buf); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; dseg = mpw.data.dseg[mpw.pkts_n]; addr = rte_pktmbuf_mtod(buf, uintptr_t); *dseg = (struct mlx5_wqe_data_seg){ .byte_count = htonl(DATA_LEN(buf)), - .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), + .lkey = mlx5_tx_mb2mr(txq, buf), .addr = htonll(addr), }; - elts_head = elts_head_next; #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) length += DATA_LEN(buf); #endif @@ -1667,7 +1500,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) /* No need to get completion as the entire packet is * copied to WQ. Free the buf right away. */ - elts_head_next = elts_head; rte_pktmbuf_free_seg(buf); mpw_room -= (inl_pad + sizeof(inl_hdr) + length); /* Add pad in the next packet if any. */ @@ -1690,8 +1522,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) dseg = (volatile void *) ((uintptr_t)mpw.data.raw + inl_pad); - elts_head_next = (elts_head + 1) & (elts_n - 1); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; addr = rte_pktmbuf_mtod(buf, uintptr_t); for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++) rte_prefetch2((void *)(addr + @@ -1699,7 +1530,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) naddr = htonll(addr); *dseg = (rte_v128u32_t) { htonl(length), - txq_mp2mr(txq, txq_mb2mp(buf)), + mlx5_tx_mb2mr(txq, buf), naddr, naddr >> 32, }; @@ -1710,7 +1541,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) mpw_room -= (inl_pad + sizeof(*dseg)); inl_pad = 0; } - elts_head = elts_head_next; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += length; @@ -1764,30 +1594,20 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) static inline uint32_t rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe) { - uint32_t pkt_type; - uint16_t flags = ntohs(cqe->hdr_type_etc); + uint8_t idx; + uint8_t pinfo = cqe->pkt_info; + uint16_t ptype = cqe->hdr_type_etc; - if (cqe->pkt_info & MLX5_CQE_RX_TUNNEL_PACKET) { - pkt_type = - TRANSPOSE(flags, - MLX5_CQE_RX_IPV4_PACKET, - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN) | - TRANSPOSE(flags, - MLX5_CQE_RX_IPV6_PACKET, - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN); - pkt_type |= ((cqe->pkt_info & MLX5_CQE_RX_OUTER_PACKET) ? - RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN); - } else { - pkt_type = - TRANSPOSE(flags, - MLX5_CQE_L3_HDR_TYPE_IPV6, - RTE_PTYPE_L3_IPV6_EXT_UNKNOWN) | - TRANSPOSE(flags, - MLX5_CQE_L3_HDR_TYPE_IPV4, - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN); - } - return pkt_type; + /* + * The index to the array should have: + * bit[1:0] = l3_hdr_type + * bit[4:2] = l4_hdr_type + * bit[5] = ip_frag + * bit[6] = tunneled + * bit[7] = outer_l3_type + */ + idx = ((pinfo & 0x3) << 6) | ((ptype & 0xfc00) >> 10); + return mlx5_ptype_table[idx]; } /** @@ -1819,7 +1639,7 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe, if (zip->ai) { volatile struct mlx5_mini_cqe8 (*mc)[8] = (volatile struct mlx5_mini_cqe8 (*)[8]) - (uintptr_t)(&(*rxq->cqes)[zip->ca & cqe_cnt]); + (uintptr_t)(&(*rxq->cqes)[zip->ca & cqe_cnt].pkt_info); len = ntohl((*mc)[zip->ai & 7].byte_cnt); *rss_hash = ntohl((*mc)[zip->ai & 7].rx_hash_result); @@ -1867,7 +1687,7 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe, volatile struct mlx5_mini_cqe8 (*mc)[8] = (volatile struct mlx5_mini_cqe8 (*)[8]) (uintptr_t)(&(*rxq->cqes)[rxq->cq_ci & - cqe_cnt]); + cqe_cnt].pkt_info); /* Fix endianness. */ zip->cqe_cnt = ntohl(cqe->byte_cnt); @@ -2018,7 +1838,7 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) pkt = seg; assert(len >= (rxq->crc_present << 2)); /* Update packet information. */ - pkt->packet_type = 0; + pkt->packet_type = rxq_cq_to_pkt_type(cqe); pkt->ol_flags = 0; if (rss_hash_res && rxq->rss_hash) { pkt->hash.rss = rss_hash_res; @@ -2036,10 +1856,8 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) mlx5_flow_mark_get(mark); } } - if (rxq->csum | rxq->csum_l2tun) { - pkt->packet_type = rxq_cq_to_pkt_type(cqe); + if (rxq->csum | rxq->csum_l2tun) pkt->ol_flags |= rxq_cq_to_ol_flags(rxq, cqe); - } if (rxq->vlan_strip && (cqe->hdr_type_etc & htons(MLX5_CQE_VLAN_STRIPPED))) { @@ -2054,9 +1872,7 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) DATA_LEN(rep) = DATA_LEN(seg); PKT_LEN(rep) = PKT_LEN(seg); SET_DATA_OFF(rep, DATA_OFF(seg)); - NB_SEGS(rep) = NB_SEGS(seg); PORT(rep) = PORT(seg); - NEXT(rep) = NULL; (*rxq->elts)[idx] = rep; /* * Fill NIC descriptor with the new buffer. The lkey and size @@ -2151,75 +1967,70 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) return 0; } -/** - * DPDK callback for rx queue interrupt enable. - * - * @param dev - * Pointer to Ethernet device structure. - * @param rx_queue_id - * RX queue number - * - * @return - * 0 on success, negative on failure. +/* + * Vectorized Rx/Tx routines are not compiled in when required vector + * instructions are not supported on a target architecture. The following null + * stubs are needed for linkage when those are not included outside of this file + * (e.g. mlx5_rxtx_vec_sse.c for x86). */ -int -mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) + +uint16_t __attribute__((weak)) +mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { -#ifdef HAVE_UPDATE_CQ_CI - struct priv *priv = mlx5_get_priv(dev); - struct rxq *rxq = (*priv->rxqs)[rx_queue_id]; - struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - struct ibv_cq *cq = rxq_ctrl->cq; - uint16_t ci = rxq->cq_ci; - int ret = 0; - - ibv_mlx5_exp_update_cq_ci(cq, ci); - ret = ibv_req_notify_cq(cq, 0); -#else - int ret = -1; - (void)dev; - (void)rx_queue_id; -#endif - if (ret) - WARN("unable to arm interrupt on rx queue %d", rx_queue_id); - return ret; + (void)dpdk_txq; + (void)pkts; + (void)pkts_n; + return 0; } -/** - * DPDK callback for rx queue interrupt disable. - * - * @param dev - * Pointer to Ethernet device structure. - * @param rx_queue_id - * RX queue number - * - * @return - * 0 on success, negative on failure. - */ -int -mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +uint16_t __attribute__((weak)) +mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { -#ifdef HAVE_UPDATE_CQ_CI - struct priv *priv = mlx5_get_priv(dev); - struct rxq *rxq = (*priv->rxqs)[rx_queue_id]; - struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - struct ibv_cq *cq = rxq_ctrl->cq; - struct ibv_cq *ev_cq; - void *ev_ctx; - int ret = 0; - - ret = ibv_get_cq_event(cq->channel, &ev_cq, &ev_ctx); - if (ret || ev_cq != cq) - ret = -1; - else - ibv_ack_cq_events(cq, 1); -#else - int ret = -1; - (void)dev; - (void)rx_queue_id; -#endif - if (ret) - WARN("unable to disable interrupt on rx queue %d", - rx_queue_id); - return ret; + (void)dpdk_txq; + (void)pkts; + (void)pkts_n; + return 0; +} + +uint16_t __attribute__((weak)) +mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + (void)dpdk_rxq; + (void)pkts; + (void)pkts_n; + return 0; +} + +int __attribute__((weak)) +priv_check_raw_vec_tx_support(struct priv *priv) +{ + (void)priv; + return -ENOTSUP; +} + +int __attribute__((weak)) +priv_check_vec_tx_support(struct priv *priv) +{ + (void)priv; + return -ENOTSUP; +} + +int __attribute__((weak)) +rxq_check_vec_support(struct rxq *rxq) +{ + (void)rxq; + return -ENOTSUP; +} + +int __attribute__((weak)) +priv_check_vec_rx_support(struct priv *priv) +{ + (void)priv; + return -ENOTSUP; +} + +void __attribute__((weak)) +priv_prep_vec_rx_function(struct priv *priv) +{ + (void)priv; } diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index 8db8eb14..7de1d108 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -115,10 +115,13 @@ struct rxq { unsigned int port_id:8; unsigned int rss_hash:1; /* RSS hash result is enabled. */ unsigned int mark:1; /* Marked flow available on the queue. */ - unsigned int :8; /* Remaining bits. */ + unsigned int pending_err:1; /* CQE error needs to be handled. */ + unsigned int trim_elts:1; /* Whether elts needs clean-up. */ + unsigned int :6; /* Remaining bits. */ volatile uint32_t *rq_db; volatile uint32_t *cq_db; uint16_t rq_ci; + uint16_t rq_pi; uint16_t cq_ci; volatile struct mlx5_wqe_data_seg(*wqes)[]; volatile struct mlx5_cqe(*cqes)[]; @@ -126,6 +129,8 @@ struct rxq { struct rte_mbuf *(*elts)[]; struct rte_mempool *mp; struct mlx5_rxq_stats stats; + uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */ + struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */ } __rte_cache_aligned; /* RX queue control descriptor. */ @@ -240,10 +245,10 @@ struct hash_rxq { }; /* TX queue descriptor. */ -RTE_STD_C11 +__extension__ struct txq { - uint16_t elts_head; /* Current index in (*elts)[]. */ - uint16_t elts_tail; /* First element awaiting completion. */ + uint16_t elts_head; /* Current counter in (*elts)[]. */ + uint16_t elts_tail; /* Counter of first element awaiting completion. */ uint16_t elts_comp; /* Counter since last completion request. */ uint16_t mpw_comp; /* WQ index since last completion request. */ uint16_t cq_ci; /* Consumer index for completion queue. */ @@ -261,16 +266,19 @@ struct txq { uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ uint16_t inline_max_packet_sz; /* Max packet size for inlining. */ uint32_t qp_num_8s; /* QP number shifted by 8. */ + uint32_t flags; /* Flags for Tx Queue. */ volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ volatile void *wqes; /* Work queue (use volatile to write into). */ volatile uint32_t *qp_db; /* Work queue doorbell. */ volatile uint32_t *cq_db; /* Completion queue doorbell. */ volatile void *bf_reg; /* Blueflame register. */ struct { - const struct rte_mempool *mp; /* Cached Memory Pool. */ + uintptr_t start; /* Start address of MR */ + uintptr_t end; /* End address of MR */ struct ibv_mr *mr; /* Memory Region (for mp). */ uint32_t lkey; /* htonl(mr->lkey) */ } mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */ + uint16_t mr_cache_idx; /* Index of last hit entry. */ struct rte_mbuf *(*elts)[]; /* TX elements. */ struct mlx5_txq_stats stats; /* TX queue counters. */ } __rte_cache_aligned; @@ -298,19 +306,17 @@ int priv_create_hash_rxqs(struct priv *); void priv_destroy_hash_rxqs(struct priv *); int priv_allow_flow_type(struct priv *, enum hash_rxq_flow_type); int priv_rehash_flows(struct priv *); -int priv_intr_efd_enable(struct priv *priv); -void priv_intr_efd_disable(struct priv *priv); -int priv_create_intr_vec(struct priv *priv); -void priv_destroy_intr_vec(struct priv *priv); void rxq_cleanup(struct rxq_ctrl *); -int rxq_rehash(struct rte_eth_dev *, struct rxq_ctrl *); -int rxq_ctrl_setup(struct rte_eth_dev *, struct rxq_ctrl *, uint16_t, - unsigned int, const struct rte_eth_rxconf *, - struct rte_mempool *); int mlx5_rx_queue_setup(struct rte_eth_dev *, uint16_t, uint16_t, unsigned int, const struct rte_eth_rxconf *, struct rte_mempool *); void mlx5_rx_queue_release(void *); uint16_t mlx5_rx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t); +int priv_rx_intr_vec_enable(struct priv *priv); +void priv_rx_intr_vec_disable(struct priv *priv); +#ifdef HAVE_UPDATE_CQ_CI +int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +#endif /* HAVE_UPDATE_CQ_CI */ /* mlx5_txq.c */ @@ -324,6 +330,9 @@ uint16_t mlx5_tx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t); /* mlx5_rxtx.c */ +extern uint32_t mlx5_ptype_table[]; + +void mlx5_set_ptype_table(void); uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t); @@ -333,8 +342,16 @@ uint16_t removed_tx_burst(void *, struct rte_mbuf **, uint16_t); uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); int mlx5_rx_descriptor_status(void *, uint16_t); int mlx5_tx_descriptor_status(void *, uint16_t); -int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); -int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); + +/* Vectorized version of mlx5_rxtx.c */ +int priv_check_raw_vec_tx_support(struct priv *); +int priv_check_vec_tx_support(struct priv *); +int rxq_check_vec_support(struct rxq *); +int priv_check_vec_rx_support(struct priv *); +void priv_prep_vec_rx_function(struct priv *); +uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t); +uint16_t mlx5_tx_burst_vec(void *, struct rte_mbuf **, uint16_t); +uint16_t mlx5_rx_burst_vec(void *, struct rte_mbuf **, uint16_t); /* mlx5_mr.c */ @@ -342,4 +359,254 @@ struct ibv_mr *mlx5_mp2mr(struct ibv_pd *, struct rte_mempool *); void txq_mp2mr_iter(struct rte_mempool *, void *); uint32_t txq_mp2mr_reg(struct txq *, struct rte_mempool *, unsigned int); +#ifndef NDEBUG +/** + * Verify or set magic value in CQE. + * + * @param cqe + * Pointer to CQE. + * + * @return + * 0 the first time. + */ +static inline int +check_cqe_seen(volatile struct mlx5_cqe *cqe) +{ + static const uint8_t magic[] = "seen"; + volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0; + int ret = 1; + unsigned int i; + + for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i) + if (!ret || (*buf)[i] != magic[i]) { + ret = 0; + (*buf)[i] = magic[i]; + } + return ret; +} +#endif /* NDEBUG */ + +/** + * Check whether CQE is valid. + * + * @param cqe + * Pointer to CQE. + * @param cqes_n + * Size of completion queue. + * @param ci + * Consumer index. + * + * @return + * 0 on success, 1 on failure. + */ +static __rte_always_inline int +check_cqe(volatile struct mlx5_cqe *cqe, + unsigned int cqes_n, const uint16_t ci) +{ + uint16_t idx = ci & cqes_n; + uint8_t op_own = cqe->op_own; + uint8_t op_owner = MLX5_CQE_OWNER(op_own); + uint8_t op_code = MLX5_CQE_OPCODE(op_own); + + if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID))) + return 1; /* No CQE. */ +#ifndef NDEBUG + if ((op_code == MLX5_CQE_RESP_ERR) || + (op_code == MLX5_CQE_REQ_ERR)) { + volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe; + uint8_t syndrome = err_cqe->syndrome; + + if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) || + (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR)) + return 0; + if (!check_cqe_seen(cqe)) + ERROR("unexpected CQE error %u (0x%02x)" + " syndrome 0x%02x", + op_code, op_code, syndrome); + return 1; + } else if ((op_code != MLX5_CQE_RESP_SEND) && + (op_code != MLX5_CQE_REQ)) { + if (!check_cqe_seen(cqe)) + ERROR("unexpected CQE opcode %u (0x%02x)", + op_code, op_code); + return 1; + } +#endif /* NDEBUG */ + return 0; +} + +/** + * Return the address of the WQE. + * + * @param txq + * Pointer to TX queue structure. + * @param wqe_ci + * WQE consumer index. + * + * @return + * WQE address. + */ +static inline uintptr_t * +tx_mlx5_wqe(struct txq *txq, uint16_t ci) +{ + ci &= ((1 << txq->wqe_n) - 1); + return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE); +} + +/** + * Manage TX completions. + * + * When sending a burst, mlx5_tx_burst() posts several WRs. + * + * @param txq + * Pointer to TX queue structure. + */ +static __rte_always_inline void +mlx5_tx_complete(struct txq *txq) +{ + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const unsigned int cqe_n = 1 << txq->cqe_n; + const unsigned int cqe_cnt = cqe_n - 1; + uint16_t elts_free = txq->elts_tail; + uint16_t elts_tail; + uint16_t cq_ci = txq->cq_ci; + volatile struct mlx5_cqe *cqe = NULL; + volatile struct mlx5_wqe_ctrl *ctrl; + struct rte_mbuf *m, *free[elts_n]; + struct rte_mempool *pool = NULL; + unsigned int blk_n = 0; + + cqe = &(*txq->cqes)[cq_ci & cqe_cnt]; + if (unlikely(check_cqe(cqe, cqe_n, cq_ci))) + return; +#ifndef NDEBUG + if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) || + (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) { + if (!check_cqe_seen(cqe)) + ERROR("unexpected error CQE, TX stopped"); + return; + } +#endif /* NDEBUG */ + ++cq_ci; + txq->wqe_pi = ntohs(cqe->wqe_counter); + ctrl = (volatile struct mlx5_wqe_ctrl *) + tx_mlx5_wqe(txq, txq->wqe_pi); + elts_tail = ctrl->ctrl3; + assert((elts_tail & elts_m) < (1 << txq->wqe_n)); + /* Free buffers. */ + while (elts_free != elts_tail) { + m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]); + if (likely(m != NULL)) { + if (likely(m->pool == pool)) { + free[blk_n++] = m; + } else { + if (likely(pool != NULL)) + rte_mempool_put_bulk(pool, + (void *)free, + blk_n); + free[0] = m; + pool = m->pool; + blk_n = 1; + } + } + } + if (blk_n) + rte_mempool_put_bulk(pool, (void *)free, blk_n); +#ifndef NDEBUG + elts_free = txq->elts_tail; + /* Poisoning. */ + while (elts_free != elts_tail) { + memset(&(*txq->elts)[elts_free & elts_m], + 0x66, + sizeof((*txq->elts)[elts_free & elts_m])); + ++elts_free; + } +#endif + txq->cq_ci = cq_ci; + txq->elts_tail = elts_tail; + /* Update the consumer index. */ + rte_wmb(); + *txq->cq_db = htonl(cq_ci); +} + +/** + * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which + * the cloned mbuf is allocated is returned instead. + * + * @param buf + * Pointer to mbuf. + * + * @return + * Memory pool where data is located for given mbuf. + */ +static struct rte_mempool * +mlx5_tx_mb2mp(struct rte_mbuf *buf) +{ + if (unlikely(RTE_MBUF_INDIRECT(buf))) + return rte_mbuf_from_indirect(buf)->pool; + return buf->pool; +} + +/** + * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[]. + * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full, + * remove an entry first. + * + * @param txq + * Pointer to TX queue structure. + * @param[in] mp + * Memory Pool for which a Memory Region lkey must be returned. + * + * @return + * mr->lkey on success, (uint32_t)-1 on failure. + */ +static __rte_always_inline uint32_t +mlx5_tx_mb2mr(struct txq *txq, struct rte_mbuf *mb) +{ + uint16_t i = txq->mr_cache_idx; + uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t); + + assert(i < RTE_DIM(txq->mp2mr)); + if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr)) + return txq->mp2mr[i].lkey; + for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) { + if (unlikely(txq->mp2mr[i].mr == NULL)) { + /* Unknown MP, add a new MR for it. */ + break; + } + if (txq->mp2mr[i].start <= addr && + txq->mp2mr[i].end >= addr) { + assert(txq->mp2mr[i].lkey != (uint32_t)-1); + assert(htonl(txq->mp2mr[i].mr->lkey) == + txq->mp2mr[i].lkey); + txq->mr_cache_idx = i; + return txq->mp2mr[i].lkey; + } + } + txq->mr_cache_idx = 0; + return txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i); +} + +/** + * Ring TX queue doorbell. + * + * @param txq + * Pointer to TX queue structure. + * @param wqe + * Pointer to the last WQE posted in the NIC. + */ +static __rte_always_inline void +mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe) +{ + uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg); + volatile uint64_t *src = ((volatile uint64_t *)wqe); + + rte_wmb(); + *txq->qp_db = htonl(txq->wqe_ci); + /* Ensure ordering between DB record and BF copy. */ + rte_wmb(); + *dst = *src; +} + #endif /* RTE_PMD_MLX5_RXTX_H_ */ diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.c b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c new file mode 100644 index 00000000..8560f745 --- /dev/null +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c @@ -0,0 +1,1417 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <assert.h> +#include <stdint.h> +#include <string.h> +#include <stdlib.h> +#include <smmintrin.h> + +/* Verbs header. */ +/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include <infiniband/verbs.h> +#include <infiniband/mlx5_hw.h> +#include <infiniband/arch.h> +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +/* DPDK headers don't like -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include <rte_mbuf.h> +#include <rte_mempool.h> +#include <rte_prefetch.h> +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include "mlx5.h" +#include "mlx5_utils.h" +#include "mlx5_rxtx.h" +#include "mlx5_autoconf.h" +#include "mlx5_defs.h" +#include "mlx5_prm.h" + +#ifndef __INTEL_COMPILER +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +/** + * Fill in buffer descriptors in a multi-packet send descriptor. + * + * @param txq + * Pointer to TX queue structure. + * @param dseg + * Pointer to buffer descriptor to be writen. + * @param pkts + * Pointer to array of packets to be sent. + * @param n + * Number of packets to be filled. + */ +static inline void +txq_wr_dseg_v(struct txq *txq, __m128i *dseg, + struct rte_mbuf **pkts, unsigned int n) +{ + unsigned int pos; + uintptr_t addr; + const __m128i shuf_mask_dseg = + _mm_set_epi8(8, 9, 10, 11, /* addr, bswap64 */ + 12, 13, 14, 15, + 7, 6, 5, 4, /* lkey */ + 0, 1, 2, 3 /* length, bswap32 */); +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t tx_byte = 0; +#endif + + for (pos = 0; pos < n; ++pos, ++dseg) { + __m128i desc; + struct rte_mbuf *pkt = pkts[pos]; + + addr = rte_pktmbuf_mtod(pkt, uintptr_t); + desc = _mm_set_epi32(addr >> 32, + addr, + mlx5_tx_mb2mr(txq, pkt), + DATA_LEN(pkt)); + desc = _mm_shuffle_epi8(desc, shuf_mask_dseg); + _mm_store_si128(dseg, desc); +#ifdef MLX5_PMD_SOFT_COUNTERS + tx_byte += DATA_LEN(pkt); +#endif + } +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.obytes += tx_byte; +#endif +} + +/** + * Count the number of continuous single segment packets. The first packet must + * be a single segment packet. + * + * @param pkts + * Pointer to array of packets. + * @param pkts_n + * Number of packets. + * + * @return + * Number of continuous single segment packets. + */ +static inline unsigned int +txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n) +{ + unsigned int pos; + + if (!pkts_n) + return 0; + assert(NB_SEGS(pkts[0]) == 1); + /* Count the number of continuous single segment packets. */ + for (pos = 1; pos < pkts_n; ++pos) + if (NB_SEGS(pkts[pos]) > 1) + break; + return pos; +} + +/** + * Count the number of packets having same ol_flags and calculate cs_flags. + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets. + * @param pkts_n + * Number of packets. + * @param cs_flags + * Pointer of flags to be returned. + * + * @return + * Number of packets having same ol_flags. + */ +static inline unsigned int +txq_calc_offload(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n, + uint8_t *cs_flags) +{ + unsigned int pos; + const uint64_t ol_mask = + PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | + PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | + PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; + + if (!pkts_n) + return 0; + /* Count the number of packets having same ol_flags. */ + for (pos = 1; pos < pkts_n; ++pos) + if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) + break; + /* Should open another MPW session for the rest. */ + if (pkts[0]->ol_flags & + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { + const uint64_t is_tunneled = + pkts[0]->ol_flags & + (PKT_TX_TUNNEL_GRE | + PKT_TX_TUNNEL_VXLAN); + + if (is_tunneled && txq->tunnel_en) { + *cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | + MLX5_ETH_WQE_L4_INNER_CSUM; + if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM) + *cs_flags |= MLX5_ETH_WQE_L3_CSUM; + } else { + *cs_flags = MLX5_ETH_WQE_L3_CSUM | + MLX5_ETH_WQE_L4_CSUM; + } + } + return pos; +} + +/** + * Send multi-segmented packets until it encounters a single segment packet in + * the pkts list. + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets to be sent. + * @param pkts_n + * Number of packets to be sent. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static uint16_t +txq_scatter_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + uint16_t elts_head = txq->elts_head; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const uint16_t wq_n = 1 << txq->wqe_n; + const uint16_t wq_mask = wq_n - 1; + const unsigned int nb_dword_per_wqebb = + MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE; + const unsigned int nb_dword_in_hdr = + sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; + unsigned int n; + volatile struct mlx5_wqe *wqe = NULL; + + assert(elts_n > pkts_n); + mlx5_tx_complete(txq); + if (unlikely(!pkts_n)) + return 0; + for (n = 0; n < pkts_n; ++n) { + struct rte_mbuf *buf = pkts[n]; + unsigned int segs_n = buf->nb_segs; + unsigned int ds = nb_dword_in_hdr; + unsigned int len = PKT_LEN(buf); + uint16_t wqe_ci = txq->wqe_ci; + const __m128i shuf_mask_ctrl = + _mm_set_epi8(15, 14, 13, 12, + 8, 9, 10, 11, /* bswap32 */ + 4, 5, 6, 7, /* bswap32 */ + 0, 1, 2, 3 /* bswap32 */); + uint8_t cs_flags = 0; + uint16_t max_elts; + uint16_t max_wqe; + __m128i *t_wqe, *dseg; + __m128i ctrl; + + assert(segs_n); + max_elts = elts_n - (elts_head - txq->elts_tail); + max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi); + /* + * A MPW session consumes 2 WQEs at most to + * include MLX5_MPW_DSEG_MAX pointers. + */ + if (segs_n == 1 || + max_elts < segs_n || max_wqe < 2) + break; + wqe = &((volatile struct mlx5_wqe64 *) + txq->wqes)[wqe_ci & wq_mask].hdr; + if (buf->ol_flags & + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { + const uint64_t is_tunneled = buf->ol_flags & + (PKT_TX_TUNNEL_GRE | + PKT_TX_TUNNEL_VXLAN); + + if (is_tunneled && txq->tunnel_en) { + cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | + MLX5_ETH_WQE_L4_INNER_CSUM; + if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM) + cs_flags |= MLX5_ETH_WQE_L3_CSUM; + } else { + cs_flags = MLX5_ETH_WQE_L3_CSUM | + MLX5_ETH_WQE_L4_CSUM; + } + } + /* Title WQEBB pointer. */ + t_wqe = (__m128i *)wqe; + dseg = (__m128i *)(wqe + 1); + do { + if (!(ds++ % nb_dword_per_wqebb)) { + dseg = (__m128i *) + &((volatile struct mlx5_wqe64 *) + txq->wqes)[++wqe_ci & wq_mask]; + } + txq_wr_dseg_v(txq, dseg++, &buf, 1); + (*txq->elts)[elts_head++ & elts_m] = buf; + buf = buf->next; + } while (--segs_n); + ++wqe_ci; + /* Fill CTRL in the header. */ + ctrl = _mm_set_epi32(0, 0, txq->qp_num_8s | ds, + MLX5_OPC_MOD_MPW << 24 | + txq->wqe_ci << 8 | MLX5_OPCODE_TSO); + ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl); + _mm_store_si128(t_wqe, ctrl); + /* Fill ESEG in the header. */ + _mm_store_si128(t_wqe + 1, + _mm_set_epi16(0, 0, 0, 0, + htons(len), cs_flags, + 0, 0)); + txq->wqe_ci = wqe_ci; + } + if (!n) + return 0; + txq->elts_comp += (uint16_t)(elts_head - txq->elts_head); + txq->elts_head = elts_head; + if (txq->elts_comp >= MLX5_TX_COMP_THRESH) { + wqe->ctrl[2] = htonl(8); + wqe->ctrl[3] = txq->elts_head; + txq->elts_comp = 0; + ++txq->cq_pi; + } +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.opackets += n; +#endif + mlx5_tx_dbrec(txq, wqe); + return n; +} + +/** + * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet, + * it returns to make it processed by txq_scatter_v(). All the packets in + * the pkts list should be single segment packets having same offload flags. + * This must be checked by txq_check_multiseg() and txq_calc_offload(). + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets to be sent. + * @param pkts_n + * Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST). + * @param cs_flags + * Checksum offload flags to be written in the descriptor. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static inline uint16_t +txq_burst_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n, + uint8_t cs_flags) +{ + struct rte_mbuf **elts; + uint16_t elts_head = txq->elts_head; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const unsigned int nb_dword_per_wqebb = + MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE; + const unsigned int nb_dword_in_hdr = + sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; + unsigned int n = 0; + unsigned int pos; + uint16_t max_elts; + uint16_t max_wqe; + uint32_t comp_req = 0; + const uint16_t wq_n = 1 << txq->wqe_n; + const uint16_t wq_mask = wq_n - 1; + uint16_t wq_idx = txq->wqe_ci & wq_mask; + volatile struct mlx5_wqe64 *wq = + &((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx]; + volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq; + const __m128i shuf_mask_ctrl = + _mm_set_epi8(15, 14, 13, 12, + 8, 9, 10, 11, /* bswap32 */ + 4, 5, 6, 7, /* bswap32 */ + 0, 1, 2, 3 /* bswap32 */); + __m128i *t_wqe, *dseg; + __m128i ctrl; + + /* Make sure all packets can fit into a single WQE. */ + assert(elts_n > pkts_n); + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); + max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); + pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts); + if (unlikely(!pkts_n)) + return 0; + elts = &(*txq->elts)[elts_head & elts_m]; + /* Loop for available tailroom first. */ + n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n); + for (pos = 0; pos < (n & -2); pos += 2) + _mm_storeu_si128((__m128i *)&elts[pos], + _mm_loadu_si128((__m128i *)&pkts[pos])); + if (n & 1) + elts[pos] = pkts[pos]; + /* Check if it crosses the end of the queue. */ + if (unlikely(n < pkts_n)) { + elts = &(*txq->elts)[0]; + for (pos = 0; pos < pkts_n - n; ++pos) + elts[pos] = pkts[n + pos]; + } + txq->elts_head += pkts_n; + /* Save title WQEBB pointer. */ + t_wqe = (__m128i *)wqe; + dseg = (__m128i *)(wqe + 1); + /* Calculate the number of entries to the end. */ + n = RTE_MIN( + (wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr, + pkts_n); + /* Fill DSEGs. */ + txq_wr_dseg_v(txq, dseg, pkts, n); + /* Check if it crosses the end of the queue. */ + if (n < pkts_n) { + dseg = (__m128i *)txq->wqes; + txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n); + } + if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) { + txq->elts_comp += pkts_n; + } else { + /* Request a completion. */ + txq->elts_comp = 0; + ++txq->cq_pi; + comp_req = 8; + } + /* Fill CTRL in the header. */ + ctrl = _mm_set_epi32(txq->elts_head, comp_req, + txq->qp_num_8s | (pkts_n + 2), + MLX5_OPC_MOD_ENHANCED_MPSW << 24 | + txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW); + ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl); + _mm_store_si128(t_wqe, ctrl); + /* Fill ESEG in the header. */ + _mm_store_si128(t_wqe + 1, + _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, cs_flags, + 0, 0, 0, 0)); +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.opackets += pkts_n; +#endif + txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) / + nb_dword_per_wqebb; + /* Ring QP doorbell. */ + mlx5_tx_dbrec(txq, wqe); + return pkts_n; +} + +/** + * DPDK callback for vectorized TX. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct txq *txq = (struct txq *)dpdk_txq; + uint16_t nb_tx = 0; + + while (pkts_n > nb_tx) { + uint16_t n; + uint16_t ret; + + n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); + ret = txq_burst_v(txq, &pkts[nb_tx], n, 0); + nb_tx += ret; + if (!ret) + break; + } + return nb_tx; +} + +/** + * DPDK callback for vectorized TX with multi-seg packets and offload. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + struct txq *txq = (struct txq *)dpdk_txq; + uint16_t nb_tx = 0; + + while (pkts_n > nb_tx) { + uint8_t cs_flags = 0; + uint16_t n; + uint16_t ret; + + /* Transmit multi-seg packets in the head of pkts list. */ + if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) && + NB_SEGS(pkts[nb_tx]) > 1) + nb_tx += txq_scatter_v(txq, + &pkts[nb_tx], + pkts_n - nb_tx); + n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); + if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS)) + n = txq_check_multiseg(&pkts[nb_tx], n); + if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) + n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags); + ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); + nb_tx += ret; + if (!ret) + break; + } + return nb_tx; +} + +/** + * Store free buffers to RX SW ring. + * + * @param rxq + * Pointer to RX queue structure. + * @param pkts + * Pointer to array of packets to be stored. + * @param pkts_n + * Number of packets to be stored. + */ +static inline void +rxq_copy_mbuf_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t n) +{ + const uint16_t q_mask = (1 << rxq->elts_n) - 1; + struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask]; + unsigned int pos; + uint16_t p = n & -2; + + for (pos = 0; pos < p; pos += 2) { + __m128i mbp; + + mbp = _mm_loadu_si128((__m128i *)&elts[pos]); + _mm_storeu_si128((__m128i *)&pkts[pos], mbp); + } + if (n & 1) + pkts[pos] = elts[pos]; +} + +/** + * Replenish buffers for RX in bulk. + * + * @param rxq + * Pointer to RX queue structure. + * @param n + * Number of buffers to be replenished. + */ +static inline void +rxq_replenish_bulk_mbuf(struct rxq *rxq, uint16_t n) +{ + const uint16_t q_n = 1 << rxq->elts_n; + const uint16_t q_mask = q_n - 1; + const uint16_t elts_idx = rxq->rq_ci & q_mask; + struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; + volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx]; + unsigned int i; + + assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH); + assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi))); + assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP); + /* Not to cross queue end. */ + n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx); + if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { + rxq->stats.rx_nombuf += n; + return; + } + for (i = 0; i < n; ++i) + wq[i].addr = htonll((uintptr_t)elts[i]->buf_addr + + RTE_PKTMBUF_HEADROOM); + rxq->rq_ci += n; + rte_wmb(); + *rxq->rq_db = htonl(rxq->rq_ci); +} + +/** + * Decompress a compressed completion and fill in mbufs in RX SW ring with data + * extracted from the title completion descriptor. + * + * @param rxq + * Pointer to RX queue structure. + * @param cq + * Pointer to completion array having a compressed completion at first. + * @param elts + * Pointer to SW ring to be filled. The first mbuf has to be pre-built from + * the title completion descriptor to be copied to the rest of mbufs. + */ +static inline void +rxq_cq_decompress_v(struct rxq *rxq, + volatile struct mlx5_cqe *cq, + struct rte_mbuf **elts) +{ + volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + 1); + struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */ + unsigned int pos; + unsigned int i; + unsigned int inv = 0; + /* Mask to shuffle from extracted mini CQE to mbuf. */ + const __m128i shuf_mask1 = + _mm_set_epi8(0, 1, 2, 3, /* rss, bswap32 */ + -1, -1, /* skip vlan_tci */ + 6, 7, /* data_len, bswap16 */ + -1, -1, 6, 7, /* pkt_len, bswap16 */ + -1, -1, -1, -1 /* skip packet_type */); + const __m128i shuf_mask2 = + _mm_set_epi8(8, 9, 10, 11, /* rss, bswap32 */ + -1, -1, /* skip vlan_tci */ + 14, 15, /* data_len, bswap16 */ + -1, -1, 14, 15, /* pkt_len, bswap16 */ + -1, -1, -1, -1 /* skip packet_type */); + /* Restore the compressed count. Must be 16 bits. */ + const uint16_t mcqe_n = t_pkt->data_len + + (rxq->crc_present * ETHER_CRC_LEN); + const __m128i rearm = + _mm_loadu_si128((__m128i *)&t_pkt->rearm_data); + const __m128i rxdf = + _mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1); + const __m128i crc_adj = + _mm_set_epi16(0, 0, 0, + rxq->crc_present * ETHER_CRC_LEN, + 0, + rxq->crc_present * ETHER_CRC_LEN, + 0, 0); + const uint32_t flow_tag = t_pkt->hash.fdir.hi; +#ifdef MLX5_PMD_SOFT_COUNTERS + const __m128i zero = _mm_setzero_si128(); + const __m128i ones = _mm_cmpeq_epi32(zero, zero); + uint32_t rcvd_byte = 0; + /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */ + const __m128i len_shuf_mask = + _mm_set_epi8(-1, -1, -1, -1, + -1, -1, -1, -1, + 14, 15, 6, 7, + 10, 11, 2, 3); +#endif + + /* Compile time sanity check for this function. */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + /* + * A. load mCQEs into a 128bit register. + * B. store rearm data to mbuf. + * C. combine data from mCQEs with rx_descriptor_fields1. + * D. store rx_descriptor_fields1. + * E. store flow tag (rte_flow mark). + */ + for (pos = 0; pos < mcqe_n; ) { + __m128i mcqe1, mcqe2; + __m128i rxdf1, rxdf2; +#ifdef MLX5_PMD_SOFT_COUNTERS + __m128i byte_cnt, invalid_mask; +#endif + + if (!(pos & 0x7) && pos + 8 < mcqe_n) + rte_prefetch0((void *)(cq + pos + 8)); + /* A.1 load mCQEs into a 128bit register. */ + mcqe1 = _mm_loadu_si128((__m128i *)&mcq[pos % 8]); + mcqe2 = _mm_loadu_si128((__m128i *)&mcq[pos % 8 + 2]); + /* B.1 store rearm data to mbuf. */ + _mm_storeu_si128((__m128i *)&elts[pos]->rearm_data, rearm); + _mm_storeu_si128((__m128i *)&elts[pos + 1]->rearm_data, rearm); + /* C.1 combine data from mCQEs with rx_descriptor_fields1. */ + rxdf1 = _mm_shuffle_epi8(mcqe1, shuf_mask1); + rxdf2 = _mm_shuffle_epi8(mcqe1, shuf_mask2); + rxdf1 = _mm_sub_epi16(rxdf1, crc_adj); + rxdf2 = _mm_sub_epi16(rxdf2, crc_adj); + rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23); + rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23); + /* D.1 store rx_descriptor_fields1. */ + _mm_storeu_si128((__m128i *) + &elts[pos]->rx_descriptor_fields1, + rxdf1); + _mm_storeu_si128((__m128i *) + &elts[pos + 1]->rx_descriptor_fields1, + rxdf2); + /* B.1 store rearm data to mbuf. */ + _mm_storeu_si128((__m128i *)&elts[pos + 2]->rearm_data, rearm); + _mm_storeu_si128((__m128i *)&elts[pos + 3]->rearm_data, rearm); + /* C.1 combine data from mCQEs with rx_descriptor_fields1. */ + rxdf1 = _mm_shuffle_epi8(mcqe2, shuf_mask1); + rxdf2 = _mm_shuffle_epi8(mcqe2, shuf_mask2); + rxdf1 = _mm_sub_epi16(rxdf1, crc_adj); + rxdf2 = _mm_sub_epi16(rxdf2, crc_adj); + rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23); + rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23); + /* D.1 store rx_descriptor_fields1. */ + _mm_storeu_si128((__m128i *) + &elts[pos + 2]->rx_descriptor_fields1, + rxdf1); + _mm_storeu_si128((__m128i *) + &elts[pos + 3]->rx_descriptor_fields1, + rxdf2); +#ifdef MLX5_PMD_SOFT_COUNTERS + invalid_mask = _mm_set_epi64x(0, + (mcqe_n - pos) * + sizeof(uint16_t) * 8); + invalid_mask = _mm_sll_epi64(ones, invalid_mask); + mcqe1 = _mm_srli_si128(mcqe1, 4); + byte_cnt = _mm_blend_epi16(mcqe1, mcqe2, 0xcc); + byte_cnt = _mm_shuffle_epi8(byte_cnt, len_shuf_mask); + byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt); + byte_cnt = _mm_hadd_epi16(byte_cnt, zero); + rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero)); +#endif + if (rxq->mark) { + /* E.1 store flow tag (rte_flow mark). */ + elts[pos]->hash.fdir.hi = flow_tag; + elts[pos + 1]->hash.fdir.hi = flow_tag; + elts[pos + 2]->hash.fdir.hi = flow_tag; + elts[pos + 3]->hash.fdir.hi = flow_tag; + } + pos += MLX5_VPMD_DESCS_PER_LOOP; + /* Move to next CQE and invalidate consumed CQEs. */ + if (!(pos & 0x7) && pos < mcqe_n) { + mcq = (void *)(cq + pos); + for (i = 0; i < 8; ++i) + cq[inv++].op_own = MLX5_CQE_INVALIDATE; + } + } + /* Invalidate the rest of CQEs. */ + for (; inv < mcqe_n; ++inv) + cq[inv].op_own = MLX5_CQE_INVALIDATE; +#ifdef MLX5_PMD_SOFT_COUNTERS + rxq->stats.ipackets += mcqe_n; + rxq->stats.ibytes += rcvd_byte; +#endif + rxq->cq_ci += mcqe_n; +} + +/** + * Calculate packet type and offload flag for mbuf and store it. + * + * @param rxq + * Pointer to RX queue structure. + * @param cqes[4] + * Array of four 16bytes completions extracted from the original completion + * descriptor. + * @param op_err + * Opcode vector having responder error status. Each field is 4B. + * @param pkts + * Pointer to array of packets to be filled. + */ +static inline void +rxq_cq_to_ptype_oflags_v(struct rxq *rxq, __m128i cqes[4], __m128i op_err, + struct rte_mbuf **pkts) +{ + __m128i pinfo0, pinfo1; + __m128i pinfo, ptype; + __m128i ol_flags = _mm_set1_epi32(rxq->rss_hash * PKT_RX_RSS_HASH); + __m128i cv_flags; + const __m128i zero = _mm_setzero_si128(); + const __m128i ptype_mask = + _mm_set_epi32(0xfd06, 0xfd06, 0xfd06, 0xfd06); + const __m128i ptype_ol_mask = + _mm_set_epi32(0x106, 0x106, 0x106, 0x106); + const __m128i pinfo_mask = + _mm_set_epi32(0x3, 0x3, 0x3, 0x3); + const __m128i cv_flag_sel = + _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, + (uint8_t)((PKT_RX_IP_CKSUM_GOOD | + PKT_RX_L4_CKSUM_GOOD) >> 1), + 0, + (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1), + 0, + (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1), + (uint8_t)(PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED), + 0); + const __m128i cv_mask = + _mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED); + const __m128i mbuf_init = + _mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer); + __m128i rearm0, rearm1, rearm2, rearm3; + + /* Extract pkt_info field. */ + pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]); + pinfo1 = _mm_unpacklo_epi32(cqes[2], cqes[3]); + pinfo = _mm_unpacklo_epi64(pinfo0, pinfo1); + /* Extract hdr_type_etc field. */ + pinfo0 = _mm_unpackhi_epi32(cqes[0], cqes[1]); + pinfo1 = _mm_unpackhi_epi32(cqes[2], cqes[3]); + ptype = _mm_unpacklo_epi64(pinfo0, pinfo1); + if (rxq->mark) { + const __m128i pinfo_ft_mask = + _mm_set_epi32(0xffffff00, 0xffffff00, + 0xffffff00, 0xffffff00); + const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR); + const __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID); + __m128i flow_tag, invalid_mask; + + flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask); + /* Check if flow tag is non-zero then set PKT_RX_FDIR. */ + invalid_mask = _mm_cmpeq_epi32(flow_tag, zero); + ol_flags = _mm_or_si128(ol_flags, + _mm_andnot_si128(invalid_mask, + fdir_flags)); + /* Mask out invalid entries. */ + flow_tag = _mm_andnot_si128(invalid_mask, flow_tag); + /* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */ + ol_flags = _mm_or_si128(ol_flags, + _mm_andnot_si128( + _mm_cmpeq_epi32(flow_tag, + pinfo_ft_mask), + fdir_id_flags)); + } + /* + * Merge the two fields to generate the following: + * bit[1] = l3_ok + * bit[2] = l4_ok + * bit[8] = cv + * bit[11:10] = l3_hdr_type + * bit[14:12] = l4_hdr_type + * bit[15] = ip_frag + * bit[16] = tunneled + * bit[17] = outer_l3_type + */ + ptype = _mm_and_si128(ptype, ptype_mask); + pinfo = _mm_and_si128(pinfo, pinfo_mask); + pinfo = _mm_slli_epi32(pinfo, 16); + /* Make pinfo has merged fields for ol_flags calculation. */ + pinfo = _mm_or_si128(ptype, pinfo); + ptype = _mm_srli_epi32(pinfo, 10); + ptype = _mm_packs_epi32(ptype, zero); + /* Errored packets will have RTE_PTYPE_ALL_MASK. */ + op_err = _mm_srli_epi16(op_err, 8); + ptype = _mm_or_si128(ptype, op_err); + pkts[0]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 0)]; + pkts[1]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 2)]; + pkts[2]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 4)]; + pkts[3]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 6)]; + /* Fill flags for checksum and VLAN. */ + pinfo = _mm_and_si128(pinfo, ptype_ol_mask); + pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo); + /* Locate checksum flags at byte[2:1] and merge with VLAN flags. */ + cv_flags = _mm_slli_epi32(pinfo, 9); + cv_flags = _mm_or_si128(pinfo, cv_flags); + /* Move back flags to start from byte[0]. */ + cv_flags = _mm_srli_epi32(cv_flags, 8); + /* Mask out garbage bits. */ + cv_flags = _mm_and_si128(cv_flags, cv_mask); + /* Merge to ol_flags. */ + ol_flags = _mm_or_si128(ol_flags, cv_flags); + /* Merge mbuf_init and ol_flags. */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30); + rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30); + rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30); + rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30); + /* Write 8B rearm_data and 8B ol_flags. */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); + _mm_store_si128((__m128i *)&pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&pkts[3]->rearm_data, rearm3); +} + +/** + * Skip error packets. + * + * @param rxq + * Pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +static uint16_t +rxq_handle_pending_error(struct rxq *rxq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + uint16_t n = 0; + unsigned int i; +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t err_bytes = 0; +#endif + + for (i = 0; i < pkts_n; ++i) { + struct rte_mbuf *pkt = pkts[i]; + + if (pkt->packet_type == RTE_PTYPE_ALL_MASK) { +#ifdef MLX5_PMD_SOFT_COUNTERS + err_bytes += PKT_LEN(pkt); +#endif + rte_pktmbuf_free_seg(pkt); + } else { + pkts[n++] = pkt; + } + } + rxq->stats.idropped += (pkts_n - n); +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Correct counters of errored completions. */ + rxq->stats.ipackets -= (pkts_n - n); + rxq->stats.ibytes -= err_bytes; +#endif + rxq->pending_err = 0; + return n; +} + +/** + * Receive burst of packets. An errored completion also consumes a mbuf, but the + * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed + * before returning to application. + * + * @param rxq + * Pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets received including errors (<= pkts_n). + */ +static inline uint16_t +rxq_burst_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + const uint16_t q_n = 1 << rxq->cqe_n; + const uint16_t q_mask = q_n - 1; + volatile struct mlx5_cqe *cq; + struct rte_mbuf **elts; + unsigned int pos; + uint64_t n; + uint16_t repl_n; + uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; + uint16_t nocmp_n = 0; + uint16_t rcvd_pkt = 0; + unsigned int cq_idx = rxq->cq_ci & q_mask; + unsigned int elts_idx; + unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1)); + const __m128i owner_check = + _mm_set_epi64x(0x0100000001000000LL, 0x0100000001000000LL); + const __m128i opcode_check = + _mm_set_epi64x(0xf0000000f0000000LL, 0xf0000000f0000000LL); + const __m128i format_check = + _mm_set_epi64x(0x0c0000000c000000LL, 0x0c0000000c000000LL); + const __m128i resp_err_check = + _mm_set_epi64x(0xe0000000e0000000LL, 0xe0000000e0000000LL); +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t rcvd_byte = 0; + /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */ + const __m128i len_shuf_mask = + _mm_set_epi8(-1, -1, -1, -1, + -1, -1, -1, -1, + 12, 13, 8, 9, + 4, 5, 0, 1); +#endif + /* Mask to shuffle from extracted CQE to mbuf. */ + const __m128i shuf_mask = + _mm_set_epi8(-1, 3, 2, 1, /* fdir.hi */ + 12, 13, 14, 15, /* rss, bswap32 */ + 10, 11, /* vlan_tci, bswap16 */ + 4, 5, /* data_len, bswap16 */ + -1, -1, /* zero out 2nd half of pkt_len */ + 4, 5 /* pkt_len, bswap16 */); + /* Mask to blend from the last Qword to the first DQword. */ + const __m128i blend_mask = + _mm_set_epi8(-1, -1, -1, -1, + -1, -1, -1, -1, + 0, 0, 0, 0, + 0, 0, 0, -1); + const __m128i zero = _mm_setzero_si128(); + const __m128i ones = _mm_cmpeq_epi32(zero, zero); + const __m128i crc_adj = + _mm_set_epi16(0, 0, 0, 0, 0, + rxq->crc_present * ETHER_CRC_LEN, + 0, + rxq->crc_present * ETHER_CRC_LEN); + const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0); + + /* Compile time sanity check for this function. */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, pkt_info) != 0); + RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rx_hash_res) != + offsetof(struct mlx5_cqe, pkt_info) + 12); + RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd1) + + sizeof(((struct mlx5_cqe *)0)->rsvd1) != + offsetof(struct mlx5_cqe, hdr_type_etc)); + RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, vlan_info) != + offsetof(struct mlx5_cqe, hdr_type_etc) + 2); + RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd2) + + sizeof(((struct mlx5_cqe *)0)->rsvd2) != + offsetof(struct mlx5_cqe, byte_cnt)); + RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, sop_drop_qpn) != + RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8)); + RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, op_own) != + offsetof(struct mlx5_cqe, sop_drop_qpn) + 7); + assert(rxq->sges_n == 0); + assert(rxq->cqe_n == rxq->elts_n); + cq = &(*rxq->cqes)[cq_idx]; + rte_prefetch0(cq); + rte_prefetch0(cq + 1); + rte_prefetch0(cq + 2); + rte_prefetch0(cq + 3); + pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); + /* + * Order of indexes: + * rq_ci >= cq_ci >= rq_pi + * Definition of indexes: + * rq_ci - cq_ci := # of buffers owned by HW (posted). + * cq_ci - rq_pi := # of buffers not returned to app (decompressed). + * N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished). + */ + repl_n = q_n - (rxq->rq_ci - rxq->rq_pi); + if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH) + rxq_replenish_bulk_mbuf(rxq, repl_n); + /* See if there're unreturned mbufs from compressed CQE. */ + rcvd_pkt = rxq->cq_ci - rxq->rq_pi; + if (rcvd_pkt > 0) { + rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); + rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt); + rxq->rq_pi += rcvd_pkt; + pkts += rcvd_pkt; + } + elts_idx = rxq->rq_pi & q_mask; + elts = &(*rxq->elts)[elts_idx]; + /* Not to overflow pkts array. */ + pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP); + /* Not to cross queue end. */ + pkts_n = RTE_MIN(pkts_n, q_n - elts_idx); + if (!pkts_n) + return rcvd_pkt; + /* At this point, there shouldn't be any remained packets. */ + assert(rxq->rq_pi == rxq->cq_ci); + /* + * A. load first Qword (8bytes) in one loop. + * B. copy 4 mbuf pointers from elts ring to returing pkts. + * C. load remained CQE data and extract necessary fields. + * Final 16bytes cqes[] extracted from original 64bytes CQE has the + * following structure: + * struct { + * uint8_t pkt_info; + * uint8_t flow_tag[3]; + * uint16_t byte_cnt; + * uint8_t rsvd4; + * uint8_t op_own; + * uint16_t hdr_type_etc; + * uint16_t vlan_info; + * uint32_t rx_has_res; + * } c; + * D. fill in mbuf. + * E. get valid CQEs. + * F. find compressed CQE. + */ + for (pos = 0; + pos < pkts_n; + pos += MLX5_VPMD_DESCS_PER_LOOP) { + __m128i cqes[MLX5_VPMD_DESCS_PER_LOOP]; + __m128i cqe_tmp1, cqe_tmp2; + __m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3; + __m128i op_own, op_own_tmp1, op_own_tmp2; + __m128i opcode, owner_mask, invalid_mask; + __m128i comp_mask; + __m128i mask; +#ifdef MLX5_PMD_SOFT_COUNTERS + __m128i byte_cnt; +#endif + __m128i mbp1, mbp2; + __m128i p = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0); + unsigned int p1, p2, p3; + + /* Prefetch next 4 CQEs. */ + if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) { + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]); + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]); + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]); + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]); + } + /* A.0 do not cross the end of CQ. */ + mask = _mm_set_epi64x(0, (pkts_n - pos) * sizeof(uint16_t) * 8); + mask = _mm_sll_epi64(ones, mask); + p = _mm_andnot_si128(mask, p); + /* A.1 load cqes. */ + p3 = _mm_extract_epi16(p, 3); + cqes[3] = _mm_loadl_epi64((__m128i *) + &cq[pos + p3].sop_drop_qpn); + rte_compiler_barrier(); + p2 = _mm_extract_epi16(p, 2); + cqes[2] = _mm_loadl_epi64((__m128i *) + &cq[pos + p2].sop_drop_qpn); + rte_compiler_barrier(); + /* B.1 load mbuf pointers. */ + mbp1 = _mm_loadu_si128((__m128i *)&elts[pos]); + mbp2 = _mm_loadu_si128((__m128i *)&elts[pos + 2]); + /* A.1 load a block having op_own. */ + p1 = _mm_extract_epi16(p, 1); + cqes[1] = _mm_loadl_epi64((__m128i *) + &cq[pos + p1].sop_drop_qpn); + rte_compiler_barrier(); + cqes[0] = _mm_loadl_epi64((__m128i *) + &cq[pos].sop_drop_qpn); + /* B.2 copy mbuf pointers. */ + _mm_storeu_si128((__m128i *)&pkts[pos], mbp1); + _mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2); + rte_compiler_barrier(); + /* C.1 load remained CQE data and extract necessary fields. */ + cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]); + cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]); + cqes[3] = _mm_blendv_epi8(cqes[3], cqe_tmp2, blend_mask); + cqes[2] = _mm_blendv_epi8(cqes[2], cqe_tmp1, blend_mask); + cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p3].rsvd1[3]); + cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos + p2].rsvd1[3]); + cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x30); + cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x30); + cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p3].rsvd2[10]); + cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos + p2].rsvd2[10]); + cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x04); + cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x04); + /* C.2 generate final structure for mbuf with swapping bytes. */ + pkt_mb3 = _mm_shuffle_epi8(cqes[3], shuf_mask); + pkt_mb2 = _mm_shuffle_epi8(cqes[2], shuf_mask); + /* C.3 adjust CRC length. */ + pkt_mb3 = _mm_sub_epi16(pkt_mb3, crc_adj); + pkt_mb2 = _mm_sub_epi16(pkt_mb2, crc_adj); + /* C.4 adjust flow mark. */ + pkt_mb3 = _mm_add_epi32(pkt_mb3, flow_mark_adj); + pkt_mb2 = _mm_add_epi32(pkt_mb2, flow_mark_adj); + /* D.1 fill in mbuf - rx_descriptor_fields1. */ + _mm_storeu_si128((void *)&pkts[pos + 3]->pkt_len, pkt_mb3); + _mm_storeu_si128((void *)&pkts[pos + 2]->pkt_len, pkt_mb2); + /* E.1 extract op_own field. */ + op_own_tmp2 = _mm_unpacklo_epi32(cqes[2], cqes[3]); + /* C.1 load remained CQE data and extract necessary fields. */ + cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p1]); + cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos]); + cqes[1] = _mm_blendv_epi8(cqes[1], cqe_tmp2, blend_mask); + cqes[0] = _mm_blendv_epi8(cqes[0], cqe_tmp1, blend_mask); + cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p1].rsvd1[3]); + cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos].rsvd1[3]); + cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x30); + cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x30); + cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p1].rsvd2[10]); + cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos].rsvd2[10]); + cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x04); + cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x04); + /* C.2 generate final structure for mbuf with swapping bytes. */ + pkt_mb1 = _mm_shuffle_epi8(cqes[1], shuf_mask); + pkt_mb0 = _mm_shuffle_epi8(cqes[0], shuf_mask); + /* C.3 adjust CRC length. */ + pkt_mb1 = _mm_sub_epi16(pkt_mb1, crc_adj); + pkt_mb0 = _mm_sub_epi16(pkt_mb0, crc_adj); + /* C.4 adjust flow mark. */ + pkt_mb1 = _mm_add_epi32(pkt_mb1, flow_mark_adj); + pkt_mb0 = _mm_add_epi32(pkt_mb0, flow_mark_adj); + /* E.1 extract op_own byte. */ + op_own_tmp1 = _mm_unpacklo_epi32(cqes[0], cqes[1]); + op_own = _mm_unpackhi_epi64(op_own_tmp1, op_own_tmp2); + /* D.1 fill in mbuf - rx_descriptor_fields1. */ + _mm_storeu_si128((void *)&pkts[pos + 1]->pkt_len, pkt_mb1); + _mm_storeu_si128((void *)&pkts[pos]->pkt_len, pkt_mb0); + /* E.2 flip owner bit to mark CQEs from last round. */ + owner_mask = _mm_and_si128(op_own, owner_check); + if (ownership) + owner_mask = _mm_xor_si128(owner_mask, owner_check); + owner_mask = _mm_cmpeq_epi32(owner_mask, owner_check); + owner_mask = _mm_packs_epi32(owner_mask, zero); + /* E.3 get mask for invalidated CQEs. */ + opcode = _mm_and_si128(op_own, opcode_check); + invalid_mask = _mm_cmpeq_epi32(opcode_check, opcode); + invalid_mask = _mm_packs_epi32(invalid_mask, zero); + /* E.4 mask out beyond boundary. */ + invalid_mask = _mm_or_si128(invalid_mask, mask); + /* E.5 merge invalid_mask with invalid owner. */ + invalid_mask = _mm_or_si128(invalid_mask, owner_mask); + /* F.1 find compressed CQE format. */ + comp_mask = _mm_and_si128(op_own, format_check); + comp_mask = _mm_cmpeq_epi32(comp_mask, format_check); + comp_mask = _mm_packs_epi32(comp_mask, zero); + /* F.2 mask out invalid entries. */ + comp_mask = _mm_andnot_si128(invalid_mask, comp_mask); + comp_idx = _mm_cvtsi128_si64(comp_mask); + /* F.3 get the first compressed CQE. */ + comp_idx = comp_idx ? + __builtin_ctzll(comp_idx) / + (sizeof(uint16_t) * 8) : + MLX5_VPMD_DESCS_PER_LOOP; + /* E.6 mask out entries after the compressed CQE. */ + mask = _mm_set_epi64x(0, comp_idx * sizeof(uint16_t) * 8); + mask = _mm_sll_epi64(ones, mask); + invalid_mask = _mm_or_si128(invalid_mask, mask); + /* E.7 count non-compressed valid CQEs. */ + n = _mm_cvtsi128_si64(invalid_mask); + n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) : + MLX5_VPMD_DESCS_PER_LOOP; + nocmp_n += n; + /* D.2 get the final invalid mask. */ + mask = _mm_set_epi64x(0, n * sizeof(uint16_t) * 8); + mask = _mm_sll_epi64(ones, mask); + invalid_mask = _mm_or_si128(invalid_mask, mask); + /* D.3 check error in opcode. */ + opcode = _mm_cmpeq_epi32(resp_err_check, opcode); + opcode = _mm_packs_epi32(opcode, zero); + opcode = _mm_andnot_si128(invalid_mask, opcode); + /* D.4 mark if any error is set */ + rxq->pending_err |= !!_mm_cvtsi128_si64(opcode); + /* D.5 fill in mbuf - rearm_data and packet_type. */ + rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]); +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Add up received bytes count. */ + byte_cnt = _mm_shuffle_epi8(op_own, len_shuf_mask); + byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt); + byte_cnt = _mm_hadd_epi16(byte_cnt, zero); + rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero)); +#endif + /* + * Break the loop unless more valid CQE is expected, or if + * there's a compressed CQE. + */ + if (n != MLX5_VPMD_DESCS_PER_LOOP) + break; + } + /* If no new CQE seen, return without updating cq_db. */ + if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) + return rcvd_pkt; + /* Update the consumer indexes for non-compressed CQEs. */ + assert(nocmp_n <= pkts_n); + rxq->cq_ci += nocmp_n; + rxq->rq_pi += nocmp_n; + rcvd_pkt += nocmp_n; +#ifdef MLX5_PMD_SOFT_COUNTERS + rxq->stats.ipackets += nocmp_n; + rxq->stats.ibytes += rcvd_byte; +#endif + /* Decompress the last CQE if compressed. */ + if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) { + assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); + rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]); + /* Return more packets if needed. */ + if (nocmp_n < pkts_n) { + uint16_t n = rxq->cq_ci - rxq->rq_pi; + + n = RTE_MIN(n, pkts_n - nocmp_n); + rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n); + rxq->rq_pi += n; + rcvd_pkt += n; + } + } + rte_wmb(); + *rxq->cq_db = htonl(rxq->cq_ci); + return rcvd_pkt; +} + +/** + * DPDK callback for vectorized RX. + * + * @param dpdk_rxq + * Generic pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +uint16_t +mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + struct rxq *rxq = dpdk_rxq; + uint16_t nb_rx; + + nb_rx = rxq_burst_v(rxq, pkts, pkts_n); + if (unlikely(rxq->pending_err)) + nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); + return nb_rx; +} + +/** + * Check Tx queue flags are set for raw vectorized Tx. + * + * @param priv + * Pointer to private structure. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +priv_check_raw_vec_tx_support(struct priv *priv) +{ + uint16_t i; + + /* All the configured queues should support. */ + for (i = 0; i < priv->txqs_n; ++i) { + struct txq *txq = (*priv->txqs)[i]; + + if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) || + !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) + break; + } + if (i != priv->txqs_n) + return -ENOTSUP; + return 1; +} + +/** + * Check a device can support vectorized TX. + * + * @param priv + * Pointer to private structure. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +priv_check_vec_tx_support(struct priv *priv) +{ + if (!priv->tx_vec_en || + priv->txqs_n > MLX5_VPMD_MIN_TXQS || + priv->mps != MLX5_MPW_ENHANCED || + priv->tso) + return -ENOTSUP; + return 1; +} + +/** + * Check a RX queue can support vectorized RX. + * + * @param rxq + * Pointer to RX queue. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +rxq_check_vec_support(struct rxq *rxq) +{ + struct rxq_ctrl *ctrl = container_of(rxq, struct rxq_ctrl, rxq); + + if (!ctrl->priv->rx_vec_en || rxq->sges_n != 0) + return -ENOTSUP; + return 1; +} + +/** + * Check a device can support vectorized RX. + * + * @param priv + * Pointer to private structure. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +priv_check_vec_rx_support(struct priv *priv) +{ + uint16_t i; + + if (!priv->rx_vec_en) + return -ENOTSUP; + /* All the configured queues should support. */ + for (i = 0; i < priv->rxqs_n; ++i) { + struct rxq *rxq = (*priv->rxqs)[i]; + + if (rxq_check_vec_support(rxq) < 0) + break; + } + if (i != priv->rxqs_n) + return -ENOTSUP; + return 1; +} + +/** + * Prepare for vectorized RX. + * + * @param priv + * Pointer to private structure. + */ +void +priv_prep_vec_rx_function(struct priv *priv) +{ + uint16_t i; + + for (i = 0; i < priv->rxqs_n; ++i) { + struct rxq *rxq = (*priv->rxqs)[i]; + struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; + const uint16_t desc = 1 << rxq->elts_n; + int j; + + assert(rxq->elts_n == rxq->cqe_n); + /* Initialize default rearm_data for vPMD. */ + mbuf_init->data_off = RTE_PKTMBUF_HEADROOM; + rte_mbuf_refcnt_set(mbuf_init, 1); + mbuf_init->nb_segs = 1; + mbuf_init->port = rxq->port_id; + /* + * prevent compiler reordering: + * rearm_data covers previous fields. + */ + rte_compiler_barrier(); + rxq->mbuf_initializer = + *(uint64_t *)&mbuf_init->rearm_data; + /* Padding with a fake mbuf for vectorized Rx. */ + for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j) + (*rxq->elts)[desc + j] = &rxq->fake_mbuf; + /* Mark that it need to be cleaned up for rxq_alloc_elts(). */ + rxq->trim_elts = 1; + } +} diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index 8c5aa691..595a9e06 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -72,6 +72,9 @@ mlx5_dev_start(struct rte_eth_dev *dev) priv_unlock(priv); return 0; } + /* Update Rx/Tx callback. */ + priv_select_tx_function(priv); + priv_select_rx_function(priv); DEBUG("%p: allocating and configuring hash RX queues", (void *)dev); err = priv_create_hash_rxqs(priv); if (!err) @@ -94,12 +97,13 @@ mlx5_dev_start(struct rte_eth_dev *dev) (void *)priv, strerror(err)); goto error; } - priv_dev_interrupt_handler_install(priv, dev); - if (dev->data->dev_conf.intr_conf.rxq) { - err = priv_intr_efd_enable(priv); - if (!err) - err = priv_create_intr_vec(priv); + err = priv_rx_intr_vec_enable(priv); + if (err) { + ERROR("%p: RX interrupt vector creation failed", + (void *)priv); + goto error; } + priv_dev_interrupt_handler_install(priv, dev); priv_xstats_init(priv); priv_unlock(priv); return 0; @@ -140,11 +144,8 @@ mlx5_dev_stop(struct rte_eth_dev *dev) priv_destroy_hash_rxqs(priv); priv_fdir_disable(priv); priv_flow_stop(priv); + priv_rx_intr_vec_disable(priv); priv_dev_interrupt_handler_uninstall(priv, dev); - if (priv->dev->data->dev_conf.intr_conf.rxq) { - priv_destroy_intr_vec(priv); - priv_intr_efd_disable(priv); - } priv->started = 0; priv_unlock(priv); } diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index bf72468d..98aaa7ca 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -64,7 +64,6 @@ #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_autoconf.h" -#include "mlx5_defs.h" /** * Allocate TX queue elements. @@ -103,9 +102,10 @@ txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n) static void txq_free_elts(struct txq_ctrl *txq_ctrl) { - unsigned int elts_n = 1 << txq_ctrl->txq.elts_n; - unsigned int elts_head = txq_ctrl->txq.elts_head; - unsigned int elts_tail = txq_ctrl->txq.elts_tail; + const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n; + const uint16_t elts_m = elts_n - 1; + uint16_t elts_head = txq_ctrl->txq.elts_head; + uint16_t elts_tail = txq_ctrl->txq.elts_tail; struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts; DEBUG("%p: freeing WRs", (void *)txq_ctrl); @@ -114,18 +114,17 @@ txq_free_elts(struct txq_ctrl *txq_ctrl) txq_ctrl->txq.elts_comp = 0; while (elts_tail != elts_head) { - struct rte_mbuf *elt = (*elts)[elts_tail]; + struct rte_mbuf *elt = (*elts)[elts_tail & elts_m]; assert(elt != NULL); rte_pktmbuf_free_seg(elt); #ifndef NDEBUG /* Poisoning. */ - memset(&(*elts)[elts_tail], + memset(&(*elts)[elts_tail & elts_m], 0x77, - sizeof((*elts)[elts_tail])); + sizeof((*elts)[elts_tail & elts_m])); #endif - if (++elts_tail == elts_n) - elts_tail = 0; + ++elts_tail; } } @@ -149,9 +148,8 @@ txq_cleanup(struct txq_ctrl *txq_ctrl) if (txq_ctrl->cq != NULL) claim_zero(ibv_destroy_cq(txq_ctrl->cq)); for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) { - if (txq_ctrl->txq.mp2mr[i].mp == NULL) + if (txq_ctrl->txq.mp2mr[i].mr == NULL) break; - assert(txq_ctrl->txq.mp2mr[i].mr != NULL); claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr)); } memset(txq_ctrl, 0, sizeof(*txq_ctrl)); @@ -244,7 +242,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set"); goto error; } - (void)conf; /* Thresholds configuration (ignored). */ + tmpl.txq.flags = conf->txq_flags; assert(desc > MLX5_TX_COMP_THRESH); tmpl.txq.elts_n = log2above(desc); if (priv->mps == MLX5_MPW_ENHANCED) @@ -497,8 +495,6 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, DEBUG("%p: adding TX queue %p to list", (void *)dev, (void *)txq_ctrl); (*priv->txqs)[idx] = &txq_ctrl->txq; - /* Update send callback. */ - priv_select_tx_function(priv); } priv_unlock(priv); return -ret; diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c index 5479fb36..92b03c4c 100644 --- a/drivers/net/nfp/nfp_net.c +++ b/drivers/net/nfp/nfp_net.c @@ -39,8 +39,6 @@ * Netronome vNIC DPDK Poll-Mode Driver: Main entry point */ -#include <math.h> - #include <rte_byteorder.h> #include <rte_common.h> #include <rte_log.h> @@ -647,7 +645,7 @@ nfp_configure_rx_interrupt(struct rte_eth_dev *dev, static int nfp_net_start(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t new_ctrl, update = 0; struct nfp_net_hw *hw; @@ -772,7 +770,7 @@ nfp_net_close(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "Close"); hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); - pci_dev = RTE_DEV_TO_PCI(dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); /* * We assume that the DPDK application is stopping all the @@ -1081,7 +1079,7 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; dev_info->min_rx_bufsize = ETHER_MIN_MTU; @@ -1173,7 +1171,7 @@ nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) * Other PMDs are just checking the DD bit in intervals of 4 * descriptors and counting all four if the first has the DD * bit on. Of course, this is not accurate but can be good for - * perfomance. But ideally that should be done in descriptors + * performance. But ideally that should be done in descriptors * chunks belonging to the same cache line */ @@ -1201,7 +1199,7 @@ nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) int base = 0; hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); - pci_dev = RTE_DEV_TO_PCI(dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) base = 1; @@ -1221,7 +1219,7 @@ nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) int base = 0; hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); - pci_dev = RTE_DEV_TO_PCI(dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) base = 1; @@ -1235,7 +1233,7 @@ nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) static void nfp_net_dev_link_status_print(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_eth_link link; memset(&link, 0, sizeof(link)); @@ -1269,7 +1267,7 @@ nfp_net_irq_unmask(struct rte_eth_dev *dev) struct rte_pci_device *pci_dev; hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); - pci_dev = RTE_DEV_TO_PCI(dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) { /* If MSI-X auto-masking is used, clear the entry */ @@ -1334,7 +1332,7 @@ nfp_net_dev_interrupt_delayed_handler(void *param) struct rte_eth_dev *dev = (struct rte_eth_dev *)param; nfp_net_link_update(dev, 0); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); nfp_net_dev_link_status_print(dev); @@ -1473,7 +1471,7 @@ nfp_net_rx_queue_setup(struct rte_eth_dev *dev, * of descriptors in log2 format */ nn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma); - nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), log2(nb_desc)); + nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc)); return 0; } @@ -1628,7 +1626,7 @@ nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, * of descriptors in log2 format */ nn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma); - nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), log2(nb_desc)); + nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc)); return 0; } @@ -2450,7 +2448,7 @@ nfp_net_init(struct rte_eth_dev *eth_dev) if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; diff --git a/drivers/net/null/rte_eth_null.c b/drivers/net/null/rte_eth_null.c index abf3ec75..5aef0591 100644 --- a/drivers/net/null/rte_eth_null.c +++ b/drivers/net/null/rte_eth_null.c @@ -49,7 +49,6 @@ static unsigned default_packet_copy; static const char *valid_arguments[] = { ETH_NULL_PACKET_SIZE_ARG, ETH_NULL_PACKET_COPY_ARG, - "driver", NULL }; @@ -139,8 +138,6 @@ eth_null_copy_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) packet_size); bufs[i]->data_len = (uint16_t)packet_size; bufs[i]->pkt_len = packet_size; - bufs[i]->nb_segs = 1; - bufs[i]->next = NULL; bufs[i]->port = h->internals->port_id; } diff --git a/drivers/net/qede/Makefile b/drivers/net/qede/Makefile index 3323914c..f03441d9 100644 --- a/drivers/net/qede/Makefile +++ b/drivers/net/qede/Makefile @@ -101,7 +101,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += bcm_osal.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_sriov.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_vf.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_ethdev.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_eth_if.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_main.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_fdir.c diff --git a/drivers/net/qede/base/bcm_osal.c b/drivers/net/qede/base/bcm_osal.c index 3f895cd4..2603a8b3 100644 --- a/drivers/net/qede/base/bcm_osal.c +++ b/drivers/net/qede/base/bcm_osal.c @@ -16,6 +16,10 @@ #include "ecore_mcp_api.h" #include "ecore_l2_api.h" +/* Array of memzone pointers */ +static const struct rte_memzone *ecore_mz_mapping[RTE_MAX_MEMZONE]; +/* Counter to track current memzone allocated */ +uint16_t ecore_mz_count; unsigned long qede_log2_align(unsigned long n) { @@ -118,6 +122,13 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev, uint32_t core_id = rte_lcore_id(); unsigned int socket_id; + if (ecore_mz_count >= RTE_MAX_MEMZONE) { + DP_ERR(p_dev, "Memzone allocation count exceeds %u\n", + RTE_MAX_MEMZONE); + *phys = 0; + return OSAL_NULL; + } + OSAL_MEM_ZERO(mz_name, sizeof(*mz_name)); snprintf(mz_name, sizeof(mz_name) - 1, "%lx", (unsigned long)rte_get_timer_cycles()); @@ -134,9 +145,11 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev, return OSAL_NULL; } *phys = mz->phys_addr; - DP_VERBOSE(p_dev, ECORE_MSG_PROBE, - "size=%zu phys=0x%" PRIx64 " virt=%p on socket=%u\n", - mz->len, mz->phys_addr, mz->addr, socket_id); + ecore_mz_mapping[ecore_mz_count++] = mz; + DP_VERBOSE(p_dev, ECORE_MSG_SP, + "Allocated dma memory size=%zu phys=0x%lx" + " virt=%p core=%d\n", + mz->len, (unsigned long)mz->phys_addr, mz->addr, core_id); return mz->addr; } @@ -148,6 +161,13 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *p_dev, uint32_t core_id = rte_lcore_id(); unsigned int socket_id; + if (ecore_mz_count >= RTE_MAX_MEMZONE) { + DP_ERR(p_dev, "Memzone allocation count exceeds %u\n", + RTE_MAX_MEMZONE); + *phys = 0; + return OSAL_NULL; + } + OSAL_MEM_ZERO(mz_name, sizeof(*mz_name)); snprintf(mz_name, sizeof(mz_name) - 1, "%lx", (unsigned long)rte_get_timer_cycles()); @@ -163,12 +183,30 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *p_dev, return OSAL_NULL; } *phys = mz->phys_addr; - DP_VERBOSE(p_dev, ECORE_MSG_PROBE, - "aligned memory size=%zu phys=0x%" PRIx64 " virt=%p core=%d\n", - mz->len, mz->phys_addr, mz->addr, core_id); + ecore_mz_mapping[ecore_mz_count++] = mz; + DP_VERBOSE(p_dev, ECORE_MSG_SP, + "Allocated aligned dma memory size=%zu phys=0x%lx" + " virt=%p core=%d\n", + mz->len, (unsigned long)mz->phys_addr, mz->addr, core_id); return mz->addr; } +void osal_dma_free_mem(struct ecore_dev *p_dev, dma_addr_t phys) +{ + uint16_t j; + + for (j = 0 ; j < ecore_mz_count; j++) { + if (phys == ecore_mz_mapping[j]->phys_addr) { + DP_VERBOSE(p_dev, ECORE_MSG_SP, + "Free memzone %s\n", ecore_mz_mapping[j]->name); + rte_memzone_free(ecore_mz_mapping[j]); + return; + } + } + + DP_ERR(p_dev, "Unexpected memory free request\n"); +} + #ifdef CONFIG_ECORE_ZIPPED_FW u32 qede_unzip_data(struct ecore_hwfn *p_hwfn, u32 input_len, u8 *input_buf, u32 max_size, u8 *unzip_buf) @@ -211,11 +249,46 @@ qede_get_mcp_proto_stats(struct ecore_dev *edev, if (type == ECORE_MCP_LAN_STATS) { ecore_get_vport_stats(edev, &lan_stats); - stats->lan_stats.ucast_rx_pkts = lan_stats.rx_ucast_pkts; - stats->lan_stats.ucast_tx_pkts = lan_stats.tx_ucast_pkts; + + /* @DPDK */ + stats->lan_stats.ucast_rx_pkts = lan_stats.common.rx_ucast_pkts; + stats->lan_stats.ucast_tx_pkts = lan_stats.common.tx_ucast_pkts; + stats->lan_stats.fcs_err = -1; } else { DP_INFO(edev, "Statistics request type %d not supported\n", type); } } + +void +qede_hw_err_notify(struct ecore_hwfn *p_hwfn, enum ecore_hw_err_type err_type) +{ + char err_str[64]; + + switch (err_type) { + case ECORE_HW_ERR_FAN_FAIL: + strcpy(err_str, "Fan Failure"); + break; + case ECORE_HW_ERR_MFW_RESP_FAIL: + strcpy(err_str, "MFW Response Failure"); + break; + case ECORE_HW_ERR_HW_ATTN: + strcpy(err_str, "HW Attention"); + break; + case ECORE_HW_ERR_DMAE_FAIL: + strcpy(err_str, "DMAE Failure"); + break; + case ECORE_HW_ERR_RAMROD_FAIL: + strcpy(err_str, "Ramrod Failure"); + break; + case ECORE_HW_ERR_FW_ASSERT: + strcpy(err_str, "FW Assertion"); + break; + default: + strcpy(err_str, "Unknown"); + } + + DP_ERR(p_hwfn, "HW error occurred [%s]\n", err_str); + ecore_int_attn_clr_enable(p_hwfn->p_dev, true); +} diff --git a/drivers/net/qede/base/bcm_osal.h b/drivers/net/qede/base/bcm_osal.h index 32c9b251..3acf8f7c 100644 --- a/drivers/net/qede/base/bcm_osal.h +++ b/drivers/net/qede/base/bcm_osal.h @@ -27,6 +27,7 @@ struct ecore_vf_acquire_sw_info; struct vf_pf_resc_request; enum ecore_mcp_protocol_type; union ecore_mcp_protocol_stats; +enum ecore_hw_err_type; void qed_link_update(struct ecore_hwfn *hwfn); @@ -107,14 +108,16 @@ void *osal_dma_alloc_coherent(struct ecore_dev *, dma_addr_t *, size_t); void *osal_dma_alloc_coherent_aligned(struct ecore_dev *, dma_addr_t *, size_t, int); +void osal_dma_free_mem(struct ecore_dev *edev, dma_addr_t phys); + #define OSAL_DMA_ALLOC_COHERENT(dev, phys, size) \ osal_dma_alloc_coherent(dev, phys, size) #define OSAL_DMA_ALLOC_COHERENT_ALIGNED(dev, phys, size, align) \ osal_dma_alloc_coherent_aligned(dev, phys, size, align) -/* TODO: */ -#define OSAL_DMA_FREE_COHERENT(dev, virt, phys, size) nothing +#define OSAL_DMA_FREE_COHERENT(dev, virt, phys, size) \ + osal_dma_free_mem(dev, phys) /* HW reads/writes */ @@ -348,6 +351,8 @@ u32 qede_unzip_data(struct ecore_hwfn *p_hwfn, u32 input_len, u8 *input_buf, u32 max_size, u8 *unzip_buf); void qede_vf_fill_driver_data(struct ecore_hwfn *, struct vf_pf_resc_request *, struct ecore_vf_acquire_sw_info *); +void qede_hw_err_notify(struct ecore_hwfn *p_hwfn, + enum ecore_hw_err_type err_type); #define OSAL_VF_FILL_ACQUIRE_RESC_REQ(_dev_p, _resc_req, _os_info) \ qede_vf_fill_driver_data(_dev_p, _resc_req, _os_info) @@ -356,7 +361,8 @@ void qede_vf_fill_driver_data(struct ecore_hwfn *, struct vf_pf_resc_request *, /* TODO: */ #define OSAL_SCHEDULE_RECOVERY_HANDLER(hwfn) nothing -#define OSAL_HW_ERROR_OCCURRED(hwfn, err_type) nothing +#define OSAL_HW_ERROR_OCCURRED(hwfn, err_type) \ + qede_hw_err_notify(hwfn, err_type) #define OSAL_NVM_IS_ACCESS_ENABLED(hwfn) (1) #define OSAL_NUM_ACTIVE_CPU() 0 @@ -421,6 +427,9 @@ void qede_get_mcp_proto_stats(struct ecore_dev *, enum ecore_mcp_protocol_type, qede_get_mcp_proto_stats(dev, type, stats) #define OSAL_SLOWPATH_IRQ_REQ(p_hwfn) (0) +#define OSAL_CRC32(crc, buf, length) 0 +#define OSAL_CRC8_POPULATE(table, polynomial) nothing +#define OSAL_CRC8(table, pdata, nbytes, crc) 0 #define OSAL_MFW_TLV_REQ(p_hwfn) (0) #define OSAL_MFW_FILL_TLV_DATA(type, buf, data) (0) #define OSAL_PF_VALIDATE_MODIFY_TUNN_CONFIG(p_hwfn, mask, b_update, tunn) 0 diff --git a/drivers/net/qede/base/common_hsi.h b/drivers/net/qede/base/common_hsi.h index cbcde227..bfe50e1f 100644 --- a/drivers/net/qede/base/common_hsi.h +++ b/drivers/net/qede/base/common_hsi.h @@ -97,8 +97,8 @@ #define FW_MAJOR_VERSION 8 -#define FW_MINOR_VERSION 18 -#define FW_REVISION_VERSION 9 +#define FW_MINOR_VERSION 20 +#define FW_REVISION_VERSION 0 #define FW_ENGINEERING_VERSION 0 /***********************/ @@ -913,24 +913,25 @@ struct db_l2_dpm_data { __le16 bd_prod /* bd producer value to update */; __le32 params; /* Size in QWORD-s of the DPM burst */ -#define DB_L2_DPM_DATA_SIZE_MASK 0x3F -#define DB_L2_DPM_DATA_SIZE_SHIFT 0 +#define DB_L2_DPM_DATA_SIZE_MASK 0x3F +#define DB_L2_DPM_DATA_SIZE_SHIFT 0 /* Type of DPM transaction (DPM_L2_INLINE or DPM_L2_BD) (use enum db_dpm_type) */ -#define DB_L2_DPM_DATA_DPM_TYPE_MASK 0x3 -#define DB_L2_DPM_DATA_DPM_TYPE_SHIFT 6 -#define DB_L2_DPM_DATA_NUM_BDS_MASK 0xFF /* number of BD-s */ -#define DB_L2_DPM_DATA_NUM_BDS_SHIFT 8 +#define DB_L2_DPM_DATA_DPM_TYPE_MASK 0x3 +#define DB_L2_DPM_DATA_DPM_TYPE_SHIFT 6 +#define DB_L2_DPM_DATA_NUM_BDS_MASK 0xFF /* number of BD-s */ +#define DB_L2_DPM_DATA_NUM_BDS_SHIFT 8 /* size of the packet to be transmitted in bytes */ -#define DB_L2_DPM_DATA_PKT_SIZE_MASK 0x7FF -#define DB_L2_DPM_DATA_PKT_SIZE_SHIFT 16 -#define DB_L2_DPM_DATA_RESERVED0_MASK 0x1 -#define DB_L2_DPM_DATA_RESERVED0_SHIFT 27 +#define DB_L2_DPM_DATA_PKT_SIZE_MASK 0x7FF +#define DB_L2_DPM_DATA_PKT_SIZE_SHIFT 16 +#define DB_L2_DPM_DATA_RESERVED0_MASK 0x1 +#define DB_L2_DPM_DATA_RESERVED0_SHIFT 27 /* In DPM_L2_BD mode: the number of SGE-s */ -#define DB_L2_DPM_DATA_SGE_NUM_MASK 0x7 -#define DB_L2_DPM_DATA_SGE_NUM_SHIFT 28 -#define DB_L2_DPM_DATA_RESERVED1_MASK 0x1 -#define DB_L2_DPM_DATA_RESERVED1_SHIFT 31 +#define DB_L2_DPM_DATA_SGE_NUM_MASK 0x7 +#define DB_L2_DPM_DATA_SGE_NUM_SHIFT 28 +/* Flag indicating whether to enable GFS search */ +#define DB_L2_DPM_DATA_RESERVED1_MASK 0x1 +#define DB_L2_DPM_DATA_RESERVED1_SHIFT 31 }; /* @@ -989,26 +990,29 @@ struct db_pwm_addr { struct db_rdma_dpm_params { __le32 params; /* Size in QWORD-s of the DPM burst */ -#define DB_RDMA_DPM_PARAMS_SIZE_MASK 0x3F -#define DB_RDMA_DPM_PARAMS_SIZE_SHIFT 0 +#define DB_RDMA_DPM_PARAMS_SIZE_MASK 0x3F +#define DB_RDMA_DPM_PARAMS_SIZE_SHIFT 0 /* Type of DPM transacation (DPM_RDMA) (use enum db_dpm_type) */ -#define DB_RDMA_DPM_PARAMS_DPM_TYPE_MASK 0x3 -#define DB_RDMA_DPM_PARAMS_DPM_TYPE_SHIFT 6 +#define DB_RDMA_DPM_PARAMS_DPM_TYPE_MASK 0x3 +#define DB_RDMA_DPM_PARAMS_DPM_TYPE_SHIFT 6 /* opcode for RDMA operation */ -#define DB_RDMA_DPM_PARAMS_OPCODE_MASK 0xFF -#define DB_RDMA_DPM_PARAMS_OPCODE_SHIFT 8 +#define DB_RDMA_DPM_PARAMS_OPCODE_MASK 0xFF +#define DB_RDMA_DPM_PARAMS_OPCODE_SHIFT 8 /* the size of the WQE payload in bytes */ -#define DB_RDMA_DPM_PARAMS_WQE_SIZE_MASK 0x7FF -#define DB_RDMA_DPM_PARAMS_WQE_SIZE_SHIFT 16 -#define DB_RDMA_DPM_PARAMS_RESERVED0_MASK 0x1 -#define DB_RDMA_DPM_PARAMS_RESERVED0_SHIFT 27 +#define DB_RDMA_DPM_PARAMS_WQE_SIZE_MASK 0x7FF +#define DB_RDMA_DPM_PARAMS_WQE_SIZE_SHIFT 16 +#define DB_RDMA_DPM_PARAMS_RESERVED0_MASK 0x1 +#define DB_RDMA_DPM_PARAMS_RESERVED0_SHIFT 27 /* RoCE completion flag */ -#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_MASK 0x1 -#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_SHIFT 28 -#define DB_RDMA_DPM_PARAMS_S_FLG_MASK 0x1 /* RoCE S flag */ -#define DB_RDMA_DPM_PARAMS_S_FLG_SHIFT 29 -#define DB_RDMA_DPM_PARAMS_RESERVED1_MASK 0x3 -#define DB_RDMA_DPM_PARAMS_RESERVED1_SHIFT 30 +#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_MASK 0x1 +#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_SHIFT 28 +#define DB_RDMA_DPM_PARAMS_S_FLG_MASK 0x1 /* RoCE S flag */ +#define DB_RDMA_DPM_PARAMS_S_FLG_SHIFT 29 +#define DB_RDMA_DPM_PARAMS_RESERVED1_MASK 0x3 +#define DB_RDMA_DPM_PARAMS_RESERVED1_SHIFT 30 +/* Connection type is iWARP */ +#define DB_RDMA_DPM_PARAMS_CONN_TYPE_IS_IWARP_MASK 0x1 +#define DB_RDMA_DPM_PARAMS_CONN_TYPE_IS_IWARP_SHIFT 31 }; /* diff --git a/drivers/net/qede/base/ecore.h b/drivers/net/qede/base/ecore.h index 80b11a4c..0d68a9bc 100644 --- a/drivers/net/qede/base/ecore.h +++ b/drivers/net/qede/base/ecore.h @@ -31,7 +31,7 @@ #define ECORE_MAJOR_VERSION 8 #define ECORE_MINOR_VERSION 18 #define ECORE_REVISION_VERSION 7 -#define ECORE_ENGINEERING_VERSION 0 +#define ECORE_ENGINEERING_VERSION 1 #define ECORE_VERSION \ ((ECORE_MAJOR_VERSION << 24) | (ECORE_MINOR_VERSION << 16) | \ @@ -770,7 +770,7 @@ struct ecore_dev { bool attn_clr_en; /* Indicates whether allowing the MFW to collect a crash dump */ - bool mdump_en; + bool allow_mdump; /* Indicates if the reg_fifo is checked after any register access */ bool chk_reg_fifo; diff --git a/drivers/net/qede/base/ecore_dev.c b/drivers/net/qede/base/ecore_dev.c index 865103c6..65b89b8f 100644 --- a/drivers/net/qede/base/ecore_dev.c +++ b/drivers/net/qede/base/ecore_dev.c @@ -1080,7 +1080,7 @@ enum _ecore_status_t ecore_final_cleanup(struct ecore_hwfn *p_hwfn, } DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "Sending final cleanup for PFVF[%d] [Command %08x\n]", + "Sending final cleanup for PFVF[%d] [Command %08x]\n", id, command); ecore_wr(p_hwfn, p_ptt, XSDM_REG_OPERATION_GEN, command); @@ -1776,13 +1776,6 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, /* perform debug configuration when chip is out of reset */ OSAL_BEFORE_PF_START((void *)p_hwfn->p_dev, p_hwfn->my_id); - /* Cleanup chip from previous driver if such remains exist */ - rc = ecore_final_cleanup(p_hwfn, p_ptt, rel_pf_id, false); - if (rc != ECORE_SUCCESS) { - ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_RAMROD_FAIL); - return rc; - } - /* PF Init sequence */ rc = ecore_init_run(p_hwfn, p_ptt, PHASE_PF, rel_pf_id, hw_mode); if (rc) @@ -1866,17 +1859,17 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, return rc; } -static enum _ecore_status_t -ecore_change_pci_hwfn(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 enable) +enum _ecore_status_t ecore_pglueb_set_pfid_enable(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + bool b_enable) { - u32 delay_idx = 0, val, set_val = enable ? 1 : 0; + u32 delay_idx = 0, val, set_val = b_enable ? 1 : 0; - /* Change PF in PXP */ + /* Configure the PF's internal FID_enable for master transactions */ ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_INTERNAL_PFID_ENABLE_MASTER, set_val); - /* wait until value is set - try for 1 second every 50us */ + /* Wait until value is set - try for 1 second every 50us */ for (delay_idx = 0; delay_idx < 20000; delay_idx++) { val = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_INTERNAL_PFID_ENABLE_MASTER); @@ -1918,14 +1911,21 @@ enum _ecore_status_t ecore_vf_start(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } +static void ecore_pglueb_clear_err(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_WAS_ERROR_PF_31_0_CLR, + 1 << p_hwfn->abs_pf_id); +} + enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, struct ecore_hw_init_params *p_params) { struct ecore_load_req_params load_req_params; - u32 load_code, param, drv_mb_param; + u32 load_code, resp, param, drv_mb_param; bool b_default_mtu = true; struct ecore_hwfn *p_hwfn; - enum _ecore_status_t rc = ECORE_SUCCESS, mfw_rc; + enum _ecore_status_t rc = ECORE_SUCCESS; int i; if ((p_params->int_mode == ECORE_INT_MODE_MSI) && @@ -1942,7 +1942,7 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, } for_each_hwfn(p_dev, i) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; + p_hwfn = &p_dev->hwfns[i]; /* If management didn't provide a default, set one of our own */ if (!p_hwfn->hw_info.mtu) { @@ -1955,11 +1955,6 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, continue; } - /* Enable DMAE in PXP */ - rc = ecore_change_pci_hwfn(p_hwfn, p_hwfn->p_main_ptt, true); - if (rc != ECORE_SUCCESS) - return rc; - rc = ecore_calc_hw_mode(p_hwfn); if (rc != ECORE_SUCCESS) return rc; @@ -2009,6 +2004,30 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, qm_lock_init = true; } + /* Clean up chip from previous driver if such remains exist. + * This is not needed when the PF is the first one on the + * engine, since afterwards we are going to init the FW. + */ + if (load_code != FW_MSG_CODE_DRV_LOAD_ENGINE) { + rc = ecore_final_cleanup(p_hwfn, p_hwfn->p_main_ptt, + p_hwfn->rel_pf_id, false); + if (rc != ECORE_SUCCESS) { + ecore_hw_err_notify(p_hwfn, + ECORE_HW_ERR_RAMROD_FAIL); + goto load_err; + } + } + + /* Log and clean previous pglue_b errors if such exist */ + ecore_pglueb_rbc_attn_handler(p_hwfn, p_hwfn->p_main_ptt); + ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt); + + /* Enable the PF's internal FID_enable in the PXP */ + rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_hwfn->p_main_ptt, + true); + if (rc != ECORE_SUCCESS) + goto load_err; + switch (load_code) { case FW_MSG_CODE_DRV_LOAD_ENGINE: rc = ecore_hw_init_common(p_hwfn, p_hwfn->p_main_ptt, @@ -2037,35 +2056,28 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, break; } - if (rc != ECORE_SUCCESS) + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, "init phase failed for loadcode 0x%x (rc %d)\n", load_code, rc); + goto load_err; + } - /* ACK mfw regardless of success or failure of initialization */ - mfw_rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, - DRV_MSG_CODE_LOAD_DONE, - 0, &load_code, ¶m); + rc = ecore_mcp_load_done(p_hwfn, p_hwfn->p_main_ptt); if (rc != ECORE_SUCCESS) return rc; - if (mfw_rc != ECORE_SUCCESS) { - DP_NOTICE(p_hwfn, true, - "Failed sending a LOAD_DONE command\n"); - return mfw_rc; - } - /* send DCBX attention request command */ DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "sending phony dcbx set command to trigger DCBx attention handling\n"); - mfw_rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, - DRV_MSG_CODE_SET_DCBX, - 1 << DRV_MB_PARAM_DCBX_NOTIFY_SHIFT, - &load_code, ¶m); - if (mfw_rc != ECORE_SUCCESS) { + rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, + DRV_MSG_CODE_SET_DCBX, + 1 << DRV_MB_PARAM_DCBX_NOTIFY_SHIFT, &resp, + ¶m); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, "Failed to send DCBX attention request\n"); - return mfw_rc; + return rc; } p_hwfn->hw_init_done = true; @@ -2076,7 +2088,7 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, drv_mb_param = STORM_FW_VERSION; rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, DRV_MSG_CODE_OV_UPDATE_STORM_FW_VER, - drv_mb_param, &load_code, ¶m); + drv_mb_param, &resp, ¶m); if (rc != ECORE_SUCCESS) DP_INFO(p_hwfn, "Failed to update firmware version\n"); @@ -2094,6 +2106,14 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, } return rc; + +load_err: + /* The MFW load lock should be released regardless of success or failure + * of initialization. + * TODO: replace this with an attempt to send cancel_load. + */ + ecore_mcp_load_done(p_hwfn, p_hwfn->p_main_ptt); + return rc; } #define ECORE_HW_STOP_RETRY_LIMIT (10) @@ -2261,18 +2281,20 @@ enum _ecore_status_t ecore_hw_stop(struct ecore_dev *p_dev) } } /* hwfn loop */ - if (IS_PF(p_dev)) { + if (IS_PF(p_dev) && !p_dev->recov_in_prog) { p_hwfn = ECORE_LEADING_HWFN(p_dev); p_ptt = ECORE_LEADING_HWFN(p_dev)->p_main_ptt; - /* Disable DMAE in PXP - in CMT, this should only be done for - * first hw-function, and only after all transactions have - * stopped for all active hw-functions. - */ - rc = ecore_change_pci_hwfn(p_hwfn, p_ptt, false); + /* Clear the PF's internal FID_enable in the PXP. + * In CMT this should only be done for first hw-function, and + * only after all transactions have stopped for all active + * hw-functions. + */ + rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_hwfn->p_main_ptt, + false); if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, - "ecore_change_pci_hwfn failed. rc = %d.\n", + "ecore_pglueb_set_pfid_enable() failed. rc = %d.\n", rc); rc2 = ECORE_UNKNOWN_ERROR; } @@ -2370,9 +2392,8 @@ static void ecore_hw_hwfn_prepare(struct ecore_hwfn *p_hwfn) PGLUE_B_REG_PGL_ADDR_94_F0_BB, 0); } - /* Clean Previous errors if such exist */ - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, - PGLUE_B_REG_WAS_ERROR_PF_31_0_CLR, 1 << p_hwfn->abs_pf_id); + /* Clean previous pglue_b errors if such exist */ + ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt); /* enable internal target-read */ ecore_wr(p_hwfn, p_hwfn->p_main_ptt, @@ -3565,6 +3586,7 @@ enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev, enum _ecore_status_t rc; p_dev->chk_reg_fifo = p_params->chk_reg_fifo; + p_dev->allow_mdump = p_params->allow_mdump; if (p_params->b_relaxed_probe) p_params->p_relaxed_res = ECORE_HW_PREPARE_SUCCESS; @@ -3718,7 +3740,7 @@ static void ecore_chain_free_pbl(struct ecore_dev *p_dev, if (!p_chain->b_external_pbl) OSAL_DMA_FREE_COHERENT(p_dev, p_chain->pbl_sp.p_virt_table, p_chain->pbl_sp.p_phys_table, pbl_size); - out: +out: OSAL_VFREE(p_dev, p_chain->pbl.pp_virt_addr_tbl); } @@ -3994,92 +4016,182 @@ enum _ecore_status_t ecore_fw_rss_eng(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u8 *p_filter) +static enum _ecore_status_t +ecore_llh_add_mac_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u32 high, u32 low, + u32 *p_entry_num) { - u32 high, low, en; + u32 en; int i; - if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) - return ECORE_SUCCESS; - - high = p_filter[1] | (p_filter[0] << 8); - low = p_filter[5] | (p_filter[4] << 8) | - (p_filter[3] << 16) | (p_filter[2] << 24); - /* Find a free entry and utilize it */ for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { en = ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32)); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32)); if (en) continue; ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32), low); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32), high); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_MODE + i * sizeof(u32), 0); + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + + i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE + + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 1); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32), 1); break; } - if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) { - DP_NOTICE(p_hwfn, false, - "Failed to find an empty LLH filter to utilize\n"); - return ECORE_INVAL; - } - DP_VERBOSE(p_hwfn, ECORE_MSG_HW, - "MAC: %x:%x:%x:%x:%x:%x is added at %d\n", - p_filter[0], p_filter[1], p_filter[2], - p_filter[3], p_filter[4], p_filter[5], i); + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + return ECORE_NORESOURCES; + + *p_entry_num = i; return ECORE_SUCCESS; } -void ecore_llh_remove_mac_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 *p_filter) +enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u8 *p_filter) { - u32 high, low; - int i; + u32 high, low, entry_num; + enum _ecore_status_t rc; if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) - return; + return ECORE_SUCCESS; high = p_filter[1] | (p_filter[0] << 8); low = p_filter[5] | (p_filter[4] << 8) | - (p_filter[3] << 16) | (p_filter[2] << 24); + (p_filter[3] << 16) | (p_filter[2] << 24); + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_add_mac_filter_bb_ah(p_hwfn, p_ptt, high, low, + &entry_num); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to find an empty LLH filter to utilize\n"); + return rc; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "MAC: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx is added at %d\n", + p_filter[0], p_filter[1], p_filter[2], p_filter[3], + p_filter[4], p_filter[5], entry_num); + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +ecore_llh_remove_mac_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u32 high, u32 low, + u32 *p_entry_num) +{ + int i; /* Find the entry and clean it */ for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32)) != low) continue; if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32)) != high) continue; ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 0); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32), 0); break; } + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + return ECORE_INVAL; + + *p_entry_num = i; + + return ECORE_SUCCESS; +} + +void ecore_llh_remove_mac_filter(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u8 *p_filter) +{ + u32 high, low, entry_num; + enum _ecore_status_t rc; + + if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) + return; + + high = p_filter[1] | (p_filter[0] << 8); + low = p_filter[5] | (p_filter[4] << 8) | + (p_filter[3] << 16) | (p_filter[2] << 24); + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_remove_mac_filter_bb_ah(p_hwfn, p_ptt, high, + low, &entry_num); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, false, "Tried to remove a non-configured filter\n"); + return; + } + + + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "MAC: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx was removed from %d\n", + p_filter[0], p_filter[1], p_filter[2], p_filter[3], + p_filter[4], p_filter[5], entry_num); +} + +static enum _ecore_status_t +ecore_llh_add_protocol_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_llh_port_filter_type_t type, + u32 high, u32 low, u32 *p_entry_num) +{ + u32 en; + int i; + + /* Find a free entry and utilize it */ + for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { + en = ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32)); + if (en) + continue; + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + 2 * i * sizeof(u32), low); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + (2 * i + 1) * sizeof(u32), high); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + + i * sizeof(u32), 1); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + + i * sizeof(u32), 1 << type); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 1); + break; + } + + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + return ECORE_NORESOURCES; + + *p_entry_num = i; + + return ECORE_SUCCESS; } enum _ecore_status_t @@ -4089,14 +4201,15 @@ ecore_llh_add_protocol_filter(struct ecore_hwfn *p_hwfn, u16 dest_port, enum ecore_llh_port_filter_type_t type) { - u32 high, low, en; - int i; + u32 high, low, entry_num; + enum _ecore_status_t rc; if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) return ECORE_SUCCESS; high = 0; low = 0; + switch (type) { case ECORE_LLH_FILTER_ETHERTYPE: high = source_port_or_eth_type; @@ -4118,67 +4231,109 @@ ecore_llh_add_protocol_filter(struct ecore_hwfn *p_hwfn, "Non valid LLH protocol filter type %d\n", type); return ECORE_INVAL; } - /* Find a free entry and utilize it */ - for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { - en = ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32)); - if (en) - continue; - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - 2 * i * sizeof(u32), low); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - (2 * i + 1) * sizeof(u32), high); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_MODE + i * sizeof(u32), 1); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE + - i * sizeof(u32), 1 << type); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 1); - break; - } - if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) { + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_add_protocol_filter_bb_ah(p_hwfn, p_ptt, type, + high, low, &entry_num); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, false, "Failed to find an empty LLH filter to utilize\n"); - return ECORE_NORESOURCES; + return rc; } switch (type) { case ECORE_LLH_FILTER_ETHERTYPE: DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "ETH type %x is added at %d\n", - source_port_or_eth_type, i); + source_port_or_eth_type, entry_num); break; case ECORE_LLH_FILTER_TCP_SRC_PORT: DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "TCP src port %x is added at %d\n", - source_port_or_eth_type, i); + source_port_or_eth_type, entry_num); break; case ECORE_LLH_FILTER_UDP_SRC_PORT: DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "UDP src port %x is added at %d\n", - source_port_or_eth_type, i); + source_port_or_eth_type, entry_num); break; case ECORE_LLH_FILTER_TCP_DEST_PORT: DP_VERBOSE(p_hwfn, ECORE_MSG_HW, - "TCP dst port %x is added at %d\n", dest_port, i); + "TCP dst port %x is added at %d\n", dest_port, + entry_num); break; case ECORE_LLH_FILTER_UDP_DEST_PORT: DP_VERBOSE(p_hwfn, ECORE_MSG_HW, - "UDP dst port %x is added at %d\n", dest_port, i); + "UDP dst port %x is added at %d\n", dest_port, + entry_num); break; case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT: DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "TCP src/dst ports %x/%x are added at %d\n", - source_port_or_eth_type, dest_port, i); + source_port_or_eth_type, dest_port, entry_num); break; case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT: DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "UDP src/dst ports %x/%x are added at %d\n", - source_port_or_eth_type, dest_port, i); + source_port_or_eth_type, dest_port, entry_num); break; } + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +ecore_llh_remove_protocol_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_llh_port_filter_type_t type, + u32 high, u32 low, u32 *p_entry_num) +{ + int i; + + /* Find the entry and clean it */ + for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { + if (!ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32))) + continue; + if (!ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + + i * sizeof(u32))) + continue; + if (!(ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + + i * sizeof(u32)) & (1 << type))) + continue; + if (ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + 2 * i * sizeof(u32)) != low) + continue; + if (ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + (2 * i + 1) * sizeof(u32)) != high) + continue; + + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + + i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + + i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + 2 * i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + (2 * i + 1) * sizeof(u32), 0); + break; + } + + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + return ECORE_INVAL; + + *p_entry_num = i; + return ECORE_SUCCESS; } @@ -4189,14 +4344,15 @@ ecore_llh_remove_protocol_filter(struct ecore_hwfn *p_hwfn, u16 dest_port, enum ecore_llh_port_filter_type_t type) { - u32 high, low; - int i; + u32 high, low, entry_num; + enum _ecore_status_t rc; if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) return; high = 0; low = 0; + switch (type) { case ECORE_LLH_FILTER_ETHERTYPE: high = source_port_or_eth_type; @@ -4219,49 +4375,24 @@ ecore_llh_remove_protocol_filter(struct ecore_hwfn *p_hwfn, return; } - for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { - if (!ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32))) - continue; - if (!ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_MODE + i * sizeof(u32))) - continue; - if (!(ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE + - i * sizeof(u32)) & (1 << type))) - continue; - if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - 2 * i * sizeof(u32)) != low) - continue; - if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - (2 * i + 1) * sizeof(u32)) != high) - continue; - - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 0); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_MODE + i * sizeof(u32), 0); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE + - i * sizeof(u32), 0); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - 2 * i * sizeof(u32), 0); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - (2 * i + 1) * sizeof(u32), 0); - break; + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_remove_protocol_filter_bb_ah(p_hwfn, p_ptt, type, + high, low, + &entry_num); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Tried to remove a non-configured filter [type %d, source_port_or_eth_type 0x%x, dest_port 0x%x]\n", + type, source_port_or_eth_type, dest_port); + return; } - if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) - DP_NOTICE(p_hwfn, false, - "Tried to remove a non-configured filter\n"); + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "Protocol filter [type %d, source_port_or_eth_type 0x%x, dest_port 0x%x] was removed from %d\n", + type, source_port_or_eth_type, dest_port, entry_num); } -void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +static void ecore_llh_clear_all_filters_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { int i; @@ -4270,16 +4401,27 @@ void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 0); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32), 0); } } +void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) + return; + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + ecore_llh_clear_all_filters_bb_ah(p_hwfn, p_ptt); +} + enum _ecore_status_t ecore_llh_set_function_as_default(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) @@ -4396,7 +4538,7 @@ enum _ecore_status_t ecore_set_rxq_coalesce(struct ecore_hwfn *p_hwfn, if (rc != ECORE_SUCCESS) goto out; - out: +out: return rc; } @@ -4434,7 +4576,7 @@ enum _ecore_status_t ecore_set_txq_coalesce(struct ecore_hwfn *p_hwfn, rc = ecore_set_coalesce(p_hwfn, p_ptt, address, ð_qzone, sizeof(struct xstorm_eth_queue_zone), timeset); - out: +out: return rc; } diff --git a/drivers/net/qede/base/ecore_dev_api.h b/drivers/net/qede/base/ecore_dev_api.h index e64a768d..9126cf95 100644 --- a/drivers/net/qede/base/ecore_dev_api.h +++ b/drivers/net/qede/base/ecore_dev_api.h @@ -186,6 +186,9 @@ struct ecore_hw_prepare_params { /* The OS Epoch time in seconds */ u32 epoch; + /* Allow the MFW to collect a crash dump */ + bool allow_mdump; + /* Allow prepare to pass even if some initializations are failing. * If set, the `p_prepare_res' field would be set with the return, * and might allow probe to pass even if there are certain issues. @@ -238,7 +241,7 @@ void ecore_ptt_release(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); #ifndef __EXTRACT__LINUX__ -struct ecore_eth_stats { +struct ecore_eth_stats_common { u64 no_buff_discards; u64 packet_too_big_discard; u64 ttl0_discard; @@ -270,11 +273,6 @@ struct ecore_eth_stats { u64 rx_256_to_511_byte_packets; u64 rx_512_to_1023_byte_packets; u64 rx_1024_to_1518_byte_packets; - u64 rx_1519_to_1522_byte_packets; - u64 rx_1519_to_2047_byte_packets; - u64 rx_2048_to_4095_byte_packets; - u64 rx_4096_to_9216_byte_packets; - u64 rx_9217_to_16383_byte_packets; u64 rx_crc_errors; u64 rx_mac_crtl_frames; u64 rx_pause_frames; @@ -291,14 +289,8 @@ struct ecore_eth_stats { u64 tx_256_to_511_byte_packets; u64 tx_512_to_1023_byte_packets; u64 tx_1024_to_1518_byte_packets; - u64 tx_1519_to_2047_byte_packets; - u64 tx_2048_to_4095_byte_packets; - u64 tx_4096_to_9216_byte_packets; - u64 tx_9217_to_16383_byte_packets; u64 tx_pause_frames; u64 tx_pfc_frames; - u64 tx_lpi_entry_count; - u64 tx_total_collisions; u64 brb_truncates; u64 brb_discards; u64 rx_mac_bytes; @@ -312,6 +304,33 @@ struct ecore_eth_stats { u64 tx_mac_bc_packets; u64 tx_mac_ctrl_frames; }; + +struct ecore_eth_stats_bb { + u64 rx_1519_to_1522_byte_packets; + u64 rx_1519_to_2047_byte_packets; + u64 rx_2048_to_4095_byte_packets; + u64 rx_4096_to_9216_byte_packets; + u64 rx_9217_to_16383_byte_packets; + u64 tx_1519_to_2047_byte_packets; + u64 tx_2048_to_4095_byte_packets; + u64 tx_4096_to_9216_byte_packets; + u64 tx_9217_to_16383_byte_packets; + u64 tx_lpi_entry_count; + u64 tx_total_collisions; +}; + +struct ecore_eth_stats_ah { + u64 rx_1519_to_max_byte_packets; + u64 tx_1519_to_max_byte_packets; +}; + +struct ecore_eth_stats { + struct ecore_eth_stats_common common; + union { + struct ecore_eth_stats_bb bb; + struct ecore_eth_stats_ah ah; + }; +}; #endif enum ecore_dmae_address_type_t { @@ -581,4 +600,16 @@ enum _ecore_status_t ecore_set_queue_coalesce(struct ecore_hwfn *p_hwfn, u16 rx_coal, u16 tx_coal, void *p_handle); +/** + * @brief ecore_pglueb_set_pfid_enable - Enable or disable PCI BUS MASTER + * + * @param p_hwfn + * @param p_ptt + * @param b_enable - true/false + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t ecore_pglueb_set_pfid_enable(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + bool b_enable); #endif diff --git a/drivers/net/qede/base/ecore_hsi_common.h b/drivers/net/qede/base/ecore_hsi_common.h index 3042ed55..5c2a08f9 100644 --- a/drivers/net/qede/base/ecore_hsi_common.h +++ b/drivers/net/qede/base/ecore_hsi_common.h @@ -856,7 +856,8 @@ struct core_rx_gsi_offload_cqe { __le16 vlan /* 802.1q VLAN tag */; __le32 src_mac_addrhi /* hi 4 bytes source mac address */; __le16 src_mac_addrlo /* lo 2 bytes of source mac address */; - u8 reserved1[2]; +/* These are the lower 16 bit of QP id in RoCE BTH header */ + __le16 qp_id; __le32 gid_dst[4] /* Gid destination address */; }; @@ -998,11 +999,9 @@ struct core_tx_bd { */ #define CORE_TX_BD_L4_HDR_OFFSET_W_MASK 0x3FFF #define CORE_TX_BD_L4_HDR_OFFSET_W_SHIFT 0 -/* Packet destination - Network, LB (use enum core_tx_dest) */ -#define CORE_TX_BD_TX_DST_MASK 0x1 +/* Packet destination - Network, Loopback or Drop (use enum core_tx_dest) */ +#define CORE_TX_BD_TX_DST_MASK 0x3 #define CORE_TX_BD_TX_DST_SHIFT 14 -#define CORE_TX_BD_RESERVED_MASK 0x1 -#define CORE_TX_BD_RESERVED_SHIFT 15 }; @@ -1011,8 +1010,10 @@ struct core_tx_bd { * Light L2 TX Destination */ enum core_tx_dest { - CORE_TX_DEST_NW /* Light L2 TX Destination to the Network */, - CORE_TX_DEST_LB /* Light L2 TX Destination to the Loopback */, + CORE_TX_DEST_NW /* TX Destination to the Network */, + CORE_TX_DEST_LB /* TX Destination to the Loopback */, + CORE_TX_DEST_RESERVED, + CORE_TX_DEST_DROP /* TX Drop */, MAX_CORE_TX_DEST }; @@ -1337,20 +1338,14 @@ struct pf_start_tunnel_config { * FW will use a default port */ u8 set_geneve_udp_port_flg; - u8 tx_enable_vxlan /* If set, enable VXLAN tunnel in TX path. */; -/* If set, enable l2 GENEVE tunnel in TX path. */ - u8 tx_enable_l2geneve; -/* If set, enable IP GENEVE tunnel in TX path. */ - u8 tx_enable_ipgeneve; - u8 tx_enable_l2gre /* If set, enable l2 GRE tunnel in TX path. */; - u8 tx_enable_ipgre /* If set, enable IP GRE tunnel in TX path. */; - u8 tunnel_clss_vxlan /* Classification scheme for VXLAN tunnel. */; -/* Classification scheme for l2 GENEVE tunnel. */ + u8 tunnel_clss_vxlan /* Rx classification scheme for VXLAN tunnel. */; +/* Rx classification scheme for l2 GENEVE tunnel. */ u8 tunnel_clss_l2geneve; -/* Classification scheme for ip GENEVE tunnel. */ +/* Rx classification scheme for ip GENEVE tunnel. */ u8 tunnel_clss_ipgeneve; - u8 tunnel_clss_l2gre /* Classification scheme for l2 GRE tunnel. */; - u8 tunnel_clss_ipgre /* Classification scheme for ip GRE tunnel. */; + u8 tunnel_clss_l2gre /* Rx classification scheme for l2 GRE tunnel. */; + u8 tunnel_clss_ipgre /* Rx classification scheme for ip GRE tunnel. */; + u8 reserved; /* VXLAN tunnel UDP destination port. Valid if set_vxlan_udp_port_flg=1 */ __le16 vxlan_udp_port; /* GENEVE tunnel UDP destination port. Valid if set_geneve_udp_port_flg=1 */ @@ -1366,6 +1361,7 @@ struct pf_start_ramrod_data { struct regpair consolid_q_pbl_addr; /* tunnel configuration. */ struct pf_start_tunnel_config tunnel_config; + __le32 reserved; __le16 event_ring_sb_id /* Status block ID */; /* All VfIds owned by Pf will be from baseVfId till baseVfId+numVfs */ u8 base_vf_id; @@ -1425,19 +1421,10 @@ struct pf_update_tunnel_config { * unicast outer MAC in NPAR mode. */ u8 update_rx_def_non_ucast_clss; -/* Update TX per PF tunnel classification scheme. used by pf update. */ - u8 update_tx_pf_clss; /* Update VXLAN tunnel UDP destination port. */ u8 set_vxlan_udp_port_flg; /* Update GENEVE tunnel UDP destination port. */ u8 set_geneve_udp_port_flg; - u8 tx_enable_vxlan /* If set, enable VXLAN tunnel in TX path. */; -/* If set, enable l2 GENEVE tunnel in TX path. */ - u8 tx_enable_l2geneve; -/* If set, enable IP GENEVE tunnel in TX path. */ - u8 tx_enable_ipgeneve; - u8 tx_enable_l2gre /* If set, enable l2 GRE tunnel in TX path. */; - u8 tx_enable_ipgre /* If set, enable IP GRE tunnel in TX path. */; u8 tunnel_clss_vxlan /* Classification scheme for VXLAN tunnel. */; /* Classification scheme for l2 GENEVE tunnel. */ u8 tunnel_clss_l2geneve; @@ -1447,7 +1434,7 @@ struct pf_update_tunnel_config { u8 tunnel_clss_ipgre /* Classification scheme for ip GRE tunnel. */; __le16 vxlan_udp_port /* VXLAN tunnel UDP destination port. */; __le16 geneve_udp_port /* GENEVE tunnel UDP destination port. */; - __le16 reserved[2]; + __le16 reserved; }; /* diff --git a/drivers/net/qede/base/ecore_hsi_debug_tools.h b/drivers/net/qede/base/ecore_hsi_debug_tools.h index 917e8f4c..7443ff9d 100644 --- a/drivers/net/qede/base/ecore_hsi_debug_tools.h +++ b/drivers/net/qede/base/ecore_hsi_debug_tools.h @@ -67,6 +67,8 @@ enum block_addr { GRCBASE_MULD = 0x4e0000, GRCBASE_YULD = 0x4c8000, GRCBASE_XYLD = 0x4c0000, + GRCBASE_PTLD = 0x590000, + GRCBASE_YPLD = 0x5b0000, GRCBASE_PRM = 0x230000, GRCBASE_PBF_PB1 = 0xda0000, GRCBASE_PBF_PB2 = 0xda4000, @@ -80,6 +82,10 @@ enum block_addr { GRCBASE_TCFC = 0x2d0000, GRCBASE_IGU = 0x180000, GRCBASE_CAU = 0x1c0000, + GRCBASE_RGFS = 0xf00000, + GRCBASE_RGSRC = 0x320000, + GRCBASE_TGFS = 0xd00000, + GRCBASE_TGSRC = 0x322000, GRCBASE_UMAC = 0x51000, GRCBASE_XMAC = 0x210000, GRCBASE_DBG = 0x10000, @@ -93,12 +99,6 @@ enum block_addr { GRCBASE_PHY_PCIE = 0x620000, GRCBASE_LED = 0x6b8000, GRCBASE_AVS_WRAP = 0x6b0000, - GRCBASE_RGFS = 0x1fa0000, - GRCBASE_RGSRC = 0x1fa8000, - GRCBASE_TGFS = 0x1fb0000, - GRCBASE_TGSRC = 0x1fb8000, - GRCBASE_PTLD = 0x1fc0000, - GRCBASE_YPLD = 0x1fe0000, GRCBASE_MISC_AEU = 0x8000, GRCBASE_BAR0_MAP = 0x1c00000, MAX_BLOCK_ADDR @@ -159,6 +159,8 @@ enum block_id { BLOCK_MULD, BLOCK_YULD, BLOCK_XYLD, + BLOCK_PTLD, + BLOCK_YPLD, BLOCK_PRM, BLOCK_PBF_PB1, BLOCK_PBF_PB2, @@ -172,6 +174,10 @@ enum block_id { BLOCK_TCFC, BLOCK_IGU, BLOCK_CAU, + BLOCK_RGFS, + BLOCK_RGSRC, + BLOCK_TGFS, + BLOCK_TGSRC, BLOCK_UMAC, BLOCK_XMAC, BLOCK_DBG, @@ -185,12 +191,6 @@ enum block_id { BLOCK_PHY_PCIE, BLOCK_LED, BLOCK_AVS_WRAP, - BLOCK_RGFS, - BLOCK_RGSRC, - BLOCK_TGFS, - BLOCK_TGSRC, - BLOCK_PTLD, - BLOCK_YPLD, BLOCK_MISC_AEU, BLOCK_BAR0_MAP, MAX_BLOCK_ID diff --git a/drivers/net/qede/base/ecore_hsi_init_func.h b/drivers/net/qede/base/ecore_hsi_init_func.h index fca74791..48b0048f 100644 --- a/drivers/net/qede/base/ecore_hsi_init_func.h +++ b/drivers/net/qede/base/ecore_hsi_init_func.h @@ -15,6 +15,10 @@ /* Number of VLAN priorities */ #define NUM_OF_VLAN_PRIORITIES 8 +/* Size of CRC8 lookup table */ +#ifndef LINUX_REMOVE +#define CRC8_TABLE_SIZE 256 +#endif /* * BRB RAM init requirements diff --git a/drivers/net/qede/base/ecore_init_fw_funcs.c b/drivers/net/qede/base/ecore_init_fw_funcs.c index 004ab351..b5ef173e 100644 --- a/drivers/net/qede/base/ecore_init_fw_funcs.c +++ b/drivers/net/qede/base/ecore_init_fw_funcs.c @@ -1590,7 +1590,8 @@ void ecore_set_rfs_mode_enable(struct ecore_hwfn *p_hwfn, /* Filters are per PF!! */ SET_FIELD(camLine.cam_line_mapped.camline, - GFT_CAM_LINE_MAPPED_PF_ID_MASK, 1); + GFT_CAM_LINE_MAPPED_PF_ID_MASK, + GFT_CAM_LINE_MAPPED_PF_ID_MASK_MASK); SET_FIELD(camLine.cam_line_mapped.camline, GFT_CAM_LINE_MAPPED_PF_ID, pf_id); @@ -1644,8 +1645,9 @@ void ecore_set_rfs_mode_enable(struct ecore_hwfn *p_hwfn, i * REG_SIZE, *(ramLinePointer + i)); /* Set default profile so that no filter match will happen */ - ramLine.lo = 0xffff; - ramLine.hi = 0xffff; + ramLine.lo = 0xffffffff; + ramLine.hi = 0x3ff; + for (i = 0; i < RAM_LINE_SIZE / REG_SIZE; i++) ecore_wr(p_hwfn, p_ptt, PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * PRS_GFT_CAM_LINES_NO_MATCH + @@ -1722,40 +1724,30 @@ u32 ecore_get_mstorm_eth_vf_prods_offset(struct ecore_hwfn *p_hwfn, return offset; } -/* Calculate CRC8 of first 4 bytes in buf */ -static u8 ecore_calc_crc8(const u8 *buf) -{ - u32 i, j, crc = 0xff << 8; - - /* CRC-8 polynomial */ - #define POLY 0x1070 - - for (j = 0; j < 4; j++, buf++) { - crc ^= (*buf << 8); - for (i = 0; i < 8; i++) { - if (crc & 0x8000) - crc ^= (POLY << 3); - - crc <<= 1; - } - } - - return (u8)(crc >> 8); -} +#ifndef LINUX_REMOVE +#define CRC8_INIT_VALUE 0xFF +#define CRC8_TABLE_SIZE 256 +#endif +static u8 cdu_crc8_table[CRC8_TABLE_SIZE]; -/* Calculate and return CDU validation byte per conneciton type / region / +/* Calculate and return CDU validation byte per connection type / region / * cid */ -static u8 ecore_calc_cdu_validation_byte(u8 conn_type, u8 region, - u32 cid) +static u8 ecore_calc_cdu_validation_byte(struct ecore_hwfn *p_hwfn, + u8 conn_type, + u8 region, u32 cid) { const u8 validation_cfg = CDU_VALIDATION_DEFAULT_CFG; + + static u8 crc8_table_valid; /*automatically initialized to 0*/ u8 crc, validation_byte = 0; u32 validation_string = 0; - const u8 *data_to_crc_rev; - u8 data_to_crc[4]; + u32 data_to_crc; - data_to_crc_rev = (const u8 *)&validation_string; + if (crc8_table_valid == 0) { + OSAL_CRC8_POPULATE(cdu_crc8_table, 0x07); + crc8_table_valid = 1; + } /* * The CRC is calculated on the String-to-compress: @@ -1772,13 +1764,22 @@ static u8 ecore_calc_cdu_validation_byte(u8 conn_type, u8 region, if ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_TYPE) & 1) validation_string |= (conn_type & 0xF); - /* Convert to big-endian (ntoh())*/ - data_to_crc[0] = data_to_crc_rev[3]; - data_to_crc[1] = data_to_crc_rev[2]; - data_to_crc[2] = data_to_crc_rev[1]; - data_to_crc[3] = data_to_crc_rev[0]; - - crc = ecore_calc_crc8(data_to_crc); + /* Convert to big-endian and calculate CRC8*/ + data_to_crc = OSAL_BE32_TO_CPU(validation_string); + + crc = OSAL_CRC8(cdu_crc8_table, (u8 *)&data_to_crc, sizeof(data_to_crc), + CRC8_INIT_VALUE); + + /* The validation byte [7:0] is composed: + * for type A validation + * [7] = active configuration bit + * [6:0] = crc[6:0] + * + * for type B validation + * [7] = active configuration bit + * [6:3] = connection_type[3:0] + * [2:0] = crc[2:0] + */ validation_byte |= ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_ACTIVE) & 1) << 7; @@ -1793,8 +1794,9 @@ static u8 ecore_calc_cdu_validation_byte(u8 conn_type, u8 region, } /* Calcualte and set validation bytes for session context */ -void ecore_calc_session_ctx_validation(void *p_ctx_mem, u16 ctx_size, - u8 ctx_type, u32 cid) +void ecore_calc_session_ctx_validation(struct ecore_hwfn *p_hwfn, + void *p_ctx_mem, + u16 ctx_size, u8 ctx_type, u32 cid) { u8 *x_val_ptr, *t_val_ptr, *u_val_ptr, *p_ctx; @@ -1805,14 +1807,14 @@ void ecore_calc_session_ctx_validation(void *p_ctx_mem, u16 ctx_size, OSAL_MEMSET(p_ctx, 0, ctx_size); - *x_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 3, cid); - *t_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 4, cid); - *u_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 5, cid); + *x_val_ptr = ecore_calc_cdu_validation_byte(p_hwfn, ctx_type, 3, cid); + *t_val_ptr = ecore_calc_cdu_validation_byte(p_hwfn, ctx_type, 4, cid); + *u_val_ptr = ecore_calc_cdu_validation_byte(p_hwfn, ctx_type, 5, cid); } /* Calcualte and set validation bytes for task context */ -void ecore_calc_task_ctx_validation(void *p_ctx_mem, u16 ctx_size, - u8 ctx_type, u32 tid) +void ecore_calc_task_ctx_validation(struct ecore_hwfn *p_hwfn, void *p_ctx_mem, + u16 ctx_size, u8 ctx_type, u32 tid) { u8 *p_ctx, *region1_val_ptr; @@ -1821,7 +1823,8 @@ void ecore_calc_task_ctx_validation(void *p_ctx_mem, u16 ctx_size, OSAL_MEMSET(p_ctx, 0, ctx_size); - *region1_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 1, tid); + *region1_val_ptr = ecore_calc_cdu_validation_byte(p_hwfn, ctx_type, + 1, tid); } /* Memset session context to 0 while preserving validation bytes */ @@ -1847,8 +1850,7 @@ void ecore_memset_session_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type) } /* Memset task context to 0 while preserving validation bytes */ -void ecore_memset_task_ctx(void *p_ctx_mem, const u32 ctx_size, - const u8 ctx_type) +void ecore_memset_task_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type) { u8 *p_ctx, *region1_val_ptr; u8 region1_val; diff --git a/drivers/net/qede/base/ecore_init_fw_funcs.h b/drivers/net/qede/base/ecore_init_fw_funcs.h index 4da3fc29..488dc005 100644 --- a/drivers/net/qede/base/ecore_init_fw_funcs.h +++ b/drivers/net/qede/base/ecore_init_fw_funcs.h @@ -424,48 +424,54 @@ void ecore_enable_context_validation(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); /** * @brief ecore_calc_session_ctx_validation - Calcualte validation byte for - * session context. + * session context. * - * - * @param p_ctx_mem - pointer to context memory. - * @param ctx_size - context size. - * @param ctx_type - context type. - * @param cid - context cid. + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - context size. + * @param ctx_type - context type. + * @param cid - context cid. */ -void ecore_calc_session_ctx_validation(void *p_ctx_mem, u16 ctx_size, - u8 ctx_type, u32 cid); +void ecore_calc_session_ctx_validation(struct ecore_hwfn *p_hwfn, + void *p_ctx_mem, + u16 ctx_size, + u8 ctx_type, + u32 cid); /** * @brief ecore_calc_task_ctx_validation - Calcualte validation byte for task - * context. - * + * context. * - * @param p_ctx_mem - pointer to context memory. - * @param ctx_size - context size. - * @param ctx_type - context type. - * @param tid - context tid. + * @param p_hwfn - HW device data + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - context size. + * @param ctx_type - context type. + * @param tid - context tid. */ -void ecore_calc_task_ctx_validation(void *p_ctx_mem, u16 ctx_size, - u8 ctx_type, u32 tid); +void ecore_calc_task_ctx_validation(struct ecore_hwfn *p_hwfn, + void *p_ctx_mem, + u16 ctx_size, + u8 ctx_type, + u32 tid); /** * @brief ecore_memset_session_ctx - Memset session context to 0 while - * preserving validation bytes. + * preserving validation bytes. * - * - * @param p_ctx_mem - pointer to context memory. - * @param ctx_size - size to initialzie. - * @param ctx_type - context type. + * @param p_hwfn - HW device data + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - size to initialzie. + * @param ctx_type - context type. */ -void ecore_memset_session_ctx(void *p_ctx_mem, u32 ctx_size, +void ecore_memset_session_ctx(void *p_ctx_mem, + u32 ctx_size, u8 ctx_type); /** - * @brief ecore_memset_task_ctx - Memset session context to 0 while preserving - * validation bytes. - * + * @brief ecore_memset_task_ctx - Memset task context to 0 while preserving + * validation bytes. * - * @param p_ctx_mem - pointer to context memory. - * @param ctx_size - size to initialzie. - * @param ctx_type - context type. + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - size to initialzie. + * @param ctx_type - context type. */ -void ecore_memset_task_ctx(void *p_ctx_mem, u32 ctx_size, +void ecore_memset_task_ctx(void *p_ctx_mem, + u32 ctx_size, u8 ctx_type); #endif diff --git a/drivers/net/qede/base/ecore_int.c b/drivers/net/qede/base/ecore_int.c index 8dc4d150..b57c510c 100644 --- a/drivers/net/qede/base/ecore_int.c +++ b/drivers/net/qede/base/ecore_int.c @@ -284,122 +284,119 @@ out: #define ECORE_PGLUE_ATTENTION_ICPL_VALID (1 << 23) #define ECORE_PGLUE_ATTENTION_ZLR_VALID (1 << 25) #define ECORE_PGLUE_ATTENTION_ILT_VALID (1 << 23) -static enum _ecore_status_t ecore_pglub_rbc_attn_cb(struct ecore_hwfn *p_hwfn) + +enum _ecore_status_t ecore_pglueb_rbc_attn_handler(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { u32 tmp; - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_TX_ERR_WR_DETAILS2); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_DETAILS2); if (tmp & ECORE_PGLUE_ATTENTION_VALID) { u32 addr_lo, addr_hi, details; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_ADD_63_32); - details = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + details = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_DETAILS); - DP_INFO(p_hwfn, - "Illegal write by chip to [%08x:%08x] blocked." - "Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x]" - " Details2 %08x [Was_error %02x BME deassert %02x" - " FID_enable deassert %02x]\n", - addr_hi, addr_lo, details, - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), - (u8)((details & ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) - ? 1 : 0), tmp, - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? 1 - : 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? 1 : - 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? 1 - : 0)); + DP_NOTICE(p_hwfn, false, + "Illegal write by chip to [%08x:%08x] blocked. Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x] Details2 %08x [Was_error %02x BME deassert %02x FID_enable deassert %02x]\n", + addr_hi, addr_lo, details, + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) ? 1 : 0), + tmp, + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? + 1 : 0)); } - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_TX_ERR_RD_DETAILS2); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_DETAILS2); if (tmp & ECORE_PGLUE_ATTENTION_RD_VALID) { u32 addr_lo, addr_hi, details; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_ADD_63_32); - details = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + details = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_DETAILS); - DP_INFO(p_hwfn, - "Illegal read by chip from [%08x:%08x] blocked." - " Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x]" - " Details2 %08x [Was_error %02x BME deassert %02x" - " FID_enable deassert %02x]\n", - addr_hi, addr_lo, details, - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), - (u8)((details & ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) - ? 1 : 0), tmp, - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? 1 - : 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? 1 : - 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? 1 - : 0)); + DP_NOTICE(p_hwfn, false, + "Illegal read by chip from [%08x:%08x] blocked. Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x] Details2 %08x [Was_error %02x BME deassert %02x FID_enable deassert %02x]\n", + addr_hi, addr_lo, details, + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) ? 1 : 0), + tmp, + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? + 1 : 0)); } - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_TX_ERR_WR_DETAILS_ICPL); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_DETAILS_ICPL); if (tmp & ECORE_PGLUE_ATTENTION_ICPL_VALID) - DP_INFO(p_hwfn, "ICPL error - %08x\n", tmp); + DP_NOTICE(p_hwfn, false, "ICPL erorr - %08x\n", tmp); - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_MASTER_ZLR_ERR_DETAILS); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_MASTER_ZLR_ERR_DETAILS); if (tmp & ECORE_PGLUE_ATTENTION_ZLR_VALID) { u32 addr_hi, addr_lo; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_MASTER_ZLR_ERR_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_MASTER_ZLR_ERR_ADD_63_32); - DP_INFO(p_hwfn, "ICPL error - %08x [Address %08x:%08x]\n", - tmp, addr_hi, addr_lo); + DP_NOTICE(p_hwfn, false, + "ICPL erorr - %08x [Address %08x:%08x]\n", + tmp, addr_hi, addr_lo); } - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_VF_ILT_ERR_DETAILS2); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_DETAILS2); if (tmp & ECORE_PGLUE_ATTENTION_ILT_VALID) { u32 addr_hi, addr_lo, details; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_ADD_63_32); - details = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + details = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_DETAILS); - DP_INFO(p_hwfn, - "ILT error - Details %08x Details2 %08x" - " [Address %08x:%08x]\n", - details, tmp, addr_hi, addr_lo); + DP_NOTICE(p_hwfn, false, + "ILT error - Details %08x Details2 %08x [Address %08x:%08x]\n", + details, tmp, addr_hi, addr_lo); } /* Clear the indications */ - ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_LATCHED_ERRORS_CLR, (1 << 2)); + ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_LATCHED_ERRORS_CLR, (1 << 2)); return ECORE_SUCCESS; } +static enum _ecore_status_t ecore_pglueb_rbc_attn_cb(struct ecore_hwfn *p_hwfn) +{ + return ecore_pglueb_rbc_attn_handler(p_hwfn, p_hwfn->p_dpc_ptt); +} + static enum _ecore_status_t ecore_fw_assertion(struct ecore_hwfn *p_hwfn) { DP_NOTICE(p_hwfn, false, "FW assertion!\n"); @@ -505,7 +502,7 @@ static struct aeu_invert_reg aeu_descs[NUM_ATTN_REGS] = { { /* After Invert 2 */ {"PGLUE config_space", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, {"PGLUE misc_flr", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, - {"PGLUE B RBC", ATTENTION_PAR_INT, ecore_pglub_rbc_attn_cb, + {"PGLUE B RBC", ATTENTION_PAR_INT, ecore_pglueb_rbc_attn_cb, BLOCK_PGLUE_B}, {"PGLUE misc_mctp", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, {"Flash event", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, @@ -827,8 +824,9 @@ ecore_int_deassertion_aeu_bit(struct ecore_hwfn *p_hwfn, ATTN_TYPE_INTERRUPT, !b_fatal); } + /* @DPDK */ /* Reach assertion if attention is fatal */ - if (b_fatal) { + if (b_fatal || (strcmp(p_bit_name, "PGLUE B RBC") == 0)) { DP_NOTICE(p_hwfn, true, "`%s': Fatal attention\n", p_bit_name); @@ -842,7 +840,7 @@ ecore_int_deassertion_aeu_bit(struct ecore_hwfn *p_hwfn, u32 mask = ~bitmask; val = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en_reg); ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en_reg, (val & mask)); - DP_INFO(p_hwfn, "`%s' - Disabled future attentions\n", + DP_ERR(p_hwfn, "`%s' - Disabled future attentions\n", p_bit_name); } diff --git a/drivers/net/qede/base/ecore_int.h b/drivers/net/qede/base/ecore_int.h index 0c8929e3..067ed605 100644 --- a/drivers/net/qede/base/ecore_int.h +++ b/drivers/net/qede/base/ecore_int.h @@ -208,4 +208,7 @@ enum _ecore_status_t ecore_int_set_timer_res(struct ecore_hwfn *p_hwfn, #define ECORE_MAPPING_MEMORY_SIZE(dev) NUM_OF_SBS(dev) #endif +enum _ecore_status_t ecore_pglueb_rbc_attn_handler(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + #endif /* __ECORE_INT_H__ */ diff --git a/drivers/net/qede/base/ecore_iro_values.h b/drivers/net/qede/base/ecore_iro_values.h index 6764bfa6..bc8df8f8 100644 --- a/drivers/net/qede/base/ecore_iro_values.h +++ b/drivers/net/qede/base/ecore_iro_values.h @@ -27,7 +27,7 @@ static const struct iro iro_arr[49] = { /* USTORM_COMMON_QUEUE_CONS_OFFSET(queue_zone_id) */ { 0x84, 0x8, 0x0, 0x0, 0x2}, /* XSTORM_INTEG_TEST_DATA_OFFSET */ - { 0x4bc0, 0x0, 0x0, 0x0, 0x78}, + { 0x4c40, 0x0, 0x0, 0x0, 0x78}, /* YSTORM_INTEG_TEST_DATA_OFFSET */ { 0x3df0, 0x0, 0x0, 0x0, 0x78}, /* PSTORM_INTEG_TEST_DATA_OFFSET */ @@ -37,13 +37,13 @@ static const struct iro iro_arr[49] = { /* MSTORM_INTEG_TEST_DATA_OFFSET */ { 0x4990, 0x0, 0x0, 0x0, 0x78}, /* USTORM_INTEG_TEST_DATA_OFFSET */ - { 0x7e48, 0x0, 0x0, 0x0, 0x78}, + { 0x7f48, 0x0, 0x0, 0x0, 0x78}, /* TSTORM_LL2_RX_PRODS_OFFSET(core_rx_queue_id) */ { 0xa28, 0x8, 0x0, 0x0, 0x8}, /* CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) */ { 0x61f8, 0x10, 0x0, 0x0, 0x10}, /* CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) */ - { 0xb820, 0x30, 0x0, 0x0, 0x30}, + { 0xbd20, 0x30, 0x0, 0x0, 0x30}, /* CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(core_tx_stats_id) */ { 0x95b8, 0x30, 0x0, 0x0, 0x30}, /* MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) */ @@ -57,9 +57,9 @@ static const struct iro iro_arr[49] = { /* MSTORM_ETH_PF_STAT_OFFSET(pf_id) */ { 0x4ba0, 0x80, 0x0, 0x0, 0x20}, /* USTORM_QUEUE_STAT_OFFSET(stat_counter_id) */ - { 0x8050, 0x40, 0x0, 0x0, 0x30}, + { 0x8150, 0x40, 0x0, 0x0, 0x30}, /* USTORM_ETH_PF_STAT_OFFSET(pf_id) */ - { 0xe770, 0x60, 0x0, 0x0, 0x60}, + { 0xec70, 0x60, 0x0, 0x0, 0x60}, /* PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) */ { 0x2b48, 0x80, 0x0, 0x0, 0x38}, /* PSTORM_ETH_PF_STAT_OFFSET(pf_id) */ @@ -89,7 +89,7 @@ static const struct iro iro_arr[49] = { /* MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) */ { 0x12988, 0x10, 0x0, 0x0, 0x8}, /* USTORM_ISCSI_RX_STATS_OFFSET(pf_id) */ - { 0x11aa0, 0x38, 0x0, 0x0, 0x18}, + { 0x11fa0, 0x38, 0x0, 0x0, 0x18}, /* XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */ { 0xa8c0, 0x38, 0x0, 0x0, 0x10}, /* YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */ diff --git a/drivers/net/qede/base/ecore_l2.c b/drivers/net/qede/base/ecore_l2.c index 4ab8fd5f..e58b8fa0 100644 --- a/drivers/net/qede/base/ecore_l2.c +++ b/drivers/net/qede/base/ecore_l2.c @@ -1714,13 +1714,20 @@ static void __ecore_get_vport_pstats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&pstats, 0, sizeof(pstats)); ecore_memcpy_from(p_hwfn, p_ptt, &pstats, pstats_addr, pstats_len); - p_stats->tx_ucast_bytes += HILO_64_REGPAIR(pstats.sent_ucast_bytes); - p_stats->tx_mcast_bytes += HILO_64_REGPAIR(pstats.sent_mcast_bytes); - p_stats->tx_bcast_bytes += HILO_64_REGPAIR(pstats.sent_bcast_bytes); - p_stats->tx_ucast_pkts += HILO_64_REGPAIR(pstats.sent_ucast_pkts); - p_stats->tx_mcast_pkts += HILO_64_REGPAIR(pstats.sent_mcast_pkts); - p_stats->tx_bcast_pkts += HILO_64_REGPAIR(pstats.sent_bcast_pkts); - p_stats->tx_err_drop_pkts += HILO_64_REGPAIR(pstats.error_drop_pkts); + p_stats->common.tx_ucast_bytes += + HILO_64_REGPAIR(pstats.sent_ucast_bytes); + p_stats->common.tx_mcast_bytes += + HILO_64_REGPAIR(pstats.sent_mcast_bytes); + p_stats->common.tx_bcast_bytes += + HILO_64_REGPAIR(pstats.sent_bcast_bytes); + p_stats->common.tx_ucast_pkts += + HILO_64_REGPAIR(pstats.sent_ucast_pkts); + p_stats->common.tx_mcast_pkts += + HILO_64_REGPAIR(pstats.sent_mcast_pkts); + p_stats->common.tx_bcast_pkts += + HILO_64_REGPAIR(pstats.sent_bcast_pkts); + p_stats->common.tx_err_drop_pkts += + HILO_64_REGPAIR(pstats.error_drop_pkts); } static void __ecore_get_vport_tstats(struct ecore_hwfn *p_hwfn, @@ -1746,10 +1753,10 @@ static void __ecore_get_vport_tstats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&tstats, 0, sizeof(tstats)); ecore_memcpy_from(p_hwfn, p_ptt, &tstats, tstats_addr, tstats_len); - p_stats->mftag_filter_discards += - HILO_64_REGPAIR(tstats.mftag_filter_discard); - p_stats->mac_filter_discards += - HILO_64_REGPAIR(tstats.eth_mac_filter_discard); + p_stats->common.mftag_filter_discards += + HILO_64_REGPAIR(tstats.mftag_filter_discard); + p_stats->common.mac_filter_discards += + HILO_64_REGPAIR(tstats.eth_mac_filter_discard); } static void __ecore_get_vport_ustats_addrlen(struct ecore_hwfn *p_hwfn, @@ -1783,12 +1790,18 @@ static void __ecore_get_vport_ustats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&ustats, 0, sizeof(ustats)); ecore_memcpy_from(p_hwfn, p_ptt, &ustats, ustats_addr, ustats_len); - p_stats->rx_ucast_bytes += HILO_64_REGPAIR(ustats.rcv_ucast_bytes); - p_stats->rx_mcast_bytes += HILO_64_REGPAIR(ustats.rcv_mcast_bytes); - p_stats->rx_bcast_bytes += HILO_64_REGPAIR(ustats.rcv_bcast_bytes); - p_stats->rx_ucast_pkts += HILO_64_REGPAIR(ustats.rcv_ucast_pkts); - p_stats->rx_mcast_pkts += HILO_64_REGPAIR(ustats.rcv_mcast_pkts); - p_stats->rx_bcast_pkts += HILO_64_REGPAIR(ustats.rcv_bcast_pkts); + p_stats->common.rx_ucast_bytes += + HILO_64_REGPAIR(ustats.rcv_ucast_bytes); + p_stats->common.rx_mcast_bytes += + HILO_64_REGPAIR(ustats.rcv_mcast_bytes); + p_stats->common.rx_bcast_bytes += + HILO_64_REGPAIR(ustats.rcv_bcast_bytes); + p_stats->common.rx_ucast_pkts += + HILO_64_REGPAIR(ustats.rcv_ucast_pkts); + p_stats->common.rx_mcast_pkts += + HILO_64_REGPAIR(ustats.rcv_mcast_pkts); + p_stats->common.rx_bcast_pkts += + HILO_64_REGPAIR(ustats.rcv_bcast_pkts); } static void __ecore_get_vport_mstats_addrlen(struct ecore_hwfn *p_hwfn, @@ -1822,23 +1835,27 @@ static void __ecore_get_vport_mstats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&mstats, 0, sizeof(mstats)); ecore_memcpy_from(p_hwfn, p_ptt, &mstats, mstats_addr, mstats_len); - p_stats->no_buff_discards += HILO_64_REGPAIR(mstats.no_buff_discard); - p_stats->packet_too_big_discard += - HILO_64_REGPAIR(mstats.packet_too_big_discard); - p_stats->ttl0_discard += HILO_64_REGPAIR(mstats.ttl0_discard); - p_stats->tpa_coalesced_pkts += - HILO_64_REGPAIR(mstats.tpa_coalesced_pkts); - p_stats->tpa_coalesced_events += - HILO_64_REGPAIR(mstats.tpa_coalesced_events); - p_stats->tpa_aborts_num += HILO_64_REGPAIR(mstats.tpa_aborts_num); - p_stats->tpa_coalesced_bytes += - HILO_64_REGPAIR(mstats.tpa_coalesced_bytes); + p_stats->common.no_buff_discards += + HILO_64_REGPAIR(mstats.no_buff_discard); + p_stats->common.packet_too_big_discard += + HILO_64_REGPAIR(mstats.packet_too_big_discard); + p_stats->common.ttl0_discard += + HILO_64_REGPAIR(mstats.ttl0_discard); + p_stats->common.tpa_coalesced_pkts += + HILO_64_REGPAIR(mstats.tpa_coalesced_pkts); + p_stats->common.tpa_coalesced_events += + HILO_64_REGPAIR(mstats.tpa_coalesced_events); + p_stats->common.tpa_aborts_num += + HILO_64_REGPAIR(mstats.tpa_aborts_num); + p_stats->common.tpa_coalesced_bytes += + HILO_64_REGPAIR(mstats.tpa_coalesced_bytes); } static void __ecore_get_vport_port_stats(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_eth_stats *p_stats) { + struct ecore_eth_stats_common *p_common = &p_stats->common; struct port_stats port_stats; int j; @@ -1849,54 +1866,75 @@ static void __ecore_get_vport_port_stats(struct ecore_hwfn *p_hwfn, OFFSETOF(struct public_port, stats), sizeof(port_stats)); - p_stats->rx_64_byte_packets += port_stats.eth.r64; - p_stats->rx_65_to_127_byte_packets += port_stats.eth.r127; - p_stats->rx_128_to_255_byte_packets += port_stats.eth.r255; - p_stats->rx_256_to_511_byte_packets += port_stats.eth.r511; - p_stats->rx_512_to_1023_byte_packets += port_stats.eth.r1023; - p_stats->rx_1024_to_1518_byte_packets += port_stats.eth.r1518; - p_stats->rx_1519_to_1522_byte_packets += port_stats.eth.r1522; - p_stats->rx_1519_to_2047_byte_packets += port_stats.eth.r2047; - p_stats->rx_2048_to_4095_byte_packets += port_stats.eth.r4095; - p_stats->rx_4096_to_9216_byte_packets += port_stats.eth.r9216; - p_stats->rx_9217_to_16383_byte_packets += port_stats.eth.r16383; - p_stats->rx_crc_errors += port_stats.eth.rfcs; - p_stats->rx_mac_crtl_frames += port_stats.eth.rxcf; - p_stats->rx_pause_frames += port_stats.eth.rxpf; - p_stats->rx_pfc_frames += port_stats.eth.rxpp; - p_stats->rx_align_errors += port_stats.eth.raln; - p_stats->rx_carrier_errors += port_stats.eth.rfcr; - p_stats->rx_oversize_packets += port_stats.eth.rovr; - p_stats->rx_jabbers += port_stats.eth.rjbr; - p_stats->rx_undersize_packets += port_stats.eth.rund; - p_stats->rx_fragments += port_stats.eth.rfrg; - p_stats->tx_64_byte_packets += port_stats.eth.t64; - p_stats->tx_65_to_127_byte_packets += port_stats.eth.t127; - p_stats->tx_128_to_255_byte_packets += port_stats.eth.t255; - p_stats->tx_256_to_511_byte_packets += port_stats.eth.t511; - p_stats->tx_512_to_1023_byte_packets += port_stats.eth.t1023; - p_stats->tx_1024_to_1518_byte_packets += port_stats.eth.t1518; - p_stats->tx_1519_to_2047_byte_packets += port_stats.eth.t2047; - p_stats->tx_2048_to_4095_byte_packets += port_stats.eth.t4095; - p_stats->tx_4096_to_9216_byte_packets += port_stats.eth.t9216; - p_stats->tx_9217_to_16383_byte_packets += port_stats.eth.t16383; - p_stats->tx_pause_frames += port_stats.eth.txpf; - p_stats->tx_pfc_frames += port_stats.eth.txpp; - p_stats->tx_lpi_entry_count += port_stats.eth.tlpiec; - p_stats->tx_total_collisions += port_stats.eth.tncl; - p_stats->rx_mac_bytes += port_stats.eth.rbyte; - p_stats->rx_mac_uc_packets += port_stats.eth.rxuca; - p_stats->rx_mac_mc_packets += port_stats.eth.rxmca; - p_stats->rx_mac_bc_packets += port_stats.eth.rxbca; - p_stats->rx_mac_frames_ok += port_stats.eth.rxpok; - p_stats->tx_mac_bytes += port_stats.eth.tbyte; - p_stats->tx_mac_uc_packets += port_stats.eth.txuca; - p_stats->tx_mac_mc_packets += port_stats.eth.txmca; - p_stats->tx_mac_bc_packets += port_stats.eth.txbca; - p_stats->tx_mac_ctrl_frames += port_stats.eth.txcf; + p_common->rx_64_byte_packets += port_stats.eth.r64; + p_common->rx_65_to_127_byte_packets += port_stats.eth.r127; + p_common->rx_128_to_255_byte_packets += port_stats.eth.r255; + p_common->rx_256_to_511_byte_packets += port_stats.eth.r511; + p_common->rx_512_to_1023_byte_packets += port_stats.eth.r1023; + p_common->rx_1024_to_1518_byte_packets += port_stats.eth.r1518; + p_common->rx_crc_errors += port_stats.eth.rfcs; + p_common->rx_mac_crtl_frames += port_stats.eth.rxcf; + p_common->rx_pause_frames += port_stats.eth.rxpf; + p_common->rx_pfc_frames += port_stats.eth.rxpp; + p_common->rx_align_errors += port_stats.eth.raln; + p_common->rx_carrier_errors += port_stats.eth.rfcr; + p_common->rx_oversize_packets += port_stats.eth.rovr; + p_common->rx_jabbers += port_stats.eth.rjbr; + p_common->rx_undersize_packets += port_stats.eth.rund; + p_common->rx_fragments += port_stats.eth.rfrg; + p_common->tx_64_byte_packets += port_stats.eth.t64; + p_common->tx_65_to_127_byte_packets += port_stats.eth.t127; + p_common->tx_128_to_255_byte_packets += port_stats.eth.t255; + p_common->tx_256_to_511_byte_packets += port_stats.eth.t511; + p_common->tx_512_to_1023_byte_packets += port_stats.eth.t1023; + p_common->tx_1024_to_1518_byte_packets += port_stats.eth.t1518; + p_common->tx_pause_frames += port_stats.eth.txpf; + p_common->tx_pfc_frames += port_stats.eth.txpp; + p_common->rx_mac_bytes += port_stats.eth.rbyte; + p_common->rx_mac_uc_packets += port_stats.eth.rxuca; + p_common->rx_mac_mc_packets += port_stats.eth.rxmca; + p_common->rx_mac_bc_packets += port_stats.eth.rxbca; + p_common->rx_mac_frames_ok += port_stats.eth.rxpok; + p_common->tx_mac_bytes += port_stats.eth.tbyte; + p_common->tx_mac_uc_packets += port_stats.eth.txuca; + p_common->tx_mac_mc_packets += port_stats.eth.txmca; + p_common->tx_mac_bc_packets += port_stats.eth.txbca; + p_common->tx_mac_ctrl_frames += port_stats.eth.txcf; for (j = 0; j < 8; j++) { - p_stats->brb_truncates += port_stats.brb.brb_truncate[j]; - p_stats->brb_discards += port_stats.brb.brb_discard[j]; + p_common->brb_truncates += port_stats.brb.brb_truncate[j]; + p_common->brb_discards += port_stats.brb.brb_discard[j]; + } + + if (ECORE_IS_BB(p_hwfn->p_dev)) { + struct ecore_eth_stats_bb *p_bb = &p_stats->bb; + + p_bb->rx_1519_to_1522_byte_packets += + port_stats.eth.u0.bb0.r1522; + p_bb->rx_1519_to_2047_byte_packets += + port_stats.eth.u0.bb0.r2047; + p_bb->rx_2048_to_4095_byte_packets += + port_stats.eth.u0.bb0.r4095; + p_bb->rx_4096_to_9216_byte_packets += + port_stats.eth.u0.bb0.r9216; + p_bb->rx_9217_to_16383_byte_packets += + port_stats.eth.u0.bb0.r16383; + p_bb->tx_1519_to_2047_byte_packets += + port_stats.eth.u1.bb1.t2047; + p_bb->tx_2048_to_4095_byte_packets += + port_stats.eth.u1.bb1.t4095; + p_bb->tx_4096_to_9216_byte_packets += + port_stats.eth.u1.bb1.t9216; + p_bb->tx_9217_to_16383_byte_packets += + port_stats.eth.u1.bb1.t16383; + p_bb->tx_lpi_entry_count += port_stats.eth.u2.bb2.tlpiec; + p_bb->tx_total_collisions += port_stats.eth.u2.bb2.tncl; + } else { + struct ecore_eth_stats_ah *p_ah = &p_stats->ah; + + p_ah->rx_1519_to_max_byte_packets += + port_stats.eth.u0.ah0.r1519_to_max; + p_ah->tx_1519_to_max_byte_packets = + port_stats.eth.u1.ah1.t1519_to_max; } } diff --git a/drivers/net/qede/base/ecore_mcp.c b/drivers/net/qede/base/ecore_mcp.c index a834ac74..88c5ceb0 100644 --- a/drivers/net/qede/base/ecore_mcp.c +++ b/drivers/net/qede/base/ecore_mcp.c @@ -893,6 +893,30 @@ enum _ecore_status_t ecore_mcp_load_req(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } +enum _ecore_status_t ecore_mcp_load_done(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 resp = 0, param = 0; + enum _ecore_status_t rc; + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_LOAD_DONE, 0, &resp, + ¶m); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to send a LOAD_DONE command, rc = %d\n", rc); + return rc; + } + +#define FW_MB_PARAM_LOAD_DONE_DID_EFUSE_ERROR (1 << 0) + + /* Check if there is a DID mismatch between nvm-cfg/efuse */ + if (param & FW_MB_PARAM_LOAD_DONE_DID_EFUSE_ERROR) + DP_NOTICE(p_hwfn, false, + "warning: device configuration is not supported on this board type. The device may not function as expected.\n"); + + return ECORE_SUCCESS; +} + enum _ecore_status_t ecore_mcp_unload_req(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { @@ -1556,10 +1580,9 @@ static void ecore_mcp_handle_critical_error(struct ecore_hwfn *p_hwfn, DP_NOTICE(p_hwfn, false, "Received a critical error notification from the MFW!\n"); - if (p_hwfn->p_dev->mdump_en) { + if (p_hwfn->p_dev->allow_mdump) { DP_NOTICE(p_hwfn, false, "Not acknowledging the notification to allow the MFW crash dump\n"); - p_hwfn->p_dev->mdump_en = false; return; } @@ -2894,6 +2917,27 @@ struct ecore_resc_alloc_out_params { u32 flags; }; +#define ECORE_RECOVERY_PROLOG_SLEEP_MS 100 + +enum _ecore_status_t ecore_recovery_prolog(struct ecore_dev *p_dev) +{ + struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); + struct ecore_ptt *p_ptt = p_hwfn->p_main_ptt; + enum _ecore_status_t rc; + + /* Allow ongoing PCIe transactions to complete */ + OSAL_MSLEEP(ECORE_RECOVERY_PROLOG_SLEEP_MS); + + /* Clear the PF's internal FID_enable in the PXP */ + rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_ptt, false); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, + "ecore_pglueb_set_pfid_enable() failed. rc = %d.\n", + rc); + + return rc; +} + static enum _ecore_status_t ecore_mcp_resc_allocation_msg(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, diff --git a/drivers/net/qede/base/ecore_mcp.h b/drivers/net/qede/base/ecore_mcp.h index 37d1835f..77fb5a3c 100644 --- a/drivers/net/qede/base/ecore_mcp.h +++ b/drivers/net/qede/base/ecore_mcp.h @@ -171,6 +171,17 @@ enum _ecore_status_t ecore_mcp_load_req(struct ecore_hwfn *p_hwfn, struct ecore_load_req_params *p_params); /** + * @brief Sends a LOAD_DONE message to the MFW + * + * @param p_hwfn + * @param p_ptt + * + * @return enum _ecore_status_t - ECORE_SUCCESS - Operation was successful. + */ +enum _ecore_status_t ecore_mcp_load_done(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** * @brief Sends a UNLOAD_REQ message to the MFW * * @param p_hwfn diff --git a/drivers/net/qede/base/ecore_mcp_api.h b/drivers/net/qede/base/ecore_mcp_api.h index 190c1352..abc190c9 100644 --- a/drivers/net/qede/base/ecore_mcp_api.h +++ b/drivers/net/qede/base/ecore_mcp_api.h @@ -736,6 +736,17 @@ enum _ecore_status_t ecore_start_recovery_process(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); /** + * @brief A recovery handler must call this function as its first step. + * It is assumed that the handler is not run from an interrupt context. + * + * @param p_dev + * @param p_ptt + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t ecore_recovery_prolog(struct ecore_dev *p_dev); + +/** * @brief Notify MFW about the change in base device properties * * @param p_hwfn diff --git a/drivers/net/qede/base/ecore_rt_defs.h b/drivers/net/qede/base/ecore_rt_defs.h index 846dc6d1..c9c23096 100644 --- a/drivers/net/qede/base/ecore_rt_defs.h +++ b/drivers/net/qede/base/ecore_rt_defs.h @@ -94,359 +94,358 @@ #define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET 6697 #define PSWRQ2_REG_VF_BASE_RT_OFFSET 6698 #define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET 6699 -#define PSWRQ2_REG_WR_MBS0_RT_OFFSET 6700 -#define PSWRQ2_REG_RD_MBS0_RT_OFFSET 6701 -#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET 6702 -#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET 6703 -#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET 6704 +#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET 6700 +#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET 6701 +#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET 6702 #define PSWRQ2_REG_ILT_MEMORY_RT_SIZE 22000 -#define PGLUE_REG_B_VF_BASE_RT_OFFSET 28704 -#define PGLUE_REG_B_MSDM_OFFSET_MASK_B_RT_OFFSET 28705 -#define PGLUE_REG_B_MSDM_VF_SHIFT_B_RT_OFFSET 28706 -#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET 28707 -#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET 28708 -#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET 28709 -#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET 28710 -#define TM_REG_VF_ENABLE_CONN_RT_OFFSET 28711 -#define TM_REG_PF_ENABLE_CONN_RT_OFFSET 28712 -#define TM_REG_PF_ENABLE_TASK_RT_OFFSET 28713 -#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET 28714 -#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET 28715 -#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET 28716 +#define PGLUE_REG_B_VF_BASE_RT_OFFSET 28702 +#define PGLUE_REG_B_MSDM_OFFSET_MASK_B_RT_OFFSET 28703 +#define PGLUE_REG_B_MSDM_VF_SHIFT_B_RT_OFFSET 28704 +#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET 28705 +#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET 28706 +#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET 28707 +#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET 28708 +#define TM_REG_VF_ENABLE_CONN_RT_OFFSET 28709 +#define TM_REG_PF_ENABLE_CONN_RT_OFFSET 28710 +#define TM_REG_PF_ENABLE_TASK_RT_OFFSET 28711 +#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET 28712 +#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET 28713 +#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET 28714 #define TM_REG_CONFIG_CONN_MEM_RT_SIZE 416 -#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET 29132 +#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET 29130 #define TM_REG_CONFIG_TASK_MEM_RT_SIZE 608 -#define QM_REG_MAXPQSIZE_0_RT_OFFSET 29740 -#define QM_REG_MAXPQSIZE_1_RT_OFFSET 29741 -#define QM_REG_MAXPQSIZE_2_RT_OFFSET 29742 -#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET 29743 -#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET 29744 -#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET 29745 -#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET 29746 -#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET 29747 -#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET 29748 -#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET 29749 -#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET 29750 -#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET 29751 -#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET 29752 -#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET 29753 -#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET 29754 -#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET 29755 -#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET 29756 -#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET 29757 -#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET 29758 -#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET 29759 -#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET 29760 -#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET 29761 -#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET 29762 -#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET 29763 -#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET 29764 -#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET 29765 -#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET 29766 -#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET 29767 -#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET 29768 -#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET 29769 -#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET 29770 -#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET 29771 -#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET 29772 -#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET 29773 -#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET 29774 -#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET 29775 -#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET 29776 -#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET 29777 -#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET 29778 -#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET 29779 -#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET 29780 -#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET 29781 -#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET 29782 -#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET 29783 -#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET 29784 -#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET 29785 -#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET 29786 -#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET 29787 -#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET 29788 -#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET 29789 -#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET 29790 -#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET 29791 -#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET 29792 -#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET 29793 -#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET 29794 -#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET 29795 -#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET 29796 -#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET 29797 -#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET 29798 -#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET 29799 -#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET 29800 -#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET 29801 -#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET 29802 -#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET 29803 -#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET 29804 -#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET 29805 -#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET 29806 -#define QM_REG_BASEADDROTHERPQ_RT_OFFSET 29807 -#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET 29935 -#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET 29936 -#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET 29937 -#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET 29938 -#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET 29939 -#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET 29940 -#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET 29941 -#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET 29942 -#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET 29943 -#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET 29944 -#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET 29945 -#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET 29946 -#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET 29947 -#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET 29948 -#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET 29949 -#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET 29950 -#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET 29951 -#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET 29952 -#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET 29953 -#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET 29954 -#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET 29955 -#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET 29956 -#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET 29957 -#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET 29958 -#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET 29959 -#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET 29960 -#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET 29961 -#define QM_REG_PQTX2PF_0_RT_OFFSET 29962 -#define QM_REG_PQTX2PF_1_RT_OFFSET 29963 -#define QM_REG_PQTX2PF_2_RT_OFFSET 29964 -#define QM_REG_PQTX2PF_3_RT_OFFSET 29965 -#define QM_REG_PQTX2PF_4_RT_OFFSET 29966 -#define QM_REG_PQTX2PF_5_RT_OFFSET 29967 -#define QM_REG_PQTX2PF_6_RT_OFFSET 29968 -#define QM_REG_PQTX2PF_7_RT_OFFSET 29969 -#define QM_REG_PQTX2PF_8_RT_OFFSET 29970 -#define QM_REG_PQTX2PF_9_RT_OFFSET 29971 -#define QM_REG_PQTX2PF_10_RT_OFFSET 29972 -#define QM_REG_PQTX2PF_11_RT_OFFSET 29973 -#define QM_REG_PQTX2PF_12_RT_OFFSET 29974 -#define QM_REG_PQTX2PF_13_RT_OFFSET 29975 -#define QM_REG_PQTX2PF_14_RT_OFFSET 29976 -#define QM_REG_PQTX2PF_15_RT_OFFSET 29977 -#define QM_REG_PQTX2PF_16_RT_OFFSET 29978 -#define QM_REG_PQTX2PF_17_RT_OFFSET 29979 -#define QM_REG_PQTX2PF_18_RT_OFFSET 29980 -#define QM_REG_PQTX2PF_19_RT_OFFSET 29981 -#define QM_REG_PQTX2PF_20_RT_OFFSET 29982 -#define QM_REG_PQTX2PF_21_RT_OFFSET 29983 -#define QM_REG_PQTX2PF_22_RT_OFFSET 29984 -#define QM_REG_PQTX2PF_23_RT_OFFSET 29985 -#define QM_REG_PQTX2PF_24_RT_OFFSET 29986 -#define QM_REG_PQTX2PF_25_RT_OFFSET 29987 -#define QM_REG_PQTX2PF_26_RT_OFFSET 29988 -#define QM_REG_PQTX2PF_27_RT_OFFSET 29989 -#define QM_REG_PQTX2PF_28_RT_OFFSET 29990 -#define QM_REG_PQTX2PF_29_RT_OFFSET 29991 -#define QM_REG_PQTX2PF_30_RT_OFFSET 29992 -#define QM_REG_PQTX2PF_31_RT_OFFSET 29993 -#define QM_REG_PQTX2PF_32_RT_OFFSET 29994 -#define QM_REG_PQTX2PF_33_RT_OFFSET 29995 -#define QM_REG_PQTX2PF_34_RT_OFFSET 29996 -#define QM_REG_PQTX2PF_35_RT_OFFSET 29997 -#define QM_REG_PQTX2PF_36_RT_OFFSET 29998 -#define QM_REG_PQTX2PF_37_RT_OFFSET 29999 -#define QM_REG_PQTX2PF_38_RT_OFFSET 30000 -#define QM_REG_PQTX2PF_39_RT_OFFSET 30001 -#define QM_REG_PQTX2PF_40_RT_OFFSET 30002 -#define QM_REG_PQTX2PF_41_RT_OFFSET 30003 -#define QM_REG_PQTX2PF_42_RT_OFFSET 30004 -#define QM_REG_PQTX2PF_43_RT_OFFSET 30005 -#define QM_REG_PQTX2PF_44_RT_OFFSET 30006 -#define QM_REG_PQTX2PF_45_RT_OFFSET 30007 -#define QM_REG_PQTX2PF_46_RT_OFFSET 30008 -#define QM_REG_PQTX2PF_47_RT_OFFSET 30009 -#define QM_REG_PQTX2PF_48_RT_OFFSET 30010 -#define QM_REG_PQTX2PF_49_RT_OFFSET 30011 -#define QM_REG_PQTX2PF_50_RT_OFFSET 30012 -#define QM_REG_PQTX2PF_51_RT_OFFSET 30013 -#define QM_REG_PQTX2PF_52_RT_OFFSET 30014 -#define QM_REG_PQTX2PF_53_RT_OFFSET 30015 -#define QM_REG_PQTX2PF_54_RT_OFFSET 30016 -#define QM_REG_PQTX2PF_55_RT_OFFSET 30017 -#define QM_REG_PQTX2PF_56_RT_OFFSET 30018 -#define QM_REG_PQTX2PF_57_RT_OFFSET 30019 -#define QM_REG_PQTX2PF_58_RT_OFFSET 30020 -#define QM_REG_PQTX2PF_59_RT_OFFSET 30021 -#define QM_REG_PQTX2PF_60_RT_OFFSET 30022 -#define QM_REG_PQTX2PF_61_RT_OFFSET 30023 -#define QM_REG_PQTX2PF_62_RT_OFFSET 30024 -#define QM_REG_PQTX2PF_63_RT_OFFSET 30025 -#define QM_REG_PQOTHER2PF_0_RT_OFFSET 30026 -#define QM_REG_PQOTHER2PF_1_RT_OFFSET 30027 -#define QM_REG_PQOTHER2PF_2_RT_OFFSET 30028 -#define QM_REG_PQOTHER2PF_3_RT_OFFSET 30029 -#define QM_REG_PQOTHER2PF_4_RT_OFFSET 30030 -#define QM_REG_PQOTHER2PF_5_RT_OFFSET 30031 -#define QM_REG_PQOTHER2PF_6_RT_OFFSET 30032 -#define QM_REG_PQOTHER2PF_7_RT_OFFSET 30033 -#define QM_REG_PQOTHER2PF_8_RT_OFFSET 30034 -#define QM_REG_PQOTHER2PF_9_RT_OFFSET 30035 -#define QM_REG_PQOTHER2PF_10_RT_OFFSET 30036 -#define QM_REG_PQOTHER2PF_11_RT_OFFSET 30037 -#define QM_REG_PQOTHER2PF_12_RT_OFFSET 30038 -#define QM_REG_PQOTHER2PF_13_RT_OFFSET 30039 -#define QM_REG_PQOTHER2PF_14_RT_OFFSET 30040 -#define QM_REG_PQOTHER2PF_15_RT_OFFSET 30041 -#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET 30042 -#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET 30043 -#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET 30044 -#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET 30045 -#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET 30046 -#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET 30047 -#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET 30048 -#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET 30049 -#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET 30050 -#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET 30051 -#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET 30052 -#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET 30053 -#define QM_REG_RLGLBLINCVAL_RT_OFFSET 30054 +#define QM_REG_MAXPQSIZE_0_RT_OFFSET 29738 +#define QM_REG_MAXPQSIZE_1_RT_OFFSET 29739 +#define QM_REG_MAXPQSIZE_2_RT_OFFSET 29740 +#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET 29741 +#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET 29742 +#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET 29743 +#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET 29744 +#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET 29745 +#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET 29746 +#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET 29747 +#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET 29748 +#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET 29749 +#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET 29750 +#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET 29751 +#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET 29752 +#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET 29753 +#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET 29754 +#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET 29755 +#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET 29756 +#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET 29757 +#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET 29758 +#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET 29759 +#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET 29760 +#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET 29761 +#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET 29762 +#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET 29763 +#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET 29764 +#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET 29765 +#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET 29766 +#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET 29767 +#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET 29768 +#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET 29769 +#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET 29770 +#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET 29771 +#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET 29772 +#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET 29773 +#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET 29774 +#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET 29775 +#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET 29776 +#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET 29777 +#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET 29778 +#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET 29779 +#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET 29780 +#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET 29781 +#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET 29782 +#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET 29783 +#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET 29784 +#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET 29785 +#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET 29786 +#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET 29787 +#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET 29788 +#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET 29789 +#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET 29790 +#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET 29791 +#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET 29792 +#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET 29793 +#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET 29794 +#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET 29795 +#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET 29796 +#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET 29797 +#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET 29798 +#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET 29799 +#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET 29800 +#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET 29801 +#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET 29802 +#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET 29803 +#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET 29804 +#define QM_REG_BASEADDROTHERPQ_RT_OFFSET 29805 +#define QM_REG_BASEADDROTHERPQ_RT_SIZE 128 +#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET 29933 +#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET 29934 +#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET 29935 +#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET 29936 +#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET 29937 +#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET 29938 +#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET 29939 +#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET 29940 +#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET 29941 +#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET 29942 +#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET 29943 +#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET 29944 +#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET 29945 +#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET 29946 +#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET 29947 +#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET 29948 +#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET 29949 +#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET 29950 +#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET 29951 +#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET 29952 +#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET 29953 +#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET 29954 +#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET 29955 +#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET 29956 +#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET 29957 +#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET 29958 +#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET 29959 +#define QM_REG_PQTX2PF_0_RT_OFFSET 29960 +#define QM_REG_PQTX2PF_1_RT_OFFSET 29961 +#define QM_REG_PQTX2PF_2_RT_OFFSET 29962 +#define QM_REG_PQTX2PF_3_RT_OFFSET 29963 +#define QM_REG_PQTX2PF_4_RT_OFFSET 29964 +#define QM_REG_PQTX2PF_5_RT_OFFSET 29965 +#define QM_REG_PQTX2PF_6_RT_OFFSET 29966 +#define QM_REG_PQTX2PF_7_RT_OFFSET 29967 +#define QM_REG_PQTX2PF_8_RT_OFFSET 29968 +#define QM_REG_PQTX2PF_9_RT_OFFSET 29969 +#define QM_REG_PQTX2PF_10_RT_OFFSET 29970 +#define QM_REG_PQTX2PF_11_RT_OFFSET 29971 +#define QM_REG_PQTX2PF_12_RT_OFFSET 29972 +#define QM_REG_PQTX2PF_13_RT_OFFSET 29973 +#define QM_REG_PQTX2PF_14_RT_OFFSET 29974 +#define QM_REG_PQTX2PF_15_RT_OFFSET 29975 +#define QM_REG_PQTX2PF_16_RT_OFFSET 29976 +#define QM_REG_PQTX2PF_17_RT_OFFSET 29977 +#define QM_REG_PQTX2PF_18_RT_OFFSET 29978 +#define QM_REG_PQTX2PF_19_RT_OFFSET 29979 +#define QM_REG_PQTX2PF_20_RT_OFFSET 29980 +#define QM_REG_PQTX2PF_21_RT_OFFSET 29981 +#define QM_REG_PQTX2PF_22_RT_OFFSET 29982 +#define QM_REG_PQTX2PF_23_RT_OFFSET 29983 +#define QM_REG_PQTX2PF_24_RT_OFFSET 29984 +#define QM_REG_PQTX2PF_25_RT_OFFSET 29985 +#define QM_REG_PQTX2PF_26_RT_OFFSET 29986 +#define QM_REG_PQTX2PF_27_RT_OFFSET 29987 +#define QM_REG_PQTX2PF_28_RT_OFFSET 29988 +#define QM_REG_PQTX2PF_29_RT_OFFSET 29989 +#define QM_REG_PQTX2PF_30_RT_OFFSET 29990 +#define QM_REG_PQTX2PF_31_RT_OFFSET 29991 +#define QM_REG_PQTX2PF_32_RT_OFFSET 29992 +#define QM_REG_PQTX2PF_33_RT_OFFSET 29993 +#define QM_REG_PQTX2PF_34_RT_OFFSET 29994 +#define QM_REG_PQTX2PF_35_RT_OFFSET 29995 +#define QM_REG_PQTX2PF_36_RT_OFFSET 29996 +#define QM_REG_PQTX2PF_37_RT_OFFSET 29997 +#define QM_REG_PQTX2PF_38_RT_OFFSET 29998 +#define QM_REG_PQTX2PF_39_RT_OFFSET 29999 +#define QM_REG_PQTX2PF_40_RT_OFFSET 30000 +#define QM_REG_PQTX2PF_41_RT_OFFSET 30001 +#define QM_REG_PQTX2PF_42_RT_OFFSET 30002 +#define QM_REG_PQTX2PF_43_RT_OFFSET 30003 +#define QM_REG_PQTX2PF_44_RT_OFFSET 30004 +#define QM_REG_PQTX2PF_45_RT_OFFSET 30005 +#define QM_REG_PQTX2PF_46_RT_OFFSET 30006 +#define QM_REG_PQTX2PF_47_RT_OFFSET 30007 +#define QM_REG_PQTX2PF_48_RT_OFFSET 30008 +#define QM_REG_PQTX2PF_49_RT_OFFSET 30009 +#define QM_REG_PQTX2PF_50_RT_OFFSET 30010 +#define QM_REG_PQTX2PF_51_RT_OFFSET 30011 +#define QM_REG_PQTX2PF_52_RT_OFFSET 30012 +#define QM_REG_PQTX2PF_53_RT_OFFSET 30013 +#define QM_REG_PQTX2PF_54_RT_OFFSET 30014 +#define QM_REG_PQTX2PF_55_RT_OFFSET 30015 +#define QM_REG_PQTX2PF_56_RT_OFFSET 30016 +#define QM_REG_PQTX2PF_57_RT_OFFSET 30017 +#define QM_REG_PQTX2PF_58_RT_OFFSET 30018 +#define QM_REG_PQTX2PF_59_RT_OFFSET 30019 +#define QM_REG_PQTX2PF_60_RT_OFFSET 30020 +#define QM_REG_PQTX2PF_61_RT_OFFSET 30021 +#define QM_REG_PQTX2PF_62_RT_OFFSET 30022 +#define QM_REG_PQTX2PF_63_RT_OFFSET 30023 +#define QM_REG_PQOTHER2PF_0_RT_OFFSET 30024 +#define QM_REG_PQOTHER2PF_1_RT_OFFSET 30025 +#define QM_REG_PQOTHER2PF_2_RT_OFFSET 30026 +#define QM_REG_PQOTHER2PF_3_RT_OFFSET 30027 +#define QM_REG_PQOTHER2PF_4_RT_OFFSET 30028 +#define QM_REG_PQOTHER2PF_5_RT_OFFSET 30029 +#define QM_REG_PQOTHER2PF_6_RT_OFFSET 30030 +#define QM_REG_PQOTHER2PF_7_RT_OFFSET 30031 +#define QM_REG_PQOTHER2PF_8_RT_OFFSET 30032 +#define QM_REG_PQOTHER2PF_9_RT_OFFSET 30033 +#define QM_REG_PQOTHER2PF_10_RT_OFFSET 30034 +#define QM_REG_PQOTHER2PF_11_RT_OFFSET 30035 +#define QM_REG_PQOTHER2PF_12_RT_OFFSET 30036 +#define QM_REG_PQOTHER2PF_13_RT_OFFSET 30037 +#define QM_REG_PQOTHER2PF_14_RT_OFFSET 30038 +#define QM_REG_PQOTHER2PF_15_RT_OFFSET 30039 +#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET 30040 +#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET 30041 +#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET 30042 +#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET 30043 +#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET 30044 +#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET 30045 +#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET 30046 +#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET 30047 +#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET 30048 +#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET 30049 +#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET 30050 +#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET 30051 +#define QM_REG_RLGLBLINCVAL_RT_OFFSET 30052 #define QM_REG_RLGLBLINCVAL_RT_SIZE 256 -#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET 30310 +#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET 30308 #define QM_REG_RLGLBLUPPERBOUND_RT_SIZE 256 -#define QM_REG_RLGLBLCRD_RT_OFFSET 30566 +#define QM_REG_RLGLBLCRD_RT_OFFSET 30564 #define QM_REG_RLGLBLCRD_RT_SIZE 256 -#define QM_REG_RLGLBLENABLE_RT_OFFSET 30822 -#define QM_REG_RLPFPERIOD_RT_OFFSET 30823 -#define QM_REG_RLPFPERIODTIMER_RT_OFFSET 30824 -#define QM_REG_RLPFINCVAL_RT_OFFSET 30825 +#define QM_REG_RLGLBLENABLE_RT_OFFSET 30820 +#define QM_REG_RLPFPERIOD_RT_OFFSET 30821 +#define QM_REG_RLPFPERIODTIMER_RT_OFFSET 30822 +#define QM_REG_RLPFINCVAL_RT_OFFSET 30823 #define QM_REG_RLPFINCVAL_RT_SIZE 16 -#define QM_REG_RLPFUPPERBOUND_RT_OFFSET 30841 +#define QM_REG_RLPFUPPERBOUND_RT_OFFSET 30839 #define QM_REG_RLPFUPPERBOUND_RT_SIZE 16 -#define QM_REG_RLPFCRD_RT_OFFSET 30857 +#define QM_REG_RLPFCRD_RT_OFFSET 30855 #define QM_REG_RLPFCRD_RT_SIZE 16 -#define QM_REG_RLPFENABLE_RT_OFFSET 30873 -#define QM_REG_RLPFVOQENABLE_RT_OFFSET 30874 -#define QM_REG_WFQPFWEIGHT_RT_OFFSET 30875 +#define QM_REG_RLPFENABLE_RT_OFFSET 30871 +#define QM_REG_RLPFVOQENABLE_RT_OFFSET 30872 +#define QM_REG_WFQPFWEIGHT_RT_OFFSET 30873 #define QM_REG_WFQPFWEIGHT_RT_SIZE 16 -#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET 30891 +#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET 30889 #define QM_REG_WFQPFUPPERBOUND_RT_SIZE 16 -#define QM_REG_WFQPFCRD_RT_OFFSET 30907 +#define QM_REG_WFQPFCRD_RT_OFFSET 30905 #define QM_REG_WFQPFCRD_RT_SIZE 256 -#define QM_REG_WFQPFENABLE_RT_OFFSET 31163 -#define QM_REG_WFQVPENABLE_RT_OFFSET 31164 -#define QM_REG_BASEADDRTXPQ_RT_OFFSET 31165 +#define QM_REG_WFQPFENABLE_RT_OFFSET 31161 +#define QM_REG_WFQVPENABLE_RT_OFFSET 31162 +#define QM_REG_BASEADDRTXPQ_RT_OFFSET 31163 #define QM_REG_BASEADDRTXPQ_RT_SIZE 512 -#define QM_REG_TXPQMAP_RT_OFFSET 31677 +#define QM_REG_TXPQMAP_RT_OFFSET 31675 #define QM_REG_TXPQMAP_RT_SIZE 512 -#define QM_REG_WFQVPWEIGHT_RT_OFFSET 32189 +#define QM_REG_WFQVPWEIGHT_RT_OFFSET 32187 #define QM_REG_WFQVPWEIGHT_RT_SIZE 512 -#define QM_REG_WFQVPCRD_RT_OFFSET 32701 +#define QM_REG_WFQVPCRD_RT_OFFSET 32699 #define QM_REG_WFQVPCRD_RT_SIZE 512 -#define QM_REG_WFQVPMAP_RT_OFFSET 33213 +#define QM_REG_WFQVPMAP_RT_OFFSET 33211 #define QM_REG_WFQVPMAP_RT_SIZE 512 -#define QM_REG_WFQPFCRD_MSB_RT_OFFSET 33725 +#define QM_REG_WFQPFCRD_MSB_RT_OFFSET 33723 #define QM_REG_WFQPFCRD_MSB_RT_SIZE 320 -#define QM_REG_VOQCRDLINE_RT_OFFSET 34045 +#define QM_REG_VOQCRDLINE_RT_OFFSET 34043 #define QM_REG_VOQCRDLINE_RT_SIZE 36 -#define QM_REG_VOQINITCRDLINE_RT_OFFSET 34081 +#define QM_REG_VOQINITCRDLINE_RT_OFFSET 34079 #define QM_REG_VOQINITCRDLINE_RT_SIZE 36 -#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET 34117 -#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET 34118 -#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET 34119 -#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET 34120 -#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET 34121 -#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET 34122 -#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET 34123 -#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET 34124 +#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET 34115 +#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET 34116 +#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET 34117 +#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET 34118 +#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET 34119 +#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET 34120 +#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET 34121 +#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET 34122 #define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE 4 -#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET 34128 +#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET 34126 #define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_SIZE 4 -#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET 34132 +#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET 34130 #define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE 4 -#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET 34136 -#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET 34137 +#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET 34134 +#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET 34135 #define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE 32 -#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET 34169 +#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET 34167 #define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE 16 -#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET 34185 +#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET 34183 #define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE 16 -#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET 34201 +#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET 34199 #define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE 16 -#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET 34217 +#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET 34215 #define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE 16 -#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET 34233 -#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET 34234 -#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET 34235 -#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET 34236 -#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET 34237 -#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET 34238 -#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET 34239 -#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET 34240 -#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET 34241 -#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET 34242 -#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET 34243 -#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET 34244 -#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET 34245 -#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET 34246 -#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET 34247 -#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET 34248 -#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET 34249 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET 34250 -#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET 34251 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET 34252 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET 34253 -#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET 34254 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET 34255 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET 34256 -#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET 34257 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET 34258 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET 34259 -#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET 34260 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET 34261 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET 34262 -#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET 34263 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET 34264 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET 34265 -#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET 34266 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET 34267 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET 34268 -#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET 34269 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET 34270 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET 34271 -#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET 34272 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET 34273 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET 34274 -#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET 34275 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET 34276 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET 34277 -#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET 34278 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET 34279 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET 34280 -#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET 34281 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET 34282 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET 34283 -#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET 34284 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET 34285 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET 34286 -#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET 34287 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET 34288 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET 34289 -#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET 34290 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET 34291 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET 34292 -#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET 34293 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET 34294 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET 34295 -#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET 34296 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET 34297 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET 34298 -#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET 34299 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET 34300 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET 34301 -#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET 34302 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET 34303 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET 34304 -#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET 34305 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET 34306 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET 34307 -#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET 34308 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET 34309 -#define XCM_REG_CON_PHY_Q3_RT_OFFSET 34310 +#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET 34231 +#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET 34232 +#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET 34233 +#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET 34234 +#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET 34235 +#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET 34236 +#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET 34237 +#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET 34238 +#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET 34239 +#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET 34240 +#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET 34241 +#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET 34242 +#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET 34243 +#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET 34244 +#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET 34245 +#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET 34246 +#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET 34247 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET 34248 +#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET 34249 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET 34250 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET 34251 +#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET 34252 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET 34253 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET 34254 +#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET 34255 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET 34256 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET 34257 +#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET 34258 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET 34259 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET 34260 +#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET 34261 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET 34262 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET 34263 +#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET 34264 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET 34265 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET 34266 +#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET 34267 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET 34268 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET 34269 +#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET 34270 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET 34271 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET 34272 +#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET 34273 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET 34274 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET 34275 +#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET 34276 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET 34277 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET 34278 +#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET 34279 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET 34280 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET 34281 +#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET 34282 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET 34283 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET 34284 +#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET 34285 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET 34286 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET 34287 +#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET 34288 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET 34289 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET 34290 +#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET 34291 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET 34292 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET 34293 +#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET 34294 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET 34295 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET 34296 +#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET 34297 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET 34298 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET 34299 +#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET 34300 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET 34301 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET 34302 +#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET 34303 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET 34304 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET 34305 +#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET 34306 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET 34307 +#define XCM_REG_CON_PHY_Q3_RT_OFFSET 34308 -#define RUNTIME_ARRAY_SIZE 34311 +#define RUNTIME_ARRAY_SIZE 34309 #endif /* __RT_DEFS_H__ */ diff --git a/drivers/net/qede/base/ecore_sp_commands.c b/drivers/net/qede/base/ecore_sp_commands.c index 8fd64d7a..d6e4b9e0 100644 --- a/drivers/net/qede/base/ecore_sp_commands.c +++ b/drivers/net/qede/base/ecore_sp_commands.c @@ -165,23 +165,19 @@ static void ecore_set_tunn_ports(struct ecore_tunnel_info *p_tun, } static void -__ecore_set_ramrod_tunnel_param(u8 *p_tunn_cls, u8 *p_enable_tx_clas, +__ecore_set_ramrod_tunnel_param(u8 *p_tunn_cls, struct ecore_tunn_update_type *tun_type) { *p_tunn_cls = tun_type->tun_cls; - - if (tun_type->b_mode_enabled) - *p_enable_tx_clas = 1; } static void -ecore_set_ramrod_tunnel_param(u8 *p_tunn_cls, u8 *p_enable_tx_clas, +ecore_set_ramrod_tunnel_param(u8 *p_tunn_cls, struct ecore_tunn_update_type *tun_type, u8 *p_update_port, __le16 *p_port, struct ecore_tunn_update_udp_port *p_udp_port) { - __ecore_set_ramrod_tunnel_param(p_tunn_cls, p_enable_tx_clas, - tun_type); + __ecore_set_ramrod_tunnel_param(p_tunn_cls, tun_type); if (p_udp_port->b_update_port) { *p_update_port = 1; *p_port = OSAL_CPU_TO_LE16(p_udp_port->port); @@ -200,33 +196,27 @@ ecore_tunn_set_pf_update_params(struct ecore_hwfn *p_hwfn, ecore_set_tunn_ports(p_tun, p_src); ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_vxlan, - &p_tunn_cfg->tx_enable_vxlan, &p_tun->vxlan, &p_tunn_cfg->set_vxlan_udp_port_flg, &p_tunn_cfg->vxlan_udp_port, &p_tun->vxlan_port); ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2geneve, - &p_tunn_cfg->tx_enable_l2geneve, &p_tun->l2_geneve, &p_tunn_cfg->set_geneve_udp_port_flg, &p_tunn_cfg->geneve_udp_port, &p_tun->geneve_port); __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgeneve, - &p_tunn_cfg->tx_enable_ipgeneve, &p_tun->ip_geneve); __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2gre, - &p_tunn_cfg->tx_enable_l2gre, &p_tun->l2_gre); __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgre, - &p_tunn_cfg->tx_enable_ipgre, &p_tun->ip_gre); p_tunn_cfg->update_rx_pf_clss = p_tun->b_update_rx_cls; - p_tunn_cfg->update_tx_pf_clss = p_tun->b_update_tx_cls; } static void ecore_set_hw_tunn_mode(struct ecore_hwfn *p_hwfn, @@ -282,29 +272,24 @@ ecore_tunn_set_pf_start_params(struct ecore_hwfn *p_hwfn, ecore_set_tunn_ports(p_tun, p_src); ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_vxlan, - &p_tunn_cfg->tx_enable_vxlan, &p_tun->vxlan, &p_tunn_cfg->set_vxlan_udp_port_flg, &p_tunn_cfg->vxlan_udp_port, &p_tun->vxlan_port); ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2geneve, - &p_tunn_cfg->tx_enable_l2geneve, &p_tun->l2_geneve, &p_tunn_cfg->set_geneve_udp_port_flg, &p_tunn_cfg->geneve_udp_port, &p_tun->geneve_port); __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgeneve, - &p_tunn_cfg->tx_enable_ipgeneve, &p_tun->ip_geneve); __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2gre, - &p_tunn_cfg->tx_enable_l2gre, &p_tun->l2_gre); __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgre, - &p_tunn_cfg->tx_enable_ipgre, &p_tun->ip_gre); } @@ -345,7 +330,7 @@ enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, /* For easier debugging */ p_ramrod->dont_log_ramrods = 0; - p_ramrod->log_type_mask = OSAL_CPU_TO_LE16(0xf); + p_ramrod->log_type_mask = OSAL_CPU_TO_LE16(0x8f); switch (mode) { case ECORE_MF_DEFAULT: diff --git a/drivers/net/qede/base/eth_common.h b/drivers/net/qede/base/eth_common.h index 6dc969b0..45a0356d 100644 --- a/drivers/net/qede/base/eth_common.h +++ b/drivers/net/qede/base/eth_common.h @@ -79,6 +79,10 @@ /* Maximum number of buffers, used for RX packet placement */ #define ETH_RX_MAX_BUFF_PER_PKT 5 +/* Minimum number of free BDs in RX ring, that guarantee receiving of at least + * one RX packet. + */ +#define ETH_RX_BD_THRESHOLD 12 /* num of MAC/VLAN filters */ #define ETH_NUM_MAC_FILTERS 512 diff --git a/drivers/net/qede/base/mcp_public.h b/drivers/net/qede/base/mcp_public.h index fcf98477..1ad8a962 100644 --- a/drivers/net/qede/base/mcp_public.h +++ b/drivers/net/qede/base/mcp_public.h @@ -132,13 +132,28 @@ struct eth_stats { u64 r1023; /* 0x04 (Offset 0x20 ) RX 512 to 1023 byte frame counter*/ /* 0x05 (Offset 0x28 ) RX 1024 to 1518 byte frame counter */ u64 r1518; + union { + struct { /* bb */ /* 0x06 (Offset 0x30 ) RX 1519 to 1522 byte VLAN-tagged frame counter */ - u64 r1522; - u64 r2047; /* 0x07 (Offset 0x38 ) RX 1519 to 2047 byte frame counter*/ - u64 r4095; /* 0x08 (Offset 0x40 ) RX 2048 to 4095 byte frame counter*/ - u64 r9216; /* 0x09 (Offset 0x48 ) RX 4096 to 9216 byte frame counter*/ + u64 r1522; +/* 0x07 (Offset 0x38 ) RX 1519 to 2047 byte frame counter*/ + u64 r2047; +/* 0x08 (Offset 0x40 ) RX 2048 to 4095 byte frame counter*/ + u64 r4095; +/* 0x09 (Offset 0x48 ) RX 4096 to 9216 byte frame counter*/ + u64 r9216; /* 0x0A (Offset 0x50 ) RX 9217 to 16383 byte frame counter */ - u64 r16383; + u64 r16383; + } bb0; + struct { /* ah */ + u64 unused1; +/* 0x07 (Offset 0x38 ) RX 1519 to max byte frame counter*/ + u64 r1519_to_max; + u64 unused2; + u64 unused3; + u64 unused4; + } ah0; + } u0; u64 rfcs; /* 0x0F (Offset 0x58 ) RX FCS error frame counter*/ u64 rxcf; /* 0x10 (Offset 0x60 ) RX control frame counter*/ u64 rxpf; /* 0x11 (Offset 0x68 ) RX pause frame counter*/ @@ -156,19 +171,40 @@ struct eth_stats { u64 t1023; /* 0x44 (Offset 0xc8 ) TX 512 to 1023 byte frame counter*/ /* 0x45 (Offset 0xd0 ) TX 1024 to 1518 byte frame counter */ u64 t1518; + union { + struct { /* bb */ /* 0x47 (Offset 0xd8 ) TX 1519 to 2047 byte frame counter */ - u64 t2047; + u64 t2047; /* 0x48 (Offset 0xe0 ) TX 2048 to 4095 byte frame counter */ - u64 t4095; + u64 t4095; /* 0x49 (Offset 0xe8 ) TX 4096 to 9216 byte frame counter */ - u64 t9216; + u64 t9216; /* 0x4A (Offset 0xf0 ) TX 9217 to 16383 byte frame counter */ - u64 t16383; + u64 t16383; + } bb1; + struct { /* ah */ +/* 0x47 (Offset 0xd8 ) TX 1519 to max byte frame counter */ + u64 t1519_to_max; + u64 unused6; + u64 unused7; + u64 unused8; + } ah1; + } u1; u64 txpf; /* 0x50 (Offset 0xf8 ) TX pause frame counter */ u64 txpp; /* 0x51 (Offset 0x100) TX PFC frame counter */ /* 0x6C (Offset 0x108) Transmit Logical Type LLFC message counter */ - u64 tlpiec; - u64 tncl; /* 0x6E (Offset 0x110) Transmit Total Collision Counter */ + union { + struct { /* bb */ +/* 0x6C (Offset 0x108) Transmit Logical Type LLFC message counter */ + u64 tlpiec; +/* 0x6E (Offset 0x110) Transmit Total Collision Counter */ + u64 tncl; + } bb2; + struct { /* ah */ + u64 unused9; + u64 unused10; + } ah2; + } u2; u64 rbyte; /* 0x3d (Offset 0x118) RX byte counter */ u64 rxuca; /* 0x0c (Offset 0x120) RX UC frame counter */ u64 rxmca; /* 0x0d (Offset 0x128) RX MC frame counter */ diff --git a/drivers/net/qede/base/reg_addr.h b/drivers/net/qede/base/reg_addr.h index f9920f37..60286545 100644 --- a/drivers/net/qede/base/reg_addr.h +++ b/drivers/net/qede/base/reg_addr.h @@ -1200,3 +1200,8 @@ #define PGLUE_B_REG_VF_BAR0_SIZE_K2_E5 0x2aaeb4UL #define PRS_REG_OUTPUT_FORMAT_4_0_BB_K2 0x1f099cUL + +#define NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 0x501a00UL +#define NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 0x501a80UL +#define NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 0x501ac0UL +#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 0x501b00UL diff --git a/drivers/net/qede/qede_eth_if.c b/drivers/net/qede/qede_eth_if.c deleted file mode 100644 index a3c0b137..00000000 --- a/drivers/net/qede/qede_eth_if.c +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Copyright (c) 2016 QLogic Corporation. - * All rights reserved. - * www.qlogic.com - * - * See LICENSE.qede_pmd for copyright and licensing details. - */ - -#include "qede_ethdev.h" - -static int -qed_start_vport(struct ecore_dev *edev, struct qed_start_vport_params *p_params) -{ - int rc, i; - - for_each_hwfn(edev, i) { - struct ecore_hwfn *p_hwfn = &edev->hwfns[i]; - u8 tx_switching = 0; - struct ecore_sp_vport_start_params start = { 0 }; - - start.tpa_mode = p_params->enable_lro ? ECORE_TPA_MODE_RSC : - ECORE_TPA_MODE_NONE; - start.remove_inner_vlan = p_params->remove_inner_vlan; - start.tx_switching = tx_switching; - start.only_untagged = false; /* untagged only */ - start.drop_ttl0 = p_params->drop_ttl0; - start.concrete_fid = p_hwfn->hw_info.concrete_fid; - start.opaque_fid = p_hwfn->hw_info.opaque_fid; - start.concrete_fid = p_hwfn->hw_info.concrete_fid; - start.handle_ptp_pkts = p_params->handle_ptp_pkts; - start.vport_id = p_params->vport_id; - start.mtu = p_params->mtu; - /* @DPDK - Disable FW placement */ - start.zero_placement_offset = 1; - - rc = ecore_sp_vport_start(p_hwfn, &start); - if (rc) { - DP_ERR(edev, "Failed to start VPORT\n"); - return rc; - } - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Started V-PORT %d with MTU %d\n", - p_params->vport_id, p_params->mtu); - } - - ecore_reset_vport_stats(edev); - - return 0; -} - -static int qed_stop_vport(struct ecore_dev *edev, uint8_t vport_id) -{ - int rc, i; - - for_each_hwfn(edev, i) { - struct ecore_hwfn *p_hwfn = &edev->hwfns[i]; - rc = ecore_sp_vport_stop(p_hwfn, - p_hwfn->hw_info.opaque_fid, vport_id); - - if (rc) { - DP_ERR(edev, "Failed to stop VPORT\n"); - return rc; - } - } - - return 0; -} - -static int -qed_update_vport(struct ecore_dev *edev, struct qed_update_vport_params *params) -{ - struct ecore_sp_vport_update_params sp_params; - struct ecore_rss_params sp_rss_params; - int rc, i; - - memset(&sp_params, 0, sizeof(sp_params)); - memset(&sp_rss_params, 0, sizeof(sp_rss_params)); - - /* Translate protocol params into sp params */ - sp_params.vport_id = params->vport_id; - sp_params.update_vport_active_rx_flg = params->update_vport_active_flg; - sp_params.update_vport_active_tx_flg = params->update_vport_active_flg; - sp_params.vport_active_rx_flg = params->vport_active_flg; - sp_params.vport_active_tx_flg = params->vport_active_flg; - sp_params.update_inner_vlan_removal_flg = - params->update_inner_vlan_removal_flg; - sp_params.inner_vlan_removal_flg = params->inner_vlan_removal_flg; - sp_params.update_tx_switching_flg = params->update_tx_switching_flg; - sp_params.tx_switching_flg = params->tx_switching_flg; - sp_params.accept_any_vlan = params->accept_any_vlan; - sp_params.update_accept_any_vlan_flg = - params->update_accept_any_vlan_flg; - sp_params.mtu = params->mtu; - sp_params.sge_tpa_params = params->sge_tpa_params; - - for_each_hwfn(edev, i) { - struct ecore_hwfn *p_hwfn = &edev->hwfns[i]; - - sp_params.opaque_fid = p_hwfn->hw_info.opaque_fid; - rc = ecore_sp_vport_update(p_hwfn, &sp_params, - ECORE_SPQ_MODE_EBLOCK, NULL); - if (rc) { - DP_ERR(edev, "Failed to update VPORT\n"); - return rc; - } - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Updated V-PORT %d: active_flag %d [update %d]\n", - params->vport_id, params->vport_active_flg, - params->update_vport_active_flg); - } - - return 0; -} - -static int -qed_start_rxq(struct ecore_dev *edev, - uint8_t rss_num, - struct ecore_queue_start_common_params *p_params, - uint16_t bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, - uint16_t cqe_pbl_size, - struct ecore_rxq_start_ret_params *ret_params) -{ - struct ecore_hwfn *p_hwfn; - int rc, hwfn_index; - - hwfn_index = rss_num % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - p_params->queue_id = p_params->queue_id / edev->num_hwfns; - p_params->stats_id = p_params->vport_id; - - rc = ecore_eth_rx_queue_start(p_hwfn, - p_hwfn->hw_info.opaque_fid, - p_params, - bd_max_bytes, - bd_chain_phys_addr, - cqe_pbl_addr, - cqe_pbl_size, - ret_params); - - if (rc) { - DP_ERR(edev, "Failed to start RXQ#%d\n", p_params->queue_id); - return rc; - } - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Started RX-Q %d [rss_num %d] on V-PORT %d and SB %d\n", - p_params->queue_id, rss_num, p_params->vport_id, - p_params->sb); - - return 0; -} - -static int -qed_stop_rxq(struct ecore_dev *edev, uint8_t rss_id, void *handle) -{ - int rc, hwfn_index; - struct ecore_hwfn *p_hwfn; - - hwfn_index = rss_id % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - rc = ecore_eth_rx_queue_stop(p_hwfn, handle, true, false); - if (rc) { - DP_ERR(edev, "Failed to stop RXQ#%02x\n", rss_id); - return rc; - } - - return 0; -} - -static int -qed_start_txq(struct ecore_dev *edev, - uint8_t rss_num, - struct ecore_queue_start_common_params *p_params, - dma_addr_t pbl_addr, - uint16_t pbl_size, - struct ecore_txq_start_ret_params *ret_params) -{ - struct ecore_hwfn *p_hwfn; - int rc, hwfn_index; - - hwfn_index = rss_num % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - p_params->queue_id = p_params->queue_id / edev->num_hwfns; - p_params->stats_id = p_params->vport_id; - - rc = ecore_eth_tx_queue_start(p_hwfn, - p_hwfn->hw_info.opaque_fid, - p_params, 0 /* tc */, - pbl_addr, pbl_size, - ret_params); - - if (rc) { - DP_ERR(edev, "Failed to start TXQ#%d\n", p_params->queue_id); - return rc; - } - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Started TX-Q %d [rss_num %d] on V-PORT %d and SB %d\n", - p_params->queue_id, rss_num, p_params->vport_id, - p_params->sb); - - return 0; -} - -static int -qed_stop_txq(struct ecore_dev *edev, uint8_t rss_id, void *handle) -{ - struct ecore_hwfn *p_hwfn; - int rc, hwfn_index; - - hwfn_index = rss_id % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - rc = ecore_eth_tx_queue_stop(p_hwfn, handle); - if (rc) { - DP_ERR(edev, "Failed to stop TXQ#%02x\n", rss_id); - return rc; - } - - return 0; -} - -static int -qed_fp_cqe_completion(struct ecore_dev *edev, - uint8_t rss_id, struct eth_slow_path_rx_cqe *cqe) -{ - return ecore_eth_cqe_completion(&edev->hwfns[rss_id % edev->num_hwfns], - cqe); -} - -static int qed_fastpath_stop(struct ecore_dev *edev) -{ - ecore_hw_stop_fastpath(edev); - - return 0; -} - -static void qed_fastpath_start(struct ecore_dev *edev) -{ - struct ecore_hwfn *p_hwfn; - int i; - - for_each_hwfn(edev, i) { - p_hwfn = &edev->hwfns[i]; - ecore_hw_start_fastpath(p_hwfn); - } -} - -static void -qed_get_vport_stats(struct ecore_dev *edev, struct ecore_eth_stats *stats) -{ - ecore_get_vport_stats(edev, stats); -} - -int qed_configure_filter_rx_mode(struct rte_eth_dev *eth_dev, - enum qed_filter_rx_mode_type type) -{ - struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); - struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - struct ecore_filter_accept_flags flags; - - memset(&flags, 0, sizeof(flags)); - - flags.update_rx_mode_config = 1; - flags.update_tx_mode_config = 1; - flags.rx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | - ECORE_ACCEPT_MCAST_MATCHED | - ECORE_ACCEPT_BCAST; - - flags.tx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | - ECORE_ACCEPT_MCAST_MATCHED | - ECORE_ACCEPT_BCAST; - - if (type == QED_FILTER_RX_MODE_TYPE_PROMISC) { - flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; - if (IS_VF(edev)) { - flags.tx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; - DP_INFO(edev, "Enabling Tx unmatched flag for VF\n"); - } - } else if (type == QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC) { - flags.rx_accept_filter |= ECORE_ACCEPT_MCAST_UNMATCHED; - } else if (type == (QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC | - QED_FILTER_RX_MODE_TYPE_PROMISC)) { - flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED | - ECORE_ACCEPT_MCAST_UNMATCHED; - } - - return ecore_filter_accept_cmd(edev, 0, flags, false, false, - ECORE_SPQ_MODE_CB, NULL); -} - -static const struct qed_eth_ops qed_eth_ops_pass = { - INIT_STRUCT_FIELD(common, &qed_common_ops_pass), - INIT_STRUCT_FIELD(fill_dev_info, &qed_fill_eth_dev_info), - INIT_STRUCT_FIELD(vport_start, &qed_start_vport), - INIT_STRUCT_FIELD(vport_stop, &qed_stop_vport), - INIT_STRUCT_FIELD(vport_update, &qed_update_vport), - INIT_STRUCT_FIELD(q_rx_start, &qed_start_rxq), - INIT_STRUCT_FIELD(q_tx_start, &qed_start_txq), - INIT_STRUCT_FIELD(q_rx_stop, &qed_stop_rxq), - INIT_STRUCT_FIELD(q_tx_stop, &qed_stop_txq), - INIT_STRUCT_FIELD(eth_cqe_completion, &qed_fp_cqe_completion), - INIT_STRUCT_FIELD(fastpath_stop, &qed_fastpath_stop), - INIT_STRUCT_FIELD(fastpath_start, &qed_fastpath_start), - INIT_STRUCT_FIELD(get_vport_stats, &qed_get_vport_stats), -}; - -const struct qed_eth_ops *qed_get_eth_ops(void) -{ - return &qed_eth_ops_pass; -} diff --git a/drivers/net/qede/qede_eth_if.h b/drivers/net/qede/qede_eth_if.h deleted file mode 100644 index 097e0257..00000000 --- a/drivers/net/qede/qede_eth_if.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2016 QLogic Corporation. - * All rights reserved. - * www.qlogic.com - * - * See LICENSE.qede_pmd for copyright and licensing details. - */ - -#ifndef _QEDE_ETH_IF_H -#define _QEDE_ETH_IF_H - -#include "qede_if.h" - -/*forward decl */ -struct eth_slow_path_rx_cqe; - -#define INIT_STRUCT_FIELD(field, value) .field = value - -#define QED_ETH_INTERFACE_VERSION 609 - -#define QEDE_MAX_MCAST_FILTERS 64 - -enum qed_filter_rx_mode_type { - QED_FILTER_RX_MODE_TYPE_REGULAR, - QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC, - QED_FILTER_RX_MODE_TYPE_PROMISC, -}; - -enum qed_filter_type { - QED_FILTER_TYPE_UCAST, - QED_FILTER_TYPE_MCAST, - QED_FILTER_TYPE_RX_MODE, - QED_MAX_FILTER_TYPES, -}; - -struct qed_dev_eth_info { - struct qed_dev_info common; - - uint8_t num_queues; - uint8_t num_tc; - - struct ether_addr port_mac; - uint16_t num_vlan_filters; - uint32_t num_mac_filters; - - /* Legacy VF - this affects the datapath */ - bool is_legacy; -}; - -struct qed_update_vport_params { - uint8_t vport_id; - uint8_t update_vport_active_flg; - uint8_t vport_active_flg; - uint8_t update_inner_vlan_removal_flg; - uint8_t inner_vlan_removal_flg; - uint8_t update_tx_switching_flg; - uint8_t tx_switching_flg; - uint8_t update_accept_any_vlan_flg; - uint8_t accept_any_vlan; - uint8_t update_rss_flg; - uint16_t mtu; - struct ecore_sge_tpa_params *sge_tpa_params; -}; - -struct qed_start_vport_params { - bool remove_inner_vlan; - bool handle_ptp_pkts; - bool enable_lro; - bool drop_ttl0; - uint8_t vport_id; - uint16_t mtu; - bool clear_stats; -}; - -struct qed_eth_ops { - const struct qed_common_ops *common; - - int (*fill_dev_info)(struct ecore_dev *edev, - struct qed_dev_eth_info *info); - - int (*vport_start)(struct ecore_dev *edev, - struct qed_start_vport_params *params); - - int (*vport_stop)(struct ecore_dev *edev, uint8_t vport_id); - - int (*vport_update)(struct ecore_dev *edev, - struct qed_update_vport_params *params); - - int (*q_rx_start)(struct ecore_dev *cdev, - uint8_t rss_num, - struct ecore_queue_start_common_params *p_params, - uint16_t bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, - uint16_t cqe_pbl_size, - struct ecore_rxq_start_ret_params *ret_params); - - int (*q_rx_stop)(struct ecore_dev *edev, - uint8_t rss_id, void *handle); - - int (*q_tx_start)(struct ecore_dev *edev, - uint8_t rss_num, - struct ecore_queue_start_common_params *p_params, - dma_addr_t pbl_addr, - uint16_t pbl_size, - struct ecore_txq_start_ret_params *ret_params); - - int (*q_tx_stop)(struct ecore_dev *edev, - uint8_t rss_id, void *handle); - - int (*eth_cqe_completion)(struct ecore_dev *edev, - uint8_t rss_id, - struct eth_slow_path_rx_cqe *cqe); - - int (*fastpath_stop)(struct ecore_dev *edev); - - void (*fastpath_start)(struct ecore_dev *edev); - - void (*get_vport_stats)(struct ecore_dev *edev, - struct ecore_eth_stats *stats); -}; - -/* externs */ - -extern const struct qed_common_ops qed_common_ops_pass; - -const struct qed_eth_ops *qed_get_eth_ops(void); - -int qed_configure_filter_rx_mode(struct rte_eth_dev *eth_dev, - enum qed_filter_rx_mode_type type); - -#endif /* _QEDE_ETH_IF_H */ diff --git a/drivers/net/qede/qede_ethdev.c b/drivers/net/qede/qede_ethdev.c index 9fae40b6..0e059898 100644 --- a/drivers/net/qede/qede_ethdev.c +++ b/drivers/net/qede/qede_ethdev.c @@ -125,143 +125,199 @@ struct rte_qede_xstats_name_off { }; static const struct rte_qede_xstats_name_off qede_xstats_strings[] = { - {"rx_unicast_bytes", offsetof(struct ecore_eth_stats, rx_ucast_bytes)}, + {"rx_unicast_bytes", + offsetof(struct ecore_eth_stats_common, rx_ucast_bytes)}, {"rx_multicast_bytes", - offsetof(struct ecore_eth_stats, rx_mcast_bytes)}, + offsetof(struct ecore_eth_stats_common, rx_mcast_bytes)}, {"rx_broadcast_bytes", - offsetof(struct ecore_eth_stats, rx_bcast_bytes)}, - {"rx_unicast_packets", offsetof(struct ecore_eth_stats, rx_ucast_pkts)}, + offsetof(struct ecore_eth_stats_common, rx_bcast_bytes)}, + {"rx_unicast_packets", + offsetof(struct ecore_eth_stats_common, rx_ucast_pkts)}, {"rx_multicast_packets", - offsetof(struct ecore_eth_stats, rx_mcast_pkts)}, + offsetof(struct ecore_eth_stats_common, rx_mcast_pkts)}, {"rx_broadcast_packets", - offsetof(struct ecore_eth_stats, rx_bcast_pkts)}, + offsetof(struct ecore_eth_stats_common, rx_bcast_pkts)}, - {"tx_unicast_bytes", offsetof(struct ecore_eth_stats, tx_ucast_bytes)}, + {"tx_unicast_bytes", + offsetof(struct ecore_eth_stats_common, tx_ucast_bytes)}, {"tx_multicast_bytes", - offsetof(struct ecore_eth_stats, tx_mcast_bytes)}, + offsetof(struct ecore_eth_stats_common, tx_mcast_bytes)}, {"tx_broadcast_bytes", - offsetof(struct ecore_eth_stats, tx_bcast_bytes)}, - {"tx_unicast_packets", offsetof(struct ecore_eth_stats, tx_ucast_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_bcast_bytes)}, + {"tx_unicast_packets", + offsetof(struct ecore_eth_stats_common, tx_ucast_pkts)}, {"tx_multicast_packets", - offsetof(struct ecore_eth_stats, tx_mcast_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_mcast_pkts)}, {"tx_broadcast_packets", - offsetof(struct ecore_eth_stats, tx_bcast_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_bcast_pkts)}, {"rx_64_byte_packets", - offsetof(struct ecore_eth_stats, rx_64_byte_packets)}, + offsetof(struct ecore_eth_stats_common, rx_64_byte_packets)}, {"rx_65_to_127_byte_packets", - offsetof(struct ecore_eth_stats, rx_65_to_127_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_65_to_127_byte_packets)}, {"rx_128_to_255_byte_packets", - offsetof(struct ecore_eth_stats, rx_128_to_255_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_128_to_255_byte_packets)}, {"rx_256_to_511_byte_packets", - offsetof(struct ecore_eth_stats, rx_256_to_511_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_256_to_511_byte_packets)}, {"rx_512_to_1023_byte_packets", - offsetof(struct ecore_eth_stats, rx_512_to_1023_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_512_to_1023_byte_packets)}, {"rx_1024_to_1518_byte_packets", - offsetof(struct ecore_eth_stats, rx_1024_to_1518_byte_packets)}, - {"rx_1519_to_1522_byte_packets", - offsetof(struct ecore_eth_stats, rx_1519_to_1522_byte_packets)}, - {"rx_1519_to_2047_byte_packets", - offsetof(struct ecore_eth_stats, rx_1519_to_2047_byte_packets)}, - {"rx_2048_to_4095_byte_packets", - offsetof(struct ecore_eth_stats, rx_2048_to_4095_byte_packets)}, - {"rx_4096_to_9216_byte_packets", - offsetof(struct ecore_eth_stats, rx_4096_to_9216_byte_packets)}, - {"rx_9217_to_16383_byte_packets", - offsetof(struct ecore_eth_stats, - rx_9217_to_16383_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_1024_to_1518_byte_packets)}, {"tx_64_byte_packets", - offsetof(struct ecore_eth_stats, tx_64_byte_packets)}, + offsetof(struct ecore_eth_stats_common, tx_64_byte_packets)}, {"tx_65_to_127_byte_packets", - offsetof(struct ecore_eth_stats, tx_65_to_127_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_65_to_127_byte_packets)}, {"tx_128_to_255_byte_packets", - offsetof(struct ecore_eth_stats, tx_128_to_255_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_128_to_255_byte_packets)}, {"tx_256_to_511_byte_packets", - offsetof(struct ecore_eth_stats, tx_256_to_511_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_256_to_511_byte_packets)}, {"tx_512_to_1023_byte_packets", - offsetof(struct ecore_eth_stats, tx_512_to_1023_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_512_to_1023_byte_packets)}, {"tx_1024_to_1518_byte_packets", - offsetof(struct ecore_eth_stats, tx_1024_to_1518_byte_packets)}, - {"trx_1519_to_1522_byte_packets", - offsetof(struct ecore_eth_stats, tx_1519_to_2047_byte_packets)}, - {"tx_2048_to_4095_byte_packets", - offsetof(struct ecore_eth_stats, tx_2048_to_4095_byte_packets)}, - {"tx_4096_to_9216_byte_packets", - offsetof(struct ecore_eth_stats, tx_4096_to_9216_byte_packets)}, - {"tx_9217_to_16383_byte_packets", - offsetof(struct ecore_eth_stats, - tx_9217_to_16383_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_1024_to_1518_byte_packets)}, {"rx_mac_crtl_frames", - offsetof(struct ecore_eth_stats, rx_mac_crtl_frames)}, + offsetof(struct ecore_eth_stats_common, rx_mac_crtl_frames)}, {"tx_mac_control_frames", - offsetof(struct ecore_eth_stats, tx_mac_ctrl_frames)}, - {"rx_pause_frames", offsetof(struct ecore_eth_stats, rx_pause_frames)}, - {"tx_pause_frames", offsetof(struct ecore_eth_stats, tx_pause_frames)}, + offsetof(struct ecore_eth_stats_common, tx_mac_ctrl_frames)}, + {"rx_pause_frames", + offsetof(struct ecore_eth_stats_common, rx_pause_frames)}, + {"tx_pause_frames", + offsetof(struct ecore_eth_stats_common, tx_pause_frames)}, {"rx_priority_flow_control_frames", - offsetof(struct ecore_eth_stats, rx_pfc_frames)}, + offsetof(struct ecore_eth_stats_common, rx_pfc_frames)}, {"tx_priority_flow_control_frames", - offsetof(struct ecore_eth_stats, tx_pfc_frames)}, + offsetof(struct ecore_eth_stats_common, tx_pfc_frames)}, - {"rx_crc_errors", offsetof(struct ecore_eth_stats, rx_crc_errors)}, - {"rx_align_errors", offsetof(struct ecore_eth_stats, rx_align_errors)}, + {"rx_crc_errors", + offsetof(struct ecore_eth_stats_common, rx_crc_errors)}, + {"rx_align_errors", + offsetof(struct ecore_eth_stats_common, rx_align_errors)}, {"rx_carrier_errors", - offsetof(struct ecore_eth_stats, rx_carrier_errors)}, + offsetof(struct ecore_eth_stats_common, rx_carrier_errors)}, {"rx_oversize_packet_errors", - offsetof(struct ecore_eth_stats, rx_oversize_packets)}, - {"rx_jabber_errors", offsetof(struct ecore_eth_stats, rx_jabbers)}, + offsetof(struct ecore_eth_stats_common, rx_oversize_packets)}, + {"rx_jabber_errors", + offsetof(struct ecore_eth_stats_common, rx_jabbers)}, {"rx_undersize_packet_errors", - offsetof(struct ecore_eth_stats, rx_undersize_packets)}, - {"rx_fragments", offsetof(struct ecore_eth_stats, rx_fragments)}, + offsetof(struct ecore_eth_stats_common, rx_undersize_packets)}, + {"rx_fragments", offsetof(struct ecore_eth_stats_common, rx_fragments)}, {"rx_host_buffer_not_available", - offsetof(struct ecore_eth_stats, no_buff_discards)}, + offsetof(struct ecore_eth_stats_common, no_buff_discards)}, /* Number of packets discarded because they are bigger than MTU */ {"rx_packet_too_big_discards", - offsetof(struct ecore_eth_stats, packet_too_big_discard)}, + offsetof(struct ecore_eth_stats_common, + packet_too_big_discard)}, {"rx_ttl_zero_discards", - offsetof(struct ecore_eth_stats, ttl0_discard)}, + offsetof(struct ecore_eth_stats_common, ttl0_discard)}, {"rx_multi_function_tag_filter_discards", - offsetof(struct ecore_eth_stats, mftag_filter_discards)}, + offsetof(struct ecore_eth_stats_common, mftag_filter_discards)}, {"rx_mac_filter_discards", - offsetof(struct ecore_eth_stats, mac_filter_discards)}, + offsetof(struct ecore_eth_stats_common, mac_filter_discards)}, {"rx_hw_buffer_truncates", - offsetof(struct ecore_eth_stats, brb_truncates)}, + offsetof(struct ecore_eth_stats_common, brb_truncates)}, {"rx_hw_buffer_discards", - offsetof(struct ecore_eth_stats, brb_discards)}, - {"tx_lpi_entry_count", - offsetof(struct ecore_eth_stats, tx_lpi_entry_count)}, - {"tx_total_collisions", - offsetof(struct ecore_eth_stats, tx_total_collisions)}, + offsetof(struct ecore_eth_stats_common, brb_discards)}, {"tx_error_drop_packets", - offsetof(struct ecore_eth_stats, tx_err_drop_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_err_drop_pkts)}, - {"rx_mac_bytes", offsetof(struct ecore_eth_stats, rx_mac_bytes)}, + {"rx_mac_bytes", offsetof(struct ecore_eth_stats_common, rx_mac_bytes)}, {"rx_mac_unicast_packets", - offsetof(struct ecore_eth_stats, rx_mac_uc_packets)}, + offsetof(struct ecore_eth_stats_common, rx_mac_uc_packets)}, {"rx_mac_multicast_packets", - offsetof(struct ecore_eth_stats, rx_mac_mc_packets)}, + offsetof(struct ecore_eth_stats_common, rx_mac_mc_packets)}, {"rx_mac_broadcast_packets", - offsetof(struct ecore_eth_stats, rx_mac_bc_packets)}, + offsetof(struct ecore_eth_stats_common, rx_mac_bc_packets)}, {"rx_mac_frames_ok", - offsetof(struct ecore_eth_stats, rx_mac_frames_ok)}, - {"tx_mac_bytes", offsetof(struct ecore_eth_stats, tx_mac_bytes)}, + offsetof(struct ecore_eth_stats_common, rx_mac_frames_ok)}, + {"tx_mac_bytes", offsetof(struct ecore_eth_stats_common, tx_mac_bytes)}, {"tx_mac_unicast_packets", - offsetof(struct ecore_eth_stats, tx_mac_uc_packets)}, + offsetof(struct ecore_eth_stats_common, tx_mac_uc_packets)}, {"tx_mac_multicast_packets", - offsetof(struct ecore_eth_stats, tx_mac_mc_packets)}, + offsetof(struct ecore_eth_stats_common, tx_mac_mc_packets)}, {"tx_mac_broadcast_packets", - offsetof(struct ecore_eth_stats, tx_mac_bc_packets)}, + offsetof(struct ecore_eth_stats_common, tx_mac_bc_packets)}, {"lro_coalesced_packets", - offsetof(struct ecore_eth_stats, tpa_coalesced_pkts)}, + offsetof(struct ecore_eth_stats_common, tpa_coalesced_pkts)}, {"lro_coalesced_events", - offsetof(struct ecore_eth_stats, tpa_coalesced_events)}, + offsetof(struct ecore_eth_stats_common, tpa_coalesced_events)}, {"lro_aborts_num", - offsetof(struct ecore_eth_stats, tpa_aborts_num)}, + offsetof(struct ecore_eth_stats_common, tpa_aborts_num)}, {"lro_not_coalesced_packets", - offsetof(struct ecore_eth_stats, tpa_not_coalesced_pkts)}, + offsetof(struct ecore_eth_stats_common, + tpa_not_coalesced_pkts)}, {"lro_coalesced_bytes", - offsetof(struct ecore_eth_stats, tpa_coalesced_bytes)}, + offsetof(struct ecore_eth_stats_common, + tpa_coalesced_bytes)}, +}; + +static const struct rte_qede_xstats_name_off qede_bb_xstats_strings[] = { + {"rx_1519_to_1522_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_1519_to_1522_byte_packets)}, + {"rx_1519_to_2047_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_1519_to_2047_byte_packets)}, + {"rx_2048_to_4095_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_2048_to_4095_byte_packets)}, + {"rx_4096_to_9216_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_4096_to_9216_byte_packets)}, + {"rx_9217_to_16383_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_9217_to_16383_byte_packets)}, + + {"tx_1519_to_2047_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_1519_to_2047_byte_packets)}, + {"tx_2048_to_4095_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_2048_to_4095_byte_packets)}, + {"tx_4096_to_9216_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_4096_to_9216_byte_packets)}, + {"tx_9217_to_16383_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_9217_to_16383_byte_packets)}, + + {"tx_lpi_entry_count", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, tx_lpi_entry_count)}, + {"tx_total_collisions", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, tx_total_collisions)}, +}; + +static const struct rte_qede_xstats_name_off qede_ah_xstats_strings[] = { + {"rx_1519_to_max_byte_packets", + offsetof(struct ecore_eth_stats, ah) + + offsetof(struct ecore_eth_stats_ah, + rx_1519_to_max_byte_packets)}, + {"tx_1519_to_max_byte_packets", + offsetof(struct ecore_eth_stats, ah) + + offsetof(struct ecore_eth_stats_ah, + tx_1519_to_max_byte_packets)}, }; static const struct rte_qede_xstats_name_off qede_rxq_xstats_strings[] = { @@ -294,7 +350,6 @@ static void qede_alloc_etherdev(struct qede_dev *qdev, struct qed_dev_eth_info *info) { rte_memcpy(&qdev->dev_info, info, sizeof(*info)); - qdev->num_tc = qdev->dev_info.num_tc; qdev->ops = qed_ops; } @@ -308,9 +363,10 @@ static void qede_print_adapter_info(struct qede_dev *qdev) DP_INFO(edev, "*********************************\n"); DP_INFO(edev, " DPDK version:%s\n", rte_version()); - DP_INFO(edev, " Chip details : %s%d\n", + DP_INFO(edev, " Chip details : %s %c%d\n", ECORE_IS_BB(edev) ? "BB" : "AH", - CHIP_REV_IS_A0(edev) ? 0 : 1); + 'A' + edev->chip_rev, + (int)edev->chip_metal); snprintf(ver_str, QEDE_PMD_DRV_VER_STR_SIZE, "%d.%d.%d.%d", info->fw_major, info->fw_minor, info->fw_rev, info->fw_eng); snprintf(drv_ver, QEDE_PMD_DRV_VER_STR_SIZE, "%s_%s", @@ -329,6 +385,178 @@ static void qede_print_adapter_info(struct qede_dev *qdev) } #endif +static int +qede_start_vport(struct qede_dev *qdev, uint16_t mtu) +{ + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_start_params params; + struct ecore_hwfn *p_hwfn; + int rc; + int i; + + memset(¶ms, 0, sizeof(params)); + params.vport_id = 0; + params.mtu = mtu; + /* @DPDK - Disable FW placement */ + params.zero_placement_offset = 1; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.concrete_fid = p_hwfn->hw_info.concrete_fid; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_start(p_hwfn, ¶ms); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Start V-PORT failed %d\n", rc); + return rc; + } + } + ecore_reset_vport_stats(edev); + DP_INFO(edev, "VPORT started with MTU = %u\n", mtu); + + return 0; +} + +static int +qede_stop_vport(struct ecore_dev *edev) +{ + struct ecore_hwfn *p_hwfn; + uint8_t vport_id; + int rc; + int i; + + vport_id = 0; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + rc = ecore_sp_vport_stop(p_hwfn, p_hwfn->hw_info.opaque_fid, + vport_id); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Stop V-PORT failed rc = %d\n", rc); + return rc; + } + } + + return 0; +} + +/* Activate or deactivate vport via vport-update */ +int qede_activate_vport(struct rte_eth_dev *eth_dev, bool flg) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + uint8_t i; + int rc = -1; + + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.update_vport_active_rx_flg = 1; + params.update_vport_active_tx_flg = 1; + params.vport_active_rx_flg = flg; + params.vport_active_tx_flg = flg; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update vport\n"); + break; + } + } + DP_INFO(edev, "vport %s\n", flg ? "activated" : "deactivated"); + return rc; +} + +static void +qede_update_sge_tpa_params(struct ecore_sge_tpa_params *sge_tpa_params, + uint16_t mtu, bool enable) +{ + /* Enable LRO in split mode */ + sge_tpa_params->tpa_ipv4_en_flg = enable; + sge_tpa_params->tpa_ipv6_en_flg = enable; + sge_tpa_params->tpa_ipv4_tunn_en_flg = false; + sge_tpa_params->tpa_ipv6_tunn_en_flg = false; + /* set if tpa enable changes */ + sge_tpa_params->update_tpa_en_flg = 1; + /* set if tpa parameters should be handled */ + sge_tpa_params->update_tpa_param_flg = enable; + + sge_tpa_params->max_buffers_per_cqe = 20; + /* Enable TPA in split mode. In this mode each TPA segment + * starts on the new BD, so there is one BD per segment. + */ + sge_tpa_params->tpa_pkt_split_flg = 1; + sge_tpa_params->tpa_hdr_data_split_flg = 0; + sge_tpa_params->tpa_gro_consistent_flg = 0; + sge_tpa_params->tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM; + sge_tpa_params->tpa_max_size = 0x7FFF; + sge_tpa_params->tpa_min_size_to_start = mtu / 2; + sge_tpa_params->tpa_min_size_to_cont = mtu / 2; +} + +/* Enable/disable LRO via vport-update */ +int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_sge_tpa_params tpa_params; + struct ecore_hwfn *p_hwfn; + int rc; + int i; + + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + memset(&tpa_params, 0, sizeof(struct ecore_sge_tpa_params)); + qede_update_sge_tpa_params(&tpa_params, qdev->mtu, flg); + params.vport_id = 0; + params.sge_tpa_params = &tpa_params; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update LRO\n"); + return -1; + } + } + + DP_INFO(edev, "LRO is %s\n", flg ? "enabled" : "disabled"); + + return 0; +} + +/* Update MTU via vport-update without doing port restart. + * The vport must be deactivated before calling this API. + */ +int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + int rc; + int i; + + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.mtu = mtu; + params.vport_id = 0; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update MTU\n"); + return -1; + } + } + DP_INFO(edev, "MTU updated to %u\n", mtu); + + return 0; +} + static void qede_set_ucast_cmn_params(struct ecore_filter_ucast *ucast) { memset(ucast, 0, sizeof(struct ecore_filter_ucast)); @@ -337,6 +565,43 @@ static void qede_set_ucast_cmn_params(struct ecore_filter_ucast *ucast) /* ucast->assert_on_error = true; - For debug */ } +static int +qed_configure_filter_rx_mode(struct rte_eth_dev *eth_dev, + enum qed_filter_rx_mode_type type) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_filter_accept_flags flags; + + memset(&flags, 0, sizeof(flags)); + + flags.update_rx_mode_config = 1; + flags.update_tx_mode_config = 1; + flags.rx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | + ECORE_ACCEPT_MCAST_MATCHED | + ECORE_ACCEPT_BCAST; + + flags.tx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | + ECORE_ACCEPT_MCAST_MATCHED | + ECORE_ACCEPT_BCAST; + + if (type == QED_FILTER_RX_MODE_TYPE_PROMISC) { + flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; + if (IS_VF(edev)) { + flags.tx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; + DP_INFO(edev, "Enabling Tx unmatched flag for VF\n"); + } + } else if (type == QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC) { + flags.rx_accept_filter |= ECORE_ACCEPT_MCAST_UNMATCHED; + } else if (type == (QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC | + QED_FILTER_RX_MODE_TYPE_PROMISC)) { + flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED | + ECORE_ACCEPT_MCAST_UNMATCHED; + } + + return ecore_filter_accept_cmd(edev, 0, flags, false, false, + ECORE_SPQ_MODE_CB, NULL); +} static void qede_set_cmn_tunn_param(struct ecore_tunnel_info *p_tunn, uint8_t clss, bool mode, bool mask) { @@ -363,6 +628,7 @@ qede_ucast_filter(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast, SLIST_FOREACH(tmp, &qdev->uc_list_head, list) { if ((memcmp(mac_addr, &tmp->mac, ETHER_ADDR_LEN) == 0) && + ucast->vni == tmp->vni && ucast->vlan == tmp->vlan) { DP_ERR(edev, "Unicast MAC is already added" " with vlan = %u, vni = %u\n", @@ -565,49 +831,57 @@ qede_mac_addr_set(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr) qede_mac_addr_add(eth_dev, mac_addr, 0, 0); } -static void qede_config_accept_any_vlan(struct qede_dev *qdev, bool action) +static void qede_config_accept_any_vlan(struct qede_dev *qdev, bool flg) { - struct ecore_dev *edev = &qdev->edev; - struct qed_update_vport_params params = { - .vport_id = 0, - .accept_any_vlan = action, - .update_accept_any_vlan_flg = 1, - }; + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + uint8_t i; int rc; - /* Proceed only if action actually needs to be performed */ - if (qdev->accept_any_vlan == action) - return; - - rc = qdev->ops->vport_update(edev, ¶ms); - if (rc) { - DP_ERR(edev, "Failed to %s accept-any-vlan\n", - action ? "enable" : "disable"); - } else { - DP_INFO(edev, "%s accept-any-vlan\n", - action ? "enabled" : "disabled"); - qdev->accept_any_vlan = action; + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.update_accept_any_vlan_flg = 1; + params.accept_any_vlan = flg; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to configure accept-any-vlan\n"); + return; + } } + + DP_INFO(edev, "%s accept-any-vlan\n", flg ? "enabled" : "disabled"); } -static int qede_vlan_stripping(struct rte_eth_dev *eth_dev, bool set_stripping) +static int qede_vlan_stripping(struct rte_eth_dev *eth_dev, bool flg) { - struct qed_update_vport_params vport_update_params; struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + uint8_t i; int rc; - memset(&vport_update_params, 0, sizeof(vport_update_params)); - vport_update_params.vport_id = 0; - vport_update_params.update_inner_vlan_removal_flg = 1; - vport_update_params.inner_vlan_removal_flg = set_stripping; - rc = qdev->ops->vport_update(edev, &vport_update_params); - if (rc) { - DP_ERR(edev, "Update V-PORT failed %d\n", rc); - return rc; + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.update_inner_vlan_removal_flg = 1; + params.inner_vlan_removal_flg = flg; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update vport\n"); + return -1; + } } - qdev->vlan_strip_flg = set_stripping; + DP_INFO(edev, "VLAN stripping %s\n", flg ? "enabled" : "disabled"); return 0; } @@ -741,33 +1015,6 @@ static void qede_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) mask, rxmode->hw_vlan_strip, rxmode->hw_vlan_filter); } -static int qede_init_vport(struct qede_dev *qdev) -{ - struct ecore_dev *edev = &qdev->edev; - struct qed_start_vport_params start = {0}; - int rc; - - start.remove_inner_vlan = 1; - start.enable_lro = qdev->enable_lro; - start.mtu = ETHER_MTU + QEDE_ETH_OVERHEAD; - start.vport_id = 0; - start.drop_ttl0 = false; - start.clear_stats = 1; - start.handle_ptp_pkts = 0; - - rc = qdev->ops->vport_start(edev, &start); - if (rc) { - DP_ERR(edev, "Start V-PORT failed %d\n", rc); - return rc; - } - - DP_INFO(edev, - "Start vport ramrod passed, vport_id = %d, MTU = %u\n", - start.vport_id, ETHER_MTU); - - return 0; -} - static void qede_prandom_bytes(uint32_t *buff) { uint8_t i; @@ -818,33 +1065,119 @@ int qede_config_rss(struct rte_eth_dev *eth_dev) return 0; } +static void qede_fastpath_start(struct ecore_dev *edev) +{ + struct ecore_hwfn *p_hwfn; + int i; + + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + ecore_hw_start_fastpath(p_hwfn); + } +} + +static int qede_dev_start(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + + PMD_INIT_FUNC_TRACE(edev); + + /* Update MTU only if it has changed */ + if (qdev->mtu != qdev->new_mtu) { + if (qede_update_mtu(eth_dev, qdev->new_mtu)) + goto err; + qdev->mtu = qdev->new_mtu; + /* If MTU has changed then update TPA too */ + if (qdev->enable_lro) + if (qede_enable_tpa(eth_dev, true)) + goto err; + } + + /* Start queues */ + if (qede_start_queues(eth_dev)) + goto err; + + /* Newer SR-IOV PF driver expects RX/TX queues to be started before + * enabling RSS. Hence RSS configuration is deferred upto this point. + * Also, we would like to retain similar behavior in PF case, so we + * don't do PF/VF specific check here. + */ + if (eth_dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) + if (qede_config_rss(eth_dev)) + goto err; + + /* Enable vport*/ + if (qede_activate_vport(eth_dev, true)) + goto err; + + /* Bring-up the link */ + qede_dev_set_link_state(eth_dev, true); + + /* Start/resume traffic */ + qede_fastpath_start(edev); + + DP_INFO(edev, "Device started\n"); + + return 0; +err: + DP_ERR(edev, "Device start fails\n"); + return -1; /* common error code is < 0 */ +} + +static void qede_dev_stop(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + + PMD_INIT_FUNC_TRACE(edev); + + /* Disable vport */ + if (qede_activate_vport(eth_dev, false)) + return; + + if (qdev->enable_lro) + qede_enable_tpa(eth_dev, false); + + /* TODO: Do we need disable LRO or RSS */ + /* Stop queues */ + qede_stop_queues(eth_dev); + + /* Disable traffic */ + ecore_hw_stop_fastpath(edev); /* TBD - loop */ + + /* Bring the link down */ + qede_dev_set_link_state(eth_dev, false); + + DP_INFO(edev, "Device is stopped\n"); +} + static int qede_dev_configure(struct rte_eth_dev *eth_dev) { - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); struct rte_eth_rxmode *rxmode = ð_dev->data->dev_conf.rxmode; - int rc; PMD_INIT_FUNC_TRACE(edev); /* Check requirements for 100G mode */ if (edev->num_hwfns > 1) { if (eth_dev->data->nb_rx_queues < 2 || - eth_dev->data->nb_tx_queues < 2) { + eth_dev->data->nb_tx_queues < 2) { DP_ERR(edev, "100G mode needs min. 2 RX/TX queues\n"); return -EINVAL; } if ((eth_dev->data->nb_rx_queues % 2 != 0) || - (eth_dev->data->nb_tx_queues % 2 != 0)) { + (eth_dev->data->nb_tx_queues % 2 != 0)) { DP_ERR(edev, - "100G mode needs even no. of RX/TX queues\n"); + "100G mode needs even no. of RX/TX queues\n"); return -EINVAL; } } /* Sanity checks and throw warnings */ - if (rxmode->enable_scatter == 1) + if (rxmode->enable_scatter) eth_dev->data->scattered_rx = 1; if (!rxmode->hw_strip_crc) @@ -852,83 +1185,77 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev) if (!rxmode->hw_ip_checksum) DP_INFO(edev, "IP/UDP/TCP checksum offload is always enabled " - "in hw\n"); - - if (rxmode->enable_lro) { - qdev->enable_lro = true; - /* Enable scatter mode for LRO */ - if (!rxmode->enable_scatter) - eth_dev->data->scattered_rx = 1; + "in hw\n"); + if (rxmode->header_split) + DP_INFO(edev, "Header split enable is not supported\n"); + if (!(rxmode->mq_mode == ETH_MQ_RX_NONE || rxmode->mq_mode == + ETH_MQ_RX_RSS)) { + DP_ERR(edev, "Unsupported multi-queue mode\n"); + return -ENOTSUP; } + /* Flow director mode check */ + if (qede_check_fdir_support(eth_dev)) + return -ENOTSUP; - /* Check for the port restart case */ - if (qdev->state != QEDE_DEV_INIT) { - rc = qdev->ops->vport_stop(edev, 0); - if (rc != 0) - return rc; + /* Deallocate resources if held previously. It is needed only if the + * queue count has been changed from previous configuration. If its + * going to change then it means RX/TX queue setup will be called + * again and the fastpath pointers will be reinitialized there. + */ + if (qdev->num_tx_queues != eth_dev->data->nb_tx_queues || + qdev->num_rx_queues != eth_dev->data->nb_rx_queues) { qede_dealloc_fp_resc(eth_dev); + /* Proceed with updated queue count */ + qdev->num_tx_queues = eth_dev->data->nb_tx_queues; + qdev->num_rx_queues = eth_dev->data->nb_rx_queues; + if (qede_alloc_fp_resc(qdev)) + return -ENOMEM; } - qdev->fp_num_tx = eth_dev->data->nb_tx_queues; - qdev->fp_num_rx = eth_dev->data->nb_rx_queues; - qdev->num_queues = qdev->fp_num_tx + qdev->fp_num_rx; - - /* Fastpath status block should be initialized before sending - * VPORT-START in the case of VF. Anyway, do it for both VF/PF. + /* VF's MTU has to be set using vport-start where as + * PF's MTU can be updated via vport-update. */ - rc = qede_alloc_fp_resc(qdev); - if (rc != 0) - return rc; - - /* Issue VPORT-START with default config values to allow - * other port configurations early on. - */ - rc = qede_init_vport(qdev); - if (rc != 0) - return rc; - - if (!(rxmode->mq_mode == ETH_MQ_RX_RSS || - rxmode->mq_mode == ETH_MQ_RX_NONE)) { - DP_ERR(edev, "Unsupported RSS mode\n"); - qdev->ops->vport_stop(edev, 0); - qede_dealloc_fp_resc(eth_dev); - return -EINVAL; + if (IS_VF(edev)) { + if (qede_start_vport(qdev, rxmode->max_rx_pkt_len)) + return -1; + } else { + if (qede_update_mtu(eth_dev, rxmode->max_rx_pkt_len)) + return -1; } - /* Flow director mode check */ - rc = qede_check_fdir_support(eth_dev); - if (rc) { - qdev->ops->vport_stop(edev, 0); - qede_dealloc_fp_resc(eth_dev); - return -EINVAL; - } - SLIST_INIT(&qdev->fdir_info.fdir_list_head); + qdev->mtu = rxmode->max_rx_pkt_len; + qdev->new_mtu = qdev->mtu; - SLIST_INIT(&qdev->vlan_list_head); + /* Configure TPA parameters */ + if (rxmode->enable_lro) { + if (qede_enable_tpa(eth_dev, true)) + return -EINVAL; + /* Enable scatter mode for LRO */ + if (!rxmode->enable_scatter) + eth_dev->data->scattered_rx = 1; + } + qdev->enable_lro = rxmode->enable_lro; /* Enable VLAN offloads by default */ qede_vlan_offload_set(eth_dev, ETH_VLAN_STRIP_MASK | - ETH_VLAN_FILTER_MASK | - ETH_VLAN_EXTEND_MASK); + ETH_VLAN_FILTER_MASK | + ETH_VLAN_EXTEND_MASK); - qdev->state = QEDE_DEV_CONFIG; - - DP_INFO(edev, "Allocated RSS=%d TSS=%d (with CoS=%d)\n", - (int)QEDE_RSS_COUNT(qdev), (int)QEDE_TSS_COUNT(qdev), - qdev->num_tc); + DP_INFO(edev, "Device configured with RSS=%d TSS=%d\n", + QEDE_RSS_COUNT(qdev), QEDE_TSS_COUNT(qdev)); return 0; } /* Info about HW descriptor ring limitations */ static const struct rte_eth_desc_lim qede_rx_desc_lim = { - .nb_max = NUM_RX_BDS_MAX, + .nb_max = 0x8000, /* 32K */ .nb_min = 128, .nb_align = 128 /* lowest common multiple */ }; static const struct rte_eth_desc_lim qede_tx_desc_lim = { - .nb_max = NUM_TX_BDS_MAX, + .nb_max = 0x8000, /* 32K */ .nb_min = 256, .nb_align = 256, .nb_seg_max = ETH_TX_MAX_BDS_PER_LSO_PACKET, @@ -946,7 +1273,7 @@ qede_dev_info_get(struct rte_eth_dev *eth_dev, PMD_INIT_FUNC_TRACE(edev); - dev_info->pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); dev_info->min_rx_bufsize = (uint32_t)QEDE_MIN_RX_BUFF_SIZE; dev_info->max_rx_pktlen = (uint32_t)ETH_TX_MAX_NON_LSO_PKT_LEN; dev_info->rx_desc_lim = qede_rx_desc_lim; @@ -1103,44 +1430,34 @@ static void qede_poll_sp_sb_cb(void *param) static void qede_dev_close(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - int rc; PMD_INIT_FUNC_TRACE(edev); - qede_fdir_dealloc_resc(eth_dev); - /* dev_stop() shall cleanup fp resources in hw but without releasing * dma memories and sw structures so that dev_start() can be called * by the app without reconfiguration. However, in dev_close() we * can release all the resources and device can be brought up newly */ - if (qdev->state != QEDE_DEV_STOP) + if (eth_dev->data->dev_started) qede_dev_stop(eth_dev); - else - DP_INFO(edev, "Device is already stopped\n"); - - rc = qdev->ops->vport_stop(edev, 0); - if (rc != 0) - DP_ERR(edev, "Failed to stop VPORT\n"); + qede_stop_vport(edev); + qede_fdir_dealloc_resc(eth_dev); qede_dealloc_fp_resc(eth_dev); - qdev->ops->common->slowpath_stop(edev); + eth_dev->data->nb_rx_queues = 0; + eth_dev->data->nb_tx_queues = 0; + qdev->ops->common->slowpath_stop(edev); qdev->ops->common->remove(edev); - rte_intr_disable(&pci_dev->intr_handle); - rte_intr_callback_unregister(&pci_dev->intr_handle, qede_interrupt_handler, (void *)eth_dev); - if (edev->num_hwfns > 1) rte_eal_alarm_cancel(qede_poll_sp_sb_cb, (void *)eth_dev); - - qdev->state = QEDE_DEV_INIT; /* Go back to init state */ } static void @@ -1153,35 +1470,36 @@ qede_get_stats(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats) unsigned int rxq_stat_cntrs, txq_stat_cntrs; struct qede_tx_queue *txq; - qdev->ops->get_vport_stats(edev, &stats); + ecore_get_vport_stats(edev, &stats); /* RX Stats */ - eth_stats->ipackets = stats.rx_ucast_pkts + - stats.rx_mcast_pkts + stats.rx_bcast_pkts; + eth_stats->ipackets = stats.common.rx_ucast_pkts + + stats.common.rx_mcast_pkts + stats.common.rx_bcast_pkts; - eth_stats->ibytes = stats.rx_ucast_bytes + - stats.rx_mcast_bytes + stats.rx_bcast_bytes; + eth_stats->ibytes = stats.common.rx_ucast_bytes + + stats.common.rx_mcast_bytes + stats.common.rx_bcast_bytes; - eth_stats->ierrors = stats.rx_crc_errors + - stats.rx_align_errors + - stats.rx_carrier_errors + - stats.rx_oversize_packets + - stats.rx_jabbers + stats.rx_undersize_packets; + eth_stats->ierrors = stats.common.rx_crc_errors + + stats.common.rx_align_errors + + stats.common.rx_carrier_errors + + stats.common.rx_oversize_packets + + stats.common.rx_jabbers + stats.common.rx_undersize_packets; - eth_stats->rx_nombuf = stats.no_buff_discards; + eth_stats->rx_nombuf = stats.common.no_buff_discards; - eth_stats->imissed = stats.mftag_filter_discards + - stats.mac_filter_discards + - stats.no_buff_discards + stats.brb_truncates + stats.brb_discards; + eth_stats->imissed = stats.common.mftag_filter_discards + + stats.common.mac_filter_discards + + stats.common.no_buff_discards + + stats.common.brb_truncates + stats.common.brb_discards; /* TX stats */ - eth_stats->opackets = stats.tx_ucast_pkts + - stats.tx_mcast_pkts + stats.tx_bcast_pkts; + eth_stats->opackets = stats.common.tx_ucast_pkts + + stats.common.tx_mcast_pkts + stats.common.tx_bcast_pkts; - eth_stats->obytes = stats.tx_ucast_bytes + - stats.tx_mcast_bytes + stats.tx_bcast_bytes; + eth_stats->obytes = stats.common.tx_ucast_bytes + + stats.common.tx_mcast_bytes + stats.common.tx_bcast_bytes; - eth_stats->oerrors = stats.tx_err_drop_pkts; + eth_stats->oerrors = stats.common.tx_err_drop_pkts; /* Queue stats */ rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev), @@ -1195,38 +1513,34 @@ qede_get_stats(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats) " RTE_ETHDEV_QUEUE_STAT_CNTRS config param" " appropriately and retry.\n"); - for (qid = 0; qid < QEDE_QUEUE_CNT(qdev); qid++) { - if (qdev->fp_array[qid].type & QEDE_FASTPATH_RX) { - eth_stats->q_ipackets[i] = - *(uint64_t *)( - ((char *)(qdev->fp_array[(qid)].rxq)) + - offsetof(struct qede_rx_queue, - rcv_pkts)); - eth_stats->q_errors[i] = - *(uint64_t *)( - ((char *)(qdev->fp_array[(qid)].rxq)) + - offsetof(struct qede_rx_queue, - rx_hw_errors)) + - *(uint64_t *)( - ((char *)(qdev->fp_array[(qid)].rxq)) + - offsetof(struct qede_rx_queue, - rx_alloc_errors)); - i++; - } + for_each_rss(qid) { + eth_stats->q_ipackets[i] = + *(uint64_t *)( + ((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, + rcv_pkts)); + eth_stats->q_errors[i] = + *(uint64_t *)( + ((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, + rx_hw_errors)) + + *(uint64_t *)( + ((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, + rx_alloc_errors)); + i++; if (i == rxq_stat_cntrs) break; } - for (qid = 0; qid < QEDE_QUEUE_CNT(qdev); qid++) { - if (qdev->fp_array[qid].type & QEDE_FASTPATH_TX) { - txq = qdev->fp_array[(qid)].txqs[0]; - eth_stats->q_opackets[j] = - *((uint64_t *)(uintptr_t) - (((uint64_t)(uintptr_t)(txq)) + - offsetof(struct qede_tx_queue, - xmit_pkts))); - j++; - } + for_each_tss(qid) { + txq = qdev->fp_array[qid].txq; + eth_stats->q_opackets[j] = + *((uint64_t *)(uintptr_t) + (((uint64_t)(uintptr_t)(txq)) + + offsetof(struct qede_tx_queue, + xmit_pkts))); + j++; if (j == txq_stat_cntrs) break; } @@ -1234,18 +1548,27 @@ qede_get_stats(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats) static unsigned qede_get_xstats_count(struct qede_dev *qdev) { - return RTE_DIM(qede_xstats_strings) + - (RTE_DIM(qede_rxq_xstats_strings) * - RTE_MIN(QEDE_RSS_COUNT(qdev), - RTE_ETHDEV_QUEUE_STAT_CNTRS)); + if (ECORE_IS_BB(&qdev->edev)) + return RTE_DIM(qede_xstats_strings) + + RTE_DIM(qede_bb_xstats_strings) + + (RTE_DIM(qede_rxq_xstats_strings) * + RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS)); + else + return RTE_DIM(qede_xstats_strings) + + RTE_DIM(qede_ah_xstats_strings) + + (RTE_DIM(qede_rxq_xstats_strings) * + RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS)); } static int -qede_get_xstats_names(__rte_unused struct rte_eth_dev *dev, +qede_get_xstats_names(struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned int limit) { struct qede_dev *qdev = dev->data->dev_private; + struct ecore_dev *edev = &qdev->edev; const unsigned int stat_cnt = qede_get_xstats_count(qdev); unsigned int i, qid, stat_idx = 0; unsigned int rxq_stat_cntrs; @@ -1259,6 +1582,24 @@ qede_get_xstats_names(__rte_unused struct rte_eth_dev *dev, stat_idx++; } + if (ECORE_IS_BB(edev)) { + for (i = 0; i < RTE_DIM(qede_bb_xstats_strings); i++) { + snprintf(xstats_names[stat_idx].name, + sizeof(xstats_names[stat_idx].name), + "%s", + qede_bb_xstats_strings[i].name); + stat_idx++; + } + } else { + for (i = 0; i < RTE_DIM(qede_ah_xstats_strings); i++) { + snprintf(xstats_names[stat_idx].name, + sizeof(xstats_names[stat_idx].name), + "%s", + qede_ah_xstats_strings[i].name); + stat_idx++; + } + } + rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev), RTE_ETHDEV_QUEUE_STAT_CNTRS); for (qid = 0; qid < rxq_stat_cntrs; qid++) { @@ -1290,7 +1631,7 @@ qede_get_xstats(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, if (n < num) return num; - qdev->ops->get_vport_stats(edev, &stats); + ecore_get_vport_stats(edev, &stats); for (i = 0; i < RTE_DIM(qede_xstats_strings); i++) { xstats[stat_idx].value = *(uint64_t *)(((char *)&stats) + @@ -1299,13 +1640,31 @@ qede_get_xstats(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, stat_idx++; } + if (ECORE_IS_BB(edev)) { + for (i = 0; i < RTE_DIM(qede_bb_xstats_strings); i++) { + xstats[stat_idx].value = + *(uint64_t *)(((char *)&stats) + + qede_bb_xstats_strings[i].offset); + xstats[stat_idx].id = stat_idx; + stat_idx++; + } + } else { + for (i = 0; i < RTE_DIM(qede_ah_xstats_strings); i++) { + xstats[stat_idx].value = + *(uint64_t *)(((char *)&stats) + + qede_ah_xstats_strings[i].offset); + xstats[stat_idx].id = stat_idx; + stat_idx++; + } + } + rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev), RTE_ETHDEV_QUEUE_STAT_CNTRS); for (qid = 0; qid < rxq_stat_cntrs; qid++) { - if (qdev->fp_array[qid].type & QEDE_FASTPATH_RX) { + for_each_rss(qid) { for (i = 0; i < RTE_DIM(qede_rxq_xstats_strings); i++) { xstats[stat_idx].value = *(uint64_t *)( - ((char *)(qdev->fp_array[(qid)].rxq)) + + ((char *)(qdev->fp_array[qid].rxq)) + qede_rxq_xstats_strings[i].offset); xstats[stat_idx].id = stat_idx; stat_idx++; @@ -1723,6 +2082,8 @@ static int qede_rss_reta_query(struct rte_eth_dev *eth_dev, return 0; } + + static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { struct qede_dev *qdev = QEDE_INIT_QDEV(dev); @@ -1756,19 +2117,17 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) rte_delay_ms(1000); qdev->mtu = mtu; /* Fix up RX buf size for all queues of the port */ - for_each_queue(i) { + for_each_rss(i) { fp = &qdev->fp_array[i]; - if (fp->type & QEDE_FASTPATH_RX) { - bufsz = (uint16_t)rte_pktmbuf_data_room_size( - fp->rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; - if (dev->data->scattered_rx) - rx_buf_size = bufsz + QEDE_ETH_OVERHEAD; - else - rx_buf_size = mtu + QEDE_ETH_OVERHEAD; - rx_buf_size = QEDE_CEIL_TO_CACHE_LINE_SIZE(rx_buf_size); - fp->rxq->rx_buf_size = rx_buf_size; - DP_INFO(edev, "buf_size adjusted to %u\n", rx_buf_size); - } + bufsz = (uint16_t)rte_pktmbuf_data_room_size( + fp->rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; + if (dev->data->scattered_rx) + rx_buf_size = bufsz + QEDE_ETH_OVERHEAD; + else + rx_buf_size = mtu + QEDE_ETH_OVERHEAD; + rx_buf_size = QEDE_CEIL_TO_CACHE_LINE_SIZE(rx_buf_size); + fp->rxq->rx_buf_size = rx_buf_size; + DP_INFO(edev, "buf_size adjusted to %u\n", rx_buf_size); } qede_dev_start(dev); if (frame_size > ETHER_MAX_LEN) @@ -1910,6 +2269,8 @@ static int qede_vxlan_tunn_config(struct rte_eth_dev *eth_dev, uint16_t filter_type; int rc, i; + PMD_INIT_FUNC_TRACE(edev); + filter_type = conf->filter_type | qdev->vxlan_filter_type; /* First determine if the given filter classification is supported */ qede_get_ecore_tunn_params(filter_type, &type, &clss, str); @@ -2163,7 +2524,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) /* Extract key data structures */ adapter = eth_dev->data->dev_private; edev = &adapter->edev; - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); pci_addr = pci_dev->addr; PMD_INIT_FUNC_TRACE(edev); @@ -2177,8 +2538,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) eth_dev->tx_pkt_prepare = qede_xmit_prep_pkts; if (rte_eal_process_type() != RTE_PROC_PRIMARY) { - DP_NOTICE(edev, false, - "Skipping device init from secondary process\n"); + DP_ERR(edev, "Skipping device init from secondary process\n"); return 0; } @@ -2195,20 +2555,15 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) } DP_INFO(edev, "Starting qede probe\n"); - - rc = qed_ops->common->probe(edev, pci_dev, QED_PROTOCOL_ETH, - dp_module, dp_level, is_vf); - + rc = qed_ops->common->probe(edev, pci_dev, dp_module, + dp_level, is_vf); if (rc != 0) { DP_ERR(edev, "qede probe failed rc %d\n", rc); return -ENODEV; } - qede_update_pf_params(edev); - rte_intr_callback_register(&pci_dev->intr_handle, qede_interrupt_handler, (void *)eth_dev); - if (rte_intr_enable(&pci_dev->intr_handle)) { DP_ERR(edev, "rte_intr_enable() failed\n"); return -ENODEV; @@ -2306,8 +2661,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) ether_addr_copy(ð_dev->data->mac_addrs[0], &adapter->primary_mac); } else { - DP_NOTICE(edev, false, - "No VF macaddr assigned\n"); + DP_ERR(edev, "No VF macaddr assigned\n"); } } } @@ -2321,17 +2675,28 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) do_once = false; } - adapter->state = QEDE_DEV_INIT; + adapter->num_tx_queues = 0; + adapter->num_rx_queues = 0; + SLIST_INIT(&adapter->fdir_info.fdir_list_head); + SLIST_INIT(&adapter->vlan_list_head); + SLIST_INIT(&adapter->uc_list_head); + adapter->mtu = ETHER_MTU; + adapter->new_mtu = ETHER_MTU; + if (!is_vf) + if (qede_start_vport(adapter, adapter->mtu)) + return -1; - DP_NOTICE(edev, false, "MAC address : %02x:%02x:%02x:%02x:%02x:%02x\n", - adapter->primary_mac.addr_bytes[0], - adapter->primary_mac.addr_bytes[1], - adapter->primary_mac.addr_bytes[2], - adapter->primary_mac.addr_bytes[3], - adapter->primary_mac.addr_bytes[4], - adapter->primary_mac.addr_bytes[5]); + DP_INFO(edev, "MAC address : %02x:%02x:%02x:%02x:%02x:%02x\n", + adapter->primary_mac.addr_bytes[0], + adapter->primary_mac.addr_bytes[1], + adapter->primary_mac.addr_bytes[2], + adapter->primary_mac.addr_bytes[3], + adapter->primary_mac.addr_bytes[4], + adapter->primary_mac.addr_bytes[5]); - return rc; + DP_INFO(edev, "Device initialized\n"); + + return 0; } static int qedevf_eth_dev_init(struct rte_eth_dev *eth_dev) @@ -2346,6 +2711,13 @@ static int qede_eth_dev_init(struct rte_eth_dev *eth_dev) static int qede_dev_common_uninit(struct rte_eth_dev *eth_dev) { +#ifdef RTE_LIBRTE_QEDE_DEBUG_INIT + struct qede_dev *qdev = eth_dev->data->dev_private; + struct ecore_dev *edev = &qdev->edev; + + PMD_INIT_FUNC_TRACE(edev); +#endif + /* only uninitialize in the primary process */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; diff --git a/drivers/net/qede/qede_ethdev.h b/drivers/net/qede/qede_ethdev.h index e4323a0d..a3254b12 100644 --- a/drivers/net/qede/qede_ethdev.h +++ b/drivers/net/qede/qede_ethdev.h @@ -41,8 +41,6 @@ #include "qede_logs.h" #include "qede_if.h" -#include "qede_eth_if.h" - #include "qede_rxtx.h" #define qede_stringify1(x...) #x @@ -51,8 +49,8 @@ /* Driver versions */ #define QEDE_PMD_VER_PREFIX "QEDE PMD" #define QEDE_PMD_VERSION_MAJOR 2 -#define QEDE_PMD_VERSION_MINOR 4 -#define QEDE_PMD_VERSION_REVISION 0 +#define QEDE_PMD_VERSION_MINOR 5 +#define QEDE_PMD_VERSION_REVISION 2 #define QEDE_PMD_VERSION_PATCH 1 #define QEDE_PMD_VERSION qede_stringify(QEDE_PMD_VERSION_MAJOR) "." \ @@ -73,12 +71,8 @@ (edev)->dev_info.num_tc) #define QEDE_QUEUE_CNT(qdev) ((qdev)->num_queues) -#define QEDE_RSS_COUNT(qdev) ((qdev)->num_queues - (qdev)->fp_num_tx) -#define QEDE_TSS_COUNT(qdev) (((qdev)->num_queues - (qdev)->fp_num_rx) * \ - (qdev)->num_tc) - -#define QEDE_FASTPATH_TX (1 << 0) -#define QEDE_FASTPATH_RX (1 << 1) +#define QEDE_RSS_COUNT(qdev) ((qdev)->num_rx_queues) +#define QEDE_TSS_COUNT(qdev) ((qdev)->num_tx_queues) #define QEDE_DUPLEX_FULL 1 #define QEDE_DUPLEX_HALF 2 @@ -138,12 +132,12 @@ extern char fw_file[]; /* Maximum number of flowdir filters */ #define QEDE_RFS_MAX_FLTR (256) -/* Port/function states */ -enum qede_dev_state { - QEDE_DEV_INIT, /* Init the chip and Slowpath */ - QEDE_DEV_CONFIG, /* Create Vport/Fastpath resources */ - QEDE_DEV_START, /* Start RX/TX queues, enable traffic */ - QEDE_DEV_STOP, /* Deactivate vport and stop traffic */ +#define QEDE_MAX_MCAST_FILTERS (64) + +enum qed_filter_rx_mode_type { + QED_FILTER_RX_MODE_TYPE_REGULAR, + QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC, + QED_FILTER_RX_MODE_TYPE_PROMISC, }; struct qede_vlan_entry { @@ -183,23 +177,20 @@ struct qede_fdir_info { */ struct qede_dev { struct ecore_dev edev; - uint8_t protocol; const struct qed_eth_ops *ops; struct qed_dev_eth_info dev_info; struct ecore_sb_info *sb_array; struct qede_fastpath *fp_array; - uint8_t num_tc; uint16_t mtu; + uint16_t new_mtu; bool rss_enable; struct rte_eth_rss_conf rss_conf; uint16_t rss_ind_table[ECORE_RSS_IND_TABLE_SIZE]; uint64_t rss_hf; uint8_t rss_key_len; bool enable_lro; - uint16_t num_queues; - uint8_t fp_num_tx; - uint8_t fp_num_rx; - enum qede_dev_state state; + uint8_t num_rx_queues; + uint8_t num_tx_queues; SLIST_HEAD(vlan_list_head, qede_vlan_entry)vlan_list_head; uint16_t configured_vlans; bool accept_any_vlan; @@ -248,4 +239,10 @@ uint16_t qede_fdir_construct_pkt(struct rte_eth_dev *eth_dev, void qede_fdir_dealloc_resc(struct rte_eth_dev *eth_dev); +int qede_activate_vport(struct rte_eth_dev *eth_dev, bool flg); + +int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu); + +int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg); + #endif /* _QEDE_ETHDEV_H_ */ diff --git a/drivers/net/qede/qede_if.h b/drivers/net/qede/qede_if.h index 405c525e..9864bb44 100644 --- a/drivers/net/qede/qede_if.h +++ b/drivers/net/qede/qede_if.h @@ -50,14 +50,26 @@ struct qed_dev_info { bool geneve_enable; }; -enum qed_sb_type { - QED_SB_TYPE_L2_QUEUE, - QED_SB_TYPE_STORAGE, - QED_SB_TYPE_CNQ, +struct qed_dev_eth_info { + struct qed_dev_info common; + + uint8_t num_queues; + uint8_t num_tc; + + struct ether_addr port_mac; + uint16_t num_vlan_filters; + uint32_t num_mac_filters; + + /* Legacy VF - this affects the datapath */ + bool is_legacy; }; -enum qed_protocol { - QED_PROTOCOL_ETH, +#define INIT_STRUCT_FIELD(field, value) .field = value + +struct qed_eth_ops { + const struct qed_common_ops *common; + int (*fill_dev_info)(struct ecore_dev *edev, + struct qed_dev_eth_info *info); }; struct qed_link_params { @@ -99,64 +111,13 @@ struct qed_slowpath_params { uint8_t name[NAME_SIZE]; }; -#define ILT_PAGE_SIZE_TCFC 0x8000 /* 32KB */ - -struct qed_eth_tlvs { - u16 feat_flags; - u8 mac[3][ETH_ALEN]; - u16 lso_maxoff; - u16 lso_minseg; - bool prom_mode; - u16 num_txqs; - u16 num_rxqs; - u16 num_netqs; - u16 flex_vlan; - u32 tcp4_offloads; - u32 tcp6_offloads; - u16 tx_avg_qdepth; - u16 rx_avg_qdepth; - u8 txqs_empty; - u8 rxqs_empty; - u8 num_txqs_full; - u8 num_rxqs_full; -}; - -struct qed_tunn_update_params { - unsigned long tunn_mode_update_mask; - unsigned long tunn_mode; - u16 vxlan_udp_port; - u16 geneve_udp_port; - u8 update_rx_pf_clss; - u8 update_tx_pf_clss; - u8 update_vxlan_udp_port; - u8 update_geneve_udp_port; - u8 tunn_clss_vxlan; - u8 tunn_clss_l2geneve; - u8 tunn_clss_ipgeneve; - u8 tunn_clss_l2gre; - u8 tunn_clss_ipgre; -}; - struct qed_common_cb_ops { void (*link_update)(void *dev, struct qed_link_output *link); - void (*get_tlv_data)(void *dev, struct qed_eth_tlvs *data); -}; - -struct qed_selftest_ops { -/** - * @brief registers - Perform register tests - * - * @param edev - * - * @return 0 on success, error otherwise. - */ - int (*registers)(struct ecore_dev *edev); }; struct qed_common_ops { int (*probe)(struct ecore_dev *edev, struct rte_pci_device *pci_dev, - enum qed_protocol protocol, uint32_t dp_module, uint8_t dp_level, bool is_vf); void (*set_name)(struct ecore_dev *edev, char name[]); enum _ecore_status_t @@ -196,7 +157,7 @@ struct qed_common_ops { struct ecore_sb_info *sb_info, void *sb_virt_addr, dma_addr_t sb_phy_addr, - uint16_t sb_id, enum qed_sb_type type); + uint16_t sb_id); int (*get_sb_info)(struct ecore_dev *edev, struct ecore_sb_info *sb, u16 qid, @@ -210,4 +171,8 @@ struct qed_common_ops { int (*send_drv_state)(struct ecore_dev *edev, bool active); }; +/* Externs */ + +const struct qed_eth_ops *qed_get_eth_ops(void); + #endif /* _QEDE_IF_H */ diff --git a/drivers/net/qede/qede_logs.h b/drivers/net/qede/qede_logs.h index 25c14d8b..15821151 100644 --- a/drivers/net/qede/qede_logs.h +++ b/drivers/net/qede/qede_logs.h @@ -16,16 +16,21 @@ (p_dev)->name ? (p_dev)->name : "", \ ##__VA_ARGS__) -#ifdef RTE_LIBRTE_QEDE_DEBUG_INFO #define DP_NOTICE(p_dev, is_assert, fmt, ...) \ - rte_log(RTE_LOG_NOTICE, RTE_LOGTYPE_PMD,\ - "[QEDE PMD: (%s)]%s:" fmt, \ - (p_dev)->name ? (p_dev)->name : "", \ - __func__, \ - ##__VA_ARGS__) -#else -#define DP_NOTICE(p_dev, fmt, ...) do { } while (0) -#endif +do { \ + if (is_assert) \ + rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD,\ + "[QEDE PMD: (%s)]%s:" fmt, \ + (p_dev)->name ? (p_dev)->name : "", \ + __func__, \ + ##__VA_ARGS__); \ + else \ + rte_log(RTE_LOG_NOTICE, RTE_LOGTYPE_PMD,\ + "[QEDE PMD: (%s)]%s:" fmt, \ + (p_dev)->name ? (p_dev)->name : "", \ + __func__, \ + ##__VA_ARGS__); \ +} while (0) #ifdef RTE_LIBRTE_QEDE_DEBUG_INFO #define DP_INFO(p_dev, fmt, ...) \ diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c index 712c03fd..a6ff7af2 100644 --- a/drivers/net/qede/qede_main.c +++ b/drivers/net/qede/qede_main.c @@ -19,7 +19,7 @@ char fw_file[PATH_MAX]; const char *QEDE_DEFAULT_FIRMWARE = - "/lib/firmware/qed/qed_init_values-8.18.9.0.bin"; + "/lib/firmware/qed/qed_init_values-8.20.0.0.bin"; static void qed_update_pf_params(struct ecore_dev *edev, struct ecore_pf_params *params) @@ -40,16 +40,14 @@ static void qed_init_pci(struct ecore_dev *edev, struct rte_pci_device *pci_dev) static int qed_probe(struct ecore_dev *edev, struct rte_pci_device *pci_dev, - enum qed_protocol protocol, uint32_t dp_module, - uint8_t dp_level, bool is_vf) + uint32_t dp_module, uint8_t dp_level, bool is_vf) { struct ecore_hw_prepare_params hw_prepare_params; - struct qede_dev *qdev = (struct qede_dev *)edev; int rc; ecore_init_struct(edev); edev->drv_type = DRV_ID_DRV_TYPE_LINUX; - qdev->protocol = protocol; + /* Protocol type is always fixed to PROTOCOL_ETH */ if (is_vf) edev->b_is_vf = true; @@ -62,6 +60,7 @@ qed_probe(struct ecore_dev *edev, struct rte_pci_device *pci_dev, hw_prepare_params.drv_resc_alloc = false; hw_prepare_params.chk_reg_fifo = false; hw_prepare_params.initiate_pf_flr = true; + hw_prepare_params.allow_mdump = false; hw_prepare_params.epoch = (u32)time(NULL); rc = ecore_hw_prepare(edev, &hw_prepare_params); if (rc) { @@ -132,12 +131,12 @@ static int qed_load_firmware_data(struct ecore_dev *edev) fd = open(fw_file, O_RDONLY); if (fd < 0) { - DP_NOTICE(edev, false, "Can't open firmware file\n"); + DP_ERR(edev, "Can't open firmware file\n"); return -ENOENT; } if (fstat(fd, &st) < 0) { - DP_NOTICE(edev, false, "Can't stat firmware file\n"); + DP_ERR(edev, "Can't stat firmware file\n"); close(fd); return -1; } @@ -145,20 +144,20 @@ static int qed_load_firmware_data(struct ecore_dev *edev) edev->firmware = rte_zmalloc("qede_fw", st.st_size, RTE_CACHE_LINE_SIZE); if (!edev->firmware) { - DP_NOTICE(edev, false, "Can't allocate memory for firmware\n"); + DP_ERR(edev, "Can't allocate memory for firmware\n"); close(fd); return -ENOMEM; } if (read(fd, edev->firmware, st.st_size) != st.st_size) { - DP_NOTICE(edev, false, "Can't read firmware data\n"); + DP_ERR(edev, "Can't read firmware data\n"); close(fd); return -1; } edev->fw_len = st.st_size; if (edev->fw_len < 104) { - DP_NOTICE(edev, false, "Invalid fw size: %" PRIu64 "\n", + DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n", edev->fw_len); close(fd); return -EINVAL; @@ -262,8 +261,7 @@ static int qed_slowpath_start(struct ecore_dev *edev, /* Allocate stream for unzipping */ rc = qed_alloc_stream_mem(edev); if (rc) { - DP_NOTICE(edev, true, - "Failed to allocate stream memory\n"); + DP_ERR(edev, "Failed to allocate stream memory\n"); goto err1; } } @@ -303,8 +301,7 @@ static int qed_slowpath_start(struct ecore_dev *edev, rc = ecore_mcp_send_drv_version(hwfn, hwfn->p_main_ptt, &drv_version); if (rc) { - DP_NOTICE(edev, true, - "Failed sending drv version command\n"); + DP_ERR(edev, "Failed sending drv version command\n"); goto err3; } } @@ -460,23 +457,14 @@ static void qed_set_name(struct ecore_dev *edev, char name[NAME_SIZE]) static uint32_t qed_sb_init(struct ecore_dev *edev, struct ecore_sb_info *sb_info, - void *sb_virt_addr, dma_addr_t sb_phy_addr, - uint16_t sb_id, enum qed_sb_type type) + void *sb_virt_addr, dma_addr_t sb_phy_addr, uint16_t sb_id) { struct ecore_hwfn *p_hwfn; int hwfn_index; uint16_t rel_sb_id; - uint8_t n_hwfns; + uint8_t n_hwfns = edev->num_hwfns; uint32_t rc; - /* RoCE uses single engine and CMT uses two engines. When using both - * we force only a single engine. Storage uses only engine 0 too. - */ - if (type == QED_SB_TYPE_L2_QUEUE) - n_hwfns = edev->num_hwfns; - else - n_hwfns = 1; - hwfn_index = sb_id % n_hwfns; p_hwfn = &edev->hwfns[hwfn_index]; rel_sb_id = sb_id / n_hwfns; @@ -617,7 +605,7 @@ static int qed_drain(struct ecore_dev *edev) hwfn = &edev->hwfns[i]; ptt = ecore_ptt_acquire(hwfn); if (!ptt) { - DP_NOTICE(hwfn, true, "Failed to drain NIG; No PTT\n"); + DP_ERR(hwfn, "Failed to drain NIG; No PTT\n"); return -EBUSY; } rc = ecore_mcp_drain(hwfn, ptt); @@ -710,7 +698,7 @@ static int qed_get_sb_info(struct ecore_dev *edev, struct ecore_sb_info *sb, ptt = ecore_ptt_acquire(hwfn); if (!ptt) { - DP_NOTICE(hwfn, true, "Can't acquire PTT\n"); + DP_ERR(hwfn, "Can't acquire PTT\n"); return -EAGAIN; } @@ -737,3 +725,13 @@ const struct qed_common_ops qed_common_ops_pass = { INIT_STRUCT_FIELD(remove, &qed_remove), INIT_STRUCT_FIELD(send_drv_state, &qed_send_drv_state), }; + +const struct qed_eth_ops qed_eth_ops_pass = { + INIT_STRUCT_FIELD(common, &qed_common_ops_pass), + INIT_STRUCT_FIELD(fill_dev_info, &qed_fill_eth_dev_info), +}; + +const struct qed_eth_ops *qed_get_eth_ops(void) +{ + return &qed_eth_ops_pass; +} diff --git a/drivers/net/qede/qede_rxtx.c b/drivers/net/qede/qede_rxtx.c index f5aa43dd..5c3613c7 100644 --- a/drivers/net/qede/qede_rxtx.c +++ b/drivers/net/qede/qede_rxtx.c @@ -37,64 +37,20 @@ static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq) return 0; } -static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq) -{ - uint16_t i; - - if (rxq->sw_rx_ring != NULL) { - for (i = 0; i < rxq->nb_rx_desc; i++) { - if (rxq->sw_rx_ring[i].mbuf != NULL) { - rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf); - rxq->sw_rx_ring[i].mbuf = NULL; - } - } - } -} - -void qede_rx_queue_release(void *rx_queue) -{ - struct qede_rx_queue *rxq = rx_queue; - - if (rxq != NULL) { - qede_rx_queue_release_mbufs(rxq); - rte_free(rxq->sw_rx_ring); - rxq->sw_rx_ring = NULL; - rte_free(rxq); - rxq = NULL; - } -} - -static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq) -{ - unsigned int i; - - PMD_TX_LOG(DEBUG, txq, "releasing %u mbufs", txq->nb_tx_desc); - - if (txq->sw_tx_ring) { - for (i = 0; i < txq->nb_tx_desc; i++) { - if (txq->sw_tx_ring[i].mbuf) { - rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf); - txq->sw_tx_ring[i].mbuf = NULL; - } - } - } -} - int qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, __rte_unused const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { - struct qede_dev *qdev = dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; + struct qede_dev *qdev = QEDE_INIT_QDEV(dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; struct qede_rx_queue *rxq; uint16_t max_rx_pkt_len; uint16_t bufsz; size_t size; int rc; - int i; PMD_INIT_FUNC_TRACE(edev); @@ -126,6 +82,7 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, rxq->nb_rx_desc = nb_desc; rxq->queue_id = queue_idx; rxq->port_id = dev->data->port_id; + max_rx_pkt_len = (uint16_t)rxmode->max_rx_pkt_len; qdev->mtu = max_rx_pkt_len; @@ -138,6 +95,7 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, dev->data->scattered_rx = 1; } } + if (dev->data->scattered_rx) rxq->rx_buf_size = bufsz + QEDE_ETH_OVERHEAD; else @@ -153,11 +111,9 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, rxq->sw_rx_ring = rte_zmalloc_socket("sw_rx_ring", size, RTE_CACHE_LINE_SIZE, socket_id); if (!rxq->sw_rx_ring) { - DP_NOTICE(edev, false, - "Unable to alloc memory for sw_rx_ring on socket %u\n", - socket_id); + DP_ERR(edev, "Memory allocation fails for sw_rx_ring on" + " socket %u\n", socket_id); rte_free(rxq); - rxq = NULL; return -ENOMEM; } @@ -172,13 +128,10 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, NULL); if (rc != ECORE_SUCCESS) { - DP_NOTICE(edev, false, - "Unable to alloc memory for rxbd ring on socket %u\n", - socket_id); + DP_ERR(edev, "Memory allocation fails for RX BD ring" + " on socket %u\n", socket_id); rte_free(rxq->sw_rx_ring); - rxq->sw_rx_ring = NULL; rte_free(rxq); - rxq = NULL; return -ENOMEM; } @@ -193,50 +146,91 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, NULL); if (rc != ECORE_SUCCESS) { - DP_NOTICE(edev, false, - "Unable to alloc memory for cqe ring on socket %u\n", - socket_id); - /* TBD: Freeing RX BD ring */ + DP_ERR(edev, "Memory allocation fails for RX CQE ring" + " on socket %u\n", socket_id); + qdev->ops->common->chain_free(edev, &rxq->rx_bd_ring); rte_free(rxq->sw_rx_ring); - rxq->sw_rx_ring = NULL; rte_free(rxq); return -ENOMEM; } - /* Allocate buffers for the Rx ring */ - for (i = 0; i < rxq->nb_rx_desc; i++) { - rc = qede_alloc_rx_buffer(rxq); - if (rc) { - DP_NOTICE(edev, false, - "RX buffer allocation failed at idx=%d\n", i); - goto err4; - } - } - dev->data->rx_queues[queue_idx] = rxq; + qdev->fp_array[queue_idx].rxq = rxq; DP_INFO(edev, "rxq %d num_desc %u rx_buf_size=%u socket %u\n", queue_idx, nb_desc, qdev->mtu, socket_id); return 0; -err4: - qede_rx_queue_release(rxq); - return -ENOMEM; } -void qede_tx_queue_release(void *tx_queue) +static void +qede_rx_queue_reset(__rte_unused struct qede_dev *qdev, + struct qede_rx_queue *rxq) { - struct qede_tx_queue *txq = tx_queue; + DP_INFO(&qdev->edev, "Reset RX queue %u\n", rxq->queue_id); + ecore_chain_reset(&rxq->rx_bd_ring); + ecore_chain_reset(&rxq->rx_comp_ring); + rxq->sw_rx_prod = 0; + rxq->sw_rx_cons = 0; + *rxq->hw_cons_ptr = 0; +} - if (txq != NULL) { - qede_tx_queue_release_mbufs(txq); - if (txq->sw_tx_ring) { - rte_free(txq->sw_tx_ring); - txq->sw_tx_ring = NULL; +static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq) +{ + uint16_t i; + + if (rxq->sw_rx_ring) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_rx_ring[i].mbuf) { + rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf); + rxq->sw_rx_ring[i].mbuf = NULL; + } } - rte_free(txq); } - txq = NULL; +} + +void qede_rx_queue_release(void *rx_queue) +{ + struct qede_rx_queue *rxq = rx_queue; + + if (rxq) { + qede_rx_queue_release_mbufs(rxq); + rte_free(rxq->sw_rx_ring); + rte_free(rxq); + } +} + +/* Stops a given RX queue in the HW */ +static int qede_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_hwfn *p_hwfn; + struct qede_rx_queue *rxq; + int hwfn_index; + int rc; + + if (rx_queue_id < eth_dev->data->nb_rx_queues) { + rxq = eth_dev->data->rx_queues[rx_queue_id]; + hwfn_index = rx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + rc = ecore_eth_rx_queue_stop(p_hwfn, rxq->handle, + true, false); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "RX queue %u stop fails\n", rx_queue_id); + return -1; + } + qede_rx_queue_release_mbufs(rxq); + qede_rx_queue_reset(qdev, rxq); + eth_dev->data->rx_queue_state[rx_queue_id] = + RTE_ETH_QUEUE_STATE_STOPPED; + DP_INFO(edev, "RX queue %u stopped\n", rx_queue_id); + } else { + DP_ERR(edev, "RX queue %u is not in range\n", rx_queue_id); + rc = -EINVAL; + } + + return rc; } int @@ -305,6 +299,7 @@ qede_tx_queue_setup(struct rte_eth_dev *dev, DP_ERR(edev, "Unable to allocate memory for txbd ring on socket %u", socket_id); + qdev->ops->common->chain_free(edev, &txq->tx_pbl); qede_tx_queue_release(txq); return -ENOMEM; } @@ -318,6 +313,7 @@ qede_tx_queue_setup(struct rte_eth_dev *dev, (txq->nb_tx_desc - QEDE_DEFAULT_TX_FREE_THRESH); dev->data->tx_queues[queue_idx] = txq; + qdev->fp_array[queue_idx].txq = txq; DP_INFO(edev, "txq %u num_desc %u tx_free_thresh %u socket %u\n", @@ -326,71 +322,40 @@ qede_tx_queue_setup(struct rte_eth_dev *dev, return 0; } -/* This function inits fp content and resets the SB, RXQ and TXQ arrays */ -static void qede_init_fp(struct qede_dev *qdev) +static void +qede_tx_queue_reset(__rte_unused struct qede_dev *qdev, + struct qede_tx_queue *txq) { - struct qede_fastpath *fp; - uint8_t i; - int fp_rx = qdev->fp_num_rx; - - memset((void *)qdev->fp_array, 0, (QEDE_QUEUE_CNT(qdev) * - sizeof(*qdev->fp_array))); - memset((void *)qdev->sb_array, 0, (QEDE_QUEUE_CNT(qdev) * - sizeof(*qdev->sb_array))); - for_each_queue(i) { - fp = &qdev->fp_array[i]; - if (fp_rx) { - fp->type = QEDE_FASTPATH_RX; - fp_rx--; - } else{ - fp->type = QEDE_FASTPATH_TX; - } - fp->qdev = qdev; - fp->id = i; - fp->sb_info = &qdev->sb_array[i]; - snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", "qdev", i); - } - + DP_INFO(&qdev->edev, "Reset TX queue %u\n", txq->queue_id); + ecore_chain_reset(&txq->tx_pbl); + txq->sw_tx_cons = 0; + txq->sw_tx_prod = 0; + *txq->hw_cons_ptr = 0; } -void qede_free_fp_arrays(struct qede_dev *qdev) +static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq) { - /* It asseumes qede_free_mem_load() is called before */ - if (qdev->fp_array != NULL) { - rte_free(qdev->fp_array); - qdev->fp_array = NULL; - } + uint16_t i; - if (qdev->sb_array != NULL) { - rte_free(qdev->sb_array); - qdev->sb_array = NULL; + if (txq->sw_tx_ring) { + for (i = 0; i < txq->nb_tx_desc; i++) { + if (txq->sw_tx_ring[i].mbuf) { + rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf); + txq->sw_tx_ring[i].mbuf = NULL; + } + } } } -static int qede_alloc_fp_array(struct qede_dev *qdev) +void qede_tx_queue_release(void *tx_queue) { - struct ecore_dev *edev = &qdev->edev; - - qdev->fp_array = rte_calloc("fp", QEDE_QUEUE_CNT(qdev), - sizeof(*qdev->fp_array), - RTE_CACHE_LINE_SIZE); - - if (!qdev->fp_array) { - DP_ERR(edev, "fp array allocation failed\n"); - return -ENOMEM; - } - - qdev->sb_array = rte_calloc("sb", QEDE_QUEUE_CNT(qdev), - sizeof(*qdev->sb_array), - RTE_CACHE_LINE_SIZE); + struct qede_tx_queue *txq = tx_queue; - if (!qdev->sb_array) { - DP_ERR(edev, "sb array allocation failed\n"); - rte_free(qdev->fp_array); - return -ENOMEM; + if (txq) { + qede_tx_queue_release_mbufs(txq); + rte_free(txq->sw_tx_ring); + rte_free(txq); } - - return 0; } /* This function allocates fast-path status block memory */ @@ -398,24 +363,23 @@ static int qede_alloc_mem_sb(struct qede_dev *qdev, struct ecore_sb_info *sb_info, uint16_t sb_id) { - struct ecore_dev *edev = &qdev->edev; + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); struct status_block *sb_virt; dma_addr_t sb_phys; int rc; - sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, sizeof(*sb_virt)); - + sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, + sizeof(struct status_block)); if (!sb_virt) { DP_ERR(edev, "Status block allocation failed\n"); return -ENOMEM; } - - rc = qdev->ops->common->sb_init(edev, sb_info, - sb_virt, sb_phys, sb_id, - QED_SB_TYPE_L2_QUEUE); + rc = qdev->ops->common->sb_init(edev, sb_info, sb_virt, + sb_phys, sb_id); if (rc) { DP_ERR(edev, "Status block initialization failed\n"); - /* TBD: No dma_free_coherent possible */ + OSAL_DMA_FREE_COHERENT(edev, sb_virt, sb_phys, + sizeof(struct status_block)); return rc; } @@ -427,9 +391,7 @@ int qede_alloc_fp_resc(struct qede_dev *qdev) struct ecore_dev *edev = &qdev->edev; struct qede_fastpath *fp; uint32_t num_sbs; - uint16_t i; uint16_t sb_idx; - int rc; if (IS_VF(edev)) ecore_vf_get_num_sbs(ECORE_LEADING_HWFN(edev), &num_sbs); @@ -442,25 +404,31 @@ int qede_alloc_fp_resc(struct qede_dev *qdev) return -EINVAL; } - if (qdev->fp_array) - qede_free_fp_arrays(qdev); + qdev->fp_array = rte_calloc("fp", QEDE_RXTX_MAX(qdev), + sizeof(*qdev->fp_array), RTE_CACHE_LINE_SIZE); - rc = qede_alloc_fp_array(qdev); - if (rc != 0) - return rc; + if (!qdev->fp_array) { + DP_ERR(edev, "fp array allocation failed\n"); + return -ENOMEM; + } - qede_init_fp(qdev); + memset((void *)qdev->fp_array, 0, QEDE_RXTX_MAX(qdev) * + sizeof(*qdev->fp_array)); - for (i = 0; i < QEDE_QUEUE_CNT(qdev); i++) { - fp = &qdev->fp_array[i]; - if (IS_VF(edev)) - sb_idx = i % num_sbs; - else - sb_idx = i; + for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) { + fp = &qdev->fp_array[sb_idx]; + fp->sb_info = rte_calloc("sb", 1, sizeof(struct ecore_sb_info), + RTE_CACHE_LINE_SIZE); + if (!fp->sb_info) { + DP_ERR(edev, "FP sb_info allocation fails\n"); + return -1; + } if (qede_alloc_mem_sb(qdev, fp->sb_info, sb_idx)) { - qede_free_fp_arrays(qdev); - return -ENOMEM; + DP_ERR(edev, "FP status block allocation fails\n"); + return -1; } + DP_INFO(edev, "sb_info idx 0x%x initialized\n", + fp->sb_info->igu_sb_id); } return 0; @@ -469,9 +437,54 @@ int qede_alloc_fp_resc(struct qede_dev *qdev) void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev) { struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct qede_fastpath *fp; + struct qede_rx_queue *rxq; + struct qede_tx_queue *txq; + uint16_t sb_idx; + uint8_t i; - qede_free_mem_load(eth_dev); - qede_free_fp_arrays(qdev); + PMD_INIT_FUNC_TRACE(edev); + + for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) { + fp = &qdev->fp_array[sb_idx]; + DP_INFO(edev, "Free sb_info index 0x%x\n", + fp->sb_info->igu_sb_id); + if (fp->sb_info) { + OSAL_DMA_FREE_COHERENT(edev, fp->sb_info->sb_virt, + fp->sb_info->sb_phys, + sizeof(struct status_block)); + rte_free(fp->sb_info); + fp->sb_info = NULL; + } + } + + /* Free packet buffers and ring memories */ + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + if (eth_dev->data->rx_queues[i]) { + qede_rx_queue_release(eth_dev->data->rx_queues[i]); + rxq = eth_dev->data->rx_queues[i]; + qdev->ops->common->chain_free(edev, + &rxq->rx_bd_ring); + qdev->ops->common->chain_free(edev, + &rxq->rx_comp_ring); + eth_dev->data->rx_queues[i] = NULL; + } + } + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + if (eth_dev->data->tx_queues[i]) { + txq = eth_dev->data->tx_queues[i]; + qede_tx_queue_release(eth_dev->data->tx_queues[i]); + qdev->ops->common->chain_free(edev, + &txq->tx_pbl); + eth_dev->data->tx_queues[i] = NULL; + } + } + + if (qdev->fp_array) + rte_free(qdev->fp_array); + qdev->fp_array = NULL; } static inline void @@ -506,165 +519,297 @@ qede_update_rx_prod(__rte_unused struct qede_dev *edev, PMD_RX_LOG(DEBUG, rxq, "bd_prod %u cqe_prod %u", bd_prod, cqe_prod); } -static void -qede_update_sge_tpa_params(struct ecore_sge_tpa_params *sge_tpa_params, - uint16_t mtu, bool enable) -{ - /* Enable LRO in split mode */ - sge_tpa_params->tpa_ipv4_en_flg = enable; - sge_tpa_params->tpa_ipv6_en_flg = enable; - sge_tpa_params->tpa_ipv4_tunn_en_flg = false; - sge_tpa_params->tpa_ipv6_tunn_en_flg = false; - /* set if tpa enable changes */ - sge_tpa_params->update_tpa_en_flg = 1; - /* set if tpa parameters should be handled */ - sge_tpa_params->update_tpa_param_flg = enable; - - sge_tpa_params->max_buffers_per_cqe = 20; - /* Enable TPA in split mode. In this mode each TPA segment - * starts on the new BD, so there is one BD per segment. - */ - sge_tpa_params->tpa_pkt_split_flg = 1; - sge_tpa_params->tpa_hdr_data_split_flg = 0; - sge_tpa_params->tpa_gro_consistent_flg = 0; - sge_tpa_params->tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM; - sge_tpa_params->tpa_max_size = 0x7FFF; - sge_tpa_params->tpa_min_size_to_start = mtu / 2; - sge_tpa_params->tpa_min_size_to_cont = mtu / 2; -} - -static int qede_start_queues(struct rte_eth_dev *eth_dev, - __rte_unused bool clear_stats) +/* Starts a given RX queue in HW */ +static int +qede_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - struct ecore_queue_start_common_params q_params; - struct qed_dev_info *qed_info = &qdev->dev_info.common; - struct qed_update_vport_params vport_update_params; - struct ecore_sge_tpa_params tpa_params; - struct qede_tx_queue *txq; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_queue_start_common_params params; + struct ecore_rxq_start_ret_params ret_params; + struct qede_rx_queue *rxq; struct qede_fastpath *fp; + struct ecore_hwfn *p_hwfn; dma_addr_t p_phys_table; - int txq_index; uint16_t page_cnt; - int rc, tc, i; - - for_each_queue(i) { - fp = &qdev->fp_array[i]; - if (fp->type & QEDE_FASTPATH_RX) { - struct ecore_rxq_start_ret_params ret_params; - - p_phys_table = - ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring); - page_cnt = - ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring); - - memset(&ret_params, 0, sizeof(ret_params)); - memset(&q_params, 0, sizeof(q_params)); - q_params.queue_id = i; - q_params.vport_id = 0; - q_params.sb = fp->sb_info->igu_sb_id; - q_params.sb_idx = RX_PI; - - ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0); - - rc = qdev->ops->q_rx_start(edev, i, &q_params, - fp->rxq->rx_buf_size, - fp->rxq->rx_bd_ring.p_phys_addr, - p_phys_table, - page_cnt, - &ret_params); + uint16_t j; + int hwfn_index; + int rc; + + if (rx_queue_id < eth_dev->data->nb_rx_queues) { + fp = &qdev->fp_array[rx_queue_id]; + rxq = eth_dev->data->rx_queues[rx_queue_id]; + /* Allocate buffers for the Rx ring */ + for (j = 0; j < rxq->nb_rx_desc; j++) { + rc = qede_alloc_rx_buffer(rxq); if (rc) { - DP_ERR(edev, "Start rxq #%d failed %d\n", - fp->rxq->queue_id, rc); - return rc; + DP_ERR(edev, "RX buffer allocation failed" + " for rxq = %u\n", rx_queue_id); + return -ENOMEM; } + } + /* disable interrupts */ + ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0); + /* Prepare ramrod */ + memset(¶ms, 0, sizeof(params)); + params.queue_id = rx_queue_id / edev->num_hwfns; + params.vport_id = 0; + params.stats_id = params.vport_id; + params.sb = fp->sb_info->igu_sb_id; + DP_INFO(edev, "rxq %u igu_sb_id 0x%x\n", + fp->rxq->queue_id, fp->sb_info->igu_sb_id); + params.sb_idx = RX_PI; + hwfn_index = rx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring); + page_cnt = ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring); + memset(&ret_params, 0, sizeof(ret_params)); + rc = ecore_eth_rx_queue_start(p_hwfn, + p_hwfn->hw_info.opaque_fid, + ¶ms, fp->rxq->rx_buf_size, + fp->rxq->rx_bd_ring.p_phys_addr, + p_phys_table, page_cnt, + &ret_params); + if (rc) { + DP_ERR(edev, "RX queue %u could not be started, rc = %d\n", + rx_queue_id, rc); + return -1; + } + /* Update with the returned parameters */ + fp->rxq->hw_rxq_prod_addr = ret_params.p_prod; + fp->rxq->handle = ret_params.p_handle; + + fp->rxq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[RX_PI]; + qede_update_rx_prod(qdev, fp->rxq); + eth_dev->data->rx_queue_state[rx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + DP_INFO(edev, "RX queue %u started\n", rx_queue_id); + } else { + DP_ERR(edev, "RX queue %u is not in range\n", rx_queue_id); + rc = -EINVAL; + } - /* Use the return parameters */ - fp->rxq->hw_rxq_prod_addr = ret_params.p_prod; - fp->rxq->handle = ret_params.p_handle; + return rc; +} - fp->rxq->hw_cons_ptr = - &fp->sb_info->sb_virt->pi_array[RX_PI]; +static int +qede_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_queue_start_common_params params; + struct ecore_txq_start_ret_params ret_params; + struct ecore_hwfn *p_hwfn; + dma_addr_t p_phys_table; + struct qede_tx_queue *txq; + struct qede_fastpath *fp; + uint16_t page_cnt; + int hwfn_index; + int rc; - qede_update_rx_prod(qdev, fp->rxq); + if (tx_queue_id < eth_dev->data->nb_tx_queues) { + txq = eth_dev->data->tx_queues[tx_queue_id]; + fp = &qdev->fp_array[tx_queue_id]; + memset(¶ms, 0, sizeof(params)); + params.queue_id = tx_queue_id / edev->num_hwfns; + params.vport_id = 0; + params.stats_id = params.vport_id; + params.sb = fp->sb_info->igu_sb_id; + DP_INFO(edev, "txq %u igu_sb_id 0x%x\n", + fp->txq->queue_id, fp->sb_info->igu_sb_id); + params.sb_idx = TX_PI(0); /* tc = 0 */ + p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl); + page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl); + hwfn_index = tx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + if (qdev->dev_info.is_legacy) + fp->txq->is_legacy = true; + rc = ecore_eth_tx_queue_start(p_hwfn, + p_hwfn->hw_info.opaque_fid, + ¶ms, 0 /* tc */, + p_phys_table, page_cnt, + &ret_params); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "TX queue %u couldn't be started, rc=%d\n", + tx_queue_id, rc); + return -1; } + txq->doorbell_addr = ret_params.p_doorbell; + txq->handle = ret_params.p_handle; + + txq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[TX_PI(0)]; + SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST, + DB_DEST_XCM); + SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, + DB_AGG_CMD_SET); + SET_FIELD(txq->tx_db.data.params, + ETH_DB_DATA_AGG_VAL_SEL, + DQ_XCM_ETH_TX_BD_PROD_CMD); + txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD; + eth_dev->data->tx_queue_state[tx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + DP_INFO(edev, "TX queue %u started\n", tx_queue_id); + } else { + DP_ERR(edev, "TX queue %u is not in range\n", tx_queue_id); + rc = -EINVAL; + } - if (!(fp->type & QEDE_FASTPATH_TX)) - continue; - for (tc = 0; tc < qdev->num_tc; tc++) { - struct ecore_txq_start_ret_params ret_params; + return rc; +} - txq = fp->txqs[tc]; - txq_index = tc * QEDE_RSS_COUNT(qdev) + i; +static inline void +qede_free_tx_pkt(struct qede_tx_queue *txq) +{ + struct rte_mbuf *mbuf; + uint16_t nb_segs; + uint16_t idx; - p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl); - page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl); + idx = TX_CONS(txq); + mbuf = txq->sw_tx_ring[idx].mbuf; + if (mbuf) { + nb_segs = mbuf->nb_segs; + PMD_TX_LOG(DEBUG, txq, "nb_segs to free %u\n", nb_segs); + while (nb_segs) { + /* It's like consuming rxbuf in recv() */ + ecore_chain_consume(&txq->tx_pbl); + txq->nb_tx_avail++; + nb_segs--; + } + rte_pktmbuf_free(mbuf); + txq->sw_tx_ring[idx].mbuf = NULL; + txq->sw_tx_cons++; + PMD_TX_LOG(DEBUG, txq, "Freed tx packet\n"); + } else { + ecore_chain_consume(&txq->tx_pbl); + txq->nb_tx_avail++; + } +} - memset(&q_params, 0, sizeof(q_params)); - memset(&ret_params, 0, sizeof(ret_params)); - q_params.queue_id = txq->queue_id; - q_params.vport_id = 0; - q_params.sb = fp->sb_info->igu_sb_id; - q_params.sb_idx = TX_PI(tc); +static inline void +qede_process_tx_compl(__rte_unused struct ecore_dev *edev, + struct qede_tx_queue *txq) +{ + uint16_t hw_bd_cons; +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + uint16_t sw_tx_cons; +#endif - rc = qdev->ops->q_tx_start(edev, i, &q_params, - p_phys_table, - page_cnt, /* **pp_doorbell */ - &ret_params); - if (rc) { - DP_ERR(edev, "Start txq %u failed %d\n", - txq_index, rc); - return rc; + rte_compiler_barrier(); + hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr); +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + sw_tx_cons = ecore_chain_get_cons_idx(&txq->tx_pbl); + PMD_TX_LOG(DEBUG, txq, "Tx Completions = %u\n", + abs(hw_bd_cons - sw_tx_cons)); +#endif + while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) + qede_free_tx_pkt(txq); +} + +static int qede_drain_txq(struct qede_dev *qdev, + struct qede_tx_queue *txq, bool allow_drain) +{ + struct ecore_dev *edev = &qdev->edev; + int rc, cnt = 1000; + + while (txq->sw_tx_cons != txq->sw_tx_prod) { + qede_process_tx_compl(edev, txq); + if (!cnt) { + if (allow_drain) { + DP_ERR(edev, "Tx queue[%u] is stuck," + "requesting MCP to drain\n", + txq->queue_id); + rc = qdev->ops->common->drain(edev); + if (rc) + return rc; + return qede_drain_txq(qdev, txq, false); } + DP_ERR(edev, "Timeout waiting for tx queue[%d]:" + "PROD=%d, CONS=%d\n", + txq->queue_id, txq->sw_tx_prod, + txq->sw_tx_cons); + return -1; + } + cnt--; + DELAY(1000); + rte_compiler_barrier(); + } - txq->doorbell_addr = ret_params.p_doorbell; - txq->handle = ret_params.p_handle; + /* FW finished processing, wait for HW to transmit all tx packets */ + DELAY(2000); - txq->hw_cons_ptr = - &fp->sb_info->sb_virt->pi_array[TX_PI(tc)]; - SET_FIELD(txq->tx_db.data.params, - ETH_DB_DATA_DEST, DB_DEST_XCM); - SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, - DB_AGG_CMD_SET); - SET_FIELD(txq->tx_db.data.params, - ETH_DB_DATA_AGG_VAL_SEL, - DQ_XCM_ETH_TX_BD_PROD_CMD); + return 0; +} + +/* Stops a given TX queue in the HW */ +static int qede_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_hwfn *p_hwfn; + struct qede_tx_queue *txq; + int hwfn_index; + int rc; - txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD; + if (tx_queue_id < eth_dev->data->nb_tx_queues) { + txq = eth_dev->data->tx_queues[tx_queue_id]; + /* Drain txq */ + if (qede_drain_txq(qdev, txq, true)) + return -1; /* For the lack of retcodes */ + /* Stop txq */ + hwfn_index = tx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + rc = ecore_eth_tx_queue_stop(p_hwfn, txq->handle); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "TX queue %u stop fails\n", tx_queue_id); + return -1; } + qede_tx_queue_release_mbufs(txq); + qede_tx_queue_reset(qdev, txq); + eth_dev->data->tx_queue_state[tx_queue_id] = + RTE_ETH_QUEUE_STATE_STOPPED; + DP_INFO(edev, "TX queue %u stopped\n", tx_queue_id); + } else { + DP_ERR(edev, "TX queue %u is not in range\n", tx_queue_id); + rc = -EINVAL; } - /* Prepare and send the vport enable */ - memset(&vport_update_params, 0, sizeof(vport_update_params)); - /* Update MTU via vport update */ - vport_update_params.mtu = qdev->mtu; - vport_update_params.vport_id = 0; - vport_update_params.update_vport_active_flg = 1; - vport_update_params.vport_active_flg = 1; - - /* @DPDK */ - if (qed_info->mf_mode == MF_NPAR && qed_info->tx_switching) { - /* TBD: Check SRIOV enabled for VF */ - vport_update_params.update_tx_switching_flg = 1; - vport_update_params.tx_switching_flg = 1; + return rc; +} + +int qede_start_queues(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + uint8_t id; + int rc; + + for_each_rss(id) { + rc = qede_rx_queue_start(eth_dev, id); + if (rc != ECORE_SUCCESS) + return -1; } - /* TPA */ - if (qdev->enable_lro) { - DP_INFO(edev, "Enabling LRO\n"); - memset(&tpa_params, 0, sizeof(struct ecore_sge_tpa_params)); - qede_update_sge_tpa_params(&tpa_params, qdev->mtu, true); - vport_update_params.sge_tpa_params = &tpa_params; + for_each_tss(id) { + rc = qede_tx_queue_start(eth_dev, id); + if (rc != ECORE_SUCCESS) + return -1; } - rc = qdev->ops->vport_update(edev, &vport_update_params); - if (rc) { - DP_ERR(edev, "Update V-PORT failed %d\n", rc); - return rc; + return rc; +} + +void qede_stop_queues(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + uint8_t id; + + /* Stopping RX/TX queues */ + for_each_tss(id) { + qede_tx_queue_stop(eth_dev, id); } - return 0; + for_each_rss(id) { + qede_rx_queue_stop(eth_dev, id); + } } static bool qede_tunn_exist(uint16_t flag) @@ -937,7 +1082,7 @@ qede_process_sg_pkts(void *p_rxq, struct rte_mbuf *rx_mb, pkt_len; if (unlikely(!cur_size)) { PMD_RX_LOG(ERR, rxq, "Length is 0 while %u BDs" - " left for mapping jumbo", num_segs); + " left for mapping jumbo\n", num_segs); qede_recycle_rx_bd_ring(rxq, qdev, num_segs); return -EINVAL; } @@ -961,7 +1106,6 @@ qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) struct qede_rx_queue *rxq = p_rxq; struct qede_dev *qdev = rxq->qdev; struct ecore_dev *edev = &qdev->edev; - struct qede_fastpath *fp = &qdev->fp_array[rxq->queue_id]; uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index; uint16_t rx_pkt = 0; union eth_rx_cqe *cqe; @@ -1047,7 +1191,8 @@ qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) goto tpa_end; case ETH_RX_CQE_TYPE_SLOW_PATH: PMD_RX_LOG(INFO, rxq, "Got unexpected slowpath CQE\n"); - qdev->ops->eth_cqe_completion(edev, fp->id, + ecore_eth_cqe_completion( + &edev->hwfns[rxq->queue_id % edev->num_hwfns], (struct eth_slow_path_rx_cqe *)cqe); /* fall-thru */ default: @@ -1235,53 +1380,6 @@ next_cqe: return rx_pkt; } -static inline void -qede_free_tx_pkt(struct qede_tx_queue *txq) -{ - struct rte_mbuf *mbuf; - uint16_t nb_segs; - uint16_t idx; - - idx = TX_CONS(txq); - mbuf = txq->sw_tx_ring[idx].mbuf; - if (mbuf) { - nb_segs = mbuf->nb_segs; - PMD_TX_LOG(DEBUG, txq, "nb_segs to free %u\n", nb_segs); - while (nb_segs) { - /* It's like consuming rxbuf in recv() */ - ecore_chain_consume(&txq->tx_pbl); - txq->nb_tx_avail++; - nb_segs--; - } - rte_pktmbuf_free(mbuf); - txq->sw_tx_ring[idx].mbuf = NULL; - txq->sw_tx_cons++; - PMD_TX_LOG(DEBUG, txq, "Freed tx packet\n"); - } else { - ecore_chain_consume(&txq->tx_pbl); - txq->nb_tx_avail++; - } -} - -static inline void -qede_process_tx_compl(__rte_unused struct ecore_dev *edev, - struct qede_tx_queue *txq) -{ - uint16_t hw_bd_cons; -#ifdef RTE_LIBRTE_QEDE_DEBUG_TX - uint16_t sw_tx_cons; -#endif - - rte_compiler_barrier(); - hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr); -#ifdef RTE_LIBRTE_QEDE_DEBUG_TX - sw_tx_cons = ecore_chain_get_cons_idx(&txq->tx_pbl); - PMD_TX_LOG(DEBUG, txq, "Tx Completions = %u\n", - abs(hw_bd_cons - sw_tx_cons)); -#endif - while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) - qede_free_tx_pkt(txq); -} /* Populate scatter gather buffer descriptor fields */ static inline uint8_t @@ -1378,7 +1476,9 @@ qede_xmit_prep_pkts(__rte_unused void *p_txq, struct rte_mbuf **tx_pkts, uint64_t ol_flags; struct rte_mbuf *m; uint16_t i; +#ifdef RTE_LIBRTE_ETHDEV_DEBUG int ret; +#endif for (i = 0; i < nb_pkts; i++) { m = tx_pkts[i]; @@ -1411,14 +1511,6 @@ qede_xmit_prep_pkts(__rte_unused void *p_txq, struct rte_mbuf **tx_pkts, break; } #endif - /* TBD: pseudo csum calcuation required iff - * ETH_TX_DATA_2ND_BD_L4_PSEUDO_CSUM_MODE not set? - */ - ret = rte_net_intel_cksum_prepare(m); - if (ret != 0) { - rte_errno = ret; - break; - } } #ifdef RTE_LIBRTE_QEDE_DEBUG_TX @@ -1429,6 +1521,27 @@ qede_xmit_prep_pkts(__rte_unused void *p_txq, struct rte_mbuf **tx_pkts, return i; } +#define MPLSINUDP_HDR_SIZE (12) + +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX +static inline void +qede_mpls_tunn_tx_sanity_check(struct rte_mbuf *mbuf, + struct qede_tx_queue *txq) +{ + if (((mbuf->outer_l2_len + mbuf->outer_l3_len) / 2) > 0xff) + PMD_TX_LOG(ERR, txq, "tunn_l4_hdr_start_offset overflow\n"); + if (((mbuf->outer_l2_len + mbuf->outer_l3_len + + MPLSINUDP_HDR_SIZE) / 2) > 0xff) + PMD_TX_LOG(ERR, txq, "tunn_hdr_size overflow\n"); + if (((mbuf->l2_len - MPLSINUDP_HDR_SIZE) / 2) > + ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_MASK) + PMD_TX_LOG(ERR, txq, "inner_l2_hdr_size overflow\n"); + if (((mbuf->l2_len - MPLSINUDP_HDR_SIZE + mbuf->l3_len) / 2) > + ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_MASK) + PMD_TX_LOG(ERR, txq, "inner_l2_hdr_size overflow\n"); +} +#endif + uint16_t qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { @@ -1443,14 +1556,30 @@ qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) uint16_t nb_frags; uint16_t nb_pkt_sent = 0; uint8_t nbds; - bool ipv6_ext_flg; bool lso_flg; - bool tunn_flg; + bool mplsoudp_flg; + __rte_unused bool tunn_flg; + bool tunn_ipv6_ext_flg; struct eth_tx_1st_bd *bd1; struct eth_tx_2nd_bd *bd2; struct eth_tx_3rd_bd *bd3; uint64_t tx_ol_flags; uint16_t hdr_size; + /* BD1 */ + uint16_t bd1_bf; + uint8_t bd1_bd_flags_bf; + uint16_t vlan; + /* BD2 */ + uint16_t bd2_bf1; + uint16_t bd2_bf2; + /* BD3 */ + uint16_t mss; + uint16_t bd3_bf; + + uint8_t tunn_l4_hdr_start_offset; + uint8_t tunn_hdr_size; + uint8_t inner_l2_hdr_size; + uint16_t inner_l4_hdr_offset; if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) { PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u", @@ -1462,14 +1591,24 @@ qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) bd_prod = rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl)); while (nb_tx_pkts--) { /* Init flags/values */ - ipv6_ext_flg = false; tunn_flg = false; lso_flg = false; nbds = 0; + vlan = 0; bd1 = NULL; bd2 = NULL; bd3 = NULL; hdr_size = 0; + bd1_bf = 0; + bd1_bd_flags_bf = 0; + bd2_bf1 = 0; + bd2_bf2 = 0; + mss = 0; + bd3_bf = 0; + mplsoudp_flg = false; + tunn_ipv6_ext_flg = false; + tunn_hdr_size = 0; + tunn_l4_hdr_start_offset = 0; mbuf = *tx_pkts++; assert(mbuf); @@ -1479,36 +1618,177 @@ qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) break; tx_ol_flags = mbuf->ol_flags; + bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; -#define RTE_ETH_IS_IPV6_HDR_EXT(ptype) ((ptype) & RTE_PTYPE_L3_IPV6_EXT) - if (RTE_ETH_IS_IPV6_HDR_EXT(mbuf->packet_type)) - ipv6_ext_flg = true; - - if (RTE_ETH_IS_TUNNEL_PKT(mbuf->packet_type)) + /* TX prepare would have already checked supported tunnel Tx + * offloads. Don't rely on pkt_type marked by Rx, instead use + * tx_ol_flags to decide. + */ + if (((tx_ol_flags & PKT_TX_TUNNEL_MASK) == + PKT_TX_TUNNEL_VXLAN) || + ((tx_ol_flags & PKT_TX_TUNNEL_MASK) == + PKT_TX_TUNNEL_MPLSINUDP)) { + /* Check against max which is Tunnel IPv6 + ext */ + if (unlikely(txq->nb_tx_avail < + ETH_TX_MIN_BDS_PER_TUNN_IPV6_WITH_EXT_PKT)) + break; tunn_flg = true; + /* First indicate its a tunnel pkt */ + bd1_bf |= ETH_TX_DATA_1ST_BD_TUNN_FLAG_MASK << + ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT; + /* Legacy FW had flipped behavior in regard to this bit + * i.e. it needed to set to prevent FW from touching + * encapsulated packets when it didn't need to. + */ + if (unlikely(txq->is_legacy)) { + bd1_bf ^= 1 << + ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT; + } - if (tx_ol_flags & PKT_TX_TCP_SEG) - lso_flg = true; + /* Outer IP checksum offload */ + if (tx_ol_flags & (PKT_TX_OUTER_IP_CKSUM | + PKT_TX_OUTER_IPV4)) { + bd1_bd_flags_bf |= + ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_MASK << + ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_SHIFT; + } - if (lso_flg) { + /** + * Currently, only inner checksum offload in MPLS-in-UDP + * tunnel with one MPLS label is supported. Both outer + * and inner layers lengths need to be provided in + * mbuf. + */ + if ((tx_ol_flags & PKT_TX_TUNNEL_MASK) == + PKT_TX_TUNNEL_MPLSINUDP) { + mplsoudp_flg = true; +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + qede_mpls_tunn_tx_sanity_check(mbuf, txq); +#endif + /* Outer L4 offset in two byte words */ + tunn_l4_hdr_start_offset = + (mbuf->outer_l2_len + mbuf->outer_l3_len) / 2; + /* Tunnel header size in two byte words */ + tunn_hdr_size = (mbuf->outer_l2_len + + mbuf->outer_l3_len + + MPLSINUDP_HDR_SIZE) / 2; + /* Inner L2 header size in two byte words */ + inner_l2_hdr_size = (mbuf->l2_len - + MPLSINUDP_HDR_SIZE) / 2; + /* Inner L4 header offset from the beggining + * of inner packet in two byte words + */ + inner_l4_hdr_offset = (mbuf->l2_len - + MPLSINUDP_HDR_SIZE + mbuf->l3_len) / 2; + + /* Inner L2 size and address type */ + bd2_bf1 |= (inner_l2_hdr_size & + ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_MASK) << + ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_SHIFT; + bd2_bf1 |= (UNICAST_ADDRESS & + ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_MASK) << + ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_SHIFT; + /* Treated as IPv6+Ext */ + bd2_bf1 |= + 1 << ETH_TX_DATA_2ND_BD_TUNN_IPV6_EXT_SHIFT; + + /* Mark inner IPv6 if present */ + if (tx_ol_flags & PKT_TX_IPV6) + bd2_bf1 |= + 1 << ETH_TX_DATA_2ND_BD_TUNN_INNER_IPV6_SHIFT; + + /* Inner L4 offsets */ + if ((tx_ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) && + (tx_ol_flags & (PKT_TX_UDP_CKSUM | + PKT_TX_TCP_CKSUM))) { + /* Determines if BD3 is needed */ + tunn_ipv6_ext_flg = true; + if ((tx_ol_flags & PKT_TX_L4_MASK) == + PKT_TX_UDP_CKSUM) { + bd2_bf1 |= + 1 << ETH_TX_DATA_2ND_BD_L4_UDP_SHIFT; + } + + /* TODO other pseudo checksum modes are + * not supported + */ + bd2_bf1 |= + ETH_L4_PSEUDO_CSUM_CORRECT_LENGTH << + ETH_TX_DATA_2ND_BD_L4_PSEUDO_CSUM_MODE_SHIFT; + bd2_bf2 |= (inner_l4_hdr_offset & + ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_MASK) << + ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_SHIFT; + } + } /* End MPLSoUDP */ + } /* End Tunnel handling */ + + if (tx_ol_flags & PKT_TX_TCP_SEG) { + lso_flg = true; if (unlikely(txq->nb_tx_avail < ETH_TX_MIN_BDS_PER_LSO_PKT)) break; + /* For LSO, packet header and payload must reside on + * buffers pointed by different BDs. Using BD1 for HDR + * and BD2 onwards for data. + */ + hdr_size = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; + bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT; + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT; + /* PKT_TX_TCP_SEG implies PKT_TX_TCP_CKSUM */ + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT; + mss = rte_cpu_to_le_16(mbuf->tso_segsz); + /* Using one header BD */ + bd3_bf |= rte_cpu_to_le_16(1 << + ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT); } else { if (unlikely(txq->nb_tx_avail < ETH_TX_MIN_BDS_PER_NON_LSO_PKT)) break; + bd1_bf |= + (mbuf->pkt_len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) + << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; } - if (tunn_flg && ipv6_ext_flg) { - if (unlikely(txq->nb_tx_avail < - ETH_TX_MIN_BDS_PER_TUNN_IPV6_WITH_EXT_PKT)) - break; + /* Descriptor based VLAN insertion */ + if (tx_ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) { + vlan = rte_cpu_to_le_16(mbuf->vlan_tci); + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT; } - if (ipv6_ext_flg) { - if (unlikely(txq->nb_tx_avail < - ETH_TX_MIN_BDS_PER_IPV6_WITH_EXT_PKT)) - break; + + /* Offload the IP checksum in the hardware */ + if (tx_ol_flags & PKT_TX_IP_CKSUM) { + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT; + /* There's no DPDK flag to request outer-L4 csum + * offload. But in the case of tunnel if inner L3 or L4 + * csum offload is requested then we need to force + * recalculation of L4 tunnel header csum also. + */ + if (tunn_flg) { + bd1_bd_flags_bf |= + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK << + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT; + } + } + + /* L4 checksum offload (tcp or udp) */ + if ((tx_ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) && + (tx_ol_flags & (PKT_TX_UDP_CKSUM | PKT_TX_TCP_CKSUM))) { + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT; + /* There's no DPDK flag to request outer-L4 csum + * offload. But in the case of tunnel if inner L3 or L4 + * csum offload is requested then we need to force + * recalculation of L4 tunnel header csum also. + */ + if (tunn_flg) { + bd1_bd_flags_bf |= + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK << + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT; + } } /* Fill the entry in the SW ring and the BDs in the FW ring */ @@ -1520,107 +1800,49 @@ qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) memset(bd1, 0, sizeof(struct eth_tx_1st_bd)); nbds++; - bd1->data.bd_flags.bitfields |= - 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; - /* FW 8.10.x specific change */ - if (!lso_flg) { - bd1->data.bitfields |= - (mbuf->pkt_len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) - << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; - /* Map MBUF linear data for DMA and set in the BD1 */ - QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_dma_addr(mbuf), - mbuf->data_len); - } else { - /* For LSO, packet header and payload must reside on - * buffers pointed by different BDs. Using BD1 for HDR - * and BD2 onwards for data. - */ - hdr_size = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; - QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_dma_addr(mbuf), - hdr_size); - } + /* Map MBUF linear data for DMA and set in the BD1 */ + QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_dma_addr(mbuf), + mbuf->data_len); + bd1->data.bitfields = rte_cpu_to_le_16(bd1_bf); + bd1->data.bd_flags.bitfields = bd1_bd_flags_bf; + bd1->data.vlan = vlan; - if (tunn_flg) { - /* First indicate its a tunnel pkt */ - bd1->data.bitfields |= - ETH_TX_DATA_1ST_BD_TUNN_FLAG_MASK << - ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT; - - /* Legacy FW had flipped behavior in regard to this bit - * i.e. it needed to set to prevent FW from touching - * encapsulated packets when it didn't need to. - */ - if (unlikely(txq->is_legacy)) - bd1->data.bitfields ^= - 1 << ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT; - - /* Outer IP checksum offload */ - if (tx_ol_flags & PKT_TX_OUTER_IP_CKSUM) { - bd1->data.bd_flags.bitfields |= - ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_MASK << - ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_SHIFT; - } - - /* Outer UDP checksum offload */ - bd1->data.bd_flags.bitfields |= - ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK << - ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT; - } - - /* Descriptor based VLAN insertion */ - if (tx_ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) { - bd1->data.vlan = rte_cpu_to_le_16(mbuf->vlan_tci); - bd1->data.bd_flags.bitfields |= - 1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT; - } - - if (lso_flg) - bd1->data.bd_flags.bitfields |= - 1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT; - - /* Offload the IP checksum in the hardware */ - if ((lso_flg) || (tx_ol_flags & PKT_TX_IP_CKSUM)) - bd1->data.bd_flags.bitfields |= - 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT; - - /* L4 checksum offload (tcp or udp) */ - if ((lso_flg) || (tx_ol_flags & (PKT_TX_TCP_CKSUM | - PKT_TX_UDP_CKSUM))) - /* PKT_TX_TCP_SEG implies PKT_TX_TCP_CKSUM */ - bd1->data.bd_flags.bitfields |= - 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT; - - /* BD2 */ - if (lso_flg || ipv6_ext_flg) { + if (lso_flg || mplsoudp_flg) { bd2 = (struct eth_tx_2nd_bd *)ecore_chain_produce (&txq->tx_pbl); memset(bd2, 0, sizeof(struct eth_tx_2nd_bd)); nbds++; - QEDE_BD_SET_ADDR_LEN(bd2, - (hdr_size + - rte_mbuf_data_dma_addr(mbuf)), - mbuf->data_len - hdr_size); - /* TBD: check pseudo csum iff tx_prepare not called? */ - if (ipv6_ext_flg) { - bd2->data.bitfields1 |= - ETH_L4_PSEUDO_CSUM_ZERO_LENGTH << - ETH_TX_DATA_2ND_BD_L4_PSEUDO_CSUM_MODE_SHIFT; - } - } - /* BD3 */ - if (lso_flg || ipv6_ext_flg) { - bd3 = (struct eth_tx_3rd_bd *)ecore_chain_produce - (&txq->tx_pbl); - memset(bd3, 0, sizeof(struct eth_tx_3rd_bd)); - nbds++; - if (lso_flg) { - bd3->data.lso_mss = - rte_cpu_to_le_16(mbuf->tso_segsz); - /* Using one header BD */ - bd3->data.bitfields |= - rte_cpu_to_le_16(1 << - ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT); + /* BD1 */ + QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_dma_addr(mbuf), + hdr_size); + /* BD2 */ + QEDE_BD_SET_ADDR_LEN(bd2, (hdr_size + + rte_mbuf_data_dma_addr(mbuf)), + mbuf->data_len - hdr_size); + bd2->data.bitfields1 = rte_cpu_to_le_16(bd2_bf1); + if (mplsoudp_flg) { + bd2->data.bitfields2 = + rte_cpu_to_le_16(bd2_bf2); + /* Outer L3 size */ + bd2->data.tunn_ip_size = + rte_cpu_to_le_16(mbuf->outer_l3_len); + } + /* BD3 */ + if (lso_flg || (mplsoudp_flg && tunn_ipv6_ext_flg)) { + bd3 = (struct eth_tx_3rd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(bd3, 0, sizeof(struct eth_tx_3rd_bd)); + nbds++; + bd3->data.bitfields = rte_cpu_to_le_16(bd3_bf); + if (lso_flg) + bd3->data.lso_mss = mss; + if (mplsoudp_flg) { + bd3->data.tunn_l4_hdr_start_offset_w = + tunn_l4_hdr_start_offset; + bd3->data.tunn_hdr_size_w = + tunn_hdr_size; + } } } @@ -1636,8 +1858,7 @@ qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl)); #ifdef RTE_LIBRTE_QEDE_DEBUG_TX print_tx_bd_info(txq, bd1, bd2, bd3, tx_ol_flags); - PMD_TX_LOG(INFO, txq, "lso=%d tunn=%d ipv6_ext=%d\n", - lso_flg, tunn_flg, ipv6_ext_flg); + PMD_TX_LOG(INFO, txq, "lso=%d tunn=%d", lso_flg, tunn_flg); #endif nb_pkt_sent++; txq->xmit_pkts++; @@ -1659,290 +1880,6 @@ qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) return nb_pkt_sent; } -static void qede_init_fp_queue(struct rte_eth_dev *eth_dev) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - struct qede_fastpath *fp; - uint8_t i, txq_index, tc; - int rxq = 0, txq = 0; - - for_each_queue(i) { - fp = &qdev->fp_array[i]; - if (fp->type & QEDE_FASTPATH_RX) { - fp->rxq = eth_dev->data->rx_queues[i]; - fp->rxq->queue_id = rxq++; - } - - if (fp->type & QEDE_FASTPATH_TX) { - for (tc = 0; tc < qdev->num_tc; tc++) { - txq_index = tc * QEDE_TSS_COUNT(qdev) + txq; - fp->txqs[tc] = - eth_dev->data->tx_queues[txq_index]; - fp->txqs[tc]->queue_id = txq_index; - if (qdev->dev_info.is_legacy) - fp->txqs[tc]->is_legacy = true; - } - txq++; - } - } -} - -int qede_dev_start(struct rte_eth_dev *eth_dev) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - int rc; - - DP_INFO(edev, "Device state is %d\n", qdev->state); - - if (qdev->state == QEDE_DEV_START) { - DP_INFO(edev, "Port is already started\n"); - return 0; - } - - if (qdev->state == QEDE_DEV_CONFIG) - qede_init_fp_queue(eth_dev); - - rc = qede_start_queues(eth_dev, true); - if (rc) { - DP_ERR(edev, "Failed to start queues\n"); - /* TBD: free */ - return rc; - } - - /* Newer SR-IOV PF driver expects RX/TX queues to be started before - * enabling RSS. Hence RSS configuration is deferred upto this point. - * Also, we would like to retain similar behavior in PF case, so we - * don't do PF/VF specific check here. - */ - if (eth_dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) - if (qede_config_rss(eth_dev)) - return -1; - - /* Bring-up the link */ - qede_dev_set_link_state(eth_dev, true); - - /* Start/resume traffic */ - qdev->ops->fastpath_start(edev); - - qdev->state = QEDE_DEV_START; - - DP_INFO(edev, "dev_state is QEDE_DEV_START\n"); - - return 0; -} - -static int qede_drain_txq(struct qede_dev *qdev, - struct qede_tx_queue *txq, bool allow_drain) -{ - struct ecore_dev *edev = &qdev->edev; - int rc, cnt = 1000; - - while (txq->sw_tx_cons != txq->sw_tx_prod) { - qede_process_tx_compl(edev, txq); - if (!cnt) { - if (allow_drain) { - DP_ERR(edev, "Tx queue[%u] is stuck," - "requesting MCP to drain\n", - txq->queue_id); - rc = qdev->ops->common->drain(edev); - if (rc) - return rc; - return qede_drain_txq(qdev, txq, false); - } - DP_ERR(edev, "Timeout waiting for tx queue[%d]:" - "PROD=%d, CONS=%d\n", - txq->queue_id, txq->sw_tx_prod, - txq->sw_tx_cons); - return -1; - } - cnt--; - DELAY(1000); - rte_compiler_barrier(); - } - - /* FW finished processing, wait for HW to transmit all tx packets */ - DELAY(2000); - - return 0; -} - -static int qede_stop_queues(struct qede_dev *qdev) -{ - struct qed_update_vport_params vport_update_params; - struct ecore_dev *edev = &qdev->edev; - struct ecore_sge_tpa_params tpa_params; - struct qede_fastpath *fp; - int rc, tc, i; - - /* Disable the vport */ - memset(&vport_update_params, 0, sizeof(vport_update_params)); - vport_update_params.vport_id = 0; - vport_update_params.update_vport_active_flg = 1; - vport_update_params.vport_active_flg = 0; - vport_update_params.update_rss_flg = 0; - /* Disable TPA */ - if (qdev->enable_lro) { - DP_INFO(edev, "Disabling LRO\n"); - memset(&tpa_params, 0, sizeof(struct ecore_sge_tpa_params)); - qede_update_sge_tpa_params(&tpa_params, qdev->mtu, false); - vport_update_params.sge_tpa_params = &tpa_params; - } - - DP_INFO(edev, "Deactivate vport\n"); - rc = qdev->ops->vport_update(edev, &vport_update_params); - if (rc) { - DP_ERR(edev, "Failed to update vport\n"); - return rc; - } - - DP_INFO(edev, "Flushing tx queues\n"); - - /* Flush Tx queues. If needed, request drain from MCP */ - for_each_queue(i) { - fp = &qdev->fp_array[i]; - - if (fp->type & QEDE_FASTPATH_TX) { - for (tc = 0; tc < qdev->num_tc; tc++) { - struct qede_tx_queue *txq = fp->txqs[tc]; - - rc = qede_drain_txq(qdev, txq, true); - if (rc) - return rc; - } - } - } - - /* Stop all Queues in reverse order */ - for (i = QEDE_QUEUE_CNT(qdev) - 1; i >= 0; i--) { - fp = &qdev->fp_array[i]; - - /* Stop the Tx Queue(s) */ - if (qdev->fp_array[i].type & QEDE_FASTPATH_TX) { - for (tc = 0; tc < qdev->num_tc; tc++) { - struct qede_tx_queue *txq = fp->txqs[tc]; - DP_INFO(edev, "Stopping tx queues\n"); - rc = qdev->ops->q_tx_stop(edev, i, txq->handle); - if (rc) { - DP_ERR(edev, "Failed to stop TXQ #%d\n", - i); - return rc; - } - } - } - - /* Stop the Rx Queue */ - if (qdev->fp_array[i].type & QEDE_FASTPATH_RX) { - DP_INFO(edev, "Stopping rx queues\n"); - rc = qdev->ops->q_rx_stop(edev, i, fp->rxq->handle); - if (rc) { - DP_ERR(edev, "Failed to stop RXQ #%d\n", i); - return rc; - } - } - } - qede_reset_fp_rings(qdev); - - return 0; -} - -int qede_reset_fp_rings(struct qede_dev *qdev) -{ - struct qede_fastpath *fp; - struct qede_tx_queue *txq; - uint8_t tc; - uint16_t id, i; - - for_each_queue(id) { - fp = &qdev->fp_array[id]; - - if (fp->type & QEDE_FASTPATH_RX) { - DP_INFO(&qdev->edev, - "Reset FP chain for RSS %u\n", id); - qede_rx_queue_release_mbufs(fp->rxq); - ecore_chain_reset(&fp->rxq->rx_bd_ring); - ecore_chain_reset(&fp->rxq->rx_comp_ring); - fp->rxq->sw_rx_prod = 0; - fp->rxq->sw_rx_cons = 0; - *fp->rxq->hw_cons_ptr = 0; - for (i = 0; i < fp->rxq->nb_rx_desc; i++) { - if (qede_alloc_rx_buffer(fp->rxq)) { - DP_ERR(&qdev->edev, - "RX buffer allocation failed\n"); - return -ENOMEM; - } - } - } - if (fp->type & QEDE_FASTPATH_TX) { - for (tc = 0; tc < qdev->num_tc; tc++) { - txq = fp->txqs[tc]; - qede_tx_queue_release_mbufs(txq); - ecore_chain_reset(&txq->tx_pbl); - txq->sw_tx_cons = 0; - txq->sw_tx_prod = 0; - *txq->hw_cons_ptr = 0; - } - } - } - - return 0; -} - -/* This function frees all memory of a single fp */ -void qede_free_mem_load(struct rte_eth_dev *eth_dev) -{ - struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); - struct qede_fastpath *fp; - uint16_t txq_idx; - uint8_t id; - uint8_t tc; - - for_each_queue(id) { - fp = &qdev->fp_array[id]; - if (fp->type & QEDE_FASTPATH_RX) { - if (!fp->rxq) - continue; - qede_rx_queue_release(fp->rxq); - eth_dev->data->rx_queues[id] = NULL; - } else { - for (tc = 0; tc < qdev->num_tc; tc++) { - if (!fp->txqs[tc]) - continue; - txq_idx = fp->txqs[tc]->queue_id; - qede_tx_queue_release(fp->txqs[tc]); - eth_dev->data->tx_queues[txq_idx] = NULL; - } - } - } -} - -void qede_dev_stop(struct rte_eth_dev *eth_dev) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - - DP_INFO(edev, "port %u\n", eth_dev->data->port_id); - - if (qdev->state != QEDE_DEV_START) { - DP_INFO(edev, "Device not yet started\n"); - return; - } - - if (qede_stop_queues(qdev)) - DP_ERR(edev, "Didn't succeed to close queues\n"); - - DP_INFO(edev, "Stopped queues\n"); - - qdev->ops->fastpath_stop(edev); - - /* Bring the link down */ - qede_dev_set_link_state(eth_dev, false); - - qdev->state = QEDE_DEV_STOP; - - DP_INFO(edev, "dev_state is QEDE_DEV_STOP\n"); -} - uint16_t qede_rxtx_pkts_dummy(__rte_unused void *p_rxq, __rte_unused struct rte_mbuf **pkts, diff --git a/drivers/net/qede/qede_rxtx.h b/drivers/net/qede/qede_rxtx.h index 21f2dacd..b551fd6a 100644 --- a/drivers/net/qede/qede_rxtx.h +++ b/drivers/net/qede/qede_rxtx.h @@ -77,10 +77,10 @@ #define QEDE_TXQ_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS) -#define MAX_NUM_TC 8 - -#define for_each_queue(i) for (i = 0; i < qdev->num_queues; i++) - +#define for_each_rss(i) for (i = 0; i < qdev->num_rx_queues; i++) +#define for_each_tss(i) for (i = 0; i < qdev->num_tx_queues; i++) +#define QEDE_RXTX_MAX(qdev) \ + (RTE_MAX(QEDE_RSS_COUNT(qdev), QEDE_TSS_COUNT(qdev))) /* Macros for non-tunnel packet types lkup table */ #define QEDE_PKT_TYPE_UNKNOWN 0x0 @@ -135,7 +135,8 @@ #define QEDE_TX_OFFLOAD_MASK (QEDE_TX_CSUM_OFFLOAD_MASK | \ PKT_TX_QINQ_PKT | \ PKT_TX_VLAN_PKT | \ - PKT_TX_TUNNEL_VXLAN) + PKT_TX_TUNNEL_VXLAN | \ + PKT_TX_TUNNEL_MPLSINUDP) #define QEDE_TX_OFFLOAD_NOTSUP_MASK \ (PKT_TX_OFFLOAD_MASK ^ QEDE_TX_OFFLOAD_MASK) @@ -165,6 +166,7 @@ struct qede_rx_queue { uint16_t *hw_cons_ptr; void OSAL_IOMEM *hw_rxq_prod_addr; struct qede_rx_entry *sw_rx_ring; + struct ecore_sb_info *sb_info; uint16_t sw_rx_cons; uint16_t sw_rx_prod; uint16_t nb_rx_desc; @@ -213,13 +215,9 @@ struct qede_tx_queue { }; struct qede_fastpath { - struct qede_dev *qdev; - u8 type; - uint8_t id; struct ecore_sb_info *sb_info; struct qede_rx_queue *rxq; - struct qede_tx_queue *txqs[MAX_NUM_TC]; - char name[80]; + struct qede_tx_queue *txq; }; /* @@ -240,16 +238,6 @@ void qede_rx_queue_release(void *rx_queue); void qede_tx_queue_release(void *tx_queue); -int qede_dev_start(struct rte_eth_dev *eth_dev); - -void qede_dev_stop(struct rte_eth_dev *eth_dev); - -int qede_reset_fp_rings(struct qede_dev *qdev); - -void qede_free_fp_arrays(struct qede_dev *qdev); - -void qede_free_mem_load(struct rte_eth_dev *eth_dev); - uint16_t qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); @@ -259,9 +247,13 @@ uint16_t qede_xmit_prep_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); -uint16_t qede_rxtx_pkts_dummy(__rte_unused void *p_rxq, - __rte_unused struct rte_mbuf **pkts, - __rte_unused uint16_t nb_pkts); +uint16_t qede_rxtx_pkts_dummy(void *p_rxq, + struct rte_mbuf **pkts, + uint16_t nb_pkts); + +int qede_start_queues(struct rte_eth_dev *eth_dev); + +void qede_stop_queues(struct rte_eth_dev *eth_dev); /* Fastpath resource alloc/dealloc helpers */ int qede_alloc_fp_resc(struct qede_dev *qdev); diff --git a/drivers/net/ring/rte_eth_ring.c b/drivers/net/ring/rte_eth_ring.c index d4dce957..464d3d38 100644 --- a/drivers/net/ring/rte_eth_ring.c +++ b/drivers/net/ring/rte_eth_ring.c @@ -45,12 +45,23 @@ #define ETH_RING_NUMA_NODE_ACTION_ARG "nodeaction" #define ETH_RING_ACTION_CREATE "CREATE" #define ETH_RING_ACTION_ATTACH "ATTACH" +#define ETH_RING_INTERNAL_ARG "internal" static const char *valid_arguments[] = { ETH_RING_NUMA_NODE_ACTION_ARG, + ETH_RING_INTERNAL_ARG, NULL }; +struct ring_internal_args { + struct rte_ring * const *rx_queues; + const unsigned int nb_rx_queues; + struct rte_ring * const *tx_queues; + const unsigned int nb_tx_queues; + const unsigned int numa_node; + void *addr; /* self addr for sanity check */ +}; + enum dev_action { DEV_CREATE, DEV_ATTACH @@ -263,11 +274,14 @@ static int do_eth_dev_ring_create(const char *name, struct rte_ring * const rx_queues[], const unsigned nb_rx_queues, struct rte_ring *const tx_queues[], const unsigned nb_tx_queues, - const unsigned numa_node, enum dev_action action) + const unsigned int numa_node, enum dev_action action, + struct rte_eth_dev **eth_dev_p) { struct rte_eth_dev_data *data = NULL; struct pmd_internals *internals = NULL; struct rte_eth_dev *eth_dev = NULL; + void **rx_queues_local = NULL; + void **tx_queues_local = NULL; unsigned i; RTE_LOG(INFO, PMD, "Creating rings-backed ethdev on numa socket %u\n", @@ -282,16 +296,16 @@ do_eth_dev_ring_create(const char *name, goto error; } - data->rx_queues = rte_zmalloc_socket(name, + rx_queues_local = rte_zmalloc_socket(name, sizeof(void *) * nb_rx_queues, 0, numa_node); - if (data->rx_queues == NULL) { + if (rx_queues_local == NULL) { rte_errno = ENOMEM; goto error; } - data->tx_queues = rte_zmalloc_socket(name, + tx_queues_local = rte_zmalloc_socket(name, sizeof(void *) * nb_tx_queues, 0, numa_node); - if (data->tx_queues == NULL) { + if (tx_queues_local == NULL) { rte_errno = ENOMEM; goto error; } @@ -318,6 +332,10 @@ do_eth_dev_ring_create(const char *name, /* NOTE: we'll replace the data element, of originally allocated eth_dev * so the rings are local per-process */ + rte_memcpy(data, eth_dev->data, sizeof(*data)); + data->rx_queues = rx_queues_local; + data->tx_queues = tx_queues_local; + internals->action = action; internals->max_rx_queues = nb_rx_queues; internals->max_tx_queues = nb_tx_queues; @@ -331,8 +349,6 @@ do_eth_dev_ring_create(const char *name, } data->dev_private = internals; - data->port_id = eth_dev->data->port_id; - memmove(data->name, eth_dev->data->name, sizeof(data->name)); data->nb_rx_queues = (uint16_t)nb_rx_queues; data->nb_tx_queues = (uint16_t)nb_tx_queues; data->dev_link = pmd_link; @@ -342,20 +358,19 @@ do_eth_dev_ring_create(const char *name, eth_dev->dev_ops = &ops; data->dev_flags = RTE_ETH_DEV_DETACHABLE; data->kdrv = RTE_KDRV_NONE; - data->drv_name = pmd_ring_drv.driver.name; data->numa_node = numa_node; /* finally assign rx and tx ops */ eth_dev->rx_pkt_burst = eth_ring_rx; eth_dev->tx_pkt_burst = eth_ring_tx; + *eth_dev_p = eth_dev; + return data->port_id; error: - if (data) { - rte_free(data->rx_queues); - rte_free(data->tx_queues); - } + rte_free(rx_queues_local); + rte_free(tx_queues_local); rte_free(data); rte_free(internals); @@ -369,6 +384,19 @@ rte_eth_from_rings(const char *name, struct rte_ring *const rx_queues[], const unsigned nb_tx_queues, const unsigned numa_node) { + struct ring_internal_args args = { + .rx_queues = rx_queues, + .nb_rx_queues = nb_rx_queues, + .tx_queues = tx_queues, + .nb_tx_queues = nb_tx_queues, + .numa_node = numa_node, + .addr = &args, + }; + char args_str[32] = { 0 }; + char ring_name[32] = { 0 }; + uint8_t port_id = RTE_MAX_ETHPORTS; + int ret; + /* do some parameter checking */ if (rx_queues == NULL && nb_rx_queues > 0) { rte_errno = EINVAL; @@ -383,8 +411,18 @@ rte_eth_from_rings(const char *name, struct rte_ring *const rx_queues[], return -1; } - return do_eth_dev_ring_create(name, rx_queues, nb_rx_queues, - tx_queues, nb_tx_queues, numa_node, DEV_ATTACH); + snprintf(args_str, 32, "%s=%p", ETH_RING_INTERNAL_ARG, &args); + snprintf(ring_name, 32, "net_ring_%s", name); + + ret = rte_vdev_init(ring_name, args_str); + if (ret) { + rte_errno = EINVAL; + return -1; + } + + rte_eth_dev_get_port_by_name(ring_name, &port_id); + + return port_id; } int @@ -396,7 +434,7 @@ rte_eth_from_ring(struct rte_ring *r) static int eth_dev_ring_create(const char *name, const unsigned numa_node, - enum dev_action action) + enum dev_action action, struct rte_eth_dev **eth_dev) { /* rx and tx are so-called from point of view of first port. * They are inverted from the point of view of second port @@ -418,7 +456,7 @@ eth_dev_ring_create(const char *name, const unsigned numa_node, } if (do_eth_dev_ring_create(name, rxtx, num_rings, rxtx, num_rings, - numa_node, action) < 0) + numa_node, action, eth_dev) < 0) return -1; return 0; @@ -450,13 +488,14 @@ static int parse_kvlist (const char *key __rte_unused, const char *value, void * ret = -EINVAL; if (!name) { - RTE_LOG(WARNING, PMD, "command line paramter is empty for ring pmd!\n"); + RTE_LOG(WARNING, PMD, "command line parameter is empty for ring pmd!\n"); goto out; } node = strchr(name, ':'); if (!node) { - RTE_LOG(WARNING, PMD, "could not parse node value from %s", name); + RTE_LOG(WARNING, PMD, "could not parse node value from %s\n", + name); goto out; } @@ -465,7 +504,8 @@ static int parse_kvlist (const char *key __rte_unused, const char *value, void * action = strchr(node, ':'); if (!action) { - RTE_LOG(WARNING, PMD, "could not action value from %s", node); + RTE_LOG(WARNING, PMD, "could not parse action value from %s\n", + node); goto out; } @@ -502,12 +542,31 @@ out: } static int +parse_internal_args(const char *key __rte_unused, const char *value, + void *data) +{ + struct ring_internal_args **internal_args = data; + void *args; + + sscanf(value, "%p", &args); + + *internal_args = args; + + if ((*internal_args)->addr != args) + return -1; + + return 0; +} + +static int rte_pmd_ring_probe(struct rte_vdev_device *dev) { const char *name, *params; struct rte_kvargs *kvlist = NULL; int ret = 0; struct node_action_list *info = NULL; + struct rte_eth_dev *eth_dev = NULL; + struct ring_internal_args *internal_args; name = rte_vdev_device_name(dev); params = rte_vdev_device_args(dev); @@ -515,30 +574,53 @@ rte_pmd_ring_probe(struct rte_vdev_device *dev) RTE_LOG(INFO, PMD, "Initializing pmd_ring for %s\n", name); if (params == NULL || params[0] == '\0') { - ret = eth_dev_ring_create(name, rte_socket_id(), DEV_CREATE); + ret = eth_dev_ring_create(name, rte_socket_id(), DEV_CREATE, + ð_dev); if (ret == -1) { RTE_LOG(INFO, PMD, "Attach to pmd_ring for %s\n", name); ret = eth_dev_ring_create(name, rte_socket_id(), - DEV_ATTACH); + DEV_ATTACH, ð_dev); } - } - else { + } else { kvlist = rte_kvargs_parse(params, valid_arguments); if (!kvlist) { RTE_LOG(INFO, PMD, "Ignoring unsupported parameters when creating" " rings-backed ethernet device\n"); ret = eth_dev_ring_create(name, rte_socket_id(), - DEV_CREATE); + DEV_CREATE, ð_dev); if (ret == -1) { RTE_LOG(INFO, PMD, "Attach to pmd_ring for %s\n", name); ret = eth_dev_ring_create(name, rte_socket_id(), - DEV_ATTACH); + DEV_ATTACH, ð_dev); } + + if (eth_dev) + eth_dev->device = &dev->device; + return ret; + } + + if (rte_kvargs_count(kvlist, ETH_RING_INTERNAL_ARG) == 1) { + ret = rte_kvargs_process(kvlist, ETH_RING_INTERNAL_ARG, + parse_internal_args, + &internal_args); + if (ret < 0) + goto out_free; + + ret = do_eth_dev_ring_create(name, + internal_args->rx_queues, + internal_args->nb_rx_queues, + internal_args->tx_queues, + internal_args->nb_tx_queues, + internal_args->numa_node, + DEV_ATTACH, + ð_dev); + if (ret >= 0) + ret = 0; } else { ret = rte_kvargs_count(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG); info = rte_zmalloc("struct node_action_list", @@ -560,7 +642,8 @@ rte_pmd_ring_probe(struct rte_vdev_device *dev) for (info->count = 0; info->count < info->total; info->count++) { ret = eth_dev_ring_create(info->list[info->count].name, info->list[info->count].node, - info->list[info->count].action); + info->list[info->count].action, + ð_dev); if ((ret == -1) && (info->list[info->count].action == DEV_CREATE)) { RTE_LOG(INFO, PMD, @@ -568,12 +651,16 @@ rte_pmd_ring_probe(struct rte_vdev_device *dev) name); ret = eth_dev_ring_create(name, info->list[info->count].node, - DEV_ATTACH); + DEV_ATTACH, + ð_dev); } } } } + if (eth_dev) + eth_dev->device = &dev->device; + out_free: rte_kvargs_free(kvlist); rte_free(info); diff --git a/drivers/net/sfc/sfc.c b/drivers/net/sfc/sfc.c index 4e241b22..6cecfc00 100644 --- a/drivers/net/sfc/sfc.c +++ b/drivers/net/sfc/sfc.c @@ -465,7 +465,7 @@ static int sfc_mem_bar_init(struct sfc_adapter *sa) { struct rte_eth_dev *eth_dev = sa->eth_dev; - struct rte_pci_device *pci_dev = SFC_DEV_TO_PCI(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); efsys_bar_t *ebp = &sa->mem_bar; unsigned int i; struct rte_mem_resource *res; @@ -666,7 +666,7 @@ sfc_detach(struct sfc_adapter *sa) int sfc_probe(struct sfc_adapter *sa) { - struct rte_pci_device *pci_dev = SFC_DEV_TO_PCI(sa->eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); efx_nic_t *enp; int rc; diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h index fad0ce04..286d1ac1 100644 --- a/drivers/net/sfc/sfc.h +++ b/drivers/net/sfc/sfc.h @@ -34,6 +34,7 @@ #include <stdbool.h> +#include <rte_pci.h> #include <rte_ethdev.h> #include <rte_kvargs.h> #include <rte_spinlock.h> @@ -46,9 +47,6 @@ extern "C" { #endif -#define SFC_DEV_TO_PCI(eth_dev) \ - RTE_DEV_TO_PCI((eth_dev)->device) - #if EFSYS_OPT_RX_SCALE /** RSS key length (bytes) */ #define SFC_RSS_KEY_SIZE 40 @@ -153,6 +151,11 @@ struct sfc_port { boolean_t flow_ctrl_autoneg; size_t pdu; + /* + * Flow API isolated mode overrides promisc and allmulti settings; + * they won't be applied if isolated mode is active + */ + boolean_t isolated; boolean_t promisc; boolean_t allmulti; @@ -162,6 +165,7 @@ struct sfc_port { rte_spinlock_t mac_stats_lock; uint64_t *mac_stats_buf; + unsigned int mac_stats_nb_supported; efsys_mem_t mac_stats_dma_mem; boolean_t mac_stats_reset_pending; uint16_t mac_stats_update_period_ms; @@ -182,6 +186,8 @@ struct sfc_adapter { */ rte_spinlock_t lock; enum sfc_adapter_state state; + struct rte_pci_addr pci_addr; + uint16_t port_id; struct rte_eth_dev *eth_dev; struct rte_kvargs *kvargs; bool debug_init; @@ -226,7 +232,18 @@ struct sfc_adapter { uint8_t rss_key[SFC_RSS_KEY_SIZE]; #endif + /* + * Shared memory copy of the Rx datapath name to be used by + * the secondary process to find Rx datapath to be used. + */ + char *dp_rx_name; const struct sfc_dp_rx *dp_rx; + + /* + * Shared memory copy of the Tx datapath name to be used by + * the secondary process to find Rx datapath to be used. + */ + char *dp_tx_name; const struct sfc_dp_tx *dp_tx; }; diff --git a/drivers/net/sfc/sfc_debug.h b/drivers/net/sfc/sfc_debug.h index c0b48677..92eba9c3 100644 --- a/drivers/net/sfc/sfc_debug.h +++ b/drivers/net/sfc/sfc_debug.h @@ -47,13 +47,12 @@ /* Log PMD message, automatically add prefix and \n */ #define sfc_panic(sa, fmt, args...) \ do { \ - const struct rte_eth_dev *_dev = (sa)->eth_dev; \ - const struct rte_pci_device *_pci_dev = SFC_DEV_TO_PCI(_dev); \ + const struct sfc_adapter *_sa = (sa); \ \ rte_panic("sfc " PCI_PRI_FMT " #%" PRIu8 ": " fmt "\n", \ - _pci_dev->addr.domain, _pci_dev->addr.bus, \ - _pci_dev->addr.devid, _pci_dev->addr.function,\ - _dev->data->port_id, ##args); \ + _sa->pci_addr.domain, _sa->pci_addr.bus, \ + _sa->pci_addr.devid, _sa->pci_addr.function, \ + _sa->port_id, ##args); \ } while (0) #endif /* _SFC_DEBUG_H_ */ diff --git a/drivers/net/sfc/sfc_dp_rx.h b/drivers/net/sfc/sfc_dp_rx.h index 9d05a4b3..a7b82784 100644 --- a/drivers/net/sfc/sfc_dp_rx.h +++ b/drivers/net/sfc/sfc_dp_rx.h @@ -161,6 +161,7 @@ struct sfc_dp_rx { unsigned int features; #define SFC_DP_RX_FEAT_SCATTER 0x1 +#define SFC_DP_RX_FEAT_MULTI_PROCESS 0x2 sfc_dp_rx_qcreate_t *qcreate; sfc_dp_rx_qdestroy_t *qdestroy; sfc_dp_rx_qstart_t *qstart; diff --git a/drivers/net/sfc/sfc_dp_tx.h b/drivers/net/sfc/sfc_dp_tx.h index 2bb9a2e7..c1c34191 100644 --- a/drivers/net/sfc/sfc_dp_tx.h +++ b/drivers/net/sfc/sfc_dp_tx.h @@ -135,6 +135,7 @@ struct sfc_dp_tx { #define SFC_DP_TX_FEAT_VLAN_INSERT 0x1 #define SFC_DP_TX_FEAT_TSO 0x2 #define SFC_DP_TX_FEAT_MULTI_SEG 0x4 +#define SFC_DP_TX_FEAT_MULTI_PROCESS 0x8 sfc_dp_tx_qcreate_t *qcreate; sfc_dp_tx_qdestroy_t *qdestroy; sfc_dp_tx_qstart_t *qstart; diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c index 1484baba..60812cbe 100644 --- a/drivers/net/sfc/sfc_ef10_rx.c +++ b/drivers/net/sfc/sfc_ef10_rx.c @@ -699,7 +699,7 @@ struct sfc_dp_rx sfc_ef10_rx = { .type = SFC_DP_RX, .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, }, - .features = 0, + .features = SFC_DP_RX_FEAT_MULTI_PROCESS, .qcreate = sfc_ef10_rx_qcreate, .qdestroy = sfc_ef10_rx_qdestroy, .qstart = sfc_ef10_rx_qstart, diff --git a/drivers/net/sfc/sfc_ef10_tx.c b/drivers/net/sfc/sfc_ef10_tx.c index bac9baa9..da2b5edb 100644 --- a/drivers/net/sfc/sfc_ef10_tx.c +++ b/drivers/net/sfc/sfc_ef10_tx.c @@ -128,14 +128,10 @@ sfc_ef10_tx_get_event(struct sfc_ef10_txq *txq, efx_qword_t *tx_ev) return true; } -static void -sfc_ef10_tx_reap(struct sfc_ef10_txq *txq) +static unsigned int +sfc_ef10_tx_process_events(struct sfc_ef10_txq *txq) { - const unsigned int old_read_ptr = txq->evq_read_ptr; - const unsigned int ptr_mask = txq->ptr_mask; - unsigned int completed = txq->completed; - unsigned int pending = completed; - const unsigned int curr_done = pending - 1; + const unsigned int curr_done = txq->completed - 1; unsigned int anew_done = curr_done; efx_qword_t tx_ev; @@ -148,7 +144,18 @@ sfc_ef10_tx_reap(struct sfc_ef10_txq *txq) /* Update the latest done descriptor */ anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX); } - pending += (anew_done - curr_done) & ptr_mask; + return (anew_done - curr_done) & txq->ptr_mask; +} + +static void +sfc_ef10_tx_reap(struct sfc_ef10_txq *txq) +{ + const unsigned int old_read_ptr = txq->evq_read_ptr; + const unsigned int ptr_mask = txq->ptr_mask; + unsigned int completed = txq->completed; + unsigned int pending = completed; + + pending += sfc_ef10_tx_process_events(txq); if (pending != completed) { do { @@ -349,6 +356,33 @@ sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) return pktp - &tx_pkts[0]; } +static void +sfc_ef10_simple_tx_reap(struct sfc_ef10_txq *txq) +{ + const unsigned int old_read_ptr = txq->evq_read_ptr; + const unsigned int ptr_mask = txq->ptr_mask; + unsigned int completed = txq->completed; + unsigned int pending = completed; + + pending += sfc_ef10_tx_process_events(txq); + + if (pending != completed) { + do { + struct sfc_ef10_tx_sw_desc *txd; + + txd = &txq->sw_ring[completed & ptr_mask]; + + rte_pktmbuf_free_seg(txd->mbuf); + } while (++completed != pending); + + txq->completed = completed; + } + + sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr, + txq->evq_read_ptr); +} + + static uint16_t sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) @@ -372,7 +406,7 @@ sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, reap_done = (dma_desc_space < RTE_MAX(txq->free_thresh, nb_pkts)); if (reap_done) { - sfc_ef10_tx_reap(txq); + sfc_ef10_simple_tx_reap(txq); dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - (added - txq->completed); } @@ -401,7 +435,7 @@ sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE if (!reap_done) - sfc_ef10_tx_reap(txq); + sfc_ef10_simple_tx_reap(txq); #endif return pktp - &tx_pkts[0]; @@ -516,12 +550,15 @@ static void sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq) { struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); - unsigned int txds; + unsigned int completed; + + for (completed = txq->completed; completed != txq->added; ++completed) { + struct sfc_ef10_tx_sw_desc *txd; - for (txds = 0; txds <= txq->ptr_mask; ++txds) { - if (txq->sw_ring[txds].mbuf != NULL) { - rte_pktmbuf_free(txq->sw_ring[txds].mbuf); - txq->sw_ring[txds].mbuf = NULL; + txd = &txq->sw_ring[completed & txq->ptr_mask]; + if (txd->mbuf != NULL) { + rte_pktmbuf_free(txd->mbuf); + txd->mbuf = NULL; } } @@ -534,7 +571,8 @@ struct sfc_dp_tx sfc_ef10_tx = { .type = SFC_DP_TX, .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, }, - .features = SFC_DP_TX_FEAT_MULTI_SEG, + .features = SFC_DP_TX_FEAT_MULTI_SEG | + SFC_DP_TX_FEAT_MULTI_PROCESS, .qcreate = sfc_ef10_tx_qcreate, .qdestroy = sfc_ef10_tx_qdestroy, .qstart = sfc_ef10_tx_qstart, @@ -549,7 +587,7 @@ struct sfc_dp_tx sfc_ef10_simple_tx = { .name = SFC_KVARG_DATAPATH_EF10_SIMPLE, .type = SFC_DP_TX, }, - .features = 0, + .features = SFC_DP_TX_FEAT_MULTI_PROCESS, .qcreate = sfc_ef10_tx_qcreate, .qdestroy = sfc_ef10_tx_qdestroy, .qstart = sfc_ef10_tx_qstart, diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c index bdb4c466..12bcd6fa 100644 --- a/drivers/net/sfc/sfc_ethdev.c +++ b/drivers/net/sfc/sfc_ethdev.c @@ -107,7 +107,7 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) sfc_log_init(sa, "entry"); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX; /* Autonegotiation may be disabled */ @@ -361,8 +361,13 @@ sfc_dev_filter_set(struct rte_eth_dev *dev, enum sfc_dev_filter_mode mode, if (*toggle != enabled) { *toggle = enabled; - if ((sa->state == SFC_ADAPTER_STARTED) && - (sfc_set_rx_mode(sa) != 0)) { + if (port->isolated) { + sfc_warn(sa, "isolated mode is active on the port"); + sfc_warn(sa, "the change is to be applied on the next " + "start provided that isolated mode is " + "disabled prior the next start"); + } else if ((sa->state == SFC_ADAPTER_STARTED) && + (sfc_set_rx_mode(sa) != 0)) { *toggle = !(enabled); sfc_warn(sa, "Failed to %s %s mode", ((enabled) ? "enable" : "disable"), desc); @@ -661,6 +666,85 @@ sfc_xstats_get_names(struct rte_eth_dev *dev, } static int +sfc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int n) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + uint64_t *mac_stats; + unsigned int nb_supported = 0; + unsigned int nb_written = 0; + unsigned int i; + int ret; + int rc; + + if (unlikely(values == NULL) || + unlikely((ids == NULL) && (n < port->mac_stats_nb_supported))) + return port->mac_stats_nb_supported; + + rte_spinlock_lock(&port->mac_stats_lock); + + rc = sfc_port_update_mac_stats(sa); + if (rc != 0) { + SFC_ASSERT(rc > 0); + ret = -rc; + goto unlock; + } + + mac_stats = port->mac_stats_buf; + + for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < n); ++i) { + if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) + continue; + + if ((ids == NULL) || (ids[nb_written] == nb_supported)) + values[nb_written++] = mac_stats[i]; + + ++nb_supported; + } + + ret = nb_written; + +unlock: + rte_spinlock_unlock(&port->mac_stats_lock); + + return ret; +} + +static int +sfc_xstats_get_names_by_id(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, unsigned int size) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + unsigned int nb_supported = 0; + unsigned int nb_written = 0; + unsigned int i; + + if (unlikely(xstats_names == NULL) || + unlikely((ids == NULL) && (size < port->mac_stats_nb_supported))) + return port->mac_stats_nb_supported; + + for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < size); ++i) { + if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) + continue; + + if ((ids == NULL) || (ids[nb_written] == nb_supported)) { + char *name = xstats_names[nb_written++].name; + + strncpy(name, efx_mac_stat_name(sa->nic, i), + sizeof(xstats_names[0].name)); + name[sizeof(xstats_names[0].name) - 1] = '\0'; + } + + ++nb_supported; + } + + return nb_written; +} + +static int sfc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { struct sfc_adapter *sa = dev->data->dev_private; @@ -826,10 +910,17 @@ sfc_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) { struct sfc_adapter *sa = dev->data->dev_private; const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + struct sfc_port *port = &sa->port; int rc; sfc_adapter_lock(sa); + if (port->isolated) { + sfc_err(sa, "isolated mode is active on the port"); + sfc_err(sa, "will not set MAC address"); + goto unlock; + } + if (sa->state != SFC_ADAPTER_STARTED) { sfc_info(sa, "the port is not started"); sfc_info(sa, "the new MAC address will be set on port start"); @@ -884,6 +975,12 @@ sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set, int rc; unsigned int i; + if (port->isolated) { + sfc_err(sa, "isolated mode is active on the port"); + sfc_err(sa, "will not set multicast address list"); + return -ENOTSUP; + } + if (mc_addrs == NULL) return -ENOBUFS; @@ -913,6 +1010,10 @@ sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set, return -rc; } +/* + * The function is used by the secondary process as well. It must not + * use any process-local pointers from the adapter data. + */ static void sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, struct rte_eth_rxq_info *qinfo) @@ -939,6 +1040,10 @@ sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, sfc_adapter_unlock(sa); } +/* + * The function is used by the secondary process as well. It must not + * use any process-local pointers from the adapter data. + */ static void sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, struct rte_eth_txq_info *qinfo) @@ -1083,8 +1188,9 @@ sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; - if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) + if ((sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) || port->isolated) return -ENOTSUP; if (sa->rss_channels == 0) @@ -1113,9 +1219,13 @@ sfc_dev_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; unsigned int efx_hash_types; int rc = 0; + if (port->isolated) + return -ENOTSUP; + if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) { sfc_err(sa, "RSS is not available"); return -ENOTSUP; @@ -1180,9 +1290,10 @@ sfc_dev_rss_reta_query(struct rte_eth_dev *dev, uint16_t reta_size) { struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; int entry; - if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) + if ((sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) || port->isolated) return -ENOTSUP; if (sa->rss_channels == 0) @@ -1212,11 +1323,15 @@ sfc_dev_rss_reta_update(struct rte_eth_dev *dev, uint16_t reta_size) { struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; unsigned int *rss_tbl_new; uint16_t entry; int rc; + if (port->isolated) + return -ENOTSUP; + if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) { sfc_err(sa, "RSS is not available"); return -ENOTSUP; @@ -1370,8 +1485,33 @@ static const struct eth_dev_ops sfc_eth_dev_ops = { .rxq_info_get = sfc_rx_queue_info_get, .txq_info_get = sfc_tx_queue_info_get, .fw_version_get = sfc_fw_version_get, + .xstats_get_by_id = sfc_xstats_get_by_id, + .xstats_get_names_by_id = sfc_xstats_get_names_by_id, }; +/** + * Duplicate a string in potentially shared memory required for + * multi-process support. + * + * strdup() allocates from process-local heap/memory. + */ +static char * +sfc_strdup(const char *str) +{ + size_t size; + char *copy; + + if (str == NULL) + return NULL; + + size = strlen(str) + 1; + copy = rte_malloc(__func__, size, 0); + if (copy != NULL) + rte_memcpy(copy, str, size); + + return copy; +} + static int sfc_eth_dev_set_ops(struct rte_eth_dev *dev) { @@ -1407,7 +1547,7 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev) "Insufficient Hw/FW capabilities to use Rx datapath %s", rx_name); rc = EINVAL; - goto fail_dp_rx; + goto fail_dp_rx_caps; } } else { sa->dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps); @@ -1419,7 +1559,13 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev) } } - sfc_info(sa, "use %s Rx datapath", sa->dp_rx->dp.name); + sa->dp_rx_name = sfc_strdup(sa->dp_rx->dp.name); + if (sa->dp_rx_name == NULL) { + rc = ENOMEM; + goto fail_dp_rx_name; + } + + sfc_info(sa, "use %s Rx datapath", sa->dp_rx_name); dev->rx_pkt_burst = sa->dp_rx->pkt_burst; @@ -1440,7 +1586,7 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev) "Insufficient Hw/FW capabilities to use Tx datapath %s", tx_name); rc = EINVAL; - goto fail_dp_tx; + goto fail_dp_tx_caps; } } else { sa->dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps); @@ -1452,7 +1598,13 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev) } } - sfc_info(sa, "use %s Tx datapath", sa->dp_tx->dp.name); + sa->dp_tx_name = sfc_strdup(sa->dp_tx->dp.name); + if (sa->dp_tx_name == NULL) { + rc = ENOMEM; + goto fail_dp_tx_name; + } + + sfc_info(sa, "use %s Tx datapath", sa->dp_tx_name); dev->tx_pkt_burst = sa->dp_tx->pkt_burst; @@ -1460,14 +1612,108 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev) return 0; +fail_dp_tx_name: +fail_dp_tx_caps: + sa->dp_tx = NULL; + fail_dp_tx: fail_kvarg_tx_datapath: + rte_free(sa->dp_rx_name); + sa->dp_rx_name = NULL; + +fail_dp_rx_name: +fail_dp_rx_caps: + sa->dp_rx = NULL; + fail_dp_rx: fail_kvarg_rx_datapath: return rc; } static void +sfc_eth_dev_clear_ops(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + dev->dev_ops = NULL; + dev->rx_pkt_burst = NULL; + dev->tx_pkt_burst = NULL; + + rte_free(sa->dp_tx_name); + sa->dp_tx_name = NULL; + sa->dp_tx = NULL; + + rte_free(sa->dp_rx_name); + sa->dp_rx_name = NULL; + sa->dp_rx = NULL; +} + +static const struct eth_dev_ops sfc_eth_dev_secondary_ops = { + .rxq_info_get = sfc_rx_queue_info_get, + .txq_info_get = sfc_tx_queue_info_get, +}; + +static int +sfc_eth_dev_secondary_set_ops(struct rte_eth_dev *dev) +{ + /* + * Device private data has really many process-local pointers. + * Below code should be extremely careful to use data located + * in shared memory only. + */ + struct sfc_adapter *sa = dev->data->dev_private; + const struct sfc_dp_rx *dp_rx; + const struct sfc_dp_tx *dp_tx; + int rc; + + dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sa->dp_rx_name); + if (dp_rx == NULL) { + sfc_err(sa, "cannot find %s Rx datapath", sa->dp_tx_name); + rc = ENOENT; + goto fail_dp_rx; + } + if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) { + sfc_err(sa, "%s Rx datapath does not support multi-process", + sa->dp_tx_name); + rc = EINVAL; + goto fail_dp_rx_multi_process; + } + + dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sa->dp_tx_name); + if (dp_tx == NULL) { + sfc_err(sa, "cannot find %s Tx datapath", sa->dp_tx_name); + rc = ENOENT; + goto fail_dp_tx; + } + if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) { + sfc_err(sa, "%s Tx datapath does not support multi-process", + sa->dp_tx_name); + rc = EINVAL; + goto fail_dp_tx_multi_process; + } + + dev->rx_pkt_burst = dp_rx->pkt_burst; + dev->tx_pkt_burst = dp_tx->pkt_burst; + dev->dev_ops = &sfc_eth_dev_secondary_ops; + + return 0; + +fail_dp_tx_multi_process: +fail_dp_tx: +fail_dp_rx_multi_process: +fail_dp_rx: + return rc; +} + +static void +sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev) +{ + dev->dev_ops = NULL; + dev->tx_pkt_burst = NULL; + dev->rx_pkt_burst = NULL; +} + +static void sfc_register_dp(void) { /* Register once */ @@ -1486,19 +1732,27 @@ static int sfc_eth_dev_init(struct rte_eth_dev *dev) { struct sfc_adapter *sa = dev->data->dev_private; - struct rte_pci_device *pci_dev = SFC_DEV_TO_PCI(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); int rc; const efx_nic_cfg_t *encp; const struct ether_addr *from; sfc_register_dp(); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -sfc_eth_dev_secondary_set_ops(dev); + /* Required for logging */ + sa->pci_addr = pci_dev->addr; + sa->port_id = dev->data->port_id; + sa->eth_dev = dev; /* Copy PCI device info to the dev->data */ rte_eth_copy_pci_info(dev, pci_dev); + dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; + rc = sfc_kvargs_parse(sa); if (rc != 0) goto fail_kvargs_parse; @@ -1549,6 +1803,8 @@ sfc_eth_dev_init(struct rte_eth_dev *dev) return 0; fail_attach: + sfc_eth_dev_clear_ops(dev); + fail_set_ops: sfc_unprobe(sa); @@ -1571,22 +1827,26 @@ fail_kvargs_parse: static int sfc_eth_dev_uninit(struct rte_eth_dev *dev) { - struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_adapter *sa; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + sfc_eth_dev_secondary_clear_ops(dev); + return 0; + } + + sa = dev->data->dev_private; sfc_log_init(sa, "entry"); sfc_adapter_lock(sa); + sfc_eth_dev_clear_ops(dev); + sfc_detach(sa); sfc_unprobe(sa); rte_free(dev->data->mac_addrs); dev->data->mac_addrs = NULL; - dev->dev_ops = NULL; - dev->rx_pkt_burst = NULL; - dev->tx_pkt_burst = NULL; - sfc_kvargs_cleanup(sa); sfc_adapter_unlock(sa); diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c index 160d39f9..a16dc27b 100644 --- a/drivers/net/sfc/sfc_ev.c +++ b/drivers/net/sfc/sfc_ev.c @@ -237,8 +237,7 @@ sfc_ev_dp_tx(void *arg, __rte_unused uint32_t label, uint32_t id) } static boolean_t -sfc_ev_exception(void *arg, __rte_unused uint32_t code, - __rte_unused uint32_t data) +sfc_ev_exception(void *arg, uint32_t code, __rte_unused uint32_t data) { struct sfc_evq *evq = arg; diff --git a/drivers/net/sfc/sfc_flow.c b/drivers/net/sfc/sfc_flow.c index c3ea43a6..110dfb89 100644 --- a/drivers/net/sfc/sfc_flow.c +++ b/drivers/net/sfc/sfc_flow.c @@ -1112,12 +1112,35 @@ sfc_flow_flush(struct rte_eth_dev *dev, return -ret; } +static int +sfc_flow_isolate(struct rte_eth_dev *dev, int enable, + struct rte_flow_error *error) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + int ret = 0; + + sfc_adapter_lock(sa); + if (sa->state != SFC_ADAPTER_INITIALIZED) { + rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "please close the port first"); + ret = -rte_errno; + } else { + port->isolated = (enable) ? B_TRUE : B_FALSE; + } + sfc_adapter_unlock(sa); + + return ret; +} + const struct rte_flow_ops sfc_flow_ops = { .validate = sfc_flow_validate, .create = sfc_flow_create, .destroy = sfc_flow_destroy, .flush = sfc_flow_flush, .query = NULL, + .isolate = sfc_flow_isolate, }; void diff --git a/drivers/net/sfc/sfc_intr.c b/drivers/net/sfc/sfc_intr.c index 7eb4b86c..ee59cb1c 100644 --- a/drivers/net/sfc/sfc_intr.c +++ b/drivers/net/sfc/sfc_intr.c @@ -77,7 +77,7 @@ sfc_intr_line_handler(void *cb_arg) boolean_t fatal; uint32_t qmask; unsigned int lsc_seq = sa->port.lsc_seq; - struct rte_pci_device *pci_dev = SFC_DEV_TO_PCI(sa->eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); sfc_log_init(sa, "entry"); @@ -111,7 +111,8 @@ exit: sa->eth_dev->data->dev_link.link_status ? "UP" : "DOWN"); _rte_eth_dev_callback_process(sa->eth_dev, - RTE_ETH_EVENT_INTR_LSC, NULL); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } } @@ -122,7 +123,7 @@ sfc_intr_message_handler(void *cb_arg) efx_nic_t *enp = sa->nic; boolean_t fatal; unsigned int lsc_seq = sa->port.lsc_seq; - struct rte_pci_device *pci_dev = SFC_DEV_TO_PCI(sa->eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); sfc_log_init(sa, "entry"); @@ -152,7 +153,8 @@ exit: if (lsc_seq != sa->port.lsc_seq) { sfc_info(sa, "link status change event"); _rte_eth_dev_callback_process(sa->eth_dev, - RTE_ETH_EVENT_INTR_LSC, NULL); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } } @@ -177,7 +179,7 @@ sfc_intr_start(struct sfc_adapter *sa) if (rc != 0) goto fail_intr_init; - pci_dev = SFC_DEV_TO_PCI(sa->eth_dev); + pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); intr_handle = &pci_dev->intr_handle; if (intr->handler != NULL) { @@ -232,7 +234,7 @@ void sfc_intr_stop(struct sfc_adapter *sa) { struct sfc_intr *intr = &sa->intr; - struct rte_pci_device *pci_dev = SFC_DEV_TO_PCI(sa->eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); sfc_log_init(sa, "entry"); @@ -306,7 +308,7 @@ int sfc_intr_attach(struct sfc_adapter *sa) { struct sfc_intr *intr = &sa->intr; - struct rte_pci_device *pci_dev = SFC_DEV_TO_PCI(sa->eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); sfc_log_init(sa, "entry"); diff --git a/drivers/net/sfc/sfc_log.h b/drivers/net/sfc/sfc_log.h index 8a5e2302..b1a9df44 100644 --- a/drivers/net/sfc/sfc_log.h +++ b/drivers/net/sfc/sfc_log.h @@ -35,17 +35,16 @@ /* Log PMD message, automatically add prefix and \n */ #define SFC_LOG(sa, level, ...) \ do { \ - const struct rte_eth_dev *_dev = (sa)->eth_dev; \ - const struct rte_pci_device *_pci_dev = SFC_DEV_TO_PCI(_dev); \ + const struct sfc_adapter *__sa = (sa); \ \ RTE_LOG(level, PMD, \ RTE_FMT("sfc_efx " PCI_PRI_FMT " #%" PRIu8 ": " \ RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ - _pci_dev->addr.domain, \ - _pci_dev->addr.bus, \ - _pci_dev->addr.devid, \ - _pci_dev->addr.function, \ - _dev->data->port_id, \ + __sa->pci_addr.domain, \ + __sa->pci_addr.bus, \ + __sa->pci_addr.devid, \ + __sa->pci_addr.function, \ + __sa->port_id, \ RTE_FMT_TAIL(__VA_ARGS__,))); \ } while (0) diff --git a/drivers/net/sfc/sfc_port.c b/drivers/net/sfc/sfc_port.c index ee96bcde..c1466a77 100644 --- a/drivers/net/sfc/sfc_port.c +++ b/drivers/net/sfc/sfc_port.c @@ -151,6 +151,7 @@ sfc_port_start(struct sfc_adapter *sa) uint32_t phy_adv_cap; const uint32_t phy_pause_caps = ((1u << EFX_PHY_CAP_PAUSE) | (1u << EFX_PHY_CAP_ASYM)); + unsigned int i; sfc_log_init(sa, "entry"); @@ -186,26 +187,29 @@ sfc_port_start(struct sfc_adapter *sa) if (rc != 0) goto fail_mac_pdu_set; - sfc_log_init(sa, "set MAC address"); - rc = efx_mac_addr_set(sa->nic, - sa->eth_dev->data->mac_addrs[0].addr_bytes); - if (rc != 0) - goto fail_mac_addr_set; - - sfc_log_init(sa, "set MAC filters"); - port->promisc = (sa->eth_dev->data->promiscuous != 0) ? - B_TRUE : B_FALSE; - port->allmulti = (sa->eth_dev->data->all_multicast != 0) ? - B_TRUE : B_FALSE; - rc = sfc_set_rx_mode(sa); - if (rc != 0) - goto fail_mac_filter_set; + if (!port->isolated) { + struct ether_addr *mac_addrs = sa->eth_dev->data->mac_addrs; - sfc_log_init(sa, "set multicast address list"); - rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs, - port->nb_mcast_addrs); - if (rc != 0) - goto fail_mcast_address_list_set; + sfc_log_init(sa, "set MAC address"); + rc = efx_mac_addr_set(sa->nic, mac_addrs[0].addr_bytes); + if (rc != 0) + goto fail_mac_addr_set; + + sfc_log_init(sa, "set MAC filters"); + port->promisc = (sa->eth_dev->data->promiscuous != 0) ? + B_TRUE : B_FALSE; + port->allmulti = (sa->eth_dev->data->all_multicast != 0) ? + B_TRUE : B_FALSE; + rc = sfc_set_rx_mode(sa); + if (rc != 0) + goto fail_mac_filter_set; + + sfc_log_init(sa, "set multicast address list"); + rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs, + port->nb_mcast_addrs); + if (rc != 0) + goto fail_mcast_address_list_set; + } if (port->mac_stats_reset_pending) { rc = sfc_port_reset_mac_stats(sa); @@ -219,6 +223,10 @@ sfc_port_start(struct sfc_adapter *sa) efx_mac_stats_get_mask(sa->nic, port->mac_stats_mask, sizeof(port->mac_stats_mask)); + for (i = 0, port->mac_stats_nb_supported = 0; i < EFX_MAC_NSTATS; ++i) + if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) + port->mac_stats_nb_supported++; + port->mac_stats_update_generation = 0; if (port->mac_stats_update_period_ms != 0) { @@ -242,6 +250,18 @@ sfc_port_start(struct sfc_adapter *sa) } } + if ((port->mac_stats_update_period_ms != 0) && + port->mac_stats_periodic_dma_supported) { + /* + * Request an explicit MAC stats upload immediately to + * preclude bogus figures readback if the user decides + * to read stats before periodic DMA is really started + */ + rc = efx_mac_stats_upload(sa->nic, &port->mac_stats_dma_mem); + if (rc != 0) + goto fail_mac_stats_upload; + } + sfc_log_init(sa, "disable MAC drain"); rc = efx_mac_drain(sa->nic, B_FALSE); if (rc != 0) @@ -262,6 +282,7 @@ fail_mac_drain: (void)efx_mac_stats_periodic(sa->nic, &port->mac_stats_dma_mem, 0, B_FALSE); +fail_mac_stats_upload: fail_mac_stats_periodic: fail_mcast_address_list_set: fail_mac_filter_set: diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c index 2ecd6f26..1bf86445 100644 --- a/drivers/net/sfc/sfc_rx.c +++ b/drivers/net/sfc/sfc_rx.c @@ -292,7 +292,7 @@ sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (desc_flags & EFX_PKT_CONT) { /* The packet is scattered, more fragments to come */ scatter_pkt = m; - /* Futher fragments have no prefix */ + /* Further fragments have no prefix */ prefix_size = 0; continue; } @@ -529,6 +529,7 @@ sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) struct sfc_rxq *rxq; unsigned int retry_count; unsigned int wait_count; + int rc; rxq = sa->rxq_info[sw_index].rxq; SFC_ASSERT(rxq->state & SFC_RXQ_STARTED); @@ -541,8 +542,10 @@ sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) ((rxq->state & SFC_RXQ_FLUSHED) == 0) && (retry_count < SFC_RX_QFLUSH_ATTEMPTS); ++retry_count) { - if (efx_rx_qflush(rxq->common) != 0) { - rxq->state |= SFC_RXQ_FLUSH_FAILED; + rc = efx_rx_qflush(rxq->common); + if (rc != 0) { + rxq->state |= (rc == EALREADY) ? + SFC_RXQ_FLUSHED : SFC_RXQ_FLUSH_FAILED; break; } rxq->state &= ~SFC_RXQ_FLUSH_FAILED; @@ -627,6 +630,7 @@ retry: int sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) { + struct sfc_port *port = &sa->port; struct sfc_rxq_info *rxq_info; struct sfc_rxq *rxq; struct sfc_evq *evq; @@ -661,7 +665,7 @@ sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) rxq->state |= SFC_RXQ_STARTED; - if (sw_index == 0) { + if ((sw_index == 0) && !port->isolated) { rc = sfc_rx_default_rxq_set_filter(sa, rxq); if (rc != 0) goto fail_mac_filter_default_rxq_set; @@ -942,7 +946,7 @@ sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, info.mem_bar = sa->mem_bar.esb_base; rc = sa->dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index, - &SFC_DEV_TO_PCI(sa->eth_dev)->addr, + &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr, socket_id, &info, &rxq->dp); if (rc != 0) goto fail_dp_rx_qcreate; diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c index 23230144..fc439cb6 100644 --- a/drivers/net/sfc/sfc_tx.c +++ b/drivers/net/sfc/sfc_tx.c @@ -185,7 +185,7 @@ sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index, info.mem_bar = sa->mem_bar.esb_base; rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index, - &SFC_DEV_TO_PCI(sa->eth_dev)->addr, + &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr, socket_id, &info, &txq->dp); if (rc != 0) goto fail_dp_tx_qinit; @@ -479,6 +479,7 @@ sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index) struct sfc_txq *txq; unsigned int retry_count; unsigned int wait_count; + int rc; sfc_log_init(sa, "TxQ = %u", sw_index); @@ -502,8 +503,10 @@ sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index) ((txq->state & SFC_TXQ_FLUSHED) == 0) && (retry_count < SFC_TX_QFLUSH_ATTEMPTS); ++retry_count) { - if (efx_tx_qflush(txq->common) != 0) { - txq->state |= SFC_TXQ_FLUSH_FAILED; + rc = efx_tx_qflush(txq->common); + if (rc != 0) { + txq->state |= (rc == EALREADY) ? + SFC_TXQ_FLUSHED : SFC_TXQ_FLUSH_FAILED; break; } diff --git a/drivers/net/szedata2/Makefile b/drivers/net/szedata2/Makefile index 836c3b2a..0e96b922 100644 --- a/drivers/net/szedata2/Makefile +++ b/drivers/net/szedata2/Makefile @@ -48,6 +48,7 @@ LIBABIVER := 1 # all source are stored in SRCS-y # SRCS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += rte_eth_szedata2.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += szedata2_iobuf.c # # Export include files diff --git a/drivers/net/szedata2/rte_eth_szedata2.c b/drivers/net/szedata2/rte_eth_szedata2.c index 54212b71..9c0d57cc 100644 --- a/drivers/net/szedata2/rte_eth_szedata2.c +++ b/drivers/net/szedata2/rte_eth_szedata2.c @@ -53,6 +53,7 @@ #include <rte_atomic.h> #include "rte_eth_szedata2.h" +#include "szedata2_iobuf.h" #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32 @@ -64,7 +65,6 @@ #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2 -#define RTE_SZEDATA2_PCI_DRIVER_NAME "rte_szedata2_pmd" #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u" @@ -1032,7 +1032,7 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct pmd_internals *internals = dev->data->dev_private; - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->if_index = 0; dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = (uint32_t)-1; @@ -1140,6 +1140,33 @@ eth_dev_close(struct rte_eth_dev *dev) dev->data->nb_tx_queues = 0; } +/** + * Function takes value from first IBUF status register. + * Values in IBUF and OBUF should be same. + * + * @param internals + * Pointer to device private structure. + * @return + * Link speed constant. + */ +static inline enum szedata2_link_speed +get_link_speed(const struct pmd_internals *internals) +{ + const volatile struct szedata2_ibuf *ibuf = + ibuf_ptr_by_index(internals->pci_rsc, 0); + uint32_t speed = (szedata2_read32(&ibuf->ibuf_st) & 0x70) >> 4; + switch (speed) { + case 0x03: + return SZEDATA2_LINK_SPEED_10G; + case 0x04: + return SZEDATA2_LINK_SPEED_40G; + case 0x05: + return SZEDATA2_LINK_SPEED_100G; + default: + return SZEDATA2_LINK_SPEED_DEFAULT; + } +} + static int eth_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) @@ -1149,11 +1176,11 @@ eth_link_update(struct rte_eth_dev *dev, struct rte_eth_link *dev_link = &dev->data->dev_link; struct pmd_internals *internals = (struct pmd_internals *) dev->data->dev_private; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); + const volatile struct szedata2_ibuf *ibuf; + uint32_t i; + bool link_is_up = false; - switch (cgmii_link_speed(ibuf)) { + switch (get_link_speed(internals)) { case SZEDATA2_LINK_SPEED_10G: link.link_speed = ETH_SPEED_NUM_10G; break; @@ -1171,8 +1198,19 @@ eth_link_update(struct rte_eth_dev *dev, /* szedata2 uses only full duplex */ link.link_duplex = ETH_LINK_FULL_DUPLEX; - link.link_status = (cgmii_ibuf_is_enabled(ibuf) && - cgmii_ibuf_is_link_up(ibuf)) ? ETH_LINK_UP : ETH_LINK_DOWN; + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf = ibuf_ptr_by_index(internals->pci_rsc, i); + /* + * Link is considered up if at least one ibuf is enabled + * and up. + */ + if (ibuf_is_enabled(ibuf) && ibuf_is_link_up(ibuf)) { + link_is_up = true; + break; + } + } + + link.link_status = (link_is_up) ? ETH_LINK_UP : ETH_LINK_DOWN; link.link_autoneg = ETH_LINK_SPEED_FIXED; @@ -1187,15 +1225,12 @@ eth_dev_set_link_up(struct rte_eth_dev *dev) { struct pmd_internals *internals = (struct pmd_internals *) dev->data->dev_private; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF, - volatile struct szedata2_cgmii_obuf *); - - cgmii_ibuf_enable(ibuf); - cgmii_obuf_enable(obuf); + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) + ibuf_enable(ibuf_ptr_by_index(internals->pci_rsc, i)); + for (i = 0; i < szedata2_obuf_count; i++) + obuf_enable(obuf_ptr_by_index(internals->pci_rsc, i)); return 0; } @@ -1204,15 +1239,12 @@ eth_dev_set_link_down(struct rte_eth_dev *dev) { struct pmd_internals *internals = (struct pmd_internals *) dev->data->dev_private; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF, - volatile struct szedata2_cgmii_obuf *); - - cgmii_ibuf_disable(ibuf); - cgmii_obuf_disable(obuf); + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) + ibuf_disable(ibuf_ptr_by_index(internals->pci_rsc, i)); + for (i = 0; i < szedata2_obuf_count; i++) + obuf_disable(obuf_ptr_by_index(internals->pci_rsc, i)); return 0; } @@ -1292,10 +1324,12 @@ eth_promiscuous_enable(struct rte_eth_dev *dev) { struct pmd_internals *internals = (struct pmd_internals *) dev->data->dev_private; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_PROMISC); + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_PROMISC); + } } static void @@ -1303,10 +1337,12 @@ eth_promiscuous_disable(struct rte_eth_dev *dev) { struct pmd_internals *internals = (struct pmd_internals *) dev->data->dev_private; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID); + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_ONLY_VALID); + } } static void @@ -1314,10 +1350,12 @@ eth_allmulticast_enable(struct rte_eth_dev *dev) { struct pmd_internals *internals = (struct pmd_internals *) dev->data->dev_private; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ALL_MULTICAST); + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_ALL_MULTICAST); + } } static void @@ -1325,10 +1363,12 @@ eth_allmulticast_disable(struct rte_eth_dev *dev) { struct pmd_internals *internals = (struct pmd_internals *) dev->data->dev_private; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID); + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_ONLY_VALID); + } } static const struct eth_dev_ops ops = { @@ -1431,7 +1471,7 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) struct szedata *szedata_temp; int ret; uint32_t szedata2_index; - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_pci_addr *pci_addr = &pci_dev->addr; struct rte_mem_resource *pci_rsc = &pci_dev->mem_resource[PCI_RESOURCE_NUMBER]; @@ -1554,7 +1594,7 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) static int rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_pci_addr *pci_addr = &pci_dev->addr; rte_free(dev->data->mac_addrs); diff --git a/drivers/net/szedata2/rte_eth_szedata2.h b/drivers/net/szedata2/rte_eth_szedata2.h index afe8a383..f25d4c59 100644 --- a/drivers/net/szedata2/rte_eth_szedata2.h +++ b/drivers/net/szedata2/rte_eth_szedata2.h @@ -34,9 +34,11 @@ #ifndef RTE_PMD_SZEDATA2_H_ #define RTE_PMD_SZEDATA2_H_ -#include <stdbool.h> +#include <stdint.h> -#include <rte_byteorder.h> +#include <libsze2.h> + +#include <rte_common.h> /* PCI Vendor ID */ #define PCI_VENDOR_ID_NETCOPE 0x1b26 @@ -58,7 +60,7 @@ * Round 'what' to the nearest larger (or equal) multiple of '8' * (szedata2 packet is aligned to 8 bytes) */ -#define RTE_SZE2_ALIGN8(what) (((what) + ((8) - 1)) & (~((8) - 1))) +#define RTE_SZE2_ALIGN8(what) RTE_ALIGN(what, 8) /*! main handle structure */ struct szedata { @@ -113,338 +115,4 @@ struct szedata { int numa_node; }; -/* - * @return Byte from PCI resource at offset "offset". - */ -static inline uint8_t -pci_resource_read8(struct rte_mem_resource *rsc, uint32_t offset) -{ - return *((uint8_t *)((uint8_t *)rsc->addr + offset)); -} - -/* - * @return Two bytes from PCI resource starting at offset "offset". - */ -static inline uint16_t -pci_resource_read16(struct rte_mem_resource *rsc, uint32_t offset) -{ - return rte_le_to_cpu_16(*((uint16_t *)((uint8_t *)rsc->addr + - offset))); -} - -/* - * @return Four bytes from PCI resource starting at offset "offset". - */ -static inline uint32_t -pci_resource_read32(struct rte_mem_resource *rsc, uint32_t offset) -{ - return rte_le_to_cpu_32(*((uint32_t *)((uint8_t *)rsc->addr + - offset))); -} - -/* - * @return Eight bytes from PCI resource starting at offset "offset". - */ -static inline uint64_t -pci_resource_read64(struct rte_mem_resource *rsc, uint32_t offset) -{ - return rte_le_to_cpu_64(*((uint64_t *)((uint8_t *)rsc->addr + - offset))); -} - -/* - * Write one byte to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write8(struct rte_mem_resource *rsc, uint32_t offset, uint8_t val) -{ - *((uint8_t *)((uint8_t *)rsc->addr + offset)) = val; -} - -/* - * Write two bytes to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write16(struct rte_mem_resource *rsc, uint32_t offset, - uint16_t val) -{ - *((uint16_t *)((uint8_t *)rsc->addr + offset)) = rte_cpu_to_le_16(val); -} - -/* - * Write four bytes to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write32(struct rte_mem_resource *rsc, uint32_t offset, - uint32_t val) -{ - *((uint32_t *)((uint8_t *)rsc->addr + offset)) = rte_cpu_to_le_32(val); -} - -/* - * Write eight bytes to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write64(struct rte_mem_resource *rsc, uint32_t offset, - uint64_t val) -{ - *((uint64_t *)((uint8_t *)rsc->addr + offset)) = rte_cpu_to_le_64(val); -} - -#define SZEDATA2_PCI_RESOURCE_PTR(rsc, offset, type) \ - ((type)(((uint8_t *)(rsc)->addr) + (offset))) - -enum szedata2_link_speed { - SZEDATA2_LINK_SPEED_DEFAULT = 0, - SZEDATA2_LINK_SPEED_10G, - SZEDATA2_LINK_SPEED_40G, - SZEDATA2_LINK_SPEED_100G, -}; - -enum szedata2_mac_check_mode { - SZEDATA2_MAC_CHMODE_PROMISC = 0x0, - SZEDATA2_MAC_CHMODE_ONLY_VALID = 0x1, - SZEDATA2_MAC_CHMODE_ALL_BROADCAST = 0x2, - SZEDATA2_MAC_CHMODE_ALL_MULTICAST = 0x3, -}; - -/* - * Structure describes CGMII IBUF address space - */ -struct szedata2_cgmii_ibuf { - /** Total Received Frames Counter low part */ - uint32_t trfcl; - /** Correct Frames Counter low part */ - uint32_t cfcl; - /** Discarded Frames Counter low part */ - uint32_t dfcl; - /** Counter of frames discarded due to buffer overflow low part */ - uint32_t bodfcl; - /** Total Received Frames Counter high part */ - uint32_t trfch; - /** Correct Frames Counter high part */ - uint32_t cfch; - /** Discarded Frames Counter high part */ - uint32_t dfch; - /** Counter of frames discarded due to buffer overflow high part */ - uint32_t bodfch; - /** IBUF enable register */ - uint32_t ibuf_en; - /** Error mask register */ - uint32_t err_mask; - /** IBUF status register */ - uint32_t ibuf_st; - /** IBUF command register */ - uint32_t ibuf_cmd; - /** Minimum frame length allowed */ - uint32_t mfla; - /** Frame MTU */ - uint32_t mtu; - /** MAC address check mode */ - uint32_t mac_chmode; - /** Octets Received OK Counter low part */ - uint32_t orocl; - /** Octets Received OK Counter high part */ - uint32_t oroch; -} __rte_packed; - -/* Offset of CGMII IBUF memory for MAC addresses */ -#define SZEDATA2_CGMII_IBUF_MAC_MEM_OFF 0x80 - -/* - * @return - * true if IBUF is enabled - * false if IBUF is disabled - */ -static inline bool -cgmii_ibuf_is_enabled(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - return ((rte_le_to_cpu_32(ibuf->ibuf_en) & 0x1) != 0) ? true : false; -} - -/* - * Enables IBUF. - */ -static inline void -cgmii_ibuf_enable(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - ibuf->ibuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(ibuf->ibuf_en) | 0x1); -} - -/* - * Disables IBUF. - */ -static inline void -cgmii_ibuf_disable(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - ibuf->ibuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(ibuf->ibuf_en) & ~0x1); -} - -/* - * @return - * true if ibuf link is up - * false if ibuf link is down - */ -static inline bool -cgmii_ibuf_is_link_up(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - return ((rte_le_to_cpu_32(ibuf->ibuf_st) & 0x80) != 0) ? true : false; -} - -/* - * @return - * MAC address check mode - */ -static inline enum szedata2_mac_check_mode -cgmii_ibuf_mac_mode_read(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - switch (rte_le_to_cpu_32(ibuf->mac_chmode) & 0x3) { - case 0x0: - return SZEDATA2_MAC_CHMODE_PROMISC; - case 0x1: - return SZEDATA2_MAC_CHMODE_ONLY_VALID; - case 0x2: - return SZEDATA2_MAC_CHMODE_ALL_BROADCAST; - case 0x3: - return SZEDATA2_MAC_CHMODE_ALL_MULTICAST; - default: - return SZEDATA2_MAC_CHMODE_PROMISC; - } -} - -/* - * Writes "mode" in MAC address check mode register. - */ -static inline void -cgmii_ibuf_mac_mode_write(volatile struct szedata2_cgmii_ibuf *ibuf, - enum szedata2_mac_check_mode mode) -{ - ibuf->mac_chmode = rte_cpu_to_le_32( - (rte_le_to_cpu_32(ibuf->mac_chmode) & ~0x3) | mode); -} - -/* - * Structure describes CGMII OBUF address space - */ -struct szedata2_cgmii_obuf { - /** Total Sent Frames Counter low part */ - uint32_t tsfcl; - /** Octets Sent Counter low part */ - uint32_t oscl; - /** Total Discarded Frames Counter low part */ - uint32_t tdfcl; - /** reserved */ - uint32_t reserved1; - /** Total Sent Frames Counter high part */ - uint32_t tsfch; - /** Octets Sent Counter high part */ - uint32_t osch; - /** Total Discarded Frames Counter high part */ - uint32_t tdfch; - /** reserved */ - uint32_t reserved2; - /** OBUF enable register */ - uint32_t obuf_en; - /** reserved */ - uint64_t reserved3; - /** OBUF control register */ - uint32_t ctrl; - /** OBUF status register */ - uint32_t obuf_st; -} __rte_packed; - -/* - * @return - * true if OBUF is enabled - * false if OBUF is disabled - */ -static inline bool -cgmii_obuf_is_enabled(volatile struct szedata2_cgmii_obuf *obuf) -{ - return ((rte_le_to_cpu_32(obuf->obuf_en) & 0x1) != 0) ? true : false; -} - -/* - * Enables OBUF. - */ -static inline void -cgmii_obuf_enable(volatile struct szedata2_cgmii_obuf *obuf) -{ - obuf->obuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(obuf->obuf_en) | 0x1); -} - -/* - * Disables OBUF. - */ -static inline void -cgmii_obuf_disable(volatile struct szedata2_cgmii_obuf *obuf) -{ - obuf->obuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(obuf->obuf_en) & ~0x1); -} - -/* - * Function takes value from IBUF status register. Values in IBUF and OBUF - * should be same. - * - * @return Link speed constant. - */ -static inline enum szedata2_link_speed -cgmii_link_speed(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - uint32_t speed = (rte_le_to_cpu_32(ibuf->ibuf_st) & 0x70) >> 4; - switch (speed) { - case 0x03: - return SZEDATA2_LINK_SPEED_10G; - case 0x04: - return SZEDATA2_LINK_SPEED_40G; - case 0x05: - return SZEDATA2_LINK_SPEED_100G; - default: - return SZEDATA2_LINK_SPEED_DEFAULT; - } -} - -/* - * IBUFs and OBUFs can generally be located at different offsets in different - * firmwares. - * This part defines base offsets of IBUFs and OBUFs through various firmwares. - * Currently one firmware type is supported. - * Type of firmware is set through configuration option - * CONFIG_RTE_LIBRTE_PMD_SZEDATA_AS. - * Possible values are: - * 0 - for firmwares: - * NIC_100G1_LR4 - * HANIC_100G1_LR4 - * HANIC_100G1_SR10 - */ -#if !defined(RTE_LIBRTE_PMD_SZEDATA2_AS) -#error "RTE_LIBRTE_PMD_SZEDATA2_AS has to be defined" -#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 0 - -/* - * CGMII IBUF offset from the beginning of PCI resource address space. - */ -#define SZEDATA2_CGMII_IBUF_BASE_OFF 0x8000 -/* - * Size of CGMII IBUF. - */ -#define SZEDATA2_CGMII_IBUF_SIZE 0x200 - -/* - * GCMII OBUF offset from the beginning of PCI resource address space. - */ -#define SZEDATA2_CGMII_OBUF_BASE_OFF 0x9000 -/* - * Size of CGMII OBUF. - */ -#define SZEDATA2_CGMII_OBUF_SIZE 0x100 - -#else -#error "RTE_LIBRTE_PMD_SZEDATA2_AS has wrong value, see comments in config file" -#endif - -#endif +#endif /* RTE_PMD_SZEDATA2_H_ */ diff --git a/drivers/net/szedata2/szedata2_iobuf.c b/drivers/net/szedata2/szedata2_iobuf.c new file mode 100644 index 00000000..3b9a71fe --- /dev/null +++ b/drivers/net/szedata2/szedata2_iobuf.c @@ -0,0 +1,203 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 CESNET + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of CESNET nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdint.h> + +#include <rte_common.h> + +#include "szedata2_iobuf.h" + +/* + * IBUFs and OBUFs can generally be located at different offsets in different + * firmwares (modes). + * This part defines base offsets of IBUFs and OBUFs for various cards + * and firmwares (modes). + * Type of firmware (mode) is set through configuration option + * CONFIG_RTE_LIBRTE_PMD_SZEDATA2_AS. + * Possible values are: + * 0 - for cards (modes): + * NFB-100G1 (100G1) + * + * 1 - for cards (modes): + * NFB-100G2Q (100G1) + * + * 2 - for cards (modes): + * NFB-40G2 (40G2) + * NFB-100G2C (100G2) + * NFB-100G2Q (40G2) + * + * 3 - for cards (modes): + * NFB-40G2 (10G8) + * NFB-100G2Q (10G8) + * + * 4 - for cards (modes): + * NFB-100G1 (10G10) + * + * 5 - for experimental firmwares and future use + */ +#if !defined(RTE_LIBRTE_PMD_SZEDATA2_AS) +#error "RTE_LIBRTE_PMD_SZEDATA2_AS has to be defined" +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 0 + +/* + * Cards (modes): + * NFB-100G1 (100G1) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 1 + +/* + * Cards (modes): + * NFB-100G2Q (100G1) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8800 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9800 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 2 + +/* + * Cards (modes): + * NFB-40G2 (40G2) + * NFB-100G2C (100G2) + * NFB-100G2Q (40G2) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8800 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000, + 0x9800 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 3 + +/* + * Cards (modes): + * NFB-40G2 (10G8) + * NFB-100G2Q (10G8) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8200, + 0x8400, + 0x8600, + 0x8800, + 0x8A00, + 0x8C00, + 0x8E00 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000, + 0x9200, + 0x9400, + 0x9600, + 0x9800, + 0x9A00, + 0x9C00, + 0x9E00 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 4 + +/* + * Cards (modes): + * NFB-100G1 (10G10) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8200, + 0x8400, + 0x8600, + 0x8800, + 0x8A00, + 0x8C00, + 0x8E00, + 0x9000, + 0x9200 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0xA000, + 0xA200, + 0xA400, + 0xA600, + 0xA800, + 0xAA00, + 0xAC00, + 0xAE00, + 0xB000, + 0xB200 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 5 + +/* + * Future use and experimental firmwares. + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8200, + 0x8400, + 0x8600, + 0x8800 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000, + 0x9200, + 0x9400, + 0x9600, + 0x9800 +}; + +#else +#error "RTE_LIBRTE_PMD_SZEDATA2_AS has wrong value, see comments in config file" +#endif + +const uint32_t szedata2_ibuf_count = RTE_DIM(szedata2_ibuf_base_table); +const uint32_t szedata2_obuf_count = RTE_DIM(szedata2_obuf_base_table); diff --git a/drivers/net/szedata2/szedata2_iobuf.h b/drivers/net/szedata2/szedata2_iobuf.h new file mode 100644 index 00000000..f1ccb3b2 --- /dev/null +++ b/drivers/net/szedata2/szedata2_iobuf.h @@ -0,0 +1,356 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 CESNET + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of CESNET nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SZEDATA2_IOBUF_H_ +#define _SZEDATA2_IOBUF_H_ + +#include <stdint.h> +#include <stdbool.h> + +#include <rte_byteorder.h> +#include <rte_io.h> +#include <rte_dev.h> + +/* IBUF offsets from the beginning of the PCI resource address space. */ +extern const uint32_t szedata2_ibuf_base_table[]; +extern const uint32_t szedata2_ibuf_count; + +/* OBUF offsets from the beginning of the PCI resource address space. */ +extern const uint32_t szedata2_obuf_base_table[]; +extern const uint32_t szedata2_obuf_count; + +enum szedata2_link_speed { + SZEDATA2_LINK_SPEED_DEFAULT = 0, + SZEDATA2_LINK_SPEED_10G, + SZEDATA2_LINK_SPEED_40G, + SZEDATA2_LINK_SPEED_100G, +}; + +enum szedata2_mac_check_mode { + SZEDATA2_MAC_CHMODE_PROMISC = 0x0, + SZEDATA2_MAC_CHMODE_ONLY_VALID = 0x1, + SZEDATA2_MAC_CHMODE_ALL_BROADCAST = 0x2, + SZEDATA2_MAC_CHMODE_ALL_MULTICAST = 0x3, +}; + +/** + * Macro takes pointer to pci resource structure (rsc) + * and returns pointer to mapped resource memory at + * specified offset (offset) typecast to the type (type). + */ +#define SZEDATA2_PCI_RESOURCE_PTR(rsc, offset, type) \ + ((type)(((uint8_t *)(rsc)->addr) + (offset))) + +/** + * Maximum possible number of MAC addresses (limited by IBUF status + * register value MAC_COUNT which has 5 bits). + */ +#define SZEDATA2_IBUF_MAX_MAC_COUNT 32 + +/** + * Structure describes IBUF address space. + */ +struct szedata2_ibuf { + /** Total Received Frames Counter low part */ + uint32_t trfcl; /**< 0x00 */ + /** Correct Frames Counter low part */ + uint32_t cfcl; /**< 0x04 */ + /** Discarded Frames Counter low part */ + uint32_t dfcl; /**< 0x08 */ + /** Counter of frames discarded due to buffer overflow low part */ + uint32_t bodfcl; /**< 0x0C */ + /** Total Received Frames Counter high part */ + uint32_t trfch; /**< 0x10 */ + /** Correct Frames Counter high part */ + uint32_t cfch; /**< 0x14 */ + /** Discarded Frames Counter high part */ + uint32_t dfch; /**< 0x18 */ + /** Counter of frames discarded due to buffer overflow high part */ + uint32_t bodfch; /**< 0x1C */ + /** IBUF enable register */ + uint32_t ibuf_en; /**< 0x20 */ + /** Error mask register */ + uint32_t err_mask; /**< 0x24 */ + /** IBUF status register */ + uint32_t ibuf_st; /**< 0x28 */ + /** IBUF command register */ + uint32_t ibuf_cmd; /**< 0x2C */ + /** Minimum frame length allowed */ + uint32_t mfla; /**< 0x30 */ + /** Frame MTU */ + uint32_t mtu; /**< 0x34 */ + /** MAC address check mode */ + uint32_t mac_chmode; /**< 0x38 */ + /** Octets Received OK Counter low part */ + uint32_t orocl; /**< 0x3C */ + /** Octets Received OK Counter high part */ + uint32_t oroch; /**< 0x40 */ + /** reserved */ + uint8_t reserved[60]; /**< 0x4C */ + /** IBUF memory for MAC addresses */ + uint32_t mac_mem[2 * SZEDATA2_IBUF_MAX_MAC_COUNT]; /**< 0x80 */ +} __rte_packed; + +/** + * Structure describes OBUF address space. + */ +struct szedata2_obuf { + /** Total Sent Frames Counter low part */ + uint32_t tsfcl; /**< 0x00 */ + /** Octets Sent Counter low part */ + uint32_t oscl; /**< 0x04 */ + /** Total Discarded Frames Counter low part */ + uint32_t tdfcl; /**< 0x08 */ + /** reserved */ + uint32_t reserved1; /**< 0x0C */ + /** Total Sent Frames Counter high part */ + uint32_t tsfch; /**< 0x10 */ + /** Octets Sent Counter high part */ + uint32_t osch; /**< 0x14 */ + /** Total Discarded Frames Counter high part */ + uint32_t tdfch; /**< 0x18 */ + /** reserved */ + uint32_t reserved2; /**< 0x1C */ + /** OBUF enable register */ + uint32_t obuf_en; /**< 0x20 */ + /** reserved */ + uint64_t reserved3; /**< 0x24 */ + /** OBUF control register */ + uint32_t ctrl; /**< 0x2C */ + /** OBUF status register */ + uint32_t obuf_st; /**< 0x30 */ +} __rte_packed; + +/** + * Wrapper for reading 4 bytes from device memory in correct endianness. + * + * @param addr + * Address for reading. + * @return + * 4 B value. + */ +static inline uint32_t +szedata2_read32(const volatile void *addr) +{ + return rte_le_to_cpu_32(rte_read32(addr)); +} + +/** + * Wrapper for writing 4 bytes to device memory in correct endianness. + * + * @param value + * Value to write. + * @param addr + * Address for writing. + */ +static inline void +szedata2_write32(uint32_t value, volatile void *addr) +{ + rte_write32(rte_cpu_to_le_32(value), addr); +} + +/** + * Get pointer to IBUF structure according to specified index. + * + * @param rsc + * Pointer to base address of memory resource. + * @param index + * Index of IBUF. + * @return + * Pointer to IBUF structure. + */ +static inline struct szedata2_ibuf * +ibuf_ptr_by_index(struct rte_mem_resource *rsc, uint32_t index) +{ + if (index >= szedata2_ibuf_count) + index = szedata2_ibuf_count - 1; + return SZEDATA2_PCI_RESOURCE_PTR(rsc, szedata2_ibuf_base_table[index], + struct szedata2_ibuf *); +} + +/** + * Get pointer to OBUF structure according to specified idnex. + * + * @param rsc + * Pointer to base address of memory resource. + * @param index + * Index of OBUF. + * @return + * Pointer to OBUF structure. + */ +static inline struct szedata2_obuf * +obuf_ptr_by_index(struct rte_mem_resource *rsc, uint32_t index) +{ + if (index >= szedata2_obuf_count) + index = szedata2_obuf_count - 1; + return SZEDATA2_PCI_RESOURCE_PTR(rsc, szedata2_obuf_base_table[index], + struct szedata2_obuf *); +} + +/** + * Checks if IBUF is enabled. + * + * @param ibuf + * Pointer to IBUF structure. + * @return + * true if IBUF is enabled. + * false if IBUF is disabled. + */ +static inline bool +ibuf_is_enabled(const volatile struct szedata2_ibuf *ibuf) +{ + return ((szedata2_read32(&ibuf->ibuf_en) & 0x1) != 0) ? true : false; +} + +/** + * Enables IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + */ +static inline void +ibuf_enable(volatile struct szedata2_ibuf *ibuf) +{ + szedata2_write32(szedata2_read32(&ibuf->ibuf_en) | 0x1, &ibuf->ibuf_en); +} + +/** + * Disables IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + */ +static inline void +ibuf_disable(volatile struct szedata2_ibuf *ibuf) +{ + szedata2_write32(szedata2_read32(&ibuf->ibuf_en) & ~0x1, + &ibuf->ibuf_en); +} + +/** + * Checks if link is up. + * + * @param ibuf + * Pointer to IBUF structure. + * @return + * true if ibuf link is up. + * false if ibuf link is down. + */ +static inline bool +ibuf_is_link_up(const volatile struct szedata2_ibuf *ibuf) +{ + return ((szedata2_read32(&ibuf->ibuf_st) & 0x80) != 0) ? true : false; +} + +/** + * Get current MAC address check mode from IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + * @return + * MAC address check mode constant. + */ +static inline enum szedata2_mac_check_mode +ibuf_mac_mode_read(const volatile struct szedata2_ibuf *ibuf) +{ + switch (szedata2_read32(&ibuf->mac_chmode) & 0x3) { + case 0x0: + return SZEDATA2_MAC_CHMODE_PROMISC; + case 0x1: + return SZEDATA2_MAC_CHMODE_ONLY_VALID; + case 0x2: + return SZEDATA2_MAC_CHMODE_ALL_BROADCAST; + case 0x3: + return SZEDATA2_MAC_CHMODE_ALL_MULTICAST; + default: + return SZEDATA2_MAC_CHMODE_PROMISC; + } +} + +/** + * Writes mode in MAC address check mode register in IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + * @param mode + * MAC address check mode to set. + */ +static inline void +ibuf_mac_mode_write(volatile struct szedata2_ibuf *ibuf, + enum szedata2_mac_check_mode mode) +{ + szedata2_write32((szedata2_read32(&ibuf->mac_chmode) & ~0x3) | mode, + &ibuf->mac_chmode); +} + +/** + * Checks if obuf is enabled. + * + * @param obuf + * Pointer to OBUF structure. + * @return + * true if OBUF is enabled. + * false if OBUF is disabled. + */ +static inline bool +obuf_is_enabled(const volatile struct szedata2_obuf *obuf) +{ + return ((szedata2_read32(&obuf->obuf_en) & 0x1) != 0) ? true : false; +} + +/** + * Enables OBUF. + * + * @param obuf + * Pointer to OBUF structure. + */ +static inline void +obuf_enable(volatile struct szedata2_obuf *obuf) +{ + szedata2_write32(szedata2_read32(&obuf->obuf_en) | 0x1, &obuf->obuf_en); +} + +/** + * Disables OBUF. + * + * @param obuf + * Pointer to OBUF structure. + */ +static inline void +obuf_disable(volatile struct szedata2_obuf *obuf) +{ + szedata2_write32(szedata2_read32(&obuf->obuf_en) & ~0x1, + &obuf->obuf_en); +} + +#endif /* _SZEDATA2_IOBUF_H_ */ diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c index e44de027..9acea839 100644 --- a/drivers/net/tap/rte_eth_tap.c +++ b/drivers/net/tap/rte_eth_tap.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,6 +33,7 @@ #include <rte_atomic.h> #include <rte_branch_prediction.h> +#include <rte_byteorder.h> #include <rte_common.h> #include <rte_mbuf.h> #include <rte_ethdev.h> @@ -41,6 +42,8 @@ #include <rte_vdev.h> #include <rte_kvargs.h> #include <rte_net.h> +#include <rte_debug.h> +#include <rte_ip.h> #include <sys/types.h> #include <sys/stat.h> @@ -72,6 +75,8 @@ #define ETH_TAP_IFACE_ARG "iface" #define ETH_TAP_SPEED_ARG "speed" #define ETH_TAP_REMOTE_ARG "remote" +#define ETH_TAP_MAC_ARG "mac" +#define ETH_TAP_MAC_FIXED "fixed" #define FLOWER_KERNEL_VERSION KERNEL_VERSION(4, 2, 0) #define FLOWER_VLAN_KERNEL_VERSION KERNEL_VERSION(4, 9, 0) @@ -82,6 +87,7 @@ static const char *valid_arguments[] = { ETH_TAP_IFACE_ARG, ETH_TAP_SPEED_ARG, ETH_TAP_REMOTE_ARG, + ETH_TAP_MAC_ARG, NULL }; @@ -110,10 +116,6 @@ enum ioctl_mode { REMOTE_ONLY, }; -static int -tap_ioctl(struct pmd_internals *pmd, unsigned long request, - struct ifreq *ifr, int set, enum ioctl_mode mode); - static int tap_intr_handle_set(struct rte_eth_dev *dev, int set); /* Tun/Tap allocation routine @@ -122,7 +124,7 @@ static int tap_intr_handle_set(struct rte_eth_dev *dev, int set); * supplied name. */ static int -tun_alloc(struct pmd_internals *pmd, uint16_t qid) +tun_alloc(struct pmd_internals *pmd) { struct ifreq ifr; #ifdef IFF_MULTI_QUEUE @@ -143,7 +145,7 @@ tun_alloc(struct pmd_internals *pmd, uint16_t qid) fd = open(TUN_TAP_DEV_PATH, O_RDWR); if (fd < 0) { - RTE_LOG(ERR, PMD, "Unable to create TAP interface"); + RTE_LOG(ERR, PMD, "Unable to create TAP interface\n"); goto error; } @@ -221,75 +223,6 @@ tun_alloc(struct pmd_internals *pmd, uint16_t qid) strerror(errno)); } - if (qid == 0) { - struct ifreq ifr; - - /* - * pmd->eth_addr contains the desired MAC, either from remote - * or from a random assignment. Sync it with the tap netdevice. - */ - ifr.ifr_hwaddr.sa_family = AF_LOCAL; - rte_memcpy(ifr.ifr_hwaddr.sa_data, &pmd->eth_addr, - ETHER_ADDR_LEN); - if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) - goto error; - - pmd->if_index = if_nametoindex(pmd->name); - if (!pmd->if_index) { - RTE_LOG(ERR, PMD, - "Could not find ifindex for %s: rte_flow won't be usable.\n", - pmd->name); - return fd; - } - if (!pmd->flower_support) - return fd; - if (qdisc_create_multiq(pmd->nlsk_fd, pmd->if_index) < 0) { - RTE_LOG(ERR, PMD, - "Could not create multiq qdisc for %s: rte_flow won't be usable.\n", - pmd->name); - return fd; - } - if (qdisc_create_ingress(pmd->nlsk_fd, pmd->if_index) < 0) { - RTE_LOG(ERR, PMD, - "Could not create multiq qdisc for %s: rte_flow won't be usable.\n", - pmd->name); - return fd; - } - if (pmd->remote_if_index) { - /* - * Flush usually returns negative value because it tries - * to delete every QDISC (and on a running device, one - * QDISC at least is needed). Ignore negative return - * value. - */ - qdisc_flush(pmd->nlsk_fd, pmd->remote_if_index); - if (qdisc_create_ingress(pmd->nlsk_fd, - pmd->remote_if_index) < 0) - goto remote_fail; - LIST_INIT(&pmd->implicit_flows); - if (tap_flow_implicit_create( - pmd, TAP_REMOTE_LOCAL_MAC) < 0) - goto remote_fail; - if (tap_flow_implicit_create( - pmd, TAP_REMOTE_BROADCAST) < 0) - goto remote_fail; - if (tap_flow_implicit_create( - pmd, TAP_REMOTE_BROADCASTV6) < 0) - goto remote_fail; - if (tap_flow_implicit_create( - pmd, TAP_REMOTE_TX) < 0) - goto remote_fail; - } - } - - return fd; - -remote_fail: - RTE_LOG(ERR, PMD, - "Could not set up remote flow rules for %s: remote disabled.\n", - pmd->name); - pmd->remote_if_index = 0; - tap_flow_implicit_flush(pmd, NULL); return fd; error: @@ -298,6 +231,60 @@ error: return -1; } +static void +tap_verify_csum(struct rte_mbuf *mbuf) +{ + uint32_t l2 = mbuf->packet_type & RTE_PTYPE_L2_MASK; + uint32_t l3 = mbuf->packet_type & RTE_PTYPE_L3_MASK; + uint32_t l4 = mbuf->packet_type & RTE_PTYPE_L4_MASK; + unsigned int l2_len = sizeof(struct ether_hdr); + unsigned int l3_len; + uint16_t cksum = 0; + void *l3_hdr; + void *l4_hdr; + + if (l2 == RTE_PTYPE_L2_ETHER_VLAN) + l2_len += 4; + else if (l2 == RTE_PTYPE_L2_ETHER_QINQ) + l2_len += 8; + /* Don't verify checksum for packets with discontinuous L2 header */ + if (unlikely(l2_len + sizeof(struct ipv4_hdr) > + rte_pktmbuf_data_len(mbuf))) + return; + l3_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len); + if (l3 == RTE_PTYPE_L3_IPV4 || l3 == RTE_PTYPE_L3_IPV4_EXT) { + struct ipv4_hdr *iph = l3_hdr; + + /* ihl contains the number of 4-byte words in the header */ + l3_len = 4 * (iph->version_ihl & 0xf); + if (unlikely(l2_len + l3_len > rte_pktmbuf_data_len(mbuf))) + return; + + cksum = ~rte_raw_cksum(iph, l3_len); + mbuf->ol_flags |= cksum ? + PKT_RX_IP_CKSUM_BAD : + PKT_RX_IP_CKSUM_GOOD; + } else if (l3 == RTE_PTYPE_L3_IPV6) { + l3_len = sizeof(struct ipv6_hdr); + } else { + /* IPv6 extensions are not supported */ + return; + } + if (l4 == RTE_PTYPE_L4_UDP || l4 == RTE_PTYPE_L4_TCP) { + l4_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len + l3_len); + /* Don't verify checksum for multi-segment packets. */ + if (mbuf->nb_segs > 1) + return; + if (l3 == RTE_PTYPE_L3_IPV4) + cksum = ~rte_ipv4_udptcp_cksum(l3_hdr, l4_hdr); + else if (l3 == RTE_PTYPE_L3_IPV6) + cksum = ~rte_ipv6_udptcp_cksum(l3_hdr, l4_hdr); + mbuf->ol_flags |= cksum ? + PKT_RX_L4_CKSUM_BAD : + PKT_RX_L4_CKSUM_GOOD; + } +} + /* Callback to handle the rx burst of packets to the correct interface and * file descriptor(s) in a multi-queue setup. */ @@ -378,6 +365,8 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) seg->next = NULL; mbuf->packet_type = rte_net_get_ptype(mbuf, NULL, RTE_PTYPE_ALL_MASK); + if (rxq->rxmode->hw_ip_checksum) + tap_verify_csum(mbuf); /* account for the receive frame */ bufs[num_rx++] = mbuf; @@ -390,6 +379,56 @@ end: return num_rx; } +static void +tap_tx_offload(char *packet, uint64_t ol_flags, unsigned int l2_len, + unsigned int l3_len) +{ + void *l3_hdr = packet + l2_len; + + if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4)) { + struct ipv4_hdr *iph = l3_hdr; + uint16_t cksum; + + iph->hdr_checksum = 0; + cksum = rte_raw_cksum(iph, l3_len); + iph->hdr_checksum = (cksum == 0xffff) ? cksum : ~cksum; + } + if (ol_flags & PKT_TX_L4_MASK) { + uint16_t l4_len; + uint32_t cksum; + uint16_t *l4_cksum; + void *l4_hdr; + + l4_hdr = packet + l2_len + l3_len; + if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM) + l4_cksum = &((struct udp_hdr *)l4_hdr)->dgram_cksum; + else if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) + l4_cksum = &((struct tcp_hdr *)l4_hdr)->cksum; + else + return; + *l4_cksum = 0; + if (ol_flags & PKT_TX_IPV4) { + struct ipv4_hdr *iph = l3_hdr; + + l4_len = rte_be_to_cpu_16(iph->total_length) - l3_len; + cksum = rte_ipv4_phdr_cksum(l3_hdr, 0); + } else { + struct ipv6_hdr *ip6h = l3_hdr; + + /* payload_len does not include ext headers */ + l4_len = rte_be_to_cpu_16(ip6h->payload_len) - + l3_len + sizeof(struct ipv6_hdr); + cksum = rte_ipv6_phdr_cksum(l3_hdr, 0); + } + cksum += rte_raw_cksum(l4_hdr, l4_len); + cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff); + cksum = (~cksum) & 0xffff; + if (cksum == 0) + cksum = 0xffff; + *l4_cksum = cksum; + } +} + /* Callback to handle sending packets from the tap interface */ static uint16_t @@ -410,6 +449,7 @@ pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) struct iovec iovecs[mbuf->nb_segs + 1]; struct tun_pi pi = { .flags = 0 }; struct rte_mbuf *seg = mbuf; + char m_copy[mbuf->data_len]; int n; int j; @@ -425,6 +465,19 @@ pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) rte_pktmbuf_mtod(seg, void *); seg = seg->next; } + if (mbuf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4) || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) { + /* Support only packets with all data in the same seg */ + if (mbuf->nb_segs > 1) + break; + /* To change checksums, work on a copy of data. */ + rte_memcpy(m_copy, rte_pktmbuf_mtod(mbuf, void *), + rte_pktmbuf_data_len(mbuf)); + tap_tx_offload(m_copy, mbuf->ol_flags, + mbuf->l2_len, mbuf->l3_len); + iovecs[1].iov_base = m_copy; + } /* copy the tx frame data */ n = writev(txq->fd, iovecs, mbuf->nb_segs + 1); if (n <= 0) @@ -442,6 +495,24 @@ pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) return num_tx; } +static const char * +tap_ioctl_req2str(unsigned long request) +{ + switch (request) { + case SIOCSIFFLAGS: + return "SIOCSIFFLAGS"; + case SIOCGIFFLAGS: + return "SIOCGIFFLAGS"; + case SIOCGIFHWADDR: + return "SIOCGIFHWADDR"; + case SIOCSIFHWADDR: + return "SIOCSIFHWADDR"; + case SIOCSIFMTU: + return "SIOCSIFMTU"; + } + return "UNKNOWN"; +} + static int tap_ioctl(struct pmd_internals *pmd, unsigned long request, struct ifreq *ifr, int set, enum ioctl_mode mode) @@ -477,9 +548,7 @@ apply: case SIOCSIFMTU: break; default: - RTE_LOG(WARNING, PMD, "%s: ioctl() called with wrong arg\n", - pmd->name); - return -EINVAL; + RTE_ASSERT(!"unsupported request type: must not happen"); } if (ioctl(pmd->ioctl_sock, request, ifr) < 0) goto error; @@ -488,8 +557,8 @@ apply: return 0; error: - RTE_LOG(ERR, PMD, "%s: ioctl(%lu) failed with error: %s\n", - ifr->ifr_name, request, strerror(errno)); + RTE_LOG(DEBUG, PMD, "%s: %s(%s) failed: %s(%d)\n", ifr->ifr_name, + __func__, tap_ioctl_req2str(request), strerror(errno), errno); return -errno; } @@ -500,7 +569,7 @@ tap_link_set_down(struct rte_eth_dev *dev) struct ifreq ifr = { .ifr_flags = IFF_UP }; dev->data->dev_link.link_status = ETH_LINK_DOWN; - return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); + return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_ONLY); } static int @@ -586,6 +655,13 @@ tap_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->min_rx_bufsize = 0; dev_info->pci_dev = NULL; dev_info->speed_capa = tap_dev_speed_capa(); + dev_info->rx_offload_capa = (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM); + dev_info->tx_offload_capa = + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM); } static void @@ -644,7 +720,7 @@ tap_stats_reset(struct rte_eth_dev *dev) } static void -tap_dev_close(struct rte_eth_dev *dev __rte_unused) +tap_dev_close(struct rte_eth_dev *dev) { int i; struct pmd_internals *internals = dev->data->dev_private; @@ -659,6 +735,12 @@ tap_dev_close(struct rte_eth_dev *dev __rte_unused) internals->rxq[i].fd = -1; internals->txq[i].fd = -1; } + + if (internals->remote_if_index) { + /* Restore initial remote state */ + ioctl(internals->ioctl_sock, SIOCSIFFLAGS, + &internals->remote_initial_flags); + } } static void @@ -718,7 +800,7 @@ tap_promisc_enable(struct rte_eth_dev *dev) dev->data->promiscuous = 1; tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); - if (pmd->remote_if_index) + if (pmd->remote_if_index && !pmd->flow_isolate) tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC); } @@ -730,7 +812,7 @@ tap_promisc_disable(struct rte_eth_dev *dev) dev->data->promiscuous = 0; tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); - if (pmd->remote_if_index) + if (pmd->remote_if_index && !pmd->flow_isolate) tap_flow_implicit_destroy(pmd, TAP_REMOTE_PROMISC); } @@ -742,7 +824,7 @@ tap_allmulti_enable(struct rte_eth_dev *dev) dev->data->all_multicast = 1; tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); - if (pmd->remote_if_index) + if (pmd->remote_if_index && !pmd->flow_isolate) tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI); } @@ -754,50 +836,51 @@ tap_allmulti_disable(struct rte_eth_dev *dev) dev->data->all_multicast = 0; tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); - if (pmd->remote_if_index) + if (pmd->remote_if_index && !pmd->flow_isolate) tap_flow_implicit_destroy(pmd, TAP_REMOTE_ALLMULTI); } - static void tap_mac_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) { struct pmd_internals *pmd = dev->data->dev_private; + enum ioctl_mode mode = LOCAL_ONLY; struct ifreq ifr; if (is_zero_ether_addr(mac_addr)) { RTE_LOG(ERR, PMD, "%s: can't set an empty MAC address\n", - dev->data->name); + dev->device->name); return; } /* Check the actual current MAC address on the tap netdevice */ - if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, LOCAL_ONLY) != 0) { - RTE_LOG(ERR, PMD, - "%s: couldn't check current tap MAC address\n", - dev->data->name); + if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) return; - } if (is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, mac_addr)) return; - + /* Check the current MAC address on the remote */ + if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY) < 0) + return; + if (!is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, + mac_addr)) + mode = LOCAL_AND_REMOTE; ifr.ifr_hwaddr.sa_family = AF_LOCAL; rte_memcpy(ifr.ifr_hwaddr.sa_data, mac_addr, ETHER_ADDR_LEN); - if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 1, LOCAL_AND_REMOTE) < 0) + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 1, mode) < 0) return; rte_memcpy(&pmd->eth_addr, mac_addr, ETHER_ADDR_LEN); - if (pmd->remote_if_index) { + if (pmd->remote_if_index && !pmd->flow_isolate) { /* Replace MAC redirection rule after a MAC change */ if (tap_flow_implicit_destroy(pmd, TAP_REMOTE_LOCAL_MAC) < 0) { RTE_LOG(ERR, PMD, "%s: Couldn't delete MAC redirection rule\n", - dev->data->name); + dev->device->name); return; } if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0) RTE_LOG(ERR, PMD, "%s: Couldn't add MAC redirection rule\n", - dev->data->name); + dev->device->name); } } @@ -809,30 +892,16 @@ tap_setup_queue(struct rte_eth_dev *dev, struct pmd_internals *pmd = dev->data->dev_private; struct rx_queue *rx = &internals->rxq[qid]; struct tx_queue *tx = &internals->txq[qid]; - int fd; + int fd = rx->fd == -1 ? tx->fd : rx->fd; - fd = rx->fd; - if (fd < 0) { - fd = tx->fd; + if (fd == -1) { + RTE_LOG(INFO, PMD, "Add queue to TAP %s for qid %d\n", + pmd->name, qid); + fd = tun_alloc(pmd); if (fd < 0) { - RTE_LOG(INFO, PMD, "Add queue to TAP %s for qid %d\n", - pmd->name, qid); - fd = tun_alloc(pmd, qid); - if (fd < 0) { - RTE_LOG(ERR, PMD, "tun_alloc(%s, %d) failed\n", - pmd->name, qid); - return -1; - } - if (qid == 0) { - struct ifreq ifr; - - ifr.ifr_mtu = dev->data->mtu; - if (tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, - LOCAL_AND_REMOTE) < 0) { - close(fd); - return -1; - } - } + RTE_LOG(ERR, PMD, "%s: tun_alloc() failed.\n", + pmd->name); + return -1; } } @@ -845,26 +914,6 @@ tap_setup_queue(struct rte_eth_dev *dev, } static int -rx_setup_queue(struct rte_eth_dev *dev, - struct pmd_internals *internals, - uint16_t qid) -{ - dev->data->rx_queues[qid] = &internals->rxq[qid]; - - return tap_setup_queue(dev, internals, qid); -} - -static int -tx_setup_queue(struct rte_eth_dev *dev, - struct pmd_internals *internals, - uint16_t qid) -{ - dev->data->tx_queues[qid] = &internals->txq[qid]; - - return tap_setup_queue(dev, internals, qid); -} - -static int tap_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, uint16_t nb_rx_desc, @@ -894,17 +943,18 @@ tap_rx_queue_setup(struct rte_eth_dev *dev, rxq->trigger_seen = 1; /* force initial burst */ rxq->in_port = dev->data->port_id; rxq->nb_rx_desc = nb_desc; - iovecs = rte_zmalloc_socket(dev->data->name, sizeof(*iovecs), 0, + iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0, socket_id); if (!iovecs) { RTE_LOG(WARNING, PMD, "%s: Couldn't allocate %d RX descriptors\n", - dev->data->name, nb_desc); + dev->device->name, nb_desc); return -ENOMEM; } rxq->iovecs = iovecs; - fd = rx_setup_queue(dev, internals, rx_queue_id); + dev->data->rx_queues[rx_queue_id] = rxq; + fd = tap_setup_queue(dev, internals, rx_queue_id); if (fd == -1) { ret = fd; goto error; @@ -918,7 +968,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev, if (!*tmp) { RTE_LOG(WARNING, PMD, "%s: couldn't allocate memory for queue %d\n", - dev->data->name, rx_queue_id); + dev->device->name, rx_queue_id); ret = -ENOMEM; goto error; } @@ -955,7 +1005,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev, if (tx_queue_id >= internals->nb_queues) return -1; - ret = tx_setup_queue(dev, internals, tx_queue_id); + dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id]; + ret = tap_setup_queue(dev, internals, tx_queue_id); if (ret == -1) return -1; @@ -1115,32 +1166,16 @@ static const struct eth_dev_ops ops = { .filter_ctrl = tap_dev_filter_ctrl, }; -static int -tap_kernel_support(struct pmd_internals *pmd) -{ - struct utsname utsname; - int ver[3]; - - if (uname(&utsname) == -1 || - sscanf(utsname.release, "%d.%d.%d", - &ver[0], &ver[1], &ver[2]) != 3) - return 0; - if (KERNEL_VERSION(ver[0], ver[1], ver[2]) >= FLOWER_KERNEL_VERSION) - pmd->flower_support = 1; - if (KERNEL_VERSION(ver[0], ver[1], ver[2]) >= - FLOWER_VLAN_KERNEL_VERSION) - pmd->flower_vlan_support = 1; - return 1; -} static int eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, - char *remote_iface) + char *remote_iface, int fixed_mac_type) { int numa_node = rte_socket_id(); struct rte_eth_dev *dev; struct pmd_internals *pmd; struct rte_eth_dev_data *data; + struct ifreq ifr; int i; RTE_LOG(DEBUG, PMD, " TAP device on numa %u\n", rte_socket_id()); @@ -1174,7 +1209,6 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, data->dev_private = pmd; data->dev_flags = RTE_ETH_DEV_DETACHABLE | RTE_ETH_DEV_INTR_LSC; data->numa_node = numa_node; - data->drv_name = pmd_tap_drv.driver.name; data->dev_link = pmd_link; data->mac_addrs = &pmd->eth_addr; @@ -1195,41 +1229,132 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, pmd->txq[i].fd = -1; } - tap_kernel_support(pmd); - if (!pmd->flower_support) - return 0; - LIST_INIT(&pmd->flows); + if (fixed_mac_type) { + /* fixed mac = 00:64:74:61:70:<iface_idx> */ + static int iface_idx; + char mac[ETHER_ADDR_LEN] = "\0dtap"; + + mac[ETHER_ADDR_LEN - 1] = iface_idx++; + rte_memcpy(&pmd->eth_addr, mac, ETHER_ADDR_LEN); + } else { + eth_random_addr((uint8_t *)&pmd->eth_addr); + } + + /* Immediately create the netdevice (this will create the 1st queue). */ + if (tap_setup_queue(dev, pmd, 0) == -1) + goto error_exit; + + ifr.ifr_mtu = dev->data->mtu; + if (tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, LOCAL_AND_REMOTE) < 0) + goto error_exit; + + memset(&ifr, 0, sizeof(struct ifreq)); + ifr.ifr_hwaddr.sa_family = AF_LOCAL; + rte_memcpy(ifr.ifr_hwaddr.sa_data, &pmd->eth_addr, ETHER_ADDR_LEN); + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) + goto error_exit; + /* - * If no netlink socket can be created, then it will fail when - * creating/destroying flow rules. + * Set up everything related to rte_flow: + * - netlink socket + * - tap / remote if_index + * - mandatory QDISCs + * - rte_flow actual/implicit lists + * - implicit rules */ pmd->nlsk_fd = nl_init(0); - if (strlen(remote_iface)) { - struct ifreq ifr; + if (pmd->nlsk_fd == -1) { + RTE_LOG(WARNING, PMD, "%s: failed to create netlink socket.\n", + pmd->name); + goto disable_rte_flow; + } + pmd->if_index = if_nametoindex(pmd->name); + if (!pmd->if_index) { + RTE_LOG(ERR, PMD, "%s: failed to get if_index.\n", pmd->name); + goto disable_rte_flow; + } + if (qdisc_create_multiq(pmd->nlsk_fd, pmd->if_index) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to create multiq qdisc.\n", + pmd->name); + goto disable_rte_flow; + } + if (qdisc_create_ingress(pmd->nlsk_fd, pmd->if_index) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to create ingress qdisc.\n", + pmd->name); + goto disable_rte_flow; + } + LIST_INIT(&pmd->flows); + if (strlen(remote_iface)) { pmd->remote_if_index = if_nametoindex(remote_iface); - snprintf(pmd->remote_iface, RTE_ETH_NAME_MAX_LEN, - "%s", remote_iface); if (!pmd->remote_if_index) { - RTE_LOG(ERR, PMD, "Could not find %s ifindex: " - "remote interface will remain unconfigured\n", - remote_iface); - return 0; + RTE_LOG(ERR, PMD, "%s: failed to get %s if_index.\n", + pmd->name, remote_iface); + goto error_remote; } + snprintf(pmd->remote_iface, RTE_ETH_NAME_MAX_LEN, + "%s", remote_iface); + + /* Save state of remote device */ + tap_ioctl(pmd, SIOCGIFFLAGS, &pmd->remote_initial_flags, 0, REMOTE_ONLY); + + /* Replicate remote MAC address */ if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY) < 0) { - RTE_LOG(ERR, PMD, "Could not get remote MAC address\n"); - goto error_exit; + RTE_LOG(ERR, PMD, "%s: failed to get %s MAC address.\n", + pmd->name, pmd->remote_iface); + goto error_remote; } rte_memcpy(&pmd->eth_addr, ifr.ifr_hwaddr.sa_data, ETHER_ADDR_LEN); - } else { - eth_random_addr((uint8_t *)&pmd->eth_addr); + /* The desired MAC is already in ifreq after SIOCGIFHWADDR. */ + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to get %s MAC address.\n", + pmd->name, remote_iface); + goto error_remote; + } + + /* + * Flush usually returns negative value because it tries to + * delete every QDISC (and on a running device, one QDISC at + * least is needed). Ignore negative return value. + */ + qdisc_flush(pmd->nlsk_fd, pmd->remote_if_index); + if (qdisc_create_ingress(pmd->nlsk_fd, + pmd->remote_if_index) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to create ingress qdisc.\n", + pmd->remote_iface); + goto error_remote; + } + LIST_INIT(&pmd->implicit_flows); + if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0) { + RTE_LOG(ERR, PMD, + "%s: failed to create implicit rules.\n", + pmd->name); + goto error_remote; + } } return 0; +disable_rte_flow: + RTE_LOG(ERR, PMD, " Disabling rte flow support: %s(%d)\n", + strerror(errno), errno); + if (strlen(remote_iface)) { + RTE_LOG(ERR, PMD, "Remote feature requires flow support.\n"); + goto error_exit; + } + return 0; + +error_remote: + RTE_LOG(ERR, PMD, " Can't set up remote feature: %s(%d)\n", + strerror(errno), errno); + tap_flow_implicit_flush(pmd, NULL); + error_exit: - RTE_LOG(DEBUG, PMD, "TAP Unable to initialize %s\n", + RTE_LOG(ERR, PMD, "TAP Unable to initialize %s\n", rte_vdev_device_name(vdev)); rte_free(data); @@ -1275,6 +1400,17 @@ set_remote_iface(const char *key __rte_unused, return 0; } +static int +set_mac_type(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + if (value && + !strncasecmp(ETH_TAP_MAC_FIXED, value, strlen(ETH_TAP_MAC_FIXED))) + *(int *)extra_args = 1; + return 0; +} + /* Open a TAP interface device. */ static int @@ -1286,6 +1422,7 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev) int speed; char tap_name[RTE_ETH_NAME_MAX_LEN]; char remote_iface[RTE_ETH_NAME_MAX_LEN]; + int fixed_mac_type = 0; name = rte_vdev_device_name(dev); params = rte_vdev_device_args(dev); @@ -1296,7 +1433,7 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev) memset(remote_iface, 0, RTE_ETH_NAME_MAX_LEN); if (params && (params[0] != '\0')) { - RTE_LOG(DEBUG, PMD, "paramaters (%s)\n", params); + RTE_LOG(DEBUG, PMD, "parameters (%s)\n", params); kvlist = rte_kvargs_parse(params, valid_arguments); if (kvlist) { @@ -1326,6 +1463,15 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev) if (ret == -1) goto leave; } + + if (rte_kvargs_count(kvlist, ETH_TAP_MAC_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_MAC_ARG, + &set_mac_type, + &fixed_mac_type); + if (ret == -1) + goto leave; + } } } pmd_link.link_speed = speed; @@ -1333,7 +1479,7 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev) RTE_LOG(NOTICE, PMD, "Initializing pmd_tap for %s as %s\n", name, tap_name); - ret = eth_dev_tap_create(dev, tap_name, remote_iface); + ret = eth_dev_tap_create(dev, tap_name, remote_iface, fixed_mac_type); leave: if (ret == -1) { @@ -1364,7 +1510,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev) return 0; internals = eth_dev->data->dev_private; - if (internals->flower_support && internals->nlsk_fd) { + if (internals->nlsk_fd) { tap_flow_flush(eth_dev, NULL); tap_flow_implicit_flush(internals, NULL); nl_final(internals->nlsk_fd); @@ -1391,4 +1537,5 @@ RTE_PMD_REGISTER_ALIAS(net_tap, eth_tap); RTE_PMD_REGISTER_PARAM_STRING(net_tap, ETH_TAP_IFACE_ARG "=<string> " ETH_TAP_SPEED_ARG "=<int> " + ETH_TAP_MAC_ARG "=" ETH_TAP_MAC_FIXED " " ETH_TAP_REMOTE_ARG "=<string>"); diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h index ad497b3d..928a0454 100644 --- a/drivers/net/tap/rte_eth_tap.h +++ b/drivers/net/tap/rte_eth_tap.h @@ -37,6 +37,7 @@ #include <sys/queue.h> #include <sys/uio.h> #include <inttypes.h> +#include <net/if.h> #include <linux/if_tun.h> @@ -83,12 +84,12 @@ struct pmd_internals { char name[RTE_ETH_NAME_MAX_LEN]; /* Internal Tap device name */ uint16_t nb_queues; /* Number of queues supported */ struct ether_addr eth_addr; /* Mac address of the device port */ + struct ifreq remote_initial_flags; /* Remote netdevice flags on init */ int remote_if_index; /* remote netdevice IF_INDEX */ int if_index; /* IF_INDEX for the port */ int ioctl_sock; /* socket for ioctl calls */ int nlsk_fd; /* Netlink socket fd */ - int flower_support; /* 1 if kernel supports, else 0 */ - int flower_vlan_support; /* 1 if kernel supports, else 0 */ + int flow_isolate; /* 1 if flow isolation is enabled */ LIST_HEAD(tap_flows, rte_flow) flows; /* rte_flow rules */ /* implicit rte_flow rules set when a remote device is active */ LIST_HEAD(tap_implicit_flows, rte_flow) implicit_flows; diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c index a0dd5048..41f73452 100644 --- a/drivers/net/tap/tap_flow.c +++ b/drivers/net/tap/tap_flow.c @@ -82,6 +82,8 @@ enum { }; #endif +#define ISOLATE_HANDLE 1 + struct rte_flow { LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */ struct rte_flow *remote_flow; /* associated remote flow */ @@ -98,6 +100,7 @@ struct convert_data { struct remote_rule { struct rte_flow_attr attr; struct rte_flow_item items[2]; + struct rte_flow_action actions[2]; int mirred; }; @@ -126,11 +129,17 @@ tap_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, struct rte_flow_error *error); +static int +tap_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error); + static const struct rte_flow_ops tap_flow_ops = { .validate = tap_flow_validate, .create = tap_flow_create, .destroy = tap_flow_destroy, .flush = tap_flow_flush, + .isolate = tap_flow_isolate, }; /* Static initializer for items. */ @@ -258,6 +267,47 @@ static const struct tap_flow_items tap_flow_items[] = { }, }; +/* + * TC rules, by growing priority + * + * Remote netdevice Tap netdevice + * +-------------+-------------+ +-------------+-------------+ + * | Ingress | Egress | | Ingress | Egress | + * |-------------|-------------| |-------------|-------------| + * | | \ / | | | REMOTE TX | prio 1 + * | | \ / | | | \ / | prio 2 + * | EXPLICIT | \ / | | EXPLICIT | \ / | . + * | | \ / | | | \ / | . + * | RULES | X | | RULES | X | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * + * .... .... .... .... + * + * | . | \ / | | . | \ / | . + * | . | \ / | | . | \ / | . + * | | \ / | | | \ / | + * | LOCAL_MAC | \ / | | \ / | \ / | last prio - 5 + * | PROMISC | X | | \ / | X | last prio - 4 + * | ALLMULTI | / \ | | X | / \ | last prio - 3 + * | BROADCAST | / \ | | / \ | / \ | last prio - 2 + * | BROADCASTV6 | / \ | | / \ | / \ | last prio - 1 + * | xx | / \ | | ISOLATE | / \ | last prio + * +-------------+-------------+ +-------------+-------------+ + * + * The implicit flow rules are stored in a list in with mandatorily the last two + * being the ISOLATE and REMOTE_TX rules. e.g.: + * + * LOCAL_MAC -> BROADCAST -> BROADCASTV6 -> REMOTE_TX -> ISOLATE -> NULL + * + * That enables tap_flow_isolate() to remove implicit rules by popping the list + * head and remove it as long as it applies on the remote netdevice. The + * implicit rule for TX redirection is not removed, as isolate concerns only + * incoming traffic. + */ + static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = { [TAP_REMOTE_LOCAL_MAC] = { .attr = { @@ -364,6 +414,19 @@ static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = { }, .mirred = TCA_EGRESS_MIRROR, }, + [TAP_ISOLATE] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_ISOLATE, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_VOID, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + }, }; /** @@ -971,16 +1034,13 @@ priv_flow_process(struct pmd_internals *pmd, if (err) goto exit_item_not_supported; if (flow && cur_item->convert) { - if (!pmd->flower_vlan_support && - cur_item->convert == tap_flow_create_vlan) - goto exit_item_not_supported; err = cur_item->convert(items, &data); if (err) goto exit_item_not_supported; } } if (flow) { - if (pmd->flower_vlan_support && data.vlan) { + if (data.vlan) { nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, htons(ETH_P_8021Q)); nlattr_add16(&flow->msg.nh, @@ -1168,7 +1228,8 @@ tap_flow_create(struct rte_eth_dev *dev, "Kernel refused TC filter rule creation (%d): %s\n", errno, strerror(errno)); rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE, - NULL, "overlapping rules"); + NULL, + "overlapping rules or Kernel too old for flower support"); goto fail; } LIST_INSERT_HEAD(&pmd->flows, flow, next); @@ -1213,7 +1274,8 @@ tap_flow_create(struct rte_eth_dev *dev, errno, strerror(errno)); rte_flow_error_set( error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, - NULL, "overlapping rules"); + NULL, + "overlapping rules or Kernel too old for flower support"); goto fail; } flow->remote_flow = remote_flow; @@ -1318,6 +1380,78 @@ tap_flow_destroy(struct rte_eth_dev *dev, } /** + * Enable/disable flow isolation. + * + * @see rte_flow_isolate() + * @see rte_flow_ops + */ +static int +tap_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error __rte_unused) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + if (set) + pmd->flow_isolate = 1; + else + pmd->flow_isolate = 0; + /* + * If netdevice is there, setup appropriate flow rules immediately. + * Otherwise it will be set when bringing up the netdevice (tun_alloc). + */ + if (!pmd->rxq[0].fd) + return 0; + if (set) { + struct rte_flow *flow; + + while (1) { + flow = LIST_FIRST(&pmd->implicit_flows); + if (!flow) + break; + /* + * Remove all implicit rules on the remote. + * Keep the local rule to redirect packets on TX. + * Keep also the last implicit local rule: ISOLATE. + */ + if (flow->msg.t.tcm_ifindex == pmd->if_index) + break; + if (tap_flow_destroy_pmd(pmd, flow, NULL) < 0) + goto error; + } + /* Switch the TC rule according to pmd->flow_isolate */ + if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1) + goto error; + } else { + /* Switch the TC rule according to pmd->flow_isolate */ + if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1) + goto error; + if (!pmd->remote_if_index) + return 0; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0) + goto error; + if (dev->data->promiscuous && + tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC) < 0) + goto error; + if (dev->data->all_multicast && + tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI) < 0) + goto error; + } + return 0; +error: + pmd->flow_isolate = 0; + return -rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "TC rule creation failed"); +} + +/** * Destroy all flows. * * @see rte_flow_flush() @@ -1351,6 +1485,13 @@ tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) int tap_flow_implicit_create(struct pmd_internals *pmd, enum implicit_rule_index idx) { + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct rte_flow_action *actions = implicit_rte_flows[idx].actions; + struct rte_flow_action isolate_actions[2] = { + [1] = { + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; struct rte_flow_item *items = implicit_rte_flows[idx].items; struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr; struct rte_flow_item_eth eth_local = { .type = 0 }; @@ -1371,12 +1512,20 @@ int tap_flow_implicit_create(struct pmd_internals *pmd, remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); if (!remote_flow) { - RTE_LOG(ERR, PMD, "Cannot allocate memory for rte_flow"); + RTE_LOG(ERR, PMD, "Cannot allocate memory for rte_flow\n"); goto fail; } msg = &remote_flow->msg; if (idx == TAP_REMOTE_TX) { if_index = pmd->if_index; + } else if (idx == TAP_ISOLATE) { + if_index = pmd->if_index; + /* Don't be exclusive for this rule, it can be changed later. */ + flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE; + isolate_actions[0].type = pmd->flow_isolate ? + RTE_FLOW_ACTION_TYPE_DROP : + RTE_FLOW_ACTION_TYPE_PASSTHRU; + actions = isolate_actions; } else if (idx == TAP_REMOTE_LOCAL_MAC) { /* * eth addr couldn't be set in implicit_rte_flows[] as it is not @@ -1385,18 +1534,25 @@ int tap_flow_implicit_create(struct pmd_internals *pmd, memcpy(ð_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr)); items = items_local; } - tc_init_msg(msg, if_index, RTM_NEWTFILTER, - NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + tc_init_msg(msg, if_index, RTM_NEWTFILTER, flags); msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); - tap_flow_set_handle(remote_flow); - if (priv_flow_process(pmd, attr, items, NULL, NULL, + /* + * The ISOLATE rule is always present and must have a static handle, as + * the action is changed whether the feature is enabled (DROP) or + * disabled (PASSTHRU). + */ + if (idx == TAP_ISOLATE) + remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE; + else + tap_flow_set_handle(remote_flow); + if (priv_flow_process(pmd, attr, items, actions, NULL, remote_flow, implicit_rte_flows[idx].mirred)) { RTE_LOG(ERR, PMD, "rte flow rule validation failed\n"); goto fail; } err = nl_send(pmd->nlsk_fd, &msg->nh); if (err < 0) { - RTE_LOG(ERR, PMD, "Failure sending nl request"); + RTE_LOG(ERR, PMD, "Failure sending nl request\n"); goto fail; } err = nl_recv_ack(pmd->nlsk_fd); @@ -1481,10 +1637,6 @@ tap_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - struct pmd_internals *pmd = dev->data->dev_private; - - if (!pmd->flower_support) - return -ENOTSUP; switch (filter_type) { case RTE_ETH_FILTER_GENERIC: if (filter_op != RTE_ETH_FILTER_GET) @@ -1492,7 +1644,7 @@ tap_dev_filter_ctrl(struct rte_eth_dev *dev, *(const void **)arg = &tap_flow_ops; return 0; default: - RTE_LOG(ERR, PMD, "%p: filter type (%d) not supported", + RTE_LOG(ERR, PMD, "%p: filter type (%d) not supported\n", (void *)dev, filter_type); } return -EINVAL; diff --git a/drivers/net/tap/tap_flow.h b/drivers/net/tap/tap_flow.h index 94414f18..9e332b03 100644 --- a/drivers/net/tap/tap_flow.h +++ b/drivers/net/tap/tap_flow.h @@ -58,6 +58,7 @@ */ enum implicit_rule_index { TAP_REMOTE_TX, + TAP_ISOLATE, TAP_REMOTE_BROADCASTV6, TAP_REMOTE_BROADCAST, TAP_REMOTE_ALLMULTI, diff --git a/drivers/net/thunderx/Makefile b/drivers/net/thunderx/Makefile index 706250b8..915ae945 100644 --- a/drivers/net/thunderx/Makefile +++ b/drivers/net/thunderx/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 Cavium Networks. All rights reserved. +# Copyright(c) 2016 Cavium, Inc. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -13,7 +13,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium Networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # diff --git a/drivers/net/thunderx/base/nicvf_bsvf.c b/drivers/net/thunderx/base/nicvf_bsvf.c index 49a2646d..a4fc04cb 100644 --- a/drivers/net/thunderx/base/nicvf_bsvf.c +++ b/drivers/net/thunderx/base/nicvf_bsvf.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/base/nicvf_bsvf.h b/drivers/net/thunderx/base/nicvf_bsvf.h index fb9b2484..2d9448aa 100644 --- a/drivers/net/thunderx/base/nicvf_bsvf.h +++ b/drivers/net/thunderx/base/nicvf_bsvf.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/base/nicvf_hw.c b/drivers/net/thunderx/base/nicvf_hw.c index 04b3b69c..2634285e 100644 --- a/drivers/net/thunderx/base/nicvf_hw.c +++ b/drivers/net/thunderx/base/nicvf_hw.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -448,7 +448,8 @@ nicvf_qsize_regbit(uint32_t len, uint32_t len_shift) { int val; - val = ((uint32_t)log2(len) - len_shift); + val = nicvf_log2_u32(len) - len_shift; + assert(val >= NICVF_QSIZE_MIN_VAL); assert(val <= NICVF_QSIZE_MAX_VAL); return val; @@ -585,6 +586,7 @@ nicvf_qset_sq_config(struct nicvf *nic, uint16_t qidx, struct nicvf_txq *txq) nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, qidx, txq->phys); /* Enable send queue & set queue size */ + sq_cfg.cq_limit = 0; sq_cfg.ena = 1; sq_cfg.reset = 0; sq_cfg.ldwb = 0; @@ -801,7 +803,7 @@ nicvf_rss_reta_update(struct nicvf *nic, uint8_t *tbl, uint32_t max_count) return NICVF_ERR_RSS_GET_SZ; assert(rss->rss_size > 0); - rss->hash_bits = (uint8_t)log2(rss->rss_size); + rss->hash_bits = (uint8_t)nicvf_log2_u32(rss->rss_size); for (idx = 0; idx < rss->rss_size && idx < max_count; idx++) rss->ind_tbl[idx] = tbl[idx]; @@ -822,7 +824,8 @@ nicvf_rss_reta_query(struct nicvf *nic, uint8_t *tbl, uint32_t max_count) return NICVF_ERR_RSS_GET_SZ; assert(rss->rss_size > 0); - rss->hash_bits = (uint8_t)log2(rss->rss_size); + rss->hash_bits = (uint8_t)nicvf_log2_u32(rss->rss_size); + for (idx = 0; idx < rss->rss_size && idx < max_count; idx++) tbl[idx] = rss->ind_tbl[idx]; diff --git a/drivers/net/thunderx/base/nicvf_hw.h b/drivers/net/thunderx/base/nicvf_hw.h index 14fb2feb..b7d0a3dc 100644 --- a/drivers/net/thunderx/base/nicvf_hw.h +++ b/drivers/net/thunderx/base/nicvf_hw.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/base/nicvf_hw_defs.h b/drivers/net/thunderx/base/nicvf_hw_defs.h index 79f83c8d..0fe673e6 100644 --- a/drivers/net/thunderx/base/nicvf_hw_defs.h +++ b/drivers/net/thunderx/base/nicvf_hw_defs.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -1084,7 +1084,8 @@ struct cq_cfg { union { struct { struct sq_cfg { union { struct { #if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN - uint64_t reserved_20_63:44; + uint64_t reserved_32_63:32; + uint64_t cq_limit:8; uint64_t ena:1; uint64_t reserved_18_18:1; uint64_t reset:1; @@ -1102,7 +1103,8 @@ struct sq_cfg { union { struct { uint64_t reset:1; uint64_t reserved_18_18:1; uint64_t ena:1; - uint64_t reserved_20_63:44; + uint64_t cq_limit:8; + uint64_t reserved_32_63:32; #endif }; uint64_t value; diff --git a/drivers/net/thunderx/base/nicvf_mbox.c b/drivers/net/thunderx/base/nicvf_mbox.c index a072f19d..e26319bb 100644 --- a/drivers/net/thunderx/base/nicvf_mbox.c +++ b/drivers/net/thunderx/base/nicvf_mbox.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/base/nicvf_mbox.h b/drivers/net/thunderx/base/nicvf_mbox.h index 8675fe8f..6724ae1c 100644 --- a/drivers/net/thunderx/base/nicvf_mbox.h +++ b/drivers/net/thunderx/base/nicvf_mbox.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/base/nicvf_plat.h b/drivers/net/thunderx/base/nicvf_plat.h index 36da1200..f821c56f 100644 --- a/drivers/net/thunderx/base/nicvf_plat.h +++ b/drivers/net/thunderx/base/nicvf_plat.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -59,6 +59,7 @@ /* utils */ #include <rte_common.h> #define nicvf_min(x, y) RTE_MIN(x, y) +#define nicvf_log2_u32(x) rte_log2_u32(x) /* byte order */ #include <rte_byteorder.h> @@ -80,7 +81,7 @@ /* ARM64 specific functions */ #if defined(RTE_ARCH_ARM64) #define nicvf_prefetch_store_keep(_ptr) ({\ - asm volatile("prfm pstl1keep, %a0\n" : : "p" (_ptr)); }) + asm volatile("prfm pstl1keep, [%x0]\n" : : "r" (_ptr)); }) #define NICVF_LOAD_PAIR(reg1, reg2, addr) ({ \ diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c index 2152029b..edc17f1d 100644 --- a/drivers/net/thunderx/nicvf_ethdev.c +++ b/drivers/net/thunderx/nicvf_ethdev.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -111,7 +111,8 @@ nicvf_interrupt(void *arg) if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) { if (dev->data->dev_conf.intr_conf.lsc) nicvf_set_eth_link_status(nic, &dev->data->dev_link); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, @@ -1239,6 +1240,13 @@ nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq) struct rte_mbuf mb_def; RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) - + offsetof(struct rte_mbuf, data_off) != 2); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) - + offsetof(struct rte_mbuf, data_off) != 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) - + offsetof(struct rte_mbuf, data_off) != 6); mb_def.nb_segs = 1; mb_def.data_off = RTE_PKTMBUF_HEADROOM; mb_def.port = rxq->port_id; @@ -1366,11 +1374,11 @@ static void nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct nicvf *nic = nicvf_pmd_priv(dev); - struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); PMD_INIT_FUNC_TRACE(); - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->min_rx_bufsize = ETHER_MIN_MTU; dev_info->max_rx_pktlen = NIC_HW_MAX_FRS; @@ -2017,7 +2025,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) } } - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); nic->device_id = pci_dev->id.device_id; @@ -2079,7 +2087,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) goto fail; } - /* Detach port by returning postive error number */ + /* Detach port by returning positive error number */ return ENOTSUP; } @@ -2164,7 +2172,8 @@ static int nicvf_eth_pci_remove(struct rte_pci_device *pci_dev) static struct rte_pci_driver rte_nicvf_pmd = { .id_table = pci_id_nicvf_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_KEEP_MAPPED_RES | + RTE_PCI_DRV_INTR_LSC, .probe = nicvf_eth_pci_probe, .remove = nicvf_eth_pci_remove, }; diff --git a/drivers/net/thunderx/nicvf_ethdev.h b/drivers/net/thunderx/nicvf_ethdev.h index a74219fa..3734430f 100644 --- a/drivers/net/thunderx/nicvf_ethdev.h +++ b/drivers/net/thunderx/nicvf_ethdev.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/nicvf_logs.h b/drivers/net/thunderx/nicvf_logs.h index 0667d468..a76d1987 100644 --- a/drivers/net/thunderx/nicvf_logs.h +++ b/drivers/net/thunderx/nicvf_logs.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/nicvf_rxtx.c b/drivers/net/thunderx/nicvf_rxtx.c index 6cae8341..e27776e6 100644 --- a/drivers/net/thunderx/nicvf_rxtx.c +++ b/drivers/net/thunderx/nicvf_rxtx.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/nicvf_rxtx.h b/drivers/net/thunderx/nicvf_rxtx.h index 3631ff22..cd1b754b 100644 --- a/drivers/net/thunderx/nicvf_rxtx.h +++ b/drivers/net/thunderx/nicvf_rxtx.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/nicvf_struct.h b/drivers/net/thunderx/nicvf_struct.h index 34c41b79..4ee6c3bb 100644 --- a/drivers/net/thunderx/nicvf_struct.h +++ b/drivers/net/thunderx/nicvf_struct.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/nicvf_svf.c b/drivers/net/thunderx/nicvf_svf.c index f746e946..a7cc28a7 100644 --- a/drivers/net/thunderx/nicvf_svf.c +++ b/drivers/net/thunderx/nicvf_svf.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/thunderx/nicvf_svf.h b/drivers/net/thunderx/nicvf_svf.h index 6471aa57..e58ab9d1 100644 --- a/drivers/net/thunderx/nicvf_svf.h +++ b/drivers/net/thunderx/nicvf_svf.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c index 257bf6d6..0dac5e60 100644 --- a/drivers/net/vhost/rte_eth_vhost.c +++ b/drivers/net/vhost/rte_eth_vhost.c @@ -607,7 +607,8 @@ new_device(int vid) RTE_LOG(INFO, PMD, "New connection established\n"); - _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); return 0; } @@ -661,7 +662,8 @@ destroy_device(int vid) RTE_LOG(INFO, PMD, "Connection closed\n"); - _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); + _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } static int @@ -690,7 +692,8 @@ vring_state_changed(int vid, uint16_t vring, int enable) RTE_LOG(INFO, PMD, "vring%u is %s\n", vring, enable ? "enabled" : "disabled"); - _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE, NULL); + _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE, + NULL, NULL); return 0; } @@ -973,6 +976,18 @@ eth_link_update(struct rte_eth_dev *dev __rte_unused, return 0; } +static uint32_t +eth_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct vhost_queue *vq; + + vq = dev->data->rx_queues[rx_queue_id]; + if (vq == NULL) + return 0; + + return rte_vhost_rx_queue_count(vq->vid, vq->virtqueue_id); +} + static const struct eth_dev_ops ops = { .dev_start = eth_dev_start, .dev_stop = eth_dev_stop, @@ -984,6 +999,7 @@ static const struct eth_dev_ops ops = { .rx_queue_release = eth_queue_release, .tx_queue_release = eth_queue_release, .tx_done_cleanup = eth_tx_done_cleanup, + .rx_queue_count = eth_rx_queue_count, .link_update = eth_link_update, .stats_get = eth_stats_get, .stats_reset = eth_stats_reset, diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index 88118f1b..e320811e 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -71,7 +71,7 @@ static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev); static void virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int virtio_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); + int wait_to_complete); static void virtio_set_hwaddr(struct virtio_hw *hw); static void virtio_get_hwaddr(struct virtio_hw *hw); @@ -89,16 +89,16 @@ static int virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); static int virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, - uint32_t index, uint32_t vmdq __rte_unused); + uint32_t index, uint32_t vmdq); static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); static void virtio_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr); static int virtio_dev_queue_stats_mapping_set( - __rte_unused struct rte_eth_dev *eth_dev, - __rte_unused uint16_t queue_id, - __rte_unused uint8_t stat_idx, - __rte_unused uint8_t is_rx); + struct rte_eth_dev *eth_dev, + uint16_t queue_id, + uint8_t stat_idx, + uint8_t is_rx); /* * The set of PCI devices this driver supports @@ -1229,7 +1229,8 @@ virtio_interrupt_handler(void *param) if (isr & VIRTIO_PCI_ISR_CONFIG) { if (virtio_dev_link_update(dev, 0) == 0) _rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_INTR_LSC, NULL); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } } @@ -1355,7 +1356,7 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) return -1; if (!hw->virtio_user_dev) { - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); } @@ -1537,8 +1538,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) if (rte_eal_process_type() == RTE_PROC_SECONDARY) { if (!hw->virtio_user_dev) { - ret = virtio_remap_pci(RTE_DEV_TO_PCI(eth_dev->device), - hw); + ret = virtio_remap_pci(RTE_ETH_DEV_TO_PCI(eth_dev), hw); if (ret) return ret; } @@ -1567,7 +1567,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) * virtio_user_eth_dev_alloc() before eth_virtio_dev_init() is called. */ if (!hw->virtio_user_dev) { - ret = vtpci_init(RTE_DEV_TO_PCI(eth_dev->device), hw); + ret = vtpci_init(RTE_ETH_DEV_TO_PCI(eth_dev), hw); if (ret) return ret; } @@ -1609,7 +1609,7 @@ eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) virtio_interrupt_handler, eth_dev); if (eth_dev->device) - rte_pci_unmap_device(RTE_DEV_TO_PCI(eth_dev->device)); + rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev)); PMD_INIT_LOG(DEBUG, "dev_uninit completed"); @@ -1659,37 +1659,26 @@ virtio_dev_configure(struct rte_eth_dev *dev) { const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; struct virtio_hw *hw = dev->data->dev_private; - uint64_t req_features; int ret; PMD_INIT_LOG(DEBUG, "configure"); - req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES; - if (rxmode->hw_ip_checksum) - req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM); - if (rxmode->enable_lro) - req_features |= - (1ULL << VIRTIO_NET_F_GUEST_TSO4) | - (1ULL << VIRTIO_NET_F_GUEST_TSO6); - - /* if request features changed, reinit the device */ - if (req_features != hw->req_guest_features) { - ret = virtio_init_device(dev, req_features); + + if (dev->data->dev_conf.intr_conf.rxq) { + ret = virtio_init_device(dev, hw->req_guest_features); if (ret < 0) return ret; } - if (rxmode->hw_ip_checksum && - !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) { + /* Virtio does L4 checksum but not L3! */ + if (rxmode->hw_ip_checksum) { PMD_DRV_LOG(NOTICE, - "rx ip checksum not available on this host"); + "virtio does not support IP checksum"); return -ENOTSUP; } - if (rxmode->enable_lro && - (!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || - !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4))) { + if (rxmode->enable_lro) { PMD_DRV_LOG(NOTICE, - "lro not available on this host"); + "virtio does not support Large Receive Offload"); return -ENOTSUP; } @@ -1894,7 +1883,7 @@ virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->speed_capa = ETH_LINK_SPEED_10G; /* fake value */ - dev_info->pci_dev = dev->device ? RTE_DEV_TO_PCI(dev->device) : NULL; + dev_info->pci_dev = dev->device ? RTE_ETH_DEV_TO_PCI(dev) : NULL; dev_info->max_rx_queues = RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES); dev_info->max_tx_queues = @@ -1915,8 +1904,6 @@ virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) } tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) | (1ULL << VIRTIO_NET_F_GUEST_TSO6); - if ((host_features & tso_mask) == tso_mask) - dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO; dev_info->tx_offload_capa = 0; if (hw->guest_features & (1ULL << VIRTIO_NET_F_CSUM)) { diff --git a/drivers/net/virtio/virtio_pci.c b/drivers/net/virtio/virtio_pci.c index b7b3d615..e6da6802 100644 --- a/drivers/net/virtio/virtio_pci.c +++ b/drivers/net/virtio/virtio_pci.c @@ -38,6 +38,7 @@ #endif #include <rte_io.h> +#include <rte_bus.h> #include "virtio_pci.h" #include "virtio_logs.h" @@ -579,6 +580,8 @@ get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap) return base + offset; } +#define PCI_MSIX_ENABLE 0x8000 + static int virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw) { @@ -605,8 +608,17 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw) break; } - if (cap.cap_vndr == PCI_CAP_ID_MSIX) - hw->use_msix = 1; + if (cap.cap_vndr == PCI_CAP_ID_MSIX) { + /* Transitional devices would also have this capability, + * that's why we also check if msix is enabled. + * 1st byte is cap ID; 2nd byte is the position of next + * cap; next two bytes are the flags. + */ + uint16_t flags = ((uint16_t *)&cap)[1]; + + if (flags & PCI_MSIX_ENABLE) + hw->use_msix = 1; + } if (cap.cap_vndr != PCI_CAP_ID_VNDR) { PMD_INIT_LOG(DEBUG, @@ -684,8 +696,8 @@ vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw) if (rte_pci_ioport_map(dev, 0, VTPCI_IO(hw)) < 0) { if (dev->kdrv == RTE_KDRV_UNKNOWN && (!dev->device.devargs || - dev->device.devargs->type != - RTE_DEVTYPE_WHITELISTED_PCI)) { + dev->device.devargs->bus != + rte_bus_find_by_name("pci"))) { PMD_INIT_LOG(INFO, "skip kernel managed virtio device."); return 1; diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index fbc96dfb..e30377c5 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -744,8 +744,9 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) virtio_rmb(); - num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts); - num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ); + num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; + if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) + num = VIRTIO_MBUF_BURST_SZ; if (likely(num > DESC_PER_CACHELINE)) num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); diff --git a/drivers/net/virtio/virtio_rxtx_simple_neon.c b/drivers/net/virtio/virtio_rxtx_simple_neon.c index ecc62ada..6b40c7f7 100644 --- a/drivers/net/virtio/virtio_rxtx_simple_neon.c +++ b/drivers/net/virtio/virtio_rxtx_simple_neon.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016 + * Copyright (C) Cavium, Inc. 2016 * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c index 450404ba..79412714 100644 --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c @@ -41,7 +41,6 @@ #include <sys/eventfd.h> #include <sys/types.h> #include <sys/stat.h> -#include <unistd.h> #include "vhost.h" #include "virtio_user_dev.h" diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c index 280406c0..c9614443 100644 --- a/drivers/net/virtio/virtio_user_ethdev.c +++ b/drivers/net/virtio/virtio_user_ethdev.c @@ -35,7 +35,6 @@ #include <sys/types.h> #include <unistd.h> #include <fcntl.h> -#include <sys/types.h> #include <sys/socket.h> #include <rte_malloc.h> @@ -388,7 +387,7 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev) } /* Dev initialization routine. Invoked once for each virtio vdev at - * EAL init time, see rte_eal_dev_init(). + * EAL init time, see rte_bus_probe(). * Returns 0 on success. */ static int @@ -556,7 +555,6 @@ virtio_user_pmd_remove(struct rte_vdev_device *vdev) virtio_user_dev_uninit(dev); rte_free(eth_dev->data->dev_private); - rte_free(eth_dev->data); rte_eth_dev_release_port(eth_dev); return 0; diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c index 2b8092d9..39109919 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethdev.c +++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c @@ -57,7 +57,6 @@ #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_ethdev_pci.h> -#include <rte_atomic.h> #include <rte_string_fns.h> #include <rte_malloc.h> #include <rte_dev.h> @@ -83,10 +82,18 @@ static void vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev); static void vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev); static void vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev); static void vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev); +static int __vmxnet3_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete); static int vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete); +static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw); static void vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); +static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats, + unsigned int n); +static int vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *xstats, unsigned int n); static void vmxnet3_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static const uint32_t * @@ -96,10 +103,8 @@ static int vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, static void vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr); +static void vmxnet3_interrupt_handler(void *param); -#if PROCESS_SYS_EVENTS == 1 -static void vmxnet3_process_events(struct vmxnet3_hw *); -#endif /* * The set of PCI devices this driver supports */ @@ -121,6 +126,8 @@ static const struct eth_dev_ops vmxnet3_eth_dev_ops = { .allmulticast_disable = vmxnet3_dev_allmulticast_disable, .link_update = vmxnet3_dev_link_update, .stats_get = vmxnet3_dev_stats_get, + .xstats_get_names = vmxnet3_dev_xstats_get_names, + .xstats_get = vmxnet3_dev_xstats_get, .mac_addr_set = vmxnet3_mac_addr_set, .dev_infos_get = vmxnet3_dev_info_get, .dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get, @@ -132,6 +139,27 @@ static const struct eth_dev_ops vmxnet3_eth_dev_ops = { .tx_queue_release = vmxnet3_dev_tx_queue_release, }; +struct vmxnet3_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + unsigned int offset; +}; + +/* tx_qX_ is prepended to the name string here */ +static const struct vmxnet3_xstats_name_off vmxnet3_txq_stat_strings[] = { + {"drop_total", offsetof(struct vmxnet3_txq_stats, drop_total)}, + {"drop_too_many_segs", offsetof(struct vmxnet3_txq_stats, drop_too_many_segs)}, + {"drop_tso", offsetof(struct vmxnet3_txq_stats, drop_tso)}, + {"tx_ring_full", offsetof(struct vmxnet3_txq_stats, tx_ring_full)}, +}; + +/* rx_qX_ is prepended to the name string here */ +static const struct vmxnet3_xstats_name_off vmxnet3_rxq_stat_strings[] = { + {"drop_total", offsetof(struct vmxnet3_rxq_stats, drop_total)}, + {"drop_err", offsetof(struct vmxnet3_rxq_stats, drop_err)}, + {"drop_fcs", offsetof(struct vmxnet3_rxq_stats, drop_fcs)}, + {"rx_buf_alloc_failure", offsetof(struct vmxnet3_rxq_stats, rx_buf_alloc_failure)}, +}; + static const struct rte_memzone * gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size, const char *post_string, int socket_id, @@ -141,7 +169,7 @@ gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size, const struct rte_memzone *mz; snprintf(z_name, sizeof(z_name), "%s_%d_%s", - dev->data->drv_name, dev->data->port_id, post_string); + dev->device->driver->name, dev->data->port_id, post_string); mz = rte_memzone_lookup(z_name); if (!reuse) { @@ -221,10 +249,22 @@ vmxnet3_disable_intr(struct vmxnet3_hw *hw) PMD_INIT_FUNC_TRACE(); hw->shared->devRead.intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL; - for (i = 0; i < VMXNET3_MAX_INTRS; i++) + for (i = 0; i < hw->num_intrs; i++) VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 1); } +static void +vmxnet3_enable_intr(struct vmxnet3_hw *hw) +{ + int i; + + PMD_INIT_FUNC_TRACE(); + + hw->shared->devRead.intrConf.intrCtrl &= ~VMXNET3_IC_DISABLE_ALL; + for (i = 0; i < hw->num_intrs; i++) + VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 0); +} + /* * Gets tx data ring descriptor size. */ @@ -259,7 +299,7 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts; eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts; eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts; - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); /* * for secondary processes, we don't initialize any further as primary @@ -351,6 +391,10 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) == hw->rxdata_desc_size); + /* clear shadow stats */ + memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats)); + memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats)); + return 0; } @@ -392,7 +436,7 @@ static int eth_vmxnet3_pci_remove(struct rte_pci_device *pci_dev) static struct rte_pci_driver rte_vmxnet3_pmd = { .id_table = pci_id_vmxnet3_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, .probe = eth_vmxnet3_pci_probe, .remove = eth_vmxnet3_pci_remove, }; @@ -615,11 +659,11 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) /* * Set number of interrupts to 1 - * PMD disables all the interrupts but this is MUST to activate device - * It needs at least one interrupt for link events to handle - * So we'll disable it later after device activation if needed + * PMD by default disables all the interrupts but this is MUST + * to activate device. It needs at least one interrupt for + * link events to handle */ - devRead->intrConf.numIntrs = 1; + hw->num_intrs = devRead->intrConf.numIntrs = 1; devRead->intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL; for (i = 0; i < hw->num_tx_queues; i++) { @@ -689,7 +733,7 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) vmxnet3_dev_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK); - vmxnet3_write_mac(hw, hw->perm_addr); + vmxnet3_write_mac(hw, dev->data->mac_addrs->addr_bytes); return VMXNET3_SUCCESS; } @@ -707,10 +751,27 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); + /* Save stats before it is reset by CMD_ACTIVATE */ + vmxnet3_hw_stats_save(hw); + ret = vmxnet3_setup_driver_shared(dev); if (ret != VMXNET3_SUCCESS) return ret; + /* check if lsc interrupt feature is enabled */ + if (dev->data->dev_conf.intr_conf.lsc) { + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + + /* Setup interrupt callback */ + rte_intr_callback_register(&pci_dev->intr_handle, + vmxnet3_interrupt_handler, dev); + + if (rte_intr_enable(&pci_dev->intr_handle) < 0) { + PMD_INIT_LOG(ERR, "interrupt enable failed"); + return -EIO; + } + } + /* Exchange shared data with device */ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(hw->sharedPA)); @@ -758,14 +819,19 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) /* Setting proper Rx Mode and issue Rx Mode Update command */ vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); - /* - * Don't need to handle events for now - */ -#if PROCESS_SYS_EVENTS == 1 - events = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_ECR); - PMD_INIT_LOG(DEBUG, "Reading events: 0x%X", events); - vmxnet3_process_events(hw); -#endif + if (dev->data->dev_conf.intr_conf.lsc) { + vmxnet3_enable_intr(hw); + + /* + * Update link state from device since this won't be + * done upon starting with lsc in use. This is done + * only after enabling interrupts to avoid any race + * where the link state could change without an + * interrupt being fired. + */ + __vmxnet3_dev_link_update(dev, 0); + } + return VMXNET3_SUCCESS; } @@ -788,6 +854,15 @@ vmxnet3_dev_stop(struct rte_eth_dev *dev) /* disable interrupts */ vmxnet3_disable_intr(hw); + if (dev->data->dev_conf.intr_conf.lsc) { + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + + rte_intr_disable(&pci_dev->intr_handle); + + rte_intr_callback_unregister(&pci_dev->intr_handle, + vmxnet3_interrupt_handler, dev); + } + /* quiesce the device first */ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0); @@ -820,47 +895,190 @@ vmxnet3_dev_close(struct rte_eth_dev *dev) } static void +vmxnet3_hw_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, + struct UPT1_TxStats *res) +{ +#define VMXNET3_UPDATE_TX_STAT(h, i, f, r) \ + ((r)->f = (h)->tqd_start[(i)].stats.f + \ + (h)->saved_tx_stats[(i)].f) + + VMXNET3_UPDATE_TX_STAT(hw, q, ucastPktsTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, mcastPktsTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, bcastPktsTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, ucastBytesTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, mcastBytesTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, bcastBytesTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxError, res); + VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxDiscard, res); + +#undef VMXNET3_UPDATE_TX_STAT +} + +static void +vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, + struct UPT1_RxStats *res) +{ +#define VMXNET3_UPDATE_RX_STAT(h, i, f, r) \ + ((r)->f = (h)->rqd_start[(i)].stats.f + \ + (h)->saved_rx_stats[(i)].f) + + VMXNET3_UPDATE_RX_STAT(hw, q, ucastPktsRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, mcastPktsRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, bcastPktsRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, ucastBytesRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, mcastBytesRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, bcastBytesRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res); + VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res); + +#undef VMXNET3_UPDATE_RX_STATS +} + +static void +vmxnet3_hw_stats_save(struct vmxnet3_hw *hw) +{ + unsigned int i; + + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); + + RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); + + for (i = 0; i < hw->num_tx_queues; i++) + vmxnet3_hw_tx_stats_get(hw, i, &hw->saved_tx_stats[i]); + for (i = 0; i < hw->num_rx_queues; i++) + vmxnet3_hw_rx_stats_get(hw, i, &hw->saved_rx_stats[i]); +} + +static int +vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int n) +{ + unsigned int i, t, count = 0; + unsigned int nstats = + dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + + dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); + + if (!xstats_names || n < nstats) + return nstats; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + if (!dev->data->rx_queues[i]) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "rx_q%u_%s", i, + vmxnet3_rxq_stat_strings[t].name); + count++; + } + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + if (!dev->data->tx_queues[i]) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "tx_q%u_%s", i, + vmxnet3_txq_stat_strings[t].name); + count++; + } + } + + return count; +} + +static int +vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + unsigned int n) +{ + unsigned int i, t, count = 0; + unsigned int nstats = + dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + + dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); + + if (n < nstats) + return nstats; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i]; + + if (rxq == NULL) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { + xstats[count].value = *(uint64_t *)(((char *)&rxq->stats) + + vmxnet3_rxq_stat_strings[t].offset); + xstats[count].id = count; + count++; + } + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; + + if (txq == NULL) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { + xstats[count].value = *(uint64_t *)(((char *)&txq->stats) + + vmxnet3_txq_stat_strings[t].offset); + xstats[count].id = count; + count++; + } + } + + return count; +} + +static void vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { unsigned int i; struct vmxnet3_hw *hw = dev->data->dev_private; + struct UPT1_TxStats txStats; + struct UPT1_RxStats rxStats; VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); for (i = 0; i < hw->num_tx_queues; i++) { - struct UPT1_TxStats *txStats = &hw->tqd_start[i].stats; + vmxnet3_hw_tx_stats_get(hw, i, &txStats); - stats->q_opackets[i] = txStats->ucastPktsTxOK + - txStats->mcastPktsTxOK + - txStats->bcastPktsTxOK; - stats->q_obytes[i] = txStats->ucastBytesTxOK + - txStats->mcastBytesTxOK + - txStats->bcastBytesTxOK; + stats->q_opackets[i] = txStats.ucastPktsTxOK + + txStats.mcastPktsTxOK + + txStats.bcastPktsTxOK; + + stats->q_obytes[i] = txStats.ucastBytesTxOK + + txStats.mcastBytesTxOK + + txStats.bcastBytesTxOK; stats->opackets += stats->q_opackets[i]; stats->obytes += stats->q_obytes[i]; - stats->oerrors += txStats->pktsTxError + txStats->pktsTxDiscard; + stats->oerrors += txStats.pktsTxError + txStats.pktsTxDiscard; } RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_RX_QUEUES); for (i = 0; i < hw->num_rx_queues; i++) { - struct UPT1_RxStats *rxStats = &hw->rqd_start[i].stats; + vmxnet3_hw_rx_stats_get(hw, i, &rxStats); - stats->q_ipackets[i] = rxStats->ucastPktsRxOK + - rxStats->mcastPktsRxOK + - rxStats->bcastPktsRxOK; + stats->q_ipackets[i] = rxStats.ucastPktsRxOK + + rxStats.mcastPktsRxOK + + rxStats.bcastPktsRxOK; - stats->q_ibytes[i] = rxStats->ucastBytesRxOK + - rxStats->mcastBytesRxOK + - rxStats->bcastBytesRxOK; + stats->q_ibytes[i] = rxStats.ucastBytesRxOK + + rxStats.mcastBytesRxOK + + rxStats.bcastBytesRxOK; stats->ipackets += stats->q_ipackets[i]; stats->ibytes += stats->q_ibytes[i]; - stats->q_errors[i] = rxStats->pktsRxError; - stats->ierrors += rxStats->pktsRxError; - stats->rx_nombuf += rxStats->pktsRxOutOfBuf; + stats->q_errors[i] = rxStats.pktsRxError; + stats->ierrors += rxStats.pktsRxError; + stats->rx_nombuf += rxStats.pktsRxOutOfBuf; } } @@ -868,7 +1086,7 @@ static void vmxnet3_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; @@ -931,17 +1149,13 @@ vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) /* return 0 means link status changed, -1 means not changed */ static int -vmxnet3_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete) +__vmxnet3_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete) { struct vmxnet3_hw *hw = dev->data->dev_private; struct rte_eth_link old = { 0 }, link; uint32_t ret; - /* Link status doesn't change for stopped dev */ - if (dev->data->dev_started == 0) - return -1; - memset(&link, 0, sizeof(link)); vmxnet3_dev_atomic_read_link_status(dev, &old); @@ -960,6 +1174,16 @@ vmxnet3_dev_link_update(struct rte_eth_dev *dev, return (old.link_status == link.link_status) ? -1 : 0; } +static int +vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + /* Link status doesn't change for stopped dev */ + if (dev->data->dev_started == 0) + return -1; + + return __vmxnet3_dev_link_update(dev, wait_to_complete); +} + /* Updating rxmode through Vmxnet3_DriverShared structure in adapter */ static void vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) @@ -995,7 +1219,10 @@ vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev) struct vmxnet3_hw *hw = dev->data->dev_private; uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; - memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); + if (dev->data->dev_conf.rxmode.hw_vlan_filter) + memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); + else + memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0); VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_VLAN_FILTERS); @@ -1076,16 +1303,14 @@ vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) } } -#if PROCESS_SYS_EVENTS == 1 static void -vmxnet3_process_events(struct vmxnet3_hw *hw) +vmxnet3_process_events(struct rte_eth_dev *dev) { + struct vmxnet3_hw *hw = dev->data->dev_private; uint32_t events = hw->shared->ecr; - if (!events) { - PMD_INIT_LOG(ERR, "No events to process"); + if (!events) return; - } /* * ECR bits when written with 1b are cleared. Hence write @@ -1094,10 +1319,13 @@ vmxnet3_process_events(struct vmxnet3_hw *hw) VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events); /* Check if link state has changed */ - if (events & VMXNET3_ECR_LINK) - PMD_INIT_LOG(ERR, - "Process events in %s(): VMXNET3_ECR_LINK event", - __func__); + if (events & VMXNET3_ECR_LINK) { + PMD_DRV_LOG(DEBUG, "Process events: VMXNET3_ECR_LINK event"); + if (vmxnet3_dev_link_update(dev, 0) == 0) + _rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } /* Check if there is an error on xmit/recv queues */ if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { @@ -1105,11 +1333,11 @@ vmxnet3_process_events(struct vmxnet3_hw *hw) VMXNET3_CMD_GET_QUEUE_STATUS); if (hw->tqd_start->status.stopped) - PMD_INIT_LOG(ERR, "tq error 0x%x", - hw->tqd_start->status.error); + PMD_DRV_LOG(ERR, "tq error 0x%x", + hw->tqd_start->status.error); if (hw->rqd_start->status.stopped) - PMD_INIT_LOG(ERR, "rq error 0x%x", + PMD_DRV_LOG(ERR, "rq error 0x%x", hw->rqd_start->status.error); /* Reset the device */ @@ -1117,12 +1345,23 @@ vmxnet3_process_events(struct vmxnet3_hw *hw) } if (events & VMXNET3_ECR_DIC) - PMD_INIT_LOG(ERR, "Device implementation change event."); + PMD_DRV_LOG(DEBUG, "Device implementation change event."); if (events & VMXNET3_ECR_DEBUG) - PMD_INIT_LOG(ERR, "Debug event generated by device."); + PMD_DRV_LOG(DEBUG, "Debug event generated by device."); +} + +static void +vmxnet3_interrupt_handler(void *param) +{ + struct rte_eth_dev *dev = param; + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + + vmxnet3_process_events(dev); + + if (rte_intr_enable(&pci_dev->intr_handle) < 0) + PMD_DRV_LOG(ERR, "interrupt enable failed"); } -#endif RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd); RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map); diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.h b/drivers/net/vmxnet3/vmxnet3_ethdev.h index 7a032629..b48058af 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethdev.h +++ b/drivers/net/vmxnet3/vmxnet3_ethdev.h @@ -106,6 +106,8 @@ struct vmxnet3_hw { uint16_t txdata_desc_size; /* tx data ring buffer size */ uint16_t rxdata_desc_size; /* rx data ring buffer size */ + uint8_t num_intrs; + Vmxnet3_TxQueueDesc *tqd_start; /* start address of all tx queue desc */ Vmxnet3_RxQueueDesc *rqd_start; /* start address of all rx queue desc */ @@ -122,6 +124,8 @@ struct vmxnet3_hw { Vmxnet3_MemRegs *memRegs; uint64_t memRegsPA; #define VMXNET3_VFT_TABLE_SIZE (VMXNET3_VFT_SIZE * sizeof(uint32_t)) + UPT1_TxStats saved_tx_stats[VMXNET3_MAX_TX_QUEUES]; + UPT1_RxStats saved_rx_stats[VMXNET3_MAX_RX_QUEUES]; }; #define VMXNET3_REV_3 2 /* Vmxnet3 Rev. 3 */ diff --git a/drivers/net/vmxnet3/vmxnet3_rxtx.c b/drivers/net/vmxnet3/vmxnet3_rxtx.c index e865c675..d9cf4373 100644 --- a/drivers/net/vmxnet3/vmxnet3_rxtx.c +++ b/drivers/net/vmxnet3/vmxnet3_rxtx.c @@ -593,24 +593,40 @@ static inline void vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id, struct rte_mbuf *mbuf) { - uint32_t val = 0; + uint32_t val; struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id]; struct Vmxnet3_RxDesc *rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill); vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill]; - if (ring_id == 0) + if (ring_id == 0) { + /* Usually: One HEAD type buf per packet + * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ? + * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD; + */ + + /* We use single packet buffer so all heads here */ val = VMXNET3_RXD_BTYPE_HEAD; - else + } else { + /* All BODY type buffers for 2nd ring */ val = VMXNET3_RXD_BTYPE_BODY; + } + /* + * Load mbuf pointer into buf_info[ring_size] + * buf_info structure is equivalent to cookie for virtio-virtqueue + */ buf_info->m = mbuf; buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM); buf_info->bufPA = rte_mbuf_data_dma_addr_default(mbuf); + /* Load Rx Descriptor with the buffer's GPA */ rxd->addr = buf_info->bufPA; + + /* After this point rxd->addr MUST not be NULL */ rxd->btype = val; rxd->len = buf_info->len; + /* Flip gen bit at the end to change ownership */ rxd->gen = ring->gen; vmxnet3_cmd_ring_adv_next2fill(ring); @@ -629,28 +645,11 @@ static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) { int err = 0; - uint32_t i = 0, val = 0; + uint32_t i = 0; struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id]; - if (ring_id == 0) { - /* Usually: One HEAD type buf per packet - * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ? - * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD; - */ - - /* We use single packet buffer so all heads here */ - val = VMXNET3_RXD_BTYPE_HEAD; - } else { - /* All BODY type buffers for 2nd ring */ - val = VMXNET3_RXD_BTYPE_BODY; - } - while (vmxnet3_cmd_ring_desc_avail(ring) > 0) { - struct Vmxnet3_RxDesc *rxd; struct rte_mbuf *mbuf; - vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill]; - - rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill); /* Allocate blank mbuf for the current Rx Descriptor */ mbuf = rte_mbuf_raw_alloc(rxq->mp); @@ -661,25 +660,7 @@ vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) break; } - /* - * Load mbuf pointer into buf_info[ring_size] - * buf_info structure is equivalent to cookie for virtio-virtqueue - */ - buf_info->m = mbuf; - buf_info->len = (uint16_t)(mbuf->buf_len - - RTE_PKTMBUF_HEADROOM); - buf_info->bufPA = rte_mbuf_data_dma_addr_default(mbuf); - - /* Load Rx Descriptor with the buffer's GPA */ - rxd->addr = buf_info->bufPA; - - /* After this point rxd->addr MUST not be NULL */ - rxd->btype = val; - rxd->len = buf_info->len; - /* Flip gen bit at the end to change ownership */ - rxd->gen = ring->gen; - - vmxnet3_cmd_ring_adv_next2fill(ring); + vmxnet3_renew_desc(rxq, ring_id, mbuf); i++; } @@ -800,6 +781,12 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) (int)(rcd - (struct Vmxnet3_RxCompDesc *) rxq->comp_ring.base), rcd->rxdIdx); rte_pktmbuf_free_seg(rxm); + if (rxq->start_seg) { + struct rte_mbuf *start = rxq->start_seg; + + rxq->start_seg = NULL; + rte_pktmbuf_free(start); + } goto rcd_done; } @@ -901,7 +888,7 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, - __rte_unused const struct rte_eth_txconf *tx_conf) + const struct rte_eth_txconf *tx_conf) { struct vmxnet3_hw *hw = dev->data->dev_private; const struct rte_memzone *mz; diff --git a/drivers/net/xenvirt/rte_eth_xenvirt.c b/drivers/net/xenvirt/rte_eth_xenvirt.c index 7bd29fae..e404b775 100644 --- a/drivers/net/xenvirt/rte_eth_xenvirt.c +++ b/drivers/net/xenvirt/rte_eth_xenvirt.c @@ -672,7 +672,6 @@ eth_dev_xenvirt_create(const char *name, const char *params, eth_dev->data->dev_flags = RTE_ETH_DEV_DETACHABLE; eth_dev->data->kdrv = RTE_KDRV_NONE; - eth_dev->data->drv_name = pmd_xenvirt_drv.driver.name; eth_dev->data->numa_node = numa_node; eth_dev->rx_pkt_burst = eth_xenvirt_rx; diff --git a/drivers/net/xenvirt/virtqueue.h b/drivers/net/xenvirt/virtqueue.h index 350eae3e..1bb6877c 100644 --- a/drivers/net/xenvirt/virtqueue.h +++ b/drivers/net/xenvirt/virtqueue.h @@ -123,7 +123,7 @@ void virtqueue_dump(struct virtqueue *vq); */ struct rte_mbuf * virtqueue_detatch_unused(struct virtqueue *vq); -static inline int __attribute__((always_inline)) +static __rte_always_inline int virtqueue_full(const struct virtqueue *vq) { return vq->vq_free_cnt == 0; @@ -131,7 +131,7 @@ virtqueue_full(const struct virtqueue *vq) #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) -static inline void __attribute__((always_inline)) +static __rte_always_inline void vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) { uint16_t avail_idx; @@ -148,7 +148,7 @@ vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) vq->vq_ring.avail->idx++; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) { struct vring_desc *dp; @@ -171,7 +171,7 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) vq->vq_desc_head_idx = desc_idx; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int virtqueue_enqueue_recv_refill(struct virtqueue *rxvq, struct rte_mbuf *cookie) { const uint16_t needed = 1; @@ -201,7 +201,7 @@ virtqueue_enqueue_recv_refill(struct virtqueue *rxvq, struct rte_mbuf *cookie) return 0; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) { @@ -242,7 +242,7 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) return 0; } -static inline uint16_t __attribute__((always_inline)) +static __rte_always_inline uint16_t virtqueue_dequeue_burst(struct virtqueue *vq, struct rte_mbuf **rx_pkts, uint32_t *len, uint16_t num) { struct vring_used_elem *uep; |