From 8d01b9cd70a67cdafd5b965a70420c3bd7fb3f82 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 1 Nov 2018 11:59:50 +0000 Subject: New upstream version 18.11-rc1 Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb Signed-off-by: Luca Boccassi --- drivers/bus/dpaa/Makefile | 4 +- drivers/bus/dpaa/base/fman/netcfg_layer.c | 2 +- drivers/bus/dpaa/base/qbman/bman_driver.c | 17 +- drivers/bus/dpaa/base/qbman/qman.c | 72 +- drivers/bus/dpaa/base/qbman/qman_driver.c | 7 +- drivers/bus/dpaa/dpaa_bus.c | 19 +- drivers/bus/dpaa/include/compat.h | 20 +- drivers/bus/dpaa/include/fsl_fman_crc64.h | 8 +- drivers/bus/dpaa/include/fsl_qman.h | 20 + drivers/bus/dpaa/include/fsl_usd.h | 6 + drivers/bus/dpaa/meson.build | 5 +- drivers/bus/dpaa/rte_bus_dpaa_version.map | 16 +- drivers/bus/dpaa/rte_dpaa_bus.h | 6 + drivers/bus/fslmc/Makefile | 3 +- drivers/bus/fslmc/fslmc_bus.c | 35 +- drivers/bus/fslmc/fslmc_vfio.c | 13 +- drivers/bus/fslmc/mc/dpbp.c | 10 + drivers/bus/fslmc/mc/dpci.c | 197 ++++++ drivers/bus/fslmc/mc/dpcon.c | 30 + drivers/bus/fslmc/mc/dpdmai.c | 14 + drivers/bus/fslmc/mc/dpio.c | 9 + drivers/bus/fslmc/mc/fsl_dpbp.h | 1 + drivers/bus/fslmc/mc/fsl_dpbp_cmd.h | 16 +- drivers/bus/fslmc/mc/fsl_dpci.h | 47 +- drivers/bus/fslmc/mc/fsl_dpci_cmd.h | 62 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 19 + drivers/bus/fslmc/mc/fsl_dpdmai.h | 5 + drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h | 20 +- drivers/bus/fslmc/mc/fsl_dpmng.h | 2 +- drivers/bus/fslmc/mc/fsl_dpopr.h | 85 +++ drivers/bus/fslmc/meson.build | 5 +- drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 7 + drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 197 +++--- drivers/bus/fslmc/portal/dpaa2_hw_dpio.h | 4 + drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 53 +- drivers/bus/fslmc/qbman/include/compat.h | 3 +- drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h | 33 +- drivers/bus/fslmc/qbman/qbman_portal.c | 764 +++++++++++++++++---- drivers/bus/fslmc/qbman/qbman_portal.h | 30 +- drivers/bus/fslmc/qbman/qbman_sys.h | 100 ++- drivers/bus/fslmc/qbman/qbman_sys_decl.h | 4 + drivers/bus/fslmc/rte_bus_fslmc_version.map | 13 + drivers/bus/ifpga/Makefile | 2 +- drivers/bus/ifpga/ifpga_bus.c | 33 +- drivers/bus/ifpga/meson.build | 2 + drivers/bus/ifpga/rte_bus_ifpga.h | 3 - drivers/bus/pci/Makefile | 5 +- drivers/bus/pci/bsd/pci.c | 6 +- drivers/bus/pci/linux/Makefile | 2 - drivers/bus/pci/linux/pci.c | 46 +- drivers/bus/pci/linux/pci_vfio.c | 268 +++++++- drivers/bus/pci/meson.build | 9 +- drivers/bus/pci/pci_common.c | 143 +++- drivers/bus/pci/pci_common_uio.c | 33 + drivers/bus/pci/pci_params.c | 78 +++ drivers/bus/pci/private.h | 39 ++ drivers/bus/pci/rte_bus_pci.h | 10 +- drivers/bus/vdev/Makefile | 5 +- drivers/bus/vdev/meson.build | 7 +- drivers/bus/vdev/vdev.c | 50 +- drivers/bus/vdev/vdev_params.c | 66 ++ drivers/bus/vdev/vdev_private.h | 26 + drivers/bus/vmbus/Makefile | 2 +- drivers/bus/vmbus/linux/vmbus_bus.c | 3 + drivers/bus/vmbus/meson.build | 2 + drivers/bus/vmbus/private.h | 6 + drivers/bus/vmbus/rte_bus_vmbus.h | 15 + drivers/bus/vmbus/rte_bus_vmbus_version.map | 7 + drivers/bus/vmbus/vmbus_channel.c | 26 + drivers/bus/vmbus/vmbus_common.c | 27 +- 70 files changed, 2459 insertions(+), 445 deletions(-) create mode 100644 drivers/bus/fslmc/mc/fsl_dpopr.h create mode 100644 drivers/bus/pci/pci_params.c create mode 100644 drivers/bus/vdev/vdev_params.c create mode 100644 drivers/bus/vdev/vdev_private.h (limited to 'drivers/bus') diff --git a/drivers/bus/dpaa/Makefile b/drivers/bus/dpaa/Makefile index bffaa9d9..800e5cd2 100644 --- a/drivers/bus/dpaa/Makefile +++ b/drivers/bus/dpaa/Makefile @@ -14,7 +14,6 @@ CFLAGS := -I$(SRCDIR) $(CFLAGS) CFLAGS += -O3 $(WERROR_FLAGS) CFLAGS += -Wno-pointer-arith CFLAGS += -Wno-cast-qual -CFLAGS += -D _GNU_SOURCE CFLAGS += -I$(RTE_BUS_DPAA)/ CFLAGS += -I$(RTE_BUS_DPAA)/include CFLAGS += -I$(RTE_BUS_DPAA)/base/qbman @@ -24,7 +23,7 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include # versioning export map EXPORT_MAP := rte_bus_dpaa_version.map -LIBABIVER := 1 +LIBABIVER := 2 # all source are stored in SRCS-y # @@ -48,5 +47,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += \ LDLIBS += -lpthread LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev +LDLIBS += -lrte_common_dpaax include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/bus/dpaa/base/fman/netcfg_layer.c b/drivers/bus/dpaa/base/fman/netcfg_layer.c index 031c6f1a..6b522420 100644 --- a/drivers/bus/dpaa/base/fman/netcfg_layer.c +++ b/drivers/bus/dpaa/base/fman/netcfg_layer.c @@ -21,7 +21,7 @@ /* This data structure contaings all configurations information * related to usages of DPA devices. */ -struct netcfg_info *netcfg; +static struct netcfg_info *netcfg; /* fd to open a socket for making ioctl request to disable/enable shared * interfaces. */ diff --git a/drivers/bus/dpaa/base/qbman/bman_driver.c b/drivers/bus/dpaa/base/qbman/bman_driver.c index b14b5905..750b756b 100644 --- a/drivers/bus/dpaa/base/qbman/bman_driver.c +++ b/drivers/bus/dpaa/base/qbman/bman_driver.c @@ -23,7 +23,7 @@ static void *bman_ccsr_map; /* Portal driver */ /*****************/ -static __thread int fd = -1; +static __thread int bmfd = -1; static __thread struct bm_portal_config pcfg; static __thread struct dpaa_ioctl_portal_map map = { .type = dpaa_portal_bman @@ -70,14 +70,14 @@ static int fsl_bman_portal_init(uint32_t idx, int is_shared) pcfg.index = map.index; bman_depletion_fill(&pcfg.mask); - fd = open(BMAN_PORTAL_IRQ_PATH, O_RDONLY); - if (fd == -1) { + bmfd = open(BMAN_PORTAL_IRQ_PATH, O_RDONLY); + if (bmfd == -1) { pr_err("BMan irq init failed"); process_portal_unmap(&map.addr); return -EBUSY; } /* Use the IRQ FD as a unique IRQ number */ - pcfg.irq = fd; + pcfg.irq = bmfd; portal = bman_create_affine_portal(&pcfg); if (!portal) { @@ -90,7 +90,7 @@ static int fsl_bman_portal_init(uint32_t idx, int is_shared) /* Set the IRQ number */ irq_map.type = dpaa_portal_bman; irq_map.portal_cinh = map.addr.cinh; - process_portal_irq_map(fd, &irq_map); + process_portal_irq_map(bmfd, &irq_map); return 0; } @@ -99,7 +99,7 @@ static int fsl_bman_portal_finish(void) __maybe_unused const struct bm_portal_config *cfg; int ret; - process_portal_irq_unmap(fd); + process_portal_irq_unmap(bmfd); cfg = bman_destroy_affine_portal(); DPAA_BUG_ON(cfg != &pcfg); @@ -109,6 +109,11 @@ static int fsl_bman_portal_finish(void) return ret; } +int bman_thread_fd(void) +{ + return bmfd; +} + int bman_thread_init(void) { /* Convert from contiguous/virtual cpu numbering to real cpu when diff --git a/drivers/bus/dpaa/base/qbman/qman.c b/drivers/bus/dpaa/base/qbman/qman.c index 7c17027f..dc64d089 100644 --- a/drivers/bus/dpaa/base/qbman/qman.c +++ b/drivers/bus/dpaa/base/qbman/qman.c @@ -852,11 +852,9 @@ mr_loop: case QM_MR_VERB_FQPN: /* Parked */ #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP - fq = get_fq_table_entry( - be32_to_cpu(msg->fq.contextB)); + fq = get_fq_table_entry(msg->fq.contextB); #else - fq = (void *)(uintptr_t) - be32_to_cpu(msg->fq.contextB); + fq = (void *)(uintptr_t)msg->fq.contextB; #endif fq_state_change(p, fq, msg, verb); if (fq->cb.fqs) @@ -967,7 +965,6 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p, *shadow = *dq; dq = shadow; shadow->fqid = be32_to_cpu(shadow->fqid); - shadow->contextB = be32_to_cpu(shadow->contextB); shadow->seqnum = be16_to_cpu(shadow->seqnum); hw_fd_to_cpu(&shadow->fd); #endif @@ -1040,6 +1037,50 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p, return limit; } +int qman_irqsource_add(u32 bits) +{ + struct qman_portal *p = get_affine_portal(); + + bits = bits & QM_PIRQ_VISIBLE; + + /* Clear any previously remaining interrupt conditions in + * QCSP_ISR. This prevents raising a false interrupt when + * interrupt conditions are enabled in QCSP_IER. + */ + qm_isr_status_clear(&p->p, bits); + dpaa_set_bits(bits, &p->irq_sources); + qm_isr_enable_write(&p->p, p->irq_sources); + + + return 0; +} + +int qman_irqsource_remove(u32 bits) +{ + struct qman_portal *p = get_affine_portal(); + u32 ier; + + /* Our interrupt handler only processes+clears status register bits that + * are in p->irq_sources. As we're trimming that mask, if one of them + * were to assert in the status register just before we remove it from + * the enable register, there would be an interrupt-storm when we + * release the IRQ lock. So we wait for the enable register update to + * take effect in h/w (by reading it back) and then clear all other bits + * in the status register. Ie. we clear them from ISR once it's certain + * IER won't allow them to reassert. + */ + + bits &= QM_PIRQ_VISIBLE; + dpaa_clear_bits(bits, &p->irq_sources); + qm_isr_enable_write(&p->p, p->irq_sources); + ier = qm_isr_enable_read(&p->p); + /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a + * data-dependency, ie. to protect against re-ordering. + */ + qm_isr_status_clear(&p->p, ~ier); + return 0; +} + u16 qman_affine_channel(int cpu) { if (cpu < 0) { @@ -1092,9 +1133,9 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit, /* SDQCR: context_b points to the FQ */ #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP - fq = qman_fq_lookup_table[be32_to_cpu(dq[rx_number]->contextB)]; + fq = qman_fq_lookup_table[dq[rx_number]->contextB]; #else - fq = (void *)be32_to_cpu(dq[rx_number]->contextB); + fq = (void *)dq[rx_number]->contextB; #endif if (fq->cb.dqrr_prepare) fq->cb.dqrr_prepare(shadow[rx_number], @@ -1114,6 +1155,14 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit, return rx_number; } +void qman_clear_irq(void) +{ + struct qman_portal *p = get_affine_portal(); + u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & + ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); + qm_isr_status_clear(&p->p, clear); +} + u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit, void **bufs) { @@ -1143,7 +1192,6 @@ u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit, *shadow = *dq; dq = shadow; shadow->fqid = be32_to_cpu(shadow->fqid); - shadow->contextB = be32_to_cpu(shadow->contextB); shadow->seqnum = be16_to_cpu(shadow->seqnum); hw_fd_to_cpu(&shadow->fd); #endif @@ -1208,7 +1256,6 @@ struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq) *shadow = *dq; dq = shadow; shadow->fqid = be32_to_cpu(shadow->fqid); - shadow->contextB = be32_to_cpu(shadow->contextB); shadow->seqnum = be16_to_cpu(shadow->seqnum); hw_fd_to_cpu(&shadow->fd); #endif @@ -1504,7 +1551,7 @@ int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP - mcc->initfq.fqd.context_b = fq->key; + mcc->initfq.fqd.context_b = cpu_to_be32(fq->key); #else mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; #endif @@ -2186,11 +2233,6 @@ int qman_enqueue_multi(struct qman_fq *fq, /* try to send as many frames as possible */ while (eqcr->available && frames_to_send--) { eq->fqid = fq->fqid_le; -#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP - eq->tag = cpu_to_be32(fq->key); -#else - eq->tag = cpu_to_be32((u32)(uintptr_t)fq); -#endif eq->fd.opaque_addr = fd->opaque_addr; eq->fd.addr = cpu_to_be40(fd->addr); eq->fd.status = cpu_to_be32(fd->status); diff --git a/drivers/bus/dpaa/base/qbman/qman_driver.c b/drivers/bus/dpaa/base/qbman/qman_driver.c index f6ecd6b2..ba153396 100644 --- a/drivers/bus/dpaa/base/qbman/qman_driver.c +++ b/drivers/bus/dpaa/base/qbman/qman_driver.c @@ -113,6 +113,11 @@ static int fsl_qman_portal_finish(void) return ret; } +int qman_thread_fd(void) +{ + return qmfd; +} + int qman_thread_init(void) { /* Convert from contiguous/virtual cpu numbering to real cpu when @@ -135,7 +140,7 @@ void qman_thread_irq(void) * rather than breaking that encapsulation I am simply hard-coding the * offset to the inhibit register here. */ - out_be32(qpcfg.addr_virt[DPAA_PORTAL_CI] + 0xe0c, 0); + out_be32(qpcfg.addr_virt[DPAA_PORTAL_CI] + 0x36C0, 0); } struct qman_portal *fsl_qman_portal_create(void) diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c index 16fabd1b..203f60dc 100644 --- a/drivers/bus/dpaa/dpaa_bus.c +++ b/drivers/bus/dpaa/dpaa_bus.c @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -46,7 +47,7 @@ int dpaa_logtype_mempool; int dpaa_logtype_pmd; int dpaa_logtype_eventdev; -struct rte_dpaa_bus rte_dpaa_bus; +static struct rte_dpaa_bus rte_dpaa_bus; struct netcfg_info *dpaa_netcfg; /* define a variable to hold the portal_key, once created.*/ @@ -165,6 +166,8 @@ dpaa_create_device_list(void) goto cleanup; } + dev->device.bus = &rte_dpaa_bus.bus; + cfg = &dpaa_netcfg->port_cfg[i]; fman_intf = cfg->fman_if; @@ -546,6 +549,9 @@ rte_dpaa_bus_probe(void) fclose(svr_file); } + /* And initialize the PA->VA translation table */ + dpaax_iova_table_populate(); + /* For each registered driver, and device, call the driver->probe */ TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) { TAILQ_FOREACH(drv, &rte_dpaa_bus.driver_list, next) { @@ -553,6 +559,9 @@ rte_dpaa_bus_probe(void) if (ret) continue; + if (rte_dev_is_probed(&dev->device)) + continue; + if (!drv->probe || (dev->device.devargs && dev->device.devargs->policy == RTE_DEV_BLACKLISTED)) @@ -563,8 +572,12 @@ rte_dpaa_bus_probe(void) dev->device.devargs->policy == RTE_DEV_WHITELISTED)) { ret = drv->probe(drv, dev); - if (ret) + if (ret) { DPAA_BUS_ERR("Unable to probe.\n"); + } else { + dev->driver = drv; + dev->device.driver = &drv->driver; + } } break; } @@ -611,7 +624,7 @@ rte_dpaa_get_iommu_class(void) return RTE_IOVA_PA; } -struct rte_dpaa_bus rte_dpaa_bus = { +static struct rte_dpaa_bus rte_dpaa_bus = { .bus = { .scan = rte_dpaa_bus_scan, .probe = rte_dpaa_bus_probe, diff --git a/drivers/bus/dpaa/include/compat.h b/drivers/bus/dpaa/include/compat.h index 92241d23..41226577 100644 --- a/drivers/bus/dpaa/include/compat.h +++ b/drivers/bus/dpaa/include/compat.h @@ -57,8 +57,9 @@ #ifndef __packed #define __packed __rte_packed #endif +#ifndef noinline #define noinline __attribute__((noinline)) - +#endif #define L1_CACHE_BYTES 64 #define ____cacheline_aligned __attribute__((aligned(L1_CACHE_BYTES))) #define __stringify_1(x) #x @@ -75,20 +76,25 @@ printf(fmt, ##args); \ fflush(stdout); \ } while (0) - +#ifndef pr_crit #define pr_crit(fmt, args...) prflush("CRIT:" fmt, ##args) +#endif +#ifndef pr_err #define pr_err(fmt, args...) prflush("ERR:" fmt, ##args) +#endif +#ifndef pr_warn #define pr_warn(fmt, args...) prflush("WARN:" fmt, ##args) +#endif +#ifndef pr_info #define pr_info(fmt, args...) prflush(fmt, ##args) - -#ifdef RTE_LIBRTE_DPAA_DEBUG_BUS -#ifdef pr_debug -#undef pr_debug #endif +#ifndef pr_debug +#ifdef RTE_LIBRTE_DPAA_DEBUG_BUS #define pr_debug(fmt, args...) printf(fmt, ##args) #else #define pr_debug(fmt, args...) {} #endif +#endif #define DPAA_BUG_ON(x) RTE_ASSERT(x) @@ -256,7 +262,9 @@ __bswap_24(uint32_t x) #define be16_to_cpu(x) rte_be_to_cpu_16(x) #define cpu_to_be64(x) rte_cpu_to_be_64(x) +#if !defined(cpu_to_be32) #define cpu_to_be32(x) rte_cpu_to_be_32(x) +#endif #define cpu_to_be16(x) rte_cpu_to_be_16(x) #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN diff --git a/drivers/bus/dpaa/include/fsl_fman_crc64.h b/drivers/bus/dpaa/include/fsl_fman_crc64.h index bf162f3a..08ad6304 100644 --- a/drivers/bus/dpaa/include/fsl_fman_crc64.h +++ b/drivers/bus/dpaa/include/fsl_fman_crc64.h @@ -42,9 +42,9 @@ struct fman_crc64_t { uint64_t initial; uint64_t table[1 << 8]; }; -extern struct fman_crc64_t FMAN_CRC64_ECMA_182; +extern struct fman_crc64_t fman_crc64_ecma_182; #define DECLARE_FMAN_CRC64_TABLE() \ -struct fman_crc64_t FMAN_CRC64_ECMA_182 = { \ +struct fman_crc64_t fman_crc64_ecma_182 = { \ 0xFFFFFFFFFFFFFFFFULL, \ { \ 0x0000000000000000ULL, 0xb32e4cbe03a75f6fULL, \ @@ -183,7 +183,7 @@ struct fman_crc64_t FMAN_CRC64_ECMA_182 = { \ */ static inline uint64_t fman_crc64_init(void) { - return FMAN_CRC64_ECMA_182.initial; + return fman_crc64_ecma_182.initial; } /* Updates the CRC with arbitrary data */ @@ -192,7 +192,7 @@ static inline uint64_t fman_crc64_update(uint64_t crc, { uint8_t *p = data; while (len--) - crc = FMAN_CRC64_ECMA_182.table[(crc ^ *(p++)) & 0xff] ^ + crc = fman_crc64_ecma_182.table[(crc ^ *(p++)) & 0xff] ^ (crc >> 8); return crc; } diff --git a/drivers/bus/dpaa/include/fsl_qman.h b/drivers/bus/dpaa/include/fsl_qman.h index b18cf037..e4384149 100644 --- a/drivers/bus/dpaa/include/fsl_qman.h +++ b/drivers/bus/dpaa/include/fsl_qman.h @@ -1315,6 +1315,26 @@ int qman_get_portal_index(void); u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit, void **bufs); +/** + * qman_irqsource_add - add processing sources to be interrupt-driven + * @bits: bitmask of QM_PIRQ_**I processing sources + * + * Adds processing sources that should be interrupt-driven (rather than + * processed via qman_poll_***() functions). Returns zero for success, or + * -EINVAL if the current CPU is sharing a portal hosted on another CPU. + */ +int qman_irqsource_add(u32 bits); + +/** + * qman_irqsource_remove - remove processing sources from being interrupt-driven + * @bits: bitmask of QM_PIRQ_**I processing sources + * + * Removes processing sources from being interrupt-driven, so that they will + * instead be processed via qman_poll_***() functions. Returns zero for success, + * or -EINVAL if the current CPU is sharing a portal hosted on another CPU. + */ +int qman_irqsource_remove(u32 bits); + /** * qman_affine_channel - return the channel ID of an portal * @cpu: the cpu whose affine portal is the subject of the query diff --git a/drivers/bus/dpaa/include/fsl_usd.h b/drivers/bus/dpaa/include/fsl_usd.h index e1836175..ec1ab7ce 100644 --- a/drivers/bus/dpaa/include/fsl_usd.h +++ b/drivers/bus/dpaa/include/fsl_usd.h @@ -55,6 +55,10 @@ int qman_free_raw_portal(struct dpaa_raw_portal *portal); int bman_allocate_raw_portal(struct dpaa_raw_portal *portal); int bman_free_raw_portal(struct dpaa_raw_portal *portal); +/* Obtain thread-local UIO file-descriptors */ +int qman_thread_fd(void); +int bman_thread_fd(void); + /* Post-process interrupts. NB, the kernel IRQ handler disables the interrupt * line before notifying us, and this post-processing re-enables it once * processing is complete. As such, it is essential to call this before going @@ -63,6 +67,8 @@ int bman_free_raw_portal(struct dpaa_raw_portal *portal); void qman_thread_irq(void); void bman_thread_irq(void); +void qman_clear_irq(void); + /* Global setup */ int qman_global_init(void); int bman_global_init(void); diff --git a/drivers/bus/dpaa/meson.build b/drivers/bus/dpaa/meson.build index d10b62c0..1fcb4e91 100644 --- a/drivers/bus/dpaa/meson.build +++ b/drivers/bus/dpaa/meson.build @@ -1,11 +1,13 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright 2018 NXP +version = 2 + if host_machine.system() != 'linux' build = false endif -deps += ['eventdev'] +deps += ['common_dpaax', 'eventdev'] sources = files('base/fman/fman.c', 'base/fman/fman_hw.c', 'base/fman/netcfg_layer.c', @@ -26,4 +28,3 @@ if cc.has_argument('-Wno-cast-qual') endif includes += include_directories('include', 'base/qbman') -cflags += ['-D_GNU_SOURCE'] diff --git a/drivers/bus/dpaa/rte_bus_dpaa_version.map b/drivers/bus/dpaa/rte_bus_dpaa_version.map index 7d6d6243..70076c7a 100644 --- a/drivers/bus/dpaa/rte_bus_dpaa_version.map +++ b/drivers/bus/dpaa/rte_bus_dpaa_version.map @@ -95,10 +95,24 @@ DPDK_18.02 { DPDK_18.08 { global: - fman_if_get_sg_enable; fman_if_set_sg; of_get_mac_address; local: *; } DPDK_18.02; + +DPDK_18.11 { + global: + bman_thread_irq; + fman_if_get_sg_enable; + fman_if_set_sg; + qman_clear_irq; + + qman_irqsource_add; + qman_irqsource_remove; + qman_thread_fd; + qman_thread_irq; + + local: *; +} DPDK_18.08; diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h index 15dc6a4a..1d580a00 100644 --- a/drivers/bus/dpaa/rte_dpaa_bus.h +++ b/drivers/bus/dpaa/rte_dpaa_bus.h @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -110,6 +111,11 @@ extern struct dpaa_memseg_list rte_dpaa_memsegs; static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr) { struct dpaa_memseg *ms; + void *va; + + va = dpaax_iova_table_get_va(paddr); + if (likely(va != NULL)) + return va; /* Check if the address is already part of the memseg list internally * maintained by the dpaa driver. diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile index 515d0f53..218d9bd2 100644 --- a/drivers/bus/fslmc/Makefile +++ b/drivers/bus/fslmc/Makefile @@ -19,12 +19,13 @@ CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev +LDLIBS += -lrte_common_dpaax # versioning export map EXPORT_MAP := rte_bus_fslmc_version.map # library version -LIBABIVER := 1 +LIBABIVER := 2 SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ qbman/qbman_portal.c \ diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c index d2900edc..89af9385 100644 --- a/drivers/bus/fslmc/fslmc_bus.c +++ b/drivers/bus/fslmc/fslmc_bus.c @@ -20,6 +20,8 @@ #include #include "fslmc_logs.h" +#include + int dpaa2_logtype_bus; #define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups" @@ -161,6 +163,8 @@ scan_one_fslmc_device(char *dev_name) return -ENOMEM; } + dev->device.bus = &rte_fslmc_bus.bus; + /* Parse the device name and ID */ t_ptr = strtok(dup_dev_name, "."); if (!t_ptr) { @@ -375,6 +379,19 @@ rte_fslmc_probe(void) probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST; + /* In case of PA, the FD addresses returned by qbman APIs are physical + * addresses, which need conversion into equivalent VA address for + * rte_mbuf. For that, a table (a serial array, in memory) is used to + * increase translation efficiency. + * This has to be done before probe as some device initialization + * (during) probe allocate memory (dpaa2_sec) which needs to be pinned + * to this table. + * + * Error is ignored as relevant logs are handled within dpaax and + * handling for unavailable dpaax table too is transparent to caller. + */ + dpaax_iova_table_populate(); + TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { ret = rte_fslmc_match(drv, dev); @@ -384,6 +401,9 @@ rte_fslmc_probe(void) if (!drv->probe) continue; + if (rte_dev_is_probed(&dev->device)) + continue; + if (dev->device.devargs && dev->device.devargs->policy == RTE_DEV_BLACKLISTED) { DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping", @@ -396,8 +416,12 @@ rte_fslmc_probe(void) dev->device.devargs->policy == RTE_DEV_WHITELISTED)) { ret = drv->probe(drv, dev); - if (ret) + if (ret) { DPAA2_BUS_ERR("Unable to probe"); + } else { + dev->driver = drv; + dev->device.driver = &drv->driver; + } } break; } @@ -450,6 +474,11 @@ rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver) fslmc_bus = driver->fslmc_bus; + /* Cleanup the PA->VA Translation table; From whereever this function + * is called from. + */ + dpaax_iova_table_depopulate(); + TAILQ_REMOVE(&fslmc_bus->driver_list, driver, next); /* Update Bus references */ driver->fslmc_bus = NULL; @@ -490,6 +519,10 @@ rte_dpaa2_get_iommu_class(void) if (TAILQ_EMPTY(&rte_fslmc_bus.device_list)) return RTE_IOVA_DC; +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + return RTE_IOVA_PA; +#endif + /* check if all devices on the bus support Virtual addressing or not */ has_iova_va = fslmc_all_device_support_iova(); diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c index 4c2cd2a8..493b6e5b 100644 --- a/drivers/bus/fslmc/fslmc_vfio.c +++ b/drivers/bus/fslmc/fslmc_vfio.c @@ -221,6 +221,13 @@ fslmc_memevent_cb(enum rte_mem_event type, const void *addr, size_t len, "alloc" : "dealloc", va, virt_addr, iova_addr, map_len); + /* iova_addr may be set to RTE_BAD_IOVA */ + if (iova_addr == RTE_BAD_IOVA) { + DPAA2_BUS_DEBUG("Segment has invalid iova, skipping\n"); + cur_len += map_len; + continue; + } + if (type == RTE_MEM_EVENT_ALLOC) ret = fslmc_map_dma(virt_addr, iova_addr, map_len); else @@ -318,11 +325,15 @@ fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len) static int fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused, - const struct rte_memseg *ms, void *arg) + const struct rte_memseg *ms, void *arg) { int *n_segs = arg; int ret; + /* if IOVA address is invalid, skip */ + if (ms->iova == RTE_BAD_IOVA) + return 0; + ret = fslmc_map_dma(ms->addr_64, ms->iova, ms->len); if (ret) DPAA2_BUS_ERR("Unable to VFIO map (addr=%p, len=%zu)", diff --git a/drivers/bus/fslmc/mc/dpbp.c b/drivers/bus/fslmc/mc/dpbp.c index 0215d22d..d9103409 100644 --- a/drivers/bus/fslmc/mc/dpbp.c +++ b/drivers/bus/fslmc/mc/dpbp.c @@ -248,6 +248,16 @@ int dpbp_reset(struct fsl_mc_io *mc_io, /* send command to mc*/ return mc_send_command(mc_io, &cmd); } +/** + * dpbp_get_attributes - Retrieve DPBP attributes. + * + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ int dpbp_get_attributes(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, diff --git a/drivers/bus/fslmc/mc/dpci.c b/drivers/bus/fslmc/mc/dpci.c index ff366bfa..95edae9d 100644 --- a/drivers/bus/fslmc/mc/dpci.c +++ b/drivers/bus/fslmc/mc/dpci.c @@ -265,6 +265,15 @@ int dpci_reset(struct fsl_mc_io *mc_io, return mc_send_command(mc_io, &cmd); } +/** + * dpci_get_attributes() - Retrieve DPCI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ int dpci_get_attributes(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, @@ -292,6 +301,94 @@ int dpci_get_attributes(struct fsl_mc_io *mc_io, return 0; } +/** + * dpci_get_peer_attributes() - Retrieve peer DPCI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @attr: Returned peer attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_peer_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_peer_attr *attr) +{ + struct dpci_rsp_get_peer_attr *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_PEER_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_peer_attr *)cmd.params; + attr->peer_id = le32_to_cpu(rsp_params->id); + attr->num_of_priorities = rsp_params->num_of_priorities; + + return 0; +} + +/** + * dpci_get_link_state() - Retrieve the DPCI link state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @up: Returned link state; returns '1' if link is up, '0' otherwise + * + * DPCI can be connected to another DPCI, together they + * create a 'link'. In order to use the DPCI Tx and Rx queues, + * both objects must be enabled. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_link_state(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *up) +{ + struct dpci_rsp_get_link_state *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_LINK_STATE, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_link_state *)cmd.params; + *up = dpci_get_field(rsp_params->up, UP); + + return 0; +} + +/** + * dpci_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @priority: Select the queue relative to number of + * priorities configured at DPCI creation; use + * DPCI_ALL_QUEUES to configure all Rx queues + * identically. + * @cfg: Rx queue configuration + * + * Return: '0' on Success; Error code otherwise. + */ int dpci_set_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, @@ -314,6 +411,9 @@ int dpci_set_rx_queue(struct fsl_mc_io *mc_io, dpci_set_field(cmd_params->dest_type, DEST_TYPE, cfg->dest_cfg.dest_type); + dpci_set_field(cmd_params->dest_type, + ORDER_PRESERVATION, + cfg->order_preservation_en); /* send command to mc*/ return mc_send_command(mc_io, &cmd); @@ -438,3 +538,100 @@ int dpci_get_api_version(struct fsl_mc_io *mc_io, return 0; } + +/** + * dpci_set_opr() - Set Order Restoration configuration. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @index: The queue index + * @options: Configuration mode options + * can be OPR_OPT_CREATE or OPR_OPT_RETIRE + * @cfg: Configuration options for the OPR + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_set_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + uint8_t options, + struct opr_cfg *cfg) +{ + struct dpci_cmd_set_opr *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_SET_OPR, + cmd_flags, + token); + cmd_params = (struct dpci_cmd_set_opr *)cmd.params; + cmd_params->index = index; + cmd_params->options = options; + cmd_params->oloe = cfg->oloe; + cmd_params->oeane = cfg->oeane; + cmd_params->olws = cfg->olws; + cmd_params->oa = cfg->oa; + cmd_params->oprrws = cfg->oprrws; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpci_get_opr() - Retrieve Order Restoration config and query. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @index: The queue index + * @cfg: Returned OPR configuration + * @qry: Returned OPR query + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + struct opr_cfg *cfg, + struct opr_qry *qry) +{ + struct dpci_rsp_get_opr *rsp_params; + struct dpci_cmd_get_opr *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_OPR, + cmd_flags, + token); + cmd_params = (struct dpci_cmd_get_opr *)cmd.params; + cmd_params->index = index; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_opr *)cmd.params; + cfg->oloe = rsp_params->oloe; + cfg->oeane = rsp_params->oeane; + cfg->olws = rsp_params->olws; + cfg->oa = rsp_params->oa; + cfg->oprrws = rsp_params->oprrws; + qry->rip = dpci_get_field(rsp_params->flags, RIP); + qry->enable = dpci_get_field(rsp_params->flags, OPR_ENABLE); + qry->nesn = le16_to_cpu(rsp_params->nesn); + qry->ndsn = le16_to_cpu(rsp_params->ndsn); + qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq); + qry->tseq_nlis = dpci_get_field(rsp_params->tseq_nlis, TSEQ_NLIS); + qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq); + qry->hseq_nlis = dpci_get_field(rsp_params->hseq_nlis, HSEQ_NLIS); + qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr); + qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr); + qry->opr_vid = le16_to_cpu(rsp_params->opr_vid); + qry->opr_id = le16_to_cpu(rsp_params->opr_id); + + return 0; +} diff --git a/drivers/bus/fslmc/mc/dpcon.c b/drivers/bus/fslmc/mc/dpcon.c index 3f6e04b9..92bd2651 100644 --- a/drivers/bus/fslmc/mc/dpcon.c +++ b/drivers/bus/fslmc/mc/dpcon.c @@ -295,6 +295,36 @@ int dpcon_get_attributes(struct fsl_mc_io *mc_io, return 0; } +/** + * dpcon_set_notification() - Set DPCON notification destination + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * @cfg: Notification parameters + * + * Return: '0' on Success; Error code otherwise + */ +int dpcon_set_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_notification_cfg *cfg) +{ + struct dpcon_cmd_set_notification *dpcon_cmd; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_SET_NOTIFICATION, + cmd_flags, + token); + dpcon_cmd = (struct dpcon_cmd_set_notification *)cmd.params; + dpcon_cmd->dpio_id = cpu_to_le32(cfg->dpio_id); + dpcon_cmd->priority = cfg->priority; + dpcon_cmd->user_ctx = cpu_to_le64(cfg->user_ctx); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + /** * dpcon_get_api_version - Get Data Path Concentrator API version * @mc_io: Pointer to MC portal's DPCON object diff --git a/drivers/bus/fslmc/mc/dpdmai.c b/drivers/bus/fslmc/mc/dpdmai.c index 528889df..dcb9d516 100644 --- a/drivers/bus/fslmc/mc/dpdmai.c +++ b/drivers/bus/fslmc/mc/dpdmai.c @@ -113,6 +113,7 @@ int dpdmai_create(struct fsl_mc_io *mc_io, cmd_flags, dprc_token); cmd_params = (struct dpdmai_cmd_create *)cmd.params; + cmd_params->num_queues = cfg->num_queues; cmd_params->priorities[0] = cfg->priorities[0]; cmd_params->priorities[1] = cfg->priorities[1]; @@ -297,6 +298,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, rsp_params = (struct dpdmai_rsp_get_attr *)cmd.params; attr->id = le32_to_cpu(rsp_params->id); attr->num_of_priorities = rsp_params->num_of_priorities; + attr->num_of_queues = rsp_params->num_of_queues; return 0; } @@ -306,6 +308,8 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: Rx queue index. Accepted values are form 0 to num_queues + * parameter provided in dpdmai_create * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation; use * DPDMAI_ALL_QUEUES to configure all Rx queues @@ -317,6 +321,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, const struct dpdmai_rx_queue_cfg *cfg) { @@ -331,6 +336,7 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); cmd_params->dest_priority = cfg->dest_cfg.priority; cmd_params->priority = priority; + cmd_params->queue_idx = queue_idx; cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); cmd_params->options = cpu_to_le32(cfg->options); dpdmai_set_field(cmd_params->dest_type, @@ -346,6 +352,8 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: Rx queue index. Accepted values are form 0 to num_queues + * parameter provided in dpdmai_create * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @attr: Returned Rx queue attributes @@ -355,6 +363,7 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_rx_queue_attr *attr) { @@ -369,6 +378,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, token); cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params; cmd_params->priority = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -392,6 +402,8 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: Tx queue index. Accepted values are form 0 to num_queues + * parameter provided in dpdmai_create * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @attr: Returned Tx queue attributes @@ -401,6 +413,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_tx_queue_attr *attr) { @@ -415,6 +428,7 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, token); cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params; cmd_params->priority = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); diff --git a/drivers/bus/fslmc/mc/dpio.c b/drivers/bus/fslmc/mc/dpio.c index 966277cc..a3382ed1 100644 --- a/drivers/bus/fslmc/mc/dpio.c +++ b/drivers/bus/fslmc/mc/dpio.c @@ -268,6 +268,15 @@ int dpio_reset(struct fsl_mc_io *mc_io, return mc_send_command(mc_io, &cmd); } +/** + * dpio_get_attributes() - Retrieve DPIO attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise + */ int dpio_get_attributes(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, diff --git a/drivers/bus/fslmc/mc/fsl_dpbp.h b/drivers/bus/fslmc/mc/fsl_dpbp.h index 11183626..9d405b42 100644 --- a/drivers/bus/fslmc/mc/fsl_dpbp.h +++ b/drivers/bus/fslmc/mc/fsl_dpbp.h @@ -82,6 +82,7 @@ int dpbp_get_attributes(struct fsl_mc_io *mc_io, /** * BPSCN write will attempt to allocate into a cache (coherent write) */ +#define DPBP_NOTIF_OPT_COHERENT_WRITE 0x00000001 int dpbp_get_api_version(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t *major_ver, diff --git a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h index 18402ced..55c9fc9b 100644 --- a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h @@ -9,13 +9,15 @@ /* DPBP Version */ #define DPBP_VER_MAJOR 3 -#define DPBP_VER_MINOR 3 +#define DPBP_VER_MINOR 4 /* Command versioning */ #define DPBP_CMD_BASE_VERSION 1 +#define DPBP_CMD_VERSION_2 2 #define DPBP_CMD_ID_OFFSET 4 #define DPBP_CMD(id) ((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_BASE_VERSION) +#define DPBP_CMD_V2(id) ((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_VERSION_2) /* Command IDs */ #define DPBP_CMDID_CLOSE DPBP_CMD(0x800) @@ -37,8 +39,8 @@ #define DPBP_CMDID_GET_IRQ_STATUS DPBP_CMD(0x016) #define DPBP_CMDID_CLEAR_IRQ_STATUS DPBP_CMD(0x017) -#define DPBP_CMDID_SET_NOTIFICATIONS DPBP_CMD(0x1b0) -#define DPBP_CMDID_GET_NOTIFICATIONS DPBP_CMD(0x1b1) +#define DPBP_CMDID_SET_NOTIFICATIONS DPBP_CMD_V2(0x1b0) +#define DPBP_CMDID_GET_NOTIFICATIONS DPBP_CMD_V2(0x1b1) #define DPBP_CMDID_GET_FREE_BUFFERS_NUM DPBP_CMD(0x1b2) @@ -68,8 +70,8 @@ struct dpbp_cmd_set_notifications { uint32_t depletion_exit; uint32_t surplus_entry; uint32_t surplus_exit; - uint16_t options; - uint16_t pad[3]; + uint32_t options; + uint16_t pad[2]; uint64_t message_ctx; uint64_t message_iova; }; @@ -79,8 +81,8 @@ struct dpbp_rsp_get_notifications { uint32_t depletion_exit; uint32_t surplus_entry; uint32_t surplus_exit; - uint16_t options; - uint16_t pad[3]; + uint32_t options; + uint16_t pad[2]; uint64_t message_ctx; uint64_t message_iova; }; diff --git a/drivers/bus/fslmc/mc/fsl_dpci.h b/drivers/bus/fslmc/mc/fsl_dpci.h index f69ed3f3..9af9097e 100644 --- a/drivers/bus/fslmc/mc/fsl_dpci.h +++ b/drivers/bus/fslmc/mc/fsl_dpci.h @@ -6,6 +6,8 @@ #ifndef __FSL_DPCI_H #define __FSL_DPCI_H +#include + /* Data Path Communication Interface API * Contains initialization APIs and runtime control APIs for DPCI */ @@ -17,7 +19,7 @@ struct fsl_mc_io; /** * Maximum number of Tx/Rx priorities per DPCI object */ -#define DPCI_PRIO_NUM 2 +#define DPCI_PRIO_NUM 4 /** * Indicates an invalid frame queue @@ -106,6 +108,27 @@ int dpci_get_attributes(struct fsl_mc_io *mc_io, uint16_t token, struct dpci_attr *attr); +/** + * struct dpci_peer_attr - Structure representing the peer DPCI attributes + * @peer_id: DPCI peer id; if no peer is connected returns (-1) + * @num_of_priorities: The pper's number of receive priorities; determines the + * number of transmit priorities for the local DPCI object + */ +struct dpci_peer_attr { + int peer_id; + uint8_t num_of_priorities; +}; + +int dpci_get_peer_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_peer_attr *attr); + +int dpci_get_link_state(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *up); + /** * enum dpci_dest - DPCI destination types * @DPCI_DEST_NONE: Unassigned destination; The queue is set in parked mode @@ -153,6 +176,11 @@ struct dpci_dest_cfg { */ #define DPCI_QUEUE_OPT_DEST 0x00000002 +/** + * Set the queue to hold active mode. + */ +#define DPCI_QUEUE_OPT_HOLD_ACTIVE 0x00000004 + /** * struct dpci_rx_queue_cfg - Structure representing RX queue configuration * @options: Flags representing the suggested modifications to the queue; @@ -163,11 +191,14 @@ struct dpci_dest_cfg { * 'options' * @dest_cfg: Queue destination parameters; * valid only if 'DPCI_QUEUE_OPT_DEST' is contained in 'options' + * @order_preservation_en: order preservation configuration for the rx queue + * valid only if 'DPCI_QUEUE_OPT_HOLD_ACTIVE' is contained in 'options' */ struct dpci_rx_queue_cfg { uint32_t options; uint64_t user_ctx; struct dpci_dest_cfg dest_cfg; + int order_preservation_en; }; int dpci_set_rx_queue(struct fsl_mc_io *mc_io, @@ -217,4 +248,18 @@ int dpci_get_api_version(struct fsl_mc_io *mc_io, uint16_t *major_ver, uint16_t *minor_ver); +int dpci_set_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + uint8_t options, + struct opr_cfg *cfg); + +int dpci_get_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + struct opr_cfg *cfg, + struct opr_qry *qry); + #endif /* __FSL_DPCI_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpci_cmd.h b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h index 634248ac..92b85a82 100644 --- a/drivers/bus/fslmc/mc/fsl_dpci_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h @@ -8,7 +8,7 @@ /* DPCI Version */ #define DPCI_VER_MAJOR 3 -#define DPCI_VER_MINOR 3 +#define DPCI_VER_MINOR 4 #define DPCI_CMD_BASE_VERSION 1 #define DPCI_CMD_BASE_VERSION_V2 2 @@ -35,6 +35,8 @@ #define DPCI_CMDID_GET_PEER_ATTR DPCI_CMD_V1(0x0e2) #define DPCI_CMDID_GET_RX_QUEUE DPCI_CMD_V1(0x0e3) #define DPCI_CMDID_GET_TX_QUEUE DPCI_CMD_V1(0x0e4) +#define DPCI_CMDID_SET_OPR DPCI_CMD_V1(0x0e5) +#define DPCI_CMDID_GET_OPR DPCI_CMD_V1(0x0e6) /* Macros for accessing command fields smaller than 1byte */ #define DPCI_MASK(field) \ @@ -90,6 +92,8 @@ struct dpci_rsp_get_link_state { #define DPCI_DEST_TYPE_SHIFT 0 #define DPCI_DEST_TYPE_SIZE 4 +#define DPCI_ORDER_PRESERVATION_SHIFT 4 +#define DPCI_ORDER_PRESERVATION_SIZE 1 struct dpci_cmd_set_rx_queue { uint32_t dest_id; @@ -128,5 +132,61 @@ struct dpci_rsp_get_api_version { uint16_t minor; }; +struct dpci_cmd_set_opr { + uint16_t pad0; + uint8_t index; + uint8_t options; + uint8_t pad1[7]; + uint8_t oloe; + uint8_t oeane; + uint8_t olws; + uint8_t oa; + uint8_t oprrws; +}; + +struct dpci_cmd_get_opr { + uint16_t pad; + uint8_t index; +}; + +#define DPCI_RIP_SHIFT 0 +#define DPCI_RIP_SIZE 1 +#define DPCI_OPR_ENABLE_SHIFT 1 +#define DPCI_OPR_ENABLE_SIZE 1 +#define DPCI_TSEQ_NLIS_SHIFT 0 +#define DPCI_TSEQ_NLIS_SIZE 1 +#define DPCI_HSEQ_NLIS_SHIFT 0 +#define DPCI_HSEQ_NLIS_SIZE 1 + +struct dpci_rsp_get_opr { + uint64_t pad0; + /* from LSB: rip:1 enable:1 */ + uint8_t flags; + uint16_t pad1; + uint8_t oloe; + uint8_t oeane; + uint8_t olws; + uint8_t oa; + uint8_t oprrws; + uint16_t nesn; + uint16_t pad8; + uint16_t ndsn; + uint16_t pad2; + uint16_t ea_tseq; + /* only the LSB */ + uint8_t tseq_nlis; + uint8_t pad3; + uint16_t ea_hseq; + /* only the LSB */ + uint8_t hseq_nlis; + uint8_t pad4; + uint16_t ea_hptr; + uint16_t pad5; + uint16_t ea_tptr; + uint16_t pad6; + uint16_t opr_vid; + uint16_t pad7; + uint16_t opr_id; +}; #pragma pack(pop) #endif /* _FSL_DPCI_CMD_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h index 36dd5f3c..fc0430dc 100644 --- a/drivers/bus/fslmc/mc/fsl_dpcon.h +++ b/drivers/bus/fslmc/mc/fsl_dpcon.h @@ -81,6 +81,25 @@ int dpcon_get_attributes(struct fsl_mc_io *mc_io, uint16_t token, struct dpcon_attr *attr); +/** + * struct dpcon_notification_cfg - Structure representing notification params + * @dpio_id: DPIO object ID; must be configured with a notification channel; + * to disable notifications set it to 'DPCON_INVALID_DPIO_ID'; + * @priority: Priority selection within the DPIO channel; valid values + * are 0-7, depending on the number of priorities in that channel + * @user_ctx: User context value provided with each CDAN message + */ +struct dpcon_notification_cfg { + int dpio_id; + uint8_t priority; + uint64_t user_ctx; +}; + +int dpcon_set_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_notification_cfg *cfg); + int dpcon_get_api_version(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t *major_ver, diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai.h b/drivers/bus/fslmc/mc/fsl_dpdmai.h index 03e46ec1..40469cc1 100644 --- a/drivers/bus/fslmc/mc/fsl_dpdmai.h +++ b/drivers/bus/fslmc/mc/fsl_dpdmai.h @@ -39,6 +39,7 @@ int dpdmai_close(struct fsl_mc_io *mc_io, * should be configured with 0 */ struct dpdmai_cfg { + uint8_t num_queues; uint8_t priorities[DPDMAI_PRIO_NUM]; }; @@ -78,6 +79,7 @@ int dpdmai_reset(struct fsl_mc_io *mc_io, struct dpdmai_attr { int id; uint8_t num_of_priorities; + uint8_t num_of_queues; }; int dpdmai_get_attributes(struct fsl_mc_io *mc_io, @@ -149,6 +151,7 @@ struct dpdmai_rx_queue_cfg { int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, const struct dpdmai_rx_queue_cfg *cfg); @@ -168,6 +171,7 @@ struct dpdmai_rx_queue_attr { int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_rx_queue_attr *attr); @@ -183,6 +187,7 @@ struct dpdmai_tx_queue_attr { int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_tx_queue_attr *attr); diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h index 618e19ea..7e122de4 100644 --- a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h @@ -7,30 +7,32 @@ /* DPDMAI Version */ #define DPDMAI_VER_MAJOR 3 -#define DPDMAI_VER_MINOR 2 +#define DPDMAI_VER_MINOR 3 /* Command versioning */ #define DPDMAI_CMD_BASE_VERSION 1 +#define DPDMAI_CMD_VERSION_2 2 #define DPDMAI_CMD_ID_OFFSET 4 #define DPDMAI_CMD(id) ((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) +#define DPDMAI_CMD_V2(id) ((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_VERSION_2) /* Command IDs */ #define DPDMAI_CMDID_CLOSE DPDMAI_CMD(0x800) #define DPDMAI_CMDID_OPEN DPDMAI_CMD(0x80E) -#define DPDMAI_CMDID_CREATE DPDMAI_CMD(0x90E) +#define DPDMAI_CMDID_CREATE DPDMAI_CMD_V2(0x90E) #define DPDMAI_CMDID_DESTROY DPDMAI_CMD(0x98E) #define DPDMAI_CMDID_GET_API_VERSION DPDMAI_CMD(0xa0E) #define DPDMAI_CMDID_ENABLE DPDMAI_CMD(0x002) #define DPDMAI_CMDID_DISABLE DPDMAI_CMD(0x003) -#define DPDMAI_CMDID_GET_ATTR DPDMAI_CMD(0x004) +#define DPDMAI_CMDID_GET_ATTR DPDMAI_CMD_V2(0x004) #define DPDMAI_CMDID_RESET DPDMAI_CMD(0x005) #define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMD(0x006) -#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMD(0x1A0) -#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMD(0x1A1) -#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMD(0x1A2) +#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMD_V2(0x1A0) +#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMD_V2(0x1A1) +#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMD_V2(0x1A2) /* Macros for accessing command fields smaller than 1byte */ #define DPDMAI_MASK(field) \ @@ -47,7 +49,7 @@ struct dpdmai_cmd_open { }; struct dpdmai_cmd_create { - uint8_t pad; + uint8_t num_queues; uint8_t priorities[2]; }; @@ -66,6 +68,7 @@ struct dpdmai_rsp_is_enabled { struct dpdmai_rsp_get_attr { uint32_t id; uint8_t num_of_priorities; + uint8_t num_of_queues; }; #define DPDMAI_DEST_TYPE_SHIFT 0 @@ -77,7 +80,7 @@ struct dpdmai_cmd_set_rx_queue { uint8_t priority; /* from LSB: dest_type:4 */ uint8_t dest_type; - uint8_t pad; + uint8_t queue_idx; uint64_t user_ctx; uint32_t options; }; @@ -85,6 +88,7 @@ struct dpdmai_cmd_set_rx_queue { struct dpdmai_cmd_get_queue { uint8_t pad[5]; uint8_t priority; + uint8_t queue_idx; }; struct dpdmai_rsp_get_rx_queue { diff --git a/drivers/bus/fslmc/mc/fsl_dpmng.h b/drivers/bus/fslmc/mc/fsl_dpmng.h index afaf9b71..8559bef8 100644 --- a/drivers/bus/fslmc/mc/fsl_dpmng.h +++ b/drivers/bus/fslmc/mc/fsl_dpmng.h @@ -18,7 +18,7 @@ struct fsl_mc_io; * Management Complex firmware version information */ #define MC_VER_MAJOR 10 -#define MC_VER_MINOR 3 +#define MC_VER_MINOR 10 /** * struct mc_version diff --git a/drivers/bus/fslmc/mc/fsl_dpopr.h b/drivers/bus/fslmc/mc/fsl_dpopr.h new file mode 100644 index 00000000..fd727e01 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpopr.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * Copyright 2018 NXP + * + */ +#ifndef __FSL_DPOPR_H_ +#define __FSL_DPOPR_H_ + +/** @addtogroup dpopr Data Path Order Restoration API + * Contains initialization APIs and runtime APIs for the Order Restoration + * @{ + */ + +/** Order Restoration properties */ + +/** + * Create a new Order Point Record option + */ +#define OPR_OPT_CREATE 0x1 +/** + * Retire an existing Order Point Record option + */ +#define OPR_OPT_RETIRE 0x2 + +/** + * struct opr_cfg - Structure representing OPR configuration + * @oprrws: Order point record (OPR) restoration window size (0 to 5) + * 0 - Window size is 32 frames. + * 1 - Window size is 64 frames. + * 2 - Window size is 128 frames. + * 3 - Window size is 256 frames. + * 4 - Window size is 512 frames. + * 5 - Window size is 1024 frames. + *@oa: OPR auto advance NESN window size (0 disabled, 1 enabled) + *@olws: OPR acceptable late arrival window size (0 to 3) + * 0 - Disabled. Late arrivals are always rejected. + * 1 - Window size is 32 frames. + * 2 - Window size is the same as the OPR restoration + * window size configured in the OPRRWS field. + * 3 - Window size is 8192 frames. + * Late arrivals are always accepted. + *@oeane: Order restoration list (ORL) resource exhaustion + * advance NESN enable (0 disabled, 1 enabled) + *@oloe: OPR loose ordering enable (0 disabled, 1 enabled) + */ +struct opr_cfg { + uint8_t oprrws; + uint8_t oa; + uint8_t olws; + uint8_t oeane; + uint8_t oloe; +}; + +/** + * struct opr_qry - Structure representing OPR configuration + * @enable: Enabled state + * @rip: Retirement In Progress + * @ndsn: Next dispensed sequence number + * @nesn: Next expected sequence number + * @ea_hseq: Early arrival head sequence number + * @hseq_nlis: HSEQ not last in sequence + * @ea_tseq: Early arrival tail sequence number + * @tseq_nlis: TSEQ not last in sequence + * @ea_tptr: Early arrival tail pointer + * @ea_hptr: Early arrival head pointer + * @opr_id: Order Point Record ID + * @opr_vid: Order Point Record Virtual ID + */ +struct opr_qry { + char enable; + char rip; + uint16_t ndsn; + uint16_t nesn; + uint16_t ea_hseq; + char hseq_nlis; + uint16_t ea_tseq; + char tseq_nlis; + uint16_t ea_tptr; + uint16_t ea_hptr; + uint16_t opr_id; + uint16_t opr_vid; +}; + +#endif /* __FSL_DPOPR_H_ */ diff --git a/drivers/bus/fslmc/meson.build b/drivers/bus/fslmc/meson.build index 22a56a6f..4b052157 100644 --- a/drivers/bus/fslmc/meson.build +++ b/drivers/bus/fslmc/meson.build @@ -1,11 +1,13 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright 2018 NXP +version = 2 + if host_machine.system() != 'linux' build = false endif -deps += ['eventdev', 'kvargs'] +deps += ['common_dpaax', 'eventdev', 'kvargs'] sources = files('fslmc_bus.c', 'fslmc_vfio.c', 'mc/dpbp.c', @@ -24,4 +26,3 @@ sources = files('fslmc_bus.c', allow_experimental_apis = true includes += include_directories('mc', 'qbman/include', 'portal') -cflags += ['-D_GNU_SOURCE'] diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c index 39c5adf9..db49d637 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c @@ -28,6 +28,13 @@ #include "portal/dpaa2_hw_pvt.h" #include "portal/dpaa2_hw_dpio.h" +/* List of all the memseg information locally maintained in dpaa2 driver. This + * is to optimize the PA_to_VA searches until a better mechanism (algo) is + * available. + */ +struct dpaa2_memseg_list rte_dpaa2_memsegs + = TAILQ_HEAD_INITIALIZER(rte_dpaa2_memsegs); + TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev); static struct dpbp_dev_list dpbp_dev_list = TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */ diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c index 99f70be1..ce069984 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright 2016 NXP + * Copyright 2016-2018 NXP * */ #include @@ -53,6 +53,11 @@ static uint32_t io_space_count; /* Variable to store DPAA2 platform type */ uint32_t dpaa2_svr_family; +/* Variable to store DPAA2 DQRR size */ +uint8_t dpaa2_dqrr_size; +/* Variable to store DPAA2 EQCR size */ +uint8_t dpaa2_eqcr_size; + /*Stashing Macros default for LS208x*/ static int dpaa2_core_cluster_base = 0x04; static int dpaa2_cluster_sz = 2; @@ -125,7 +130,7 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id) cpu_mask, token); ret = system(command); if (ret < 0) - DPAA2_BUS_WARN( + DPAA2_BUS_DEBUG( "Failed to affine interrupts on respective core"); else DPAA2_BUS_DEBUG(" %s command is executed", command); @@ -177,68 +182,6 @@ static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev) } #endif -static int -configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) -{ - struct qbman_swp_desc p_des; - struct dpio_attr attr; - - dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io)); - if (!dpio_dev->dpio) { - DPAA2_BUS_ERR("Memory allocation failure"); - return -1; - } - - dpio_dev->dpio->regs = dpio_dev->mc_portal; - if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id, - &dpio_dev->token)) { - DPAA2_BUS_ERR("Failed to allocate IO space"); - free(dpio_dev->dpio); - return -1; - } - - if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { - DPAA2_BUS_ERR("Failed to reset dpio"); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { - DPAA2_BUS_ERR("Failed to Enable dpio"); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW, - dpio_dev->token, &attr)) { - DPAA2_BUS_ERR("DPIO Get attribute failed"); - dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - /* Configure & setup SW portal */ - p_des.block = NULL; - p_des.idx = attr.qbman_portal_id; - p_des.cena_bar = (void *)(dpio_dev->qbman_portal_ce_paddr); - p_des.cinh_bar = (void *)(dpio_dev->qbman_portal_ci_paddr); - p_des.irq = -1; - p_des.qman_version = attr.qbman_version; - - dpio_dev->sw_portal = qbman_swp_init(&p_des); - if (dpio_dev->sw_portal == NULL) { - DPAA2_BUS_ERR("QBMan SW Portal Init failed"); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - return 0; -} - static int dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id) { @@ -402,15 +345,17 @@ dpaa2_create_dpio_device(int vdev_fd, struct vfio_device_info *obj_info, int object_id) { - struct dpaa2_dpio_dev *dpio_dev; + struct dpaa2_dpio_dev *dpio_dev = NULL; struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)}; + struct qbman_swp_desc p_des; + struct dpio_attr attr; if (obj_info->num_regions < NUM_DPIO_REGIONS) { DPAA2_BUS_ERR("Not sufficient number of DPIO regions"); return -1; } - dpio_dev = rte_malloc(NULL, sizeof(struct dpaa2_dpio_dev), + dpio_dev = rte_zmalloc(NULL, sizeof(struct dpaa2_dpio_dev), RTE_CACHE_LINE_SIZE); if (!dpio_dev) { DPAA2_BUS_ERR("Memory allocation failed for DPIO Device"); @@ -423,45 +368,33 @@ dpaa2_create_dpio_device(int vdev_fd, /* Using single portal for all devices */ dpio_dev->mc_portal = rte_mcp_ptr_list[MC_PORTAL_INDEX]; - reg_info.index = 0; - if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { - DPAA2_BUS_ERR("vfio: error getting region info"); - rte_free(dpio_dev); - return -1; + dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io)); + if (!dpio_dev->dpio) { + DPAA2_BUS_ERR("Memory allocation failure"); + goto err; } - dpio_dev->ce_size = reg_info.size; - dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size, - PROT_WRITE | PROT_READ, MAP_SHARED, - vdev_fd, reg_info.offset); - - reg_info.index = 1; - if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { - DPAA2_BUS_ERR("vfio: error getting region info"); - rte_free(dpio_dev); - return -1; + dpio_dev->dpio->regs = dpio_dev->mc_portal; + if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id, + &dpio_dev->token)) { + DPAA2_BUS_ERR("Failed to allocate IO space"); + goto err; } - dpio_dev->ci_size = reg_info.size; - dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size, - PROT_WRITE | PROT_READ, MAP_SHARED, - vdev_fd, reg_info.offset); - - if (configure_dpio_qbman_swp(dpio_dev)) { - DPAA2_BUS_ERR( - "Fail to configure the dpio qbman portal for %d", - dpio_dev->hw_id); - rte_free(dpio_dev); - return -1; + if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { + DPAA2_BUS_ERR("Failed to reset dpio"); + goto err; } - io_space_count++; - dpio_dev->index = io_space_count; + if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { + DPAA2_BUS_ERR("Failed to Enable dpio"); + goto err; + } - if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) { - DPAA2_BUS_ERR("Fail to setup interrupt for %d", - dpio_dev->hw_id); - rte_free(dpio_dev); + if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW, + dpio_dev->token, &attr)) { + DPAA2_BUS_ERR("DPIO Get attribute failed"); + goto err; } /* find the SoC type for the first time */ @@ -481,11 +414,77 @@ dpaa2_create_dpio_device(int vdev_fd, DPAA2_BUS_DEBUG("LX2160 Platform Detected"); } dpaa2_svr_family = (mc_plat_info.svr & 0xffff0000); + + if (dpaa2_svr_family == SVR_LX2160A) { + dpaa2_dqrr_size = DPAA2_LX2_DQRR_RING_SIZE; + dpaa2_eqcr_size = DPAA2_LX2_EQCR_RING_SIZE; + } else { + dpaa2_dqrr_size = DPAA2_DQRR_RING_SIZE; + dpaa2_eqcr_size = DPAA2_EQCR_RING_SIZE; + } + } + + if (dpaa2_svr_family == SVR_LX2160A) + reg_info.index = DPAA2_SWP_CENA_MEM_REGION; + else + reg_info.index = DPAA2_SWP_CENA_REGION; + + if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + DPAA2_BUS_ERR("vfio: error getting region info"); + goto err; + } + + dpio_dev->ce_size = reg_info.size; + dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size, + PROT_WRITE | PROT_READ, MAP_SHARED, + vdev_fd, reg_info.offset); + + reg_info.index = DPAA2_SWP_CINH_REGION; + if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + DPAA2_BUS_ERR("vfio: error getting region info"); + goto err; + } + + dpio_dev->ci_size = reg_info.size; + dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size, + PROT_WRITE | PROT_READ, MAP_SHARED, + vdev_fd, reg_info.offset); + + /* Configure & setup SW portal */ + p_des.block = NULL; + p_des.idx = attr.qbman_portal_id; + p_des.cena_bar = (void *)(dpio_dev->qbman_portal_ce_paddr); + p_des.cinh_bar = (void *)(dpio_dev->qbman_portal_ci_paddr); + p_des.irq = -1; + p_des.qman_version = attr.qbman_version; + + dpio_dev->sw_portal = qbman_swp_init(&p_des); + if (dpio_dev->sw_portal == NULL) { + DPAA2_BUS_ERR("QBMan SW Portal Init failed"); + goto err; + } + + io_space_count++; + dpio_dev->index = io_space_count; + + if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) { + DPAA2_BUS_ERR("Fail to setup interrupt for %d", + dpio_dev->hw_id); + goto err; } TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next); return 0; + +err: + if (dpio_dev->dpio) { + dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + free(dpio_dev->dpio); + } + rte_free(dpio_dev); + return -1; } void @@ -506,7 +505,7 @@ dpaa2_alloc_dq_storage(struct queue_storage_info_t *q_storage) for (i = 0; i < NUM_DQS_PER_QUEUE; i++) { q_storage->dq_storage[i] = rte_malloc(NULL, - DPAA2_DQRR_RING_SIZE * sizeof(struct qbman_result), + dpaa2_dqrr_size * sizeof(struct qbman_result), RTE_CACHE_LINE_SIZE); if (!q_storage->dq_storage[i]) goto fail; diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h index d593eea7..462501a2 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h @@ -30,6 +30,10 @@ RTE_DECLARE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io); /* Variable to store DPAA2 platform type */ extern uint32_t dpaa2_svr_family; +/* Variable to store DPAA2 DQRR size */ +extern uint8_t dpaa2_dqrr_size; +/* Variable to store DPAA2 EQCR size */ +extern uint8_t dpaa2_eqcr_size; extern struct dpaa2_io_portal_t dpaa2_io_portal[RTE_MAX_LCORE]; diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h index 82075936..efbeebef 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright 2016 NXP + * Copyright 2016-2018 NXP * */ @@ -9,6 +9,7 @@ #define _DPAA2_HW_PVT_H_ #include +#include #include #include @@ -31,11 +32,27 @@ #define VLAN_TAG_SIZE 4 /** < Vlan Header Length */ #endif -#define MAX_TX_RING_SLOTS 8 - /** > 3) = 8 for LS2/LS2 */ +#define DPAA2_EQCR_SHIFT 3 +/* EQCR shift to get EQCR size for LX2 (2 >> 5) = 32 for LX2 */ +#define DPAA2_LX2_EQCR_SHIFT 5 + +#define DPAA2_SWP_CENA_REGION 0 +#define DPAA2_SWP_CINH_REGION 1 +#define DPAA2_SWP_CENA_MEM_REGION 2 #define MC_PORTAL_INDEX 0 #define NUM_DPIO_REGIONS 2 @@ -193,6 +210,12 @@ enum qbman_fd_format { #define DPAA2_RESET_FD_CTRL(fd) ((fd)->simple.ctrl = 0) #define DPAA2_SET_FD_ASAL(fd, asal) ((fd)->simple.ctrl |= (asal << 16)) + +#define DPAA2_RESET_FD_FLC(fd) do { \ + (fd)->simple.flc_lo = 0; \ + (fd)->simple.flc_hi = 0; \ +} while (0) + #define DPAA2_SET_FD_FLC(fd, addr) do { \ (fd)->simple.flc_lo = lower_32_bits((size_t)(addr)); \ (fd)->simple.flc_hi = upper_32_bits((uint64_t)(addr)); \ @@ -275,28 +298,26 @@ extern struct dpaa2_memseg_list rte_dpaa2_memsegs; #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA extern uint8_t dpaa2_virt_mode; static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused)); -/* todo - this is costly, need to write a fast coversion routine */ + static void *dpaa2_mem_ptov(phys_addr_t paddr) { - struct dpaa2_memseg *ms; + void *va; if (dpaa2_virt_mode) return (void *)(size_t)paddr; - /* Check if the address is already part of the memseg list internally - * maintained by the dpaa2 driver. - */ - TAILQ_FOREACH(ms, &rte_dpaa2_memsegs, next) { - if (paddr >= ms->iova && paddr < - ms->iova + ms->len) - return RTE_PTR_ADD(ms->vaddr, (uintptr_t)(paddr - ms->iova)); - } + va = (void *)dpaax_iova_table_get_va(paddr); + if (likely(va != NULL)) + return va; /* If not, Fallback to full memseg list searching */ - return rte_mem_iova2virt(paddr); + va = rte_mem_iova2virt(paddr); + + return va; } static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused)); + static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) { const struct rte_memseg *memseg; diff --git a/drivers/bus/fslmc/qbman/include/compat.h b/drivers/bus/fslmc/qbman/include/compat.h index 7be8f54c..655bff4b 100644 --- a/drivers/bus/fslmc/qbman/include/compat.h +++ b/drivers/bus/fslmc/qbman/include/compat.h @@ -78,13 +78,14 @@ do { \ #define lower_32_bits(x) ((uint32_t)(x)) #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) - #define __iomem #define __raw_readb(p) (*(const volatile unsigned char *)(p)) #define __raw_readl(p) (*(const volatile unsigned int *)(p)) #define __raw_writel(v, p) {*(volatile unsigned int *)(p) = (v); } +#define dma_wmb() rte_smp_mb() + #define atomic_t rte_atomic32_t #define atomic_read(v) rte_atomic32_read(v) #define atomic_set(v, i) rte_atomic32_set(v, i) diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h index 3e63db3a..10c72e04 100644 --- a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h +++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h @@ -42,6 +42,15 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d); */ void qbman_swp_finish(struct qbman_swp *p); +/** + * qbman_swp_invalidate() - Invalidate the cache enabled area of the QBMan + * portal. This is required to be called if a portal moved to another core + * because the QBMan portal area is non coherent + * @p: the qbman_swp object to be invalidated + * + */ +void qbman_swp_invalidate(struct qbman_swp *p); + /** * qbman_swp_get_desc() - Get the descriptor of the given portal object. * @p: the given portal object. @@ -172,7 +181,7 @@ void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit); /** * struct qbman_result - structure for qbman dequeue response and/or * notification. - * @donot_manipulate_directly: the 16 32bit data to represent the whole + * @dont_manipulate_directly: the 16 32bit data to represent the whole * possible qbman dequeue result. */ struct qbman_result { @@ -262,7 +271,7 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable); */ struct qbman_pull_desc { union { - uint32_t donot_manipulate_directly[16]; + uint32_t dont_manipulate_directly[16]; struct pull { uint8_t verb; uint8_t numf; @@ -355,6 +364,14 @@ void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid, void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, enum qbman_pull_type_e dct); +/** + * qbman_pull_desc_set_rad() - Decide whether reschedule the fq after dequeue + * + * @rad: 1 = Reschedule the FQ after dequeue. + * 0 = Allow the FQ to remain active after dequeue. + */ +void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad); + /** * qbman_swp_pull() - Issue the pull dequeue command * @s: the software portal object. @@ -775,7 +792,7 @@ uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn); /* struct qbman_eq_desc - structure of enqueue descriptor */ struct qbman_eq_desc { union { - uint32_t donot_manipulate_directly[8]; + uint32_t dont_manipulate_directly[8]; struct eq { uint8_t verb; uint8_t dca; @@ -796,11 +813,11 @@ struct qbman_eq_desc { /** * struct qbman_eq_response - structure of enqueue response - * @donot_manipulate_directly: the 16 32bit data to represent the whole + * @dont_manipulate_directly: the 16 32bit data to represent the whole * enqueue response. */ struct qbman_eq_response { - uint32_t donot_manipulate_directly[16]; + uint32_t dont_manipulate_directly[16]; }; /** @@ -958,6 +975,7 @@ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, * @s: the software portal used for enqueue. * @d: the enqueue descriptor. * @fd: the frame descriptor to be enqueued. + * @flags: bit-mask of QBMAN_ENQUEUE_FLAG_*** options * @num_frames: the number of the frames to be enqueued. * * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. @@ -973,7 +991,6 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, * @s: the software portal used for enqueue. * @d: the enqueue descriptor. * @fd: the frame descriptor to be enqueued. - * @flags: bit-mask of QBMAN_ENQUEUE_FLAG_*** options * @num_frames: the number of the frames to be enqueued. * * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. @@ -998,12 +1015,12 @@ int qbman_swp_enqueue_thresh(struct qbman_swp *s, unsigned int thresh); /*******************/ /** * struct qbman_release_desc - The structure for buffer release descriptor - * @donot_manipulate_directly: the 32bit data to represent the whole + * @dont_manipulate_directly: the 32bit data to represent the whole * possible settings of qbman release descriptor. */ struct qbman_release_desc { union { - uint32_t donot_manipulate_directly[16]; + uint32_t dont_manipulate_directly[16]; struct br { uint8_t verb; uint8_t reserved; diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c index 07145005..3380e54f 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.c +++ b/drivers/bus/fslmc/qbman/qbman_portal.c @@ -1,39 +1,17 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * Copyright 2018 NXP * */ +#include "qbman_sys.h" #include "qbman_portal.h" /* QBMan portal management command codes */ #define QBMAN_MC_ACQUIRE 0x30 #define QBMAN_WQCHAN_CONFIGURE 0x46 -/* CINH register offsets */ -#define QBMAN_CINH_SWP_EQCR_PI 0x800 -#define QBMAN_CINH_SWP_EQCR_CI 0x840 -#define QBMAN_CINH_SWP_EQAR 0x8c0 -#define QBMAN_CINH_SWP_DQPI 0xa00 -#define QBMAN_CINH_SWP_DCAP 0xac0 -#define QBMAN_CINH_SWP_SDQCR 0xb00 -#define QBMAN_CINH_SWP_RAR 0xcc0 -#define QBMAN_CINH_SWP_ISR 0xe00 -#define QBMAN_CINH_SWP_IER 0xe40 -#define QBMAN_CINH_SWP_ISDR 0xe80 -#define QBMAN_CINH_SWP_IIR 0xec0 -#define QBMAN_CINH_SWP_DQRR_ITR 0xa80 -#define QBMAN_CINH_SWP_ITPR 0xf40 - -/* CENA register offsets */ -#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6)) -#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6)) -#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((uint32_t)(n) << 6)) -#define QBMAN_CENA_SWP_CR 0x600 -#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((uint32_t)(vb) >> 1)) -#define QBMAN_CENA_SWP_VDQCR 0x780 -#define QBMAN_CENA_SWP_EQCR_CI 0x840 - /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */ #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6) @@ -83,6 +61,102 @@ enum qbman_sdqcr_fc { #define MAX_QBMAN_PORTALS 64 static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS]; +/* Internal Function declaration */ +static int +qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); +static int +qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); + +static int +qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); +static int +qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); + +static int +qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames); +static int +qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames); + +static int +qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); +static int +qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); + +static int +qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d); +static int +qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d); + +const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s); +const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s); + +static int +qbman_swp_release_direct(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers); +static int +qbman_swp_release_mem_back(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers); + +/* Function pointers */ +static int (*qbman_swp_enqueue_array_mode_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) + = qbman_swp_enqueue_array_mode_direct; + +static int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) + = qbman_swp_enqueue_ring_mode_direct; + +static int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) + = qbman_swp_enqueue_multiple_direct; + +static int (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) + = qbman_swp_enqueue_multiple_desc_direct; + +static int (*qbman_swp_pull_ptr)(struct qbman_swp *s, + struct qbman_pull_desc *d) + = qbman_swp_pull_direct; + +const struct qbman_result *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s) + = qbman_swp_dqrr_next_direct; + +static int (*qbman_swp_release_ptr)(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers) + = qbman_swp_release_direct; + /*********************************/ /* Portal constructor/destructor */ /*********************************/ @@ -104,25 +178,30 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) { int ret; uint32_t eqcr_pi; + uint32_t mask_size; struct qbman_swp *p = malloc(sizeof(*p)); if (!p) return NULL; + + memset(p, 0, sizeof(struct qbman_swp)); + p->desc = *d; #ifdef QBMAN_CHECKING p->mc.check = swp_mc_can_start; #endif p->mc.valid_bit = QB_VALID_BIT; - p->sdq = 0; p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) + p->mr.valid_bit = QB_VALID_BIT; atomic_set(&p->vdq.busy, 1); p->vdq.valid_bit = QB_VALID_BIT; - p->dqrr.next_idx = 0; p->dqrr.valid_bit = QB_VALID_BIT; - if ((p->desc.qman_version & 0xFFFF0000) < QMAN_REV_4100) { + qman_version = p->desc.qman_version; + if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) { p->dqrr.dqrr_size = 4; p->dqrr.reset_bug = 1; } else { @@ -136,18 +215,54 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) pr_err("qbman_swp_sys_init() failed %d\n", ret); return NULL; } + + /* Verify that the DQRRPI is 0 - if it is not the portal isn't + * in default state which is an error + */ + if (qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQPI) & 0xF) { + pr_err("qbman DQRR PI is not zero, portal is not clean\n"); + free(p); + return NULL; + } + /* SDQCR needs to be initialized to 0 when no channels are * being dequeued from or else the QMan HW will indicate an * error. The values that were calculated above will be * applied when dequeues from a specific channel are enabled. */ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0); + + p->eqcr.pi_ring_size = 8; + if ((qman_version & 0xFFFF0000) >= QMAN_REV_5000) { + p->eqcr.pi_ring_size = 32; + qbman_swp_enqueue_array_mode_ptr = + qbman_swp_enqueue_array_mode_mem_back; + qbman_swp_enqueue_ring_mode_ptr = + qbman_swp_enqueue_ring_mode_mem_back; + qbman_swp_enqueue_multiple_ptr = + qbman_swp_enqueue_multiple_mem_back; + qbman_swp_enqueue_multiple_desc_ptr = + qbman_swp_enqueue_multiple_desc_mem_back; + qbman_swp_pull_ptr = qbman_swp_pull_mem_back; + qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back; + qbman_swp_release_ptr = qbman_swp_release_mem_back; + } + + for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) + p->eqcr.pi_mask = (p->eqcr.pi_mask<<1) + 1; eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI); - p->eqcr.pi = eqcr_pi & 0xF; + p->eqcr.pi = eqcr_pi & p->eqcr.pi_mask; p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT; - p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI) & 0xF; - p->eqcr.available = QBMAN_EQCR_SIZE - qm_cyc_diff(QBMAN_EQCR_SIZE, - p->eqcr.ci, p->eqcr.pi); + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) + p->eqcr.ci = qbman_cinh_read(&p->sys, + QBMAN_CINH_SWP_EQCR_CI) & p->eqcr.pi_mask; + else + p->eqcr.ci = qbman_cinh_read(&p->sys, + QBMAN_CINH_SWP_EQCR_PI) & p->eqcr.pi_mask; + p->eqcr.available = p->eqcr.pi_ring_size - + qm_cyc_diff(p->eqcr.pi_ring_size, + p->eqcr.ci & (p->eqcr.pi_mask<<1), + p->eqcr.pi & (p->eqcr.pi_mask<<1)); portal_idx_map[p->desc.idx] = p; return p; @@ -229,7 +344,8 @@ int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p) void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit) { - qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0); + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, + inhibit ? 0xffffffff : 0); } /***********************/ @@ -246,7 +362,10 @@ void *qbman_swp_mc_start(struct qbman_swp *p) #ifdef QBMAN_CHECKING QBMAN_BUG_ON(p->mc.check != swp_mc_can_start); #endif - ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR); + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) + ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR); + else + ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR_MEM); #ifdef QBMAN_CHECKING if (!ret) p->mc.check = swp_mc_can_submit; @@ -266,8 +385,17 @@ void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb) * caller wants to OR but has forgotten to do so. */ QBMAN_BUG_ON((*v & cmd_verb) != *v); - *v = cmd_verb | p->mc.valid_bit; - qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd); + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { + dma_wmb(); + *v = cmd_verb | p->mc.valid_bit; + qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd); + clean(cmd); + } else { + *v = cmd_verb | p->mr.valid_bit; + qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR_MEM, cmd); + dma_wmb(); + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE); + } #ifdef QBMAN_CHECKING p->mc.check = swp_mc_can_poll; #endif @@ -279,17 +407,34 @@ void *qbman_swp_mc_result(struct qbman_swp *p) #ifdef QBMAN_CHECKING QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll); #endif - qbman_cena_invalidate_prefetch(&p->sys, - QBMAN_CENA_SWP_RR(p->mc.valid_bit)); - ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); - /* Remove the valid-bit - command completed if the rest is non-zero */ - verb = ret[0] & ~QB_VALID_BIT; - if (!verb) - return NULL; + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { + qbman_cena_invalidate_prefetch(&p->sys, + QBMAN_CENA_SWP_RR(p->mc.valid_bit)); + ret = qbman_cena_read(&p->sys, + QBMAN_CENA_SWP_RR(p->mc.valid_bit)); + /* Remove the valid-bit - + * command completed iff the rest is non-zero + */ + verb = ret[0] & ~QB_VALID_BIT; + if (!verb) + return NULL; + p->mc.valid_bit ^= QB_VALID_BIT; + } else { + ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR_MEM); + /* Command completed if the valid bit is toggled */ + if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT)) + return NULL; + /* Remove the valid-bit - + * command completed iff the rest is non-zero + */ + verb = ret[0] & ~QB_VALID_BIT; + if (!verb) + return NULL; + p->mr.valid_bit ^= QB_VALID_BIT; + } #ifdef QBMAN_CHECKING p->mc.check = swp_mc_can_start; #endif - p->mc.valid_bit ^= QB_VALID_BIT; return ret; } @@ -417,13 +562,26 @@ void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, } } -#define EQAR_IDX(eqar) ((eqar) & 0x7) +#define EQAR_IDX(eqar) ((eqar) & 0x1f) #define EQAR_VB(eqar) ((eqar) & 0x80) #define EQAR_SUCCESS(eqar) ((eqar) & 0x100) -static int qbman_swp_enqueue_array_mode(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd) +static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p, + uint8_t idx) +{ + if (idx < 16) + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4, + QMAN_RT_MODE); + else + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT2 + + (idx - 16) * 4, + QMAN_RT_MODE); +} + + +static int qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) { uint32_t *p; const uint32_t *cl = qb_cl(d); @@ -433,39 +591,69 @@ static int qbman_swp_enqueue_array_mode(struct qbman_swp *s, if (!EQAR_SUCCESS(eqar)) return -EBUSY; p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], fd, sizeof(*fd)); + /* Set the verb byte, have to substitute in the valid-bit */ - lwsync(); + dma_wmb(); p[0] = cl[0] | EQAR_VB(eqar); qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); return 0; } +static int qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR); -static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd) + pr_debug("EQAR=%08x\n", eqar); + if (!EQAR_SUCCESS(eqar)) + return -EBUSY; + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); + + /* Set the verb byte, have to substitute in the valid-bit */ + p[0] = cl[0] | EQAR_VB(eqar); + dma_wmb(); + qbman_write_eqcr_am_rt_register(s, EQAR_IDX(eqar)); + return 0; +} + +static inline int qbman_swp_enqueue_array_mode(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + return qbman_swp_enqueue_array_mode_ptr(s, d, fd); +} + +static int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) { uint32_t *p; const uint32_t *cl = qb_cl(d); - uint32_t eqcr_ci; - uint8_t diff; + uint32_t eqcr_ci, full_mask, half_mask; + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; if (!s->eqcr.available) { eqcr_ci = s->eqcr.ci; s->eqcr.ci = qbman_cena_read_reg(&s->sys, - QBMAN_CENA_SWP_EQCR_CI) & 0xF; - diff = qm_cyc_diff(QBMAN_EQCR_SIZE, - eqcr_ci, s->eqcr.ci); - s->eqcr.available += diff; - if (!diff) + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) return -EBUSY; } p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], fd, sizeof(*fd)); lwsync(); @@ -473,16 +661,61 @@ static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, /* Set the verb byte, have to substitute in the valid-bit */ p[0] = cl[0] | s->eqcr.pi_vb; qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); s->eqcr.pi++; - s->eqcr.pi &= 0xF; + s->eqcr.pi &= full_mask; s->eqcr.available--; - if (!(s->eqcr.pi & 7)) + if (!(s->eqcr.pi & half_mask)) s->eqcr.pi_vb ^= QB_VALID_BIT; return 0; } +static int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqcr_ci, full_mask, half_mask; + + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cinh_read(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) + return -EBUSY; + } + + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); + + /* Set the verb byte, have to substitute in the valid-bit */ + p[0] = cl[0] | s->eqcr.pi_vb; + s->eqcr.pi++; + s->eqcr.pi &= full_mask; + s->eqcr.available--; + if (!(s->eqcr.pi & half_mask)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); + return 0; +} + +static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + return qbman_swp_enqueue_ring_mode_ptr(s, d, fd); +} + int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, const struct qbman_fd *fd) { @@ -492,27 +725,27 @@ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, return qbman_swp_enqueue_ring_mode(s, d, fd); } -int qbman_swp_enqueue_multiple(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - uint32_t *flags, - int num_frames) +static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) { - uint32_t *p; + uint32_t *p = NULL; const uint32_t *cl = qb_cl(d); - uint32_t eqcr_ci, eqcr_pi; - uint8_t diff; + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; int i, num_enqueued = 0; uint64_t addr_cena; + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; if (!s->eqcr.available) { eqcr_ci = s->eqcr.ci; s->eqcr.ci = qbman_cena_read_reg(&s->sys, - QBMAN_CENA_SWP_EQCR_CI) & 0xF; - diff = qm_cyc_diff(QBMAN_EQCR_SIZE, - eqcr_ci, s->eqcr.ci); - s->eqcr.available += diff; - if (!diff) + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) return 0; } @@ -523,11 +756,10 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, /* Fill in the EQCR ring */ for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], &fd[i], sizeof(*fd)); eqcr_pi++; - eqcr_pi &= 0xF; } lwsync(); @@ -536,7 +768,7 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); p[0] = cl[0] | s->eqcr.pi_vb; if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; @@ -545,8 +777,7 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); } eqcr_pi++; - eqcr_pi &= 0xF; - if (!(eqcr_pi & 7)) + if (!(eqcr_pi & half_mask)) s->eqcr.pi_vb ^= QB_VALID_BIT; } @@ -554,35 +785,104 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; addr_cena = (size_t)s->sys.addr_cena; for (i = 0; i < num_enqueued; i++) { - dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + dcbf((uintptr_t)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask))); eqcr_pi++; - eqcr_pi &= 0xF; } - s->eqcr.pi = eqcr_pi; + s->eqcr.pi = eqcr_pi & full_mask; return num_enqueued; } -int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - int num_frames) +static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) +{ + uint32_t *p = NULL; + const uint32_t *cl = qb_cl(d); + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; + int i, num_enqueued = 0; + + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cinh_read(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) + return 0; + } + + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + } + + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + p[0] = cl[0] | s->eqcr.pi_vb; + if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { + struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; + + d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | + ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); + } + eqcr_pi++; + if (!(eqcr_pi & half_mask)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + } + s->eqcr.pi = eqcr_pi & full_mask; + + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); + return num_enqueued; +} + +inline int qbman_swp_enqueue_multiple(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) +{ + return qbman_swp_enqueue_multiple_ptr(s, d, fd, flags, num_frames); +} + +static int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) { uint32_t *p; const uint32_t *cl; - uint32_t eqcr_ci, eqcr_pi; - uint8_t diff; + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; int i, num_enqueued = 0; uint64_t addr_cena; + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; if (!s->eqcr.available) { eqcr_ci = s->eqcr.ci; s->eqcr.ci = qbman_cena_read_reg(&s->sys, - QBMAN_CENA_SWP_EQCR_CI) & 0xF; - diff = qm_cyc_diff(QBMAN_EQCR_SIZE, - eqcr_ci, s->eqcr.ci); - s->eqcr.available += diff; - if (!diff) + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) return 0; } @@ -593,12 +893,11 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, /* Fill in the EQCR ring */ for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); cl = qb_cl(&d[i]); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], &fd[i], sizeof(*fd)); eqcr_pi++; - eqcr_pi &= 0xF; } lwsync(); @@ -607,12 +906,11 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); cl = qb_cl(&d[i]); p[0] = cl[0] | s->eqcr.pi_vb; eqcr_pi++; - eqcr_pi &= 0xF; - if (!(eqcr_pi & 7)) + if (!(eqcr_pi & half_mask)) s->eqcr.pi_vb ^= QB_VALID_BIT; } @@ -620,14 +918,78 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; addr_cena = (size_t)s->sys.addr_cena; for (i = 0; i < num_enqueued; i++) { - dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + dcbf((uintptr_t)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask))); + eqcr_pi++; + } + s->eqcr.pi = eqcr_pi & full_mask; + + return num_enqueued; +} + +static int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) +{ + uint32_t *p; + const uint32_t *cl; + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; + int i, num_enqueued = 0; + + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cinh_read(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) + return 0; + } + + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + cl = qb_cl(&d[i]); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + } + + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + cl = qb_cl(&d[i]); + p[0] = cl[0] | s->eqcr.pi_vb; eqcr_pi++; - eqcr_pi &= 0xF; + if (!(eqcr_pi & half_mask)) + s->eqcr.pi_vb ^= QB_VALID_BIT; } - s->eqcr.pi = eqcr_pi; + + s->eqcr.pi = eqcr_pi & full_mask; + + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); return num_enqueued; } +inline int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) +{ + return qbman_swp_enqueue_multiple_desc_ptr(s, d, fd, num_frames); +} /*************************/ /* Static (push) dequeue */ @@ -670,6 +1032,7 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable) #define QB_VDQCR_VERB_DT_SHIFT 2 #define QB_VDQCR_VERB_RLS_SHIFT 4 #define QB_VDQCR_VERB_WAE_SHIFT 5 +#define QB_VDQCR_VERB_RAD_SHIFT 6 enum qb_pull_dt_e { qb_pull_dt_channel, @@ -702,7 +1065,8 @@ void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, d->pull.rsp_addr = storage_phys; } -void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, uint8_t numframes) +void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, + uint8_t numframes) { d->pull.numf = numframes - 1; } @@ -735,7 +1099,20 @@ void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, d->pull.dq_src = chid; } -int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) +void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad) +{ + if (d->pull.verb & (1 << QB_VDQCR_VERB_RLS_SHIFT)) { + if (rad) + d->pull.verb |= 1 << QB_VDQCR_VERB_RAD_SHIFT; + else + d->pull.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT); + } else { + printf("The RAD feature is not valid when RLS = 0\n"); + } +} + +static int qbman_swp_pull_direct(struct qbman_swp *s, + struct qbman_pull_desc *d) { uint32_t *p; uint32_t *cl = qb_cl(d); @@ -759,6 +1136,36 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) return 0; } +static int qbman_swp_pull_mem_back(struct qbman_swp *s, + struct qbman_pull_desc *d) +{ + uint32_t *p; + uint32_t *cl = qb_cl(d); + + if (!atomic_dec_and_test(&s->vdq.busy)) { + atomic_inc(&s->vdq.busy); + return -EBUSY; + } + + d->pull.tok = s->sys.idx + 1; + s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt; + p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR_MEM); + memcpy(&p[1], &cl[1], 12); + + /* Set the verb byte, have to substitute in the valid-bit */ + p[0] = cl[0] | s->vdq.valid_bit; + s->vdq.valid_bit ^= QB_VALID_BIT; + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE); + + return 0; +} + +inline int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) +{ + return qbman_swp_pull_ptr(s, d); +} + /****************/ /* Polling DQRR */ /****************/ @@ -791,7 +1198,12 @@ void qbman_swp_prefetch_dqrr_next(struct qbman_swp *s) * only once, so repeated calls can return a sequence of DQRR entries, without * requiring they be consumed immediately or in any particular order. */ -const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) +inline const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) +{ + return qbman_swp_dqrr_next_ptr(s); +} + +const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s) { uint32_t verb; uint32_t response_verb; @@ -801,7 +1213,7 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) /* Before using valid-bit to detect if something is there, we have to * handle the case of the DQRR reset bug... */ - if (unlikely(s->dqrr.reset_bug)) { + if (s->dqrr.reset_bug) { /* We pick up new entries by cache-inhibited producer index, * which means that a non-coherent mapping would require us to * invalidate and read *only* once that PI has indicated that @@ -833,7 +1245,8 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); } p = qbman_cena_read_wo_shadow(&s->sys, - QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + verb = p->dq.verb; /* If the valid-bit isn't of the expected polarity, nothing there. Note, @@ -867,11 +1280,54 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) return p; } +const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s) +{ + uint32_t verb; + uint32_t response_verb; + uint32_t flags; + const struct qbman_result *p; + + p = qbman_cena_read_wo_shadow(&s->sys, + QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx)); + + verb = p->dq.verb; + + /* If the valid-bit isn't of the expected polarity, nothing there. Note, + * in the DQRR reset bug workaround, we shouldn't need to skip these + * check, because we've already determined that a new entry is available + * and we've invalidated the cacheline before reading it, so the + * valid-bit behaviour is repaired and should tell us what we already + * knew from reading PI. + */ + if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) + return NULL; + + /* There's something there. Move "next_idx" attention to the next ring + * entry (and prefetch it) before returning what we found. + */ + s->dqrr.next_idx++; + if (s->dqrr.next_idx == s->dqrr.dqrr_size) { + s->dqrr.next_idx = 0; + s->dqrr.valid_bit ^= QB_VALID_BIT; + } + /* If this is the final response to a volatile dequeue command + * indicate that the vdq is no longer busy + */ + flags = p->dq.stat; + response_verb = verb & QBMAN_RESPONSE_VERB_MASK; + if ((response_verb == QBMAN_RESULT_DQ) && + (flags & QBMAN_DQ_STAT_VOLATILE) && + (flags & QBMAN_DQ_STAT_EXPIRED)) + atomic_inc(&s->vdq.busy); + return p; +} + /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct qbman_result *dq) { - qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); + qbman_cinh_write(&s->sys, + QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); } /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ @@ -884,6 +1340,7 @@ void qbman_swp_dqrr_idx_consume(struct qbman_swp *s, /*********************************/ /* Polling user-provided storage */ /*********************************/ + int qbman_result_has_new_result(struct qbman_swp *s, struct qbman_result *dq) { @@ -898,11 +1355,11 @@ int qbman_result_has_new_result(struct qbman_swp *s, ((struct qbman_result *)dq)->dq.tok = 0; /* - * VDQCR "no longer busy" hook - not quite the same as DQRR, because the - * fact "VDQCR" shows busy doesn't mean that we hold the result that - * makes it available. Eg. we may be looking at our 10th dequeue result, - * having released VDQCR after the 1st result and it is now busy due to - * some other command! + * VDQCR "no longer busy" hook - not quite the same as DQRR, because + * the fact "VDQCR" shows busy doesn't mean that we hold the result + * that makes it available. Eg. we may be looking at our 10th dequeue + * result, having released VDQCR after the 1st result and it is now + * busy due to some other command! */ if (s->vdq.storage == dq) { s->vdq.storage = NULL; @@ -936,11 +1393,11 @@ int qbman_check_command_complete(struct qbman_result *dq) s = portal_idx_map[dq->dq.tok - 1]; /* - * VDQCR "no longer busy" hook - not quite the same as DQRR, because the - * fact "VDQCR" shows busy doesn't mean that we hold the result that - * makes it available. Eg. we may be looking at our 10th dequeue result, - * having released VDQCR after the 1st result and it is now busy due to - * some other command! + * VDQCR "no longer busy" hook - not quite the same as DQRR, because + * the fact "VDQCR" shows busy doesn't mean that we hold the result + * that makes it available. Eg. we may be looking at our 10th dequeue + * result, having released VDQCR after the 1st result and it is now + * busy due to some other command! */ if (s->vdq.storage == dq) { s->vdq.storage = NULL; @@ -1142,8 +1599,10 @@ void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable) #define RAR_VB(rar) ((rar) & 0x80) #define RAR_SUCCESS(rar) ((rar) & 0x100) -int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, - const uint64_t *buffers, unsigned int num_buffers) +static int qbman_swp_release_direct(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, + unsigned int num_buffers) { uint32_t *p; const uint32_t *cl = qb_cl(d); @@ -1157,22 +1616,63 @@ int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, /* Start the release command */ p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); /* Copy the caller's buffer pointers to the command */ u64_to_le32_copy(&p[2], buffers, num_buffers); - /* Set the verb byte, have to substitute in the valid-bit and the number - * of buffers. + /* Set the verb byte, have to substitute in the valid-bit and the + * number of buffers. */ lwsync(); p[0] = cl[0] | RAR_VB(rar) | num_buffers; qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); return 0; } +static int qbman_swp_release_mem_back(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, + unsigned int num_buffers) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR); + + pr_debug("RAR=%08x\n", rar); + if (!RAR_SUCCESS(rar)) + return -EBUSY; + + QBMAN_BUG_ON(!num_buffers || (num_buffers > 7)); + + /* Start the release command */ + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar))); + + /* Copy the caller's buffer pointers to the command */ + u64_to_le32_copy(&p[2], buffers, num_buffers); + + /* Set the verb byte, have to substitute in the valid-bit and the + * number of buffers. + */ + p[0] = cl[0] | RAR_VB(rar) | num_buffers; + lwsync(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_RCR_AM_RT + + RAR_IDX(rar) * 4, QMAN_RT_MODE); + + return 0; +} + +inline int qbman_swp_release(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, + unsigned int num_buffers) +{ + return qbman_swp_release_ptr(s, d, buffers, num_buffers); +} + /*******************/ /* Buffer acquires */ /*******************/ @@ -1214,7 +1714,7 @@ int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, /* Complete the management command */ r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE); - if (unlikely(!r)) { + if (!r) { pr_err("qbman: acquire from BPID %d failed, no response\n", bpid); return -EIO; @@ -1224,7 +1724,7 @@ int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE); /* Determine success or failure */ - if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + if (r->rslt != QBMAN_MC_RSLT_OK) { pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n", bpid, r->rslt); return -EIO; @@ -1271,7 +1771,7 @@ static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid, /* Complete the management command */ r = qbman_swp_mc_complete(s, p, alt_fq_verb); - if (unlikely(!r)) { + if (!r) { pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n", alt_fq_verb); return -EIO; @@ -1281,7 +1781,7 @@ static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid, QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb); /* Determine success or failure */ - if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + if (r->rslt != QBMAN_MC_RSLT_OK) { pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n", fqid, alt_fq_verb, r->rslt); return -EIO; @@ -1362,7 +1862,7 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, /* Complete the management command */ r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE); - if (unlikely(!r)) { + if (!r) { pr_err("qbman: wqchan config failed, no response\n"); return -EIO; } @@ -1372,7 +1872,7 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, != QBMAN_WQCHAN_CONFIGURE); /* Determine success or failure */ - if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + if (r->rslt != QBMAN_MC_RSLT_OK) { pr_err("CDAN cQID %d failed: code = 0x%02x\n", channelid, r->rslt); return -EIO; diff --git a/drivers/bus/fslmc/qbman/qbman_portal.h b/drivers/bus/fslmc/qbman/qbman_portal.h index dbea22a1..3b0fc540 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.h +++ b/drivers/bus/fslmc/qbman/qbman_portal.h @@ -1,12 +1,17 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * Copyright 2018 NXP * */ +#ifndef _QBMAN_PORTAL_H_ +#define _QBMAN_PORTAL_H_ + #include "qbman_sys.h" #include +uint32_t qman_version; #define QMAN_REV_4000 0x04000000 #define QMAN_REV_4100 0x04010000 #define QMAN_REV_4101 0x04010001 @@ -14,13 +19,14 @@ /* All QBMan command and result structures use this "valid bit" encoding */ #define QB_VALID_BIT ((uint32_t)0x80) +/* All QBMan command use this "Read trigger bit" encoding */ +#define QB_RT_BIT ((uint32_t)0x100) + /* Management command result codes */ #define QBMAN_MC_RSLT_OK 0xf0 /* QBMan DQRR size is set at runtime in qbman_portal.c */ -#define QBMAN_EQCR_SIZE 8 - static inline uint8_t qm_cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last) { @@ -51,6 +57,10 @@ struct qbman_swp { #endif uint32_t valid_bit; /* 0x00 or 0x80 */ } mc; + /* Management response */ + struct { + uint32_t valid_bit; /* 0x00 or 0x80 */ + } mr; /* Push dequeues */ uint32_t sdq; /* Volatile dequeues */ @@ -87,6 +97,8 @@ struct qbman_swp { struct { uint32_t pi; uint32_t pi_vb; + uint32_t pi_ring_size; + uint32_t pi_mask; uint32_t ci; int available; } eqcr; @@ -141,4 +153,16 @@ static inline void *qbman_swp_mc_complete(struct qbman_swp *swp, void *cmd, * an inline) is necessary to work with different descriptor types and to work * correctly with const and non-const inputs (and similarly-qualified outputs). */ -#define qb_cl(d) (&(d)->donot_manipulate_directly[0]) +#define qb_cl(d) (&(d)->dont_manipulate_directly[0]) + +#ifdef RTE_ARCH_ARM64 + #define clean(p) \ + { asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); } + #define invalidate(p) \ + { asm volatile("dc ivac, %0" : : "r"(p) : "memory"); } +#else + #define clean(p) + #define invalidate(p) +#endif + +#endif diff --git a/drivers/bus/fslmc/qbman/qbman_sys.h b/drivers/bus/fslmc/qbman/qbman_sys.h index 2bd33ea5..d41af835 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys.h +++ b/drivers/bus/fslmc/qbman/qbman_sys.h @@ -18,11 +18,51 @@ * *not* to provide linux compatibility. */ +#ifndef _QBMAN_SYS_H_ +#define _QBMAN_SYS_H_ + #include "qbman_sys_decl.h" #define CENA_WRITE_ENABLE 0 #define CINH_WRITE_ENABLE 1 +/* CINH register offsets */ +#define QBMAN_CINH_SWP_EQCR_PI 0x800 +#define QBMAN_CINH_SWP_EQCR_CI 0x840 +#define QBMAN_CINH_SWP_EQAR 0x8c0 +#define QBMAN_CINH_SWP_CR_RT 0x900 +#define QBMAN_CINH_SWP_VDQCR_RT 0x940 +#define QBMAN_CINH_SWP_EQCR_AM_RT 0x980 +#define QBMAN_CINH_SWP_RCR_AM_RT 0x9c0 +#define QBMAN_CINH_SWP_DQPI 0xa00 +#define QBMAN_CINH_SWP_DQRR_ITR 0xa80 +#define QBMAN_CINH_SWP_DCAP 0xac0 +#define QBMAN_CINH_SWP_SDQCR 0xb00 +#define QBMAN_CINH_SWP_EQCR_AM_RT2 0xb40 +#define QBMAN_CINH_SWP_RCR_PI 0xc00 +#define QBMAN_CINH_SWP_RAR 0xcc0 +#define QBMAN_CINH_SWP_ISR 0xe00 +#define QBMAN_CINH_SWP_IER 0xe40 +#define QBMAN_CINH_SWP_ISDR 0xe80 +#define QBMAN_CINH_SWP_IIR 0xec0 +#define QBMAN_CINH_SWP_ITPR 0xf40 + +/* CENA register offsets */ +#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_CR 0x600 +#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((uint32_t)(vb) >> 1)) +#define QBMAN_CENA_SWP_VDQCR 0x780 +#define QBMAN_CENA_SWP_EQCR_CI 0x840 + +/* CENA register offsets in memory-backed mode */ +#define QBMAN_CENA_SWP_DQRR_MEM(n) (0x800 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_RCR_MEM(n) (0x1400 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_CR_MEM 0x1600 +#define QBMAN_CENA_SWP_RR_MEM 0x1680 +#define QBMAN_CENA_SWP_VDQCR_MEM 0x1780 + /* Debugging assists */ static inline void __hexdump(unsigned long start, unsigned long end, unsigned long p, size_t sz, const unsigned char *c) @@ -125,8 +165,8 @@ struct qbman_swp_sys { * place-holder. */ uint8_t *cena; - uint8_t __iomem *addr_cena; - uint8_t __iomem *addr_cinh; + uint8_t *addr_cena; + uint8_t *addr_cinh; uint32_t idx; enum qbman_eqcr_mode eqcr_mode; }; @@ -292,13 +332,16 @@ static inline void qbman_cena_prefetch(struct qbman_swp_sys *s, * qbman_portal.c. So use of it is declared locally here. */ #define QBMAN_CINH_SWP_CFG 0xd00 -#define QBMAN_CINH_SWP_CFG 0xd00 + #define SWP_CFG_DQRR_MF_SHIFT 20 #define SWP_CFG_EST_SHIFT 16 +#define SWP_CFG_CPBS_SHIFT 15 #define SWP_CFG_WN_SHIFT 14 #define SWP_CFG_RPM_SHIFT 12 #define SWP_CFG_DCM_SHIFT 10 #define SWP_CFG_EPM_SHIFT 8 +#define SWP_CFG_VPM_SHIFT 7 +#define SWP_CFG_CPM_SHIFT 6 #define SWP_CFG_SD_SHIFT 5 #define SWP_CFG_SP_SHIFT 4 #define SWP_CFG_SE_SHIFT 3 @@ -329,11 +372,20 @@ static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn, return reg; } +#define QMAN_RT_MODE 0x00000100 + +#define QMAN_REV_4000 0x04000000 +#define QMAN_REV_4100 0x04010000 +#define QMAN_REV_4101 0x04010001 +#define QMAN_REV_5000 0x05000000 +#define QMAN_REV_MASK 0xffff0000 + static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, const struct qbman_swp_desc *d, uint8_t dqrr_size) { uint32_t reg; + int i; #ifdef RTE_ARCH_64 uint8_t wn = CENA_WRITE_ENABLE; #else @@ -343,7 +395,7 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, s->addr_cena = d->cena_bar; s->addr_cinh = d->cinh_bar; s->idx = (uint32_t)d->idx; - s->cena = malloc(4096); + s->cena = malloc(64*1024); if (!s->cena) { pr_err("Could not allocate page for cena shadow\n"); return -1; @@ -358,12 +410,34 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); QBMAN_BUG_ON(reg); #endif + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) + memset(s->addr_cena, 0, 64*1024); + else { + /* Invalidate the portal memory. + * This ensures no stale cache lines + */ + for (i = 0; i < 0x1000; i += 64) + dccivac(s->addr_cena + i); + } + if (s->eqcr_mode == qman_eqcr_vb_array) - reg = qbman_set_swp_cfg(dqrr_size, wn, 0, 3, 2, 3, 1, 1, 1, 1, - 1, 1); - else - reg = qbman_set_swp_cfg(dqrr_size, wn, 1, 3, 2, 2, 1, 1, 1, 1, - 1, 1); + reg = qbman_set_swp_cfg(dqrr_size, wn, + 0, 3, 2, 3, 1, 1, 1, 1, 1, 1); + else { + if ((d->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) + reg = qbman_set_swp_cfg(dqrr_size, wn, + 1, 3, 2, 2, 1, 1, 1, 1, 1, 1); + else + reg = qbman_set_swp_cfg(dqrr_size, wn, + 1, 3, 2, 0, 1, 1, 1, 1, 1, 1); + } + + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { + reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */ + 1 << SWP_CFG_VPM_SHIFT | /* VDQCR read triggered mode */ + 1 << SWP_CFG_CPM_SHIFT; /* CR read triggered mode */ + } + qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg); reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); if (!reg) { @@ -371,6 +445,12 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, free(s->cena); return -1; } + + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { + qbman_cinh_write(s, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE); + qbman_cinh_write(s, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE); + } + return 0; } @@ -378,3 +458,5 @@ static inline void qbman_swp_sys_finish(struct qbman_swp_sys *s) { free(s->cena); } + +#endif /* _QBMAN_SYS_H_ */ diff --git a/drivers/bus/fslmc/qbman/qbman_sys_decl.h b/drivers/bus/fslmc/qbman/qbman_sys_decl.h index fa6977fe..a29f5b46 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys_decl.h +++ b/drivers/bus/fslmc/qbman/qbman_sys_decl.h @@ -3,6 +3,9 @@ * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. * */ +#ifndef _QBMAN_SYS_DECL_H_ +#define _QBMAN_SYS_DECL_H_ + #include #include @@ -51,3 +54,4 @@ static inline void prefetch_for_store(void *p) RTE_SET_USED(p); } #endif +#endif /* _QBMAN_SYS_DECL_H_ */ diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map index fe45a113..dcc4e082 100644 --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map @@ -114,5 +114,18 @@ DPDK_18.05 { dpdmai_open; dpdmai_set_rx_queue; rte_dpaa2_free_dpci_dev; + rte_dpaa2_memsegs; } DPDK_18.02; + +DPDK_18.11 { + global: + + dpaa2_dqrr_size; + dpaa2_eqcr_size; + dpci_get_link_state; + dpci_get_opr; + dpci_get_peer_attributes; + dpci_set_opr; + +} DPDK_18.05; diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile index 3ff3bdb8..514452b3 100644 --- a/drivers/bus/ifpga/Makefile +++ b/drivers/bus/ifpga/Makefile @@ -19,7 +19,7 @@ LDLIBS += -lrte_kvargs EXPORT_MAP := rte_bus_ifpga_version.map # library version -LIBABIVER := 1 +LIBABIVER := 2 SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_bus.c SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_common.c diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c index b324872e..5f23ed8b 100644 --- a/drivers/bus/ifpga/ifpga_bus.c +++ b/drivers/bus/ifpga/ifpga_bus.c @@ -142,6 +142,7 @@ ifpga_scan_one(struct rte_rawdev *rawdev, if (!afu_dev) goto end; + afu_dev->device.bus = &rte_ifpga_bus; afu_dev->device.devargs = devargs; afu_dev->device.numa_node = SOCKET_ID_ANY; afu_dev->device.name = devargs->name; @@ -279,14 +280,13 @@ ifpga_probe_one_driver(struct rte_afu_driver *drv, /* reference driver structure */ afu_dev->driver = drv; - afu_dev->device.driver = &drv->driver; /* call the driver probe() function */ ret = drv->probe(afu_dev); - if (ret) { + if (ret) afu_dev->driver = NULL; - afu_dev->device.driver = NULL; - } + else + afu_dev->device.driver = &drv->driver; return ret; } @@ -301,8 +301,11 @@ ifpga_probe_all_drivers(struct rte_afu_device *afu_dev) return -1; /* Check if a driver is already loaded */ - if (afu_dev->driver != NULL) - return 0; + if (rte_dev_is_probed(&afu_dev->device)) { + IFPGA_BUS_DEBUG("Device %s is already probed\n", + rte_ifpga_device_name(afu_dev)); + return -EEXIST; + } TAILQ_FOREACH(drv, &ifpga_afu_drv_list, next) { if (ifpga_probe_one_driver(drv, afu_dev)) { @@ -325,14 +328,13 @@ ifpga_probe(void) int ret = 0; TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) { - if (afu_dev->device.driver) - continue; - ret = ifpga_probe_all_drivers(afu_dev); + if (ret == -EEXIST) + continue; if (ret < 0) IFPGA_BUS_ERR("failed to initialize %s device\n", rte_ifpga_device_name(afu_dev)); - } + } return ret; } @@ -347,23 +349,20 @@ static int ifpga_remove_driver(struct rte_afu_device *afu_dev) { const char *name; - const struct rte_afu_driver *driver; name = rte_ifpga_device_name(afu_dev); - if (!afu_dev->device.driver) { + if (afu_dev->driver == NULL) { IFPGA_BUS_DEBUG("no driver attach to device %s\n", name); return 1; } - driver = RTE_DRV_TO_AFU_CONST(afu_dev->device.driver); - return driver->remove(afu_dev); + return afu_dev->driver->remove(afu_dev); } static int ifpga_unplug(struct rte_device *dev) { struct rte_afu_device *afu_dev = NULL; - struct rte_devargs *devargs = NULL; int ret; if (dev == NULL) @@ -373,15 +372,13 @@ ifpga_unplug(struct rte_device *dev) if (!afu_dev) return -ENOENT; - devargs = dev->devargs; - ret = ifpga_remove_driver(afu_dev); if (ret) return ret; TAILQ_REMOVE(&ifpga_afu_dev_list, afu_dev, next); - rte_devargs_remove(devargs->bus->name, devargs->name); + rte_devargs_remove(dev->devargs); free(afu_dev); return 0; diff --git a/drivers/bus/ifpga/meson.build b/drivers/bus/ifpga/meson.build index c9b08c86..0b5c38d5 100644 --- a/drivers/bus/ifpga/meson.build +++ b/drivers/bus/ifpga/meson.build @@ -1,6 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2010-2018 Intel Corporation +version = 2 + deps += ['pci', 'kvargs', 'rawdev'] install_headers('rte_bus_ifpga.h') sources = files('ifpga_common.c', 'ifpga_bus.c') diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h index 51d5ae0d..d53c0f48 100644 --- a/drivers/bus/ifpga/rte_bus_ifpga.h +++ b/drivers/bus/ifpga/rte_bus_ifpga.h @@ -83,9 +83,6 @@ struct rte_afu_device { #define RTE_DEV_TO_AFU(ptr) \ container_of(ptr, struct rte_afu_device, device) -#define RTE_DRV_TO_AFU_CONST(ptr) \ - container_of(ptr, const struct rte_afu_driver, driver) - /** * Initialization function for the driver called during FPGA BUS probing. */ diff --git a/drivers/bus/pci/Makefile b/drivers/bus/pci/Makefile index cf373068..f33e0120 100644 --- a/drivers/bus/pci/Makefile +++ b/drivers/bus/pci/Makefile @@ -4,7 +4,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_bus_pci.a -LIBABIVER := 1 +LIBABIVER := 2 EXPORT_MAP := rte_bus_pci_version.map CFLAGS := -I$(SRCDIR) $(CFLAGS) @@ -26,10 +26,11 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal CFLAGS += -DALLOW_EXPERIMENTAL_API LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring -LDLIBS += -lrte_ethdev -lrte_pci +LDLIBS += -lrte_ethdev -lrte_pci -lrte_kvargs include $(RTE_SDK)/drivers/bus/pci/$(SYSTEM)/Makefile SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) := $(addprefix $(SYSTEM)/,$(SRCS)) +SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_params.c SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_common.c SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_common_uio.c diff --git a/drivers/bus/pci/bsd/pci.c b/drivers/bus/pci/bsd/pci.c index 655b34b7..d09f8ee5 100644 --- a/drivers/bus/pci/bsd/pci.c +++ b/drivers/bus/pci/bsd/pci.c @@ -223,6 +223,8 @@ pci_scan_one(int dev_pci_fd, struct pci_conf *conf) } memset(dev, 0, sizeof(*dev)); + dev->device.bus = &rte_pci_bus.bus; + dev->addr.domain = conf->pc_sel.pc_domain; dev->addr.bus = conf->pc_sel.pc_bus; dev->addr.devid = conf->pc_sel.pc_dev; @@ -439,6 +441,8 @@ int rte_pci_read_config(const struct rte_pci_device *dev, { int fd = -1; int size; + /* Copy Linux implementation's behaviour */ + const int return_len = len; struct pci_io pi = { .pi_sel = { .pc_domain = dev->addr.domain, @@ -469,7 +473,7 @@ int rte_pci_read_config(const struct rte_pci_device *dev, } close(fd); - return 0; + return return_len; error: if (fd >= 0) diff --git a/drivers/bus/pci/linux/Makefile b/drivers/bus/pci/linux/Makefile index 96ea1d54..90404468 100644 --- a/drivers/bus/pci/linux/Makefile +++ b/drivers/bus/pci/linux/Makefile @@ -4,5 +4,3 @@ SRCS += pci.c SRCS += pci_uio.c SRCS += pci_vfio.c - -CFLAGS += -D_GNU_SOURCE diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index 04648ac9..45c24ef7 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -119,7 +119,7 @@ rte_pci_unmap_device(struct rte_pci_device *dev) static int find_max_end_va(const struct rte_memseg_list *msl, void *arg) { - size_t sz = msl->memseg_arr.len * msl->page_sz; + size_t sz = msl->len; void *end_va = RTE_PTR_ADD(msl->base_va, sz); void **max_va = arg; @@ -228,6 +228,7 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr) return -1; memset(dev, 0, sizeof(*dev)); + dev->device.bus = &rte_pci_bus.bus; dev->addr = *addr; /* get vendor id */ @@ -588,10 +589,8 @@ pci_one_device_iommu_support_va(struct rte_pci_device *dev) fclose(fp); mgaw = ((vtd_cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1; - if (mgaw < X86_VA_WIDTH) - return false; - return true; + return rte_eal_check_dma_mask(mgaw) == 0 ? true : false; } #elif defined(RTE_ARCH_PPC_64) static bool @@ -620,8 +619,11 @@ pci_devices_iommu_support_va(void) FOREACH_DEVICE_ON_PCIBUS(dev) { if (!rte_pci_match(drv, dev)) continue; - if (!pci_one_device_iommu_support_va(dev)) - return false; + /* + * just one PCI device needs to be checked out because + * the IOMMU hardware is the same for all of them. + */ + return pci_one_device_iommu_support_va(dev); } } return true; @@ -672,23 +674,21 @@ rte_pci_get_iommu_class(void) int rte_pci_read_config(const struct rte_pci_device *device, void *buf, size_t len, off_t offset) { + char devname[RTE_DEV_NAME_MAX_LEN] = ""; const struct rte_intr_handle *intr_handle = &device->intr_handle; - switch (intr_handle->type) { - case RTE_INTR_HANDLE_UIO: - case RTE_INTR_HANDLE_UIO_INTX: + switch (device->kdrv) { + case RTE_KDRV_IGB_UIO: return pci_uio_read_config(intr_handle, buf, len, offset); - #ifdef VFIO_PRESENT - case RTE_INTR_HANDLE_VFIO_MSIX: - case RTE_INTR_HANDLE_VFIO_MSI: - case RTE_INTR_HANDLE_VFIO_LEGACY: + case RTE_KDRV_VFIO: return pci_vfio_read_config(intr_handle, buf, len, offset); #endif default: + rte_pci_device_name(&device->addr, devname, + RTE_DEV_NAME_MAX_LEN); RTE_LOG(ERR, EAL, - "Unknown handle type of fd %d\n", - intr_handle->fd); + "Unknown driver type for %s\n", devname); return -1; } } @@ -697,23 +697,21 @@ int rte_pci_read_config(const struct rte_pci_device *device, int rte_pci_write_config(const struct rte_pci_device *device, const void *buf, size_t len, off_t offset) { + char devname[RTE_DEV_NAME_MAX_LEN] = ""; const struct rte_intr_handle *intr_handle = &device->intr_handle; - switch (intr_handle->type) { - case RTE_INTR_HANDLE_UIO: - case RTE_INTR_HANDLE_UIO_INTX: + switch (device->kdrv) { + case RTE_KDRV_IGB_UIO: return pci_uio_write_config(intr_handle, buf, len, offset); - #ifdef VFIO_PRESENT - case RTE_INTR_HANDLE_VFIO_MSIX: - case RTE_INTR_HANDLE_VFIO_MSI: - case RTE_INTR_HANDLE_VFIO_LEGACY: + case RTE_KDRV_VFIO: return pci_vfio_write_config(intr_handle, buf, len, offset); #endif default: + rte_pci_device_name(&device->addr, devname, + RTE_DEV_NAME_MAX_LEN); RTE_LOG(ERR, EAL, - "Unknown handle type of fd %d\n", - intr_handle->fd); + "Unknown driver type for %s\n", devname); return -1; } } diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c index 686386d6..305cc060 100644 --- a/drivers/bus/pci/linux/pci_vfio.c +++ b/drivers/bus/pci/linux/pci_vfio.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include "eal_filesystem.h" @@ -35,7 +37,9 @@ #ifdef VFIO_PRESENT +#ifndef PAGE_SIZE #define PAGE_SIZE (sysconf(_SC_PAGESIZE)) +#endif #define PAGE_MASK (~(PAGE_SIZE - 1)) static struct rte_tailq_elem rte_vfio_tailq = { @@ -277,6 +281,114 @@ pci_vfio_setup_interrupts(struct rte_pci_device *dev, int vfio_dev_fd) return -1; } +#ifdef HAVE_VFIO_DEV_REQ_INTERFACE +static void +pci_vfio_req_handler(void *param) +{ + struct rte_bus *bus; + int ret; + struct rte_device *device = (struct rte_device *)param; + + bus = rte_bus_find_by_device(device); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Cannot find bus for device (%s)\n", + device->name); + return; + } + + /* + * vfio kernel module request user space to release allocated + * resources before device be deleted in kernel, so it can directly + * call the vfio bus hot-unplug handler to process it. + */ + ret = bus->hot_unplug_handler(device); + if (ret) + RTE_LOG(ERR, EAL, + "Can not handle hot-unplug for device (%s)\n", + device->name); +} + +/* enable notifier (only enable req now) */ +static int +pci_vfio_enable_notifier(struct rte_pci_device *dev, int vfio_dev_fd) +{ + int ret; + int fd = -1; + + /* set up an eventfd for req notifier */ + fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot set up eventfd, error %i (%s)\n", + errno, strerror(errno)); + return -1; + } + + dev->vfio_req_intr_handle.fd = fd; + dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_VFIO_REQ; + dev->vfio_req_intr_handle.vfio_dev_fd = vfio_dev_fd; + + ret = rte_intr_callback_register(&dev->vfio_req_intr_handle, + pci_vfio_req_handler, + (void *)&dev->device); + if (ret) { + RTE_LOG(ERR, EAL, "Fail to register req notifier handler.\n"); + goto error; + } + + ret = rte_intr_enable(&dev->vfio_req_intr_handle); + if (ret) { + RTE_LOG(ERR, EAL, "Fail to enable req notifier.\n"); + ret = rte_intr_callback_unregister(&dev->vfio_req_intr_handle, + pci_vfio_req_handler, + (void *)&dev->device); + if (ret < 0) + RTE_LOG(ERR, EAL, + "Fail to unregister req notifier handler.\n"); + goto error; + } + + return 0; +error: + close(fd); + + dev->vfio_req_intr_handle.fd = -1; + dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + dev->vfio_req_intr_handle.vfio_dev_fd = -1; + + return -1; +} + +/* disable notifier (only disable req now) */ +static int +pci_vfio_disable_notifier(struct rte_pci_device *dev) +{ + int ret; + + ret = rte_intr_disable(&dev->vfio_req_intr_handle); + if (ret) { + RTE_LOG(ERR, EAL, "fail to disable req notifier.\n"); + return -1; + } + + ret = rte_intr_callback_unregister(&dev->vfio_req_intr_handle, + pci_vfio_req_handler, + (void *)&dev->device); + if (ret < 0) { + RTE_LOG(ERR, EAL, + "fail to unregister req notifier handler.\n"); + return -1; + } + + close(dev->vfio_req_intr_handle.fd); + + dev->vfio_req_intr_handle.fd = -1; + dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + dev->vfio_req_intr_handle.vfio_dev_fd = -1; + + return 0; +} +#endif + static int pci_vfio_is_ioport_bar(int vfio_dev_fd, int bar_index) { @@ -415,6 +527,93 @@ pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res, return 0; } +/* + * region info may contain capability headers, so we need to keep reallocating + * the memory until we match allocated memory size with argsz. + */ +static int +pci_vfio_get_region_info(int vfio_dev_fd, struct vfio_region_info **info, + int region) +{ + struct vfio_region_info *ri; + size_t argsz = sizeof(*ri); + int ret; + + ri = malloc(sizeof(*ri)); + if (ri == NULL) { + RTE_LOG(ERR, EAL, "Cannot allocate memory for region info\n"); + return -1; + } +again: + memset(ri, 0, argsz); + ri->argsz = argsz; + ri->index = region; + + ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ri); + if (ret < 0) { + free(ri); + return ret; + } + if (ri->argsz != argsz) { + struct vfio_region_info *tmp; + + argsz = ri->argsz; + tmp = realloc(ri, argsz); + + if (tmp == NULL) { + /* realloc failed but the ri is still there */ + free(ri); + RTE_LOG(ERR, EAL, "Cannot reallocate memory for region info\n"); + return -1; + } + ri = tmp; + goto again; + } + *info = ri; + + return 0; +} + +static struct vfio_info_cap_header * +pci_vfio_info_cap(struct vfio_region_info *info, int cap) +{ + struct vfio_info_cap_header *h; + size_t offset; + + if ((info->flags & RTE_VFIO_INFO_FLAG_CAPS) == 0) { + /* VFIO info does not advertise capabilities */ + return NULL; + } + + offset = VFIO_CAP_OFFSET(info); + while (offset != 0) { + h = RTE_PTR_ADD(info, offset); + if (h->id == cap) + return h; + offset = h->next; + } + return NULL; +} + +static int +pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) +{ + struct vfio_region_info *info; + int ret; + + ret = pci_vfio_get_region_info(vfio_dev_fd, &info, msix_region); + if (ret < 0) + return -1; + + ret = pci_vfio_info_cap(info, RTE_VFIO_CAP_MSIX_MAPPABLE) != NULL; + + /* cleanup */ + free(info); + + return ret; +} + + static int pci_vfio_map_resource_primary(struct rte_pci_device *dev) { @@ -430,6 +629,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev) struct pci_map *maps; dev->intr_handle.fd = -1; +#ifdef HAVE_VFIO_DEV_REQ_INTERFACE + dev->vfio_req_intr_handle.fd = -1; +#endif /* store PCI address string */ snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, @@ -464,56 +666,75 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev) if (ret < 0) { RTE_LOG(ERR, EAL, " %s cannot get MSI-X BAR number!\n", pci_addr); - goto err_vfio_dev_fd; + goto err_vfio_res; + } + /* if we found our MSI-X BAR region, check if we can mmap it */ + if (vfio_res->msix_table.bar_index != -1) { + int ret = pci_vfio_msix_is_mappable(vfio_dev_fd, + vfio_res->msix_table.bar_index); + if (ret < 0) { + RTE_LOG(ERR, EAL, "Couldn't check if MSI-X BAR is mappable\n"); + goto err_vfio_res; + } else if (ret != 0) { + /* we can map it, so we don't care where it is */ + RTE_LOG(DEBUG, EAL, "VFIO reports MSI-X BAR as mappable\n"); + vfio_res->msix_table.bar_index = -1; + } } for (i = 0; i < (int) vfio_res->nb_maps; i++) { - struct vfio_region_info reg = { .argsz = sizeof(reg) }; + struct vfio_region_info *reg = NULL; void *bar_addr; - reg.index = i; - - ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®); - if (ret) { + ret = pci_vfio_get_region_info(vfio_dev_fd, ®, i); + if (ret < 0) { RTE_LOG(ERR, EAL, " %s cannot get device region info " - "error %i (%s)\n", pci_addr, errno, strerror(errno)); + "error %i (%s)\n", pci_addr, errno, + strerror(errno)); goto err_vfio_res; } /* chk for io port region */ ret = pci_vfio_is_ioport_bar(vfio_dev_fd, i); - if (ret < 0) + if (ret < 0) { + free(reg); goto err_vfio_res; - else if (ret) { + } else if (ret) { RTE_LOG(INFO, EAL, "Ignore mapping IO port bar(%d)\n", i); + free(reg); continue; } /* skip non-mmapable BARs */ - if ((reg.flags & VFIO_REGION_INFO_FLAG_MMAP) == 0) + if ((reg->flags & VFIO_REGION_INFO_FLAG_MMAP) == 0) { + free(reg); continue; + } /* try mapping somewhere close to the end of hugepages */ if (pci_map_addr == NULL) pci_map_addr = pci_find_max_end_va(); bar_addr = pci_map_addr; - pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg.size); + pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg->size); maps[i].addr = bar_addr; - maps[i].offset = reg.offset; - maps[i].size = reg.size; + maps[i].offset = reg->offset; + maps[i].size = reg->size; maps[i].path = NULL; /* vfio doesn't have per-resource paths */ ret = pci_vfio_mmap_bar(vfio_dev_fd, vfio_res, i, 0); if (ret < 0) { RTE_LOG(ERR, EAL, " %s mapping BAR%i failed: %s\n", pci_addr, i, strerror(errno)); + free(reg); goto err_vfio_res; } dev->mem_resource[i].addr = maps[i].addr; + + free(reg); } if (pci_rte_vfio_setup_device(dev, vfio_dev_fd) < 0) { @@ -521,6 +742,13 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev) goto err_vfio_res; } +#ifdef HAVE_VFIO_DEV_REQ_INTERFACE + if (pci_vfio_enable_notifier(dev, vfio_dev_fd) != 0) { + RTE_LOG(ERR, EAL, "Error setting up notifier!\n"); + goto err_vfio_res; + } + +#endif TAILQ_INSERT_TAIL(vfio_res_list, vfio_res, next); return 0; @@ -546,6 +774,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev) struct pci_map *maps; dev->intr_handle.fd = -1; +#ifdef HAVE_VFIO_DEV_REQ_INTERFACE + dev->vfio_req_intr_handle.fd = -1; +#endif /* store PCI address string */ snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, @@ -586,6 +817,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev) /* we need save vfio_dev_fd, so it can be used during release */ dev->intr_handle.vfio_dev_fd = vfio_dev_fd; +#ifdef HAVE_VFIO_DEV_REQ_INTERFACE + dev->vfio_req_intr_handle.vfio_dev_fd = vfio_dev_fd; +#endif return 0; err_vfio_dev_fd: @@ -658,6 +892,14 @@ pci_vfio_unmap_resource_primary(struct rte_pci_device *dev) snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, loc->domain, loc->bus, loc->devid, loc->function); +#ifdef HAVE_VFIO_DEV_REQ_INTERFACE + ret = pci_vfio_disable_notifier(dev); + if (ret) { + RTE_LOG(ERR, EAL, "fail to disable req notifier.\n"); + return -1; + } + +#endif if (close(dev->intr_handle.fd) < 0) { RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n", pci_addr); diff --git a/drivers/bus/pci/meson.build b/drivers/bus/pci/meson.build index 72939e59..a3140ff9 100644 --- a/drivers/bus/pci/meson.build +++ b/drivers/bus/pci/meson.build @@ -1,15 +1,18 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2017 Intel Corporation +version = 2 + deps += ['pci'] install_headers('rte_bus_pci.h') -sources = files('pci_common.c', 'pci_common_uio.c') +sources = files('pci_common.c', + 'pci_common_uio.c', + 'pci_params.c') if host_machine.system() == 'linux' sources += files('linux/pci.c', 'linux/pci_uio.c', 'linux/pci_vfio.c') includes += include_directories('linux') - cflags += ['-D_GNU_SOURCE'] else sources += files('bsd/pci.c') includes += include_directories('bsd') @@ -17,3 +20,5 @@ endif # memseg walk is not part of stable API yet allow_experimental_apis = true + +deps += ['kvargs'] diff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c index 7736b3f9..6276e5d6 100644 --- a/drivers/bus/pci/pci_common.c +++ b/drivers/bus/pci/pci_common.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -23,12 +24,11 @@ #include #include #include +#include #include "private.h" -extern struct rte_pci_bus rte_pci_bus; - #define SYSFS_PCI_DEVICES "/sys/bus/pci/devices" const char *rte_pci_get_sysfs_path(void) @@ -123,6 +123,7 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, struct rte_pci_device *dev) { int ret; + bool already_probed; struct rte_pci_addr *loc; if ((dr == NULL) || (dev == NULL)) @@ -153,6 +154,13 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, dev->device.numa_node = 0; } + already_probed = rte_dev_is_probed(&dev->device); + if (already_probed && !(dr->drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { + RTE_LOG(DEBUG, EAL, "Device %s is already probed\n", + dev->device.name); + return -EEXIST; + } + RTE_LOG(INFO, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id, dev->id.device_id, dr->driver.name); @@ -161,24 +169,24 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, * This needs to be before rte_pci_map_device(), as it enables to use * driver flags for adjusting configuration. */ - dev->driver = dr; - dev->device.driver = &dr->driver; + if (!already_probed) + dev->driver = dr; - if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) { + if (!already_probed && (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING)) { /* map resources for devices that use igb_uio */ ret = rte_pci_map_device(dev); if (ret != 0) { dev->driver = NULL; - dev->device.driver = NULL; return ret; } } /* call the driver probe() function */ ret = dr->probe(dr, dev); + if (already_probed) + return ret; /* no rollback if already succeeded earlier */ if (ret) { dev->driver = NULL; - dev->device.driver = NULL; if ((dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) && /* Don't unmap if device is unsupported and * driver needs mapped resources. @@ -186,6 +194,8 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, !(ret > 0 && (dr->drv_flags & RTE_PCI_DRV_KEEP_MAPPED_RES))) rte_pci_unmap_device(dev); + } else { + dev->device.driver = &dr->driver; } return ret; @@ -233,7 +243,7 @@ rte_pci_detach_dev(struct rte_pci_device *dev) /* * If vendor/device ID match, call the probe() function of all - * registered driver for the given device. Return -1 if initialization + * registered driver for the given device. Return < 0 if initialization * failed, return 1 if no driver is found for this device. */ static int @@ -243,17 +253,13 @@ pci_probe_all_drivers(struct rte_pci_device *dev) int rc = 0; if (dev == NULL) - return -1; - - /* Check if a driver is already loaded */ - if (dev->driver != NULL) - return 0; + return -EINVAL; FOREACH_DRIVER_ON_PCIBUS(dr) { rc = rte_pci_probe_one_driver(dr, dev); if (rc < 0) /* negative value is an error */ - return -1; + return rc; if (rc > 0) /* positive value means driver doesn't support it */ continue; @@ -290,11 +296,14 @@ rte_pci_probe(void) devargs->policy == RTE_DEV_WHITELISTED) ret = pci_probe_all_drivers(dev); if (ret < 0) { - RTE_LOG(ERR, EAL, "Requested device " PCI_PRI_FMT - " cannot be used\n", dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - rte_errno = errno; - failed++; + if (ret != -EEXIST) { + RTE_LOG(ERR, EAL, "Requested device " + PCI_PRI_FMT " cannot be used\n", + dev->addr.domain, dev->addr.bus, + dev->addr.devid, dev->addr.function); + rte_errno = errno; + failed++; + } ret = 0; } } @@ -405,6 +414,98 @@ pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, return NULL; } +/* + * find the device which encounter the failure, by iterate over all device on + * PCI bus to check if the memory failure address is located in the range + * of the BARs of the device. + */ +static struct rte_pci_device * +pci_find_device_by_addr(const void *failure_addr) +{ + struct rte_pci_device *pdev = NULL; + uint64_t check_point, start, end, len; + int i; + + check_point = (uint64_t)(uintptr_t)failure_addr; + + FOREACH_DEVICE_ON_PCIBUS(pdev) { + for (i = 0; i != RTE_DIM(pdev->mem_resource); i++) { + start = (uint64_t)(uintptr_t)pdev->mem_resource[i].addr; + len = pdev->mem_resource[i].len; + end = start + len; + if (check_point >= start && check_point < end) { + RTE_LOG(DEBUG, EAL, "Failure address %16.16" + PRIx64" belongs to device %s!\n", + check_point, pdev->device.name); + return pdev; + } + } + } + return NULL; +} + +static int +pci_hot_unplug_handler(struct rte_device *dev) +{ + struct rte_pci_device *pdev = NULL; + int ret = 0; + + pdev = RTE_DEV_TO_PCI(dev); + if (!pdev) + return -1; + + switch (pdev->kdrv) { +#ifdef HAVE_VFIO_DEV_REQ_INTERFACE + case RTE_KDRV_VFIO: + /* + * vfio kernel module guaranty the pci device would not be + * deleted until the user space release the resource, so no + * need to remap BARs resource here, just directly notify + * the req event to the user space to handle it. + */ + rte_dev_event_callback_process(dev->name, + RTE_DEV_EVENT_REMOVE); + break; +#endif + case RTE_KDRV_IGB_UIO: + case RTE_KDRV_UIO_GENERIC: + case RTE_KDRV_NIC_UIO: + /* BARs resource is invalid, remap it to be safe. */ + ret = pci_uio_remap_resource(pdev); + break; + default: + RTE_LOG(DEBUG, EAL, + "Not managed by a supported kernel driver, skipped\n"); + ret = -1; + break; + } + + return ret; +} + +static int +pci_sigbus_handler(const void *failure_addr) +{ + struct rte_pci_device *pdev = NULL; + int ret = 0; + + pdev = pci_find_device_by_addr(failure_addr); + if (!pdev) { + /* It is a generic sigbus error, no bus would handle it. */ + ret = 1; + } else { + /* The sigbus error is caused of hot-unplug. */ + ret = pci_hot_unplug_handler(&pdev->device); + if (ret) { + RTE_LOG(ERR, EAL, + "Failed to handle hot-unplug for device %s", + pdev->name); + ret = -1; + } + } + return ret; +} + static int pci_plug(struct rte_device *dev) { @@ -421,6 +522,7 @@ pci_unplug(struct rte_device *dev) ret = rte_pci_detach_dev(pdev); if (ret == 0) { rte_pci_remove_device(pdev); + rte_devargs_remove(dev->devargs); free(pdev); } return ret; @@ -435,6 +537,9 @@ struct rte_pci_bus rte_pci_bus = { .unplug = pci_unplug, .parse = pci_parse, .get_iommu_class = rte_pci_get_iommu_class, + .dev_iterate = rte_pci_dev_iterate, + .hot_unplug_handler = pci_hot_unplug_handler, + .sigbus_handler = pci_sigbus_handler, }, .device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list), .driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list), diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c index 54bc20b5..7ea73dbc 100644 --- a/drivers/bus/pci/pci_common_uio.c +++ b/drivers/bus/pci/pci_common_uio.c @@ -146,6 +146,39 @@ pci_uio_unmap(struct mapped_pci_resource *uio_res) } } +/* remap the PCI resource of a PCI device in anonymous virtual memory */ +int +pci_uio_remap_resource(struct rte_pci_device *dev) +{ + int i; + void *map_address; + + if (dev == NULL) + return -1; + + /* Remap all BARs */ + for (i = 0; i != PCI_MAX_RESOURCE; i++) { + /* skip empty BAR */ + if (dev->mem_resource[i].phys_addr == 0) + continue; + map_address = mmap(dev->mem_resource[i].addr, + (size_t)dev->mem_resource[i].len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (map_address == MAP_FAILED) { + RTE_LOG(ERR, EAL, + "Cannot remap resource for device %s\n", + dev->name); + return -1; + } + RTE_LOG(INFO, EAL, + "Successful remap resource for device %s\n", + dev->name); + } + + return 0; +} + static struct mapped_pci_resource * pci_uio_find_resource(struct rte_pci_device *dev) { diff --git a/drivers/bus/pci/pci_params.c b/drivers/bus/pci/pci_params.c new file mode 100644 index 00000000..3192e9c9 --- /dev/null +++ b/drivers/bus/pci/pci_params.c @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 Gaëtan Rivet + */ + +#include +#include +#include +#include +#include +#include + +#include "private.h" + +enum pci_params { + RTE_PCI_PARAM_ADDR, + RTE_PCI_PARAM_MAX, +}; + +static const char * const pci_params_keys[] = { + [RTE_PCI_PARAM_ADDR] = "addr", + [RTE_PCI_PARAM_MAX] = NULL, +}; + +static int +pci_addr_kv_cmp(const char *key __rte_unused, + const char *value, + void *_addr2) +{ + struct rte_pci_addr _addr1; + struct rte_pci_addr *addr1 = &_addr1; + struct rte_pci_addr *addr2 = _addr2; + + if (rte_pci_addr_parse(value, addr1)) + return -1; + return -abs(rte_pci_addr_cmp(addr1, addr2)); +} + +static int +pci_dev_match(const struct rte_device *dev, + const void *_kvlist) +{ + const struct rte_kvargs *kvlist = _kvlist; + const struct rte_pci_device *pdev; + + if (kvlist == NULL) + /* Empty string matches everything. */ + return 0; + pdev = RTE_DEV_TO_PCI_CONST(dev); + /* if any field does not match. */ + if (rte_kvargs_process(kvlist, pci_params_keys[RTE_PCI_PARAM_ADDR], + &pci_addr_kv_cmp, + (void *)(intptr_t)&pdev->addr)) + return 1; + return 0; +} + +void * +rte_pci_dev_iterate(const void *start, + const char *str, + const struct rte_dev_iterator *it __rte_unused) +{ + rte_bus_find_device_t find_device; + struct rte_kvargs *kvargs = NULL; + struct rte_device *dev; + + if (str != NULL) { + kvargs = rte_kvargs_parse(str, pci_params_keys); + if (kvargs == NULL) { + RTE_LOG(ERR, EAL, "cannot parse argument list\n"); + rte_errno = EINVAL; + return NULL; + } + } + find_device = rte_pci_bus.bus.find_device; + dev = find_device(start, pci_dev_match, kvargs); + rte_kvargs_free(kvargs); + return dev; +} diff --git a/drivers/bus/pci/private.h b/drivers/bus/pci/private.h index 8ddd03e1..13c3324b 100644 --- a/drivers/bus/pci/private.h +++ b/drivers/bus/pci/private.h @@ -10,9 +10,13 @@ #include #include +extern struct rte_pci_bus rte_pci_bus; + struct rte_pci_driver; struct rte_pci_device; +extern struct rte_pci_bus rte_pci_bus; + /** * Probe the PCI bus * @@ -122,6 +126,18 @@ int pci_uio_alloc_resource(struct rte_pci_device *dev, void pci_uio_free_resource(struct rte_pci_device *dev, struct mapped_pci_resource *uio_res); +/** + * Remap the PCI resource of a PCI device in anonymous virtual memory. + * + * @param dev + * Point to the struct rte pci device. + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int +pci_uio_remap_resource(struct rte_pci_device *dev); + /** * Map device memory to uio resource * @@ -166,4 +182,27 @@ rte_pci_match(const struct rte_pci_driver *pci_drv, enum rte_iova_mode rte_pci_get_iommu_class(void); +/* + * Iterate over internal devices, + * matching any device against the provided + * string. + * + * @param start + * Iteration starting point. + * + * @param str + * Device string to match against. + * + * @param it + * (unused) iterator structure. + * + * @return + * A pointer to the next matching device if any. + * NULL otherwise. + */ +void * +rte_pci_dev_iterate(const void *start, + const char *str, + const struct rte_dev_iterator *it); + #endif /* _PCI_PRIVATE_H_ */ diff --git a/drivers/bus/pci/rte_bus_pci.h b/drivers/bus/pci/rte_bus_pci.h index 0d1955ff..f0d6d81c 100644 --- a/drivers/bus/pci/rte_bus_pci.h +++ b/drivers/bus/pci/rte_bus_pci.h @@ -62,10 +62,12 @@ struct rte_pci_device { struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE]; /**< PCI Memory Resource */ struct rte_intr_handle intr_handle; /**< Interrupt handle */ - struct rte_pci_driver *driver; /**< Associated driver */ + struct rte_pci_driver *driver; /**< PCI driver used in probing */ uint16_t max_vfs; /**< sriov enable if not zero */ enum rte_kernel_driver kdrv; /**< Kernel driver passthrough */ char name[PCI_PRI_STR_SIZE+1]; /**< PCI location (ASCII) */ + struct rte_intr_handle vfio_req_intr_handle; + /**< Handler of VFIO request interrupt */ }; /** @@ -121,7 +123,7 @@ struct rte_pci_driver { pci_probe_t *probe; /**< Device Probe function. */ pci_remove_t *remove; /**< Device Remove function. */ const struct rte_pci_id *id_table; /**< ID table, NULL terminated. */ - uint32_t drv_flags; /**< Flags contolling handling of device. */ + uint32_t drv_flags; /**< Flags RTE_PCI_DRV_*. */ }; /** @@ -137,6 +139,8 @@ struct rte_pci_bus { #define RTE_PCI_DRV_NEED_MAPPING 0x0001 /** Device needs PCI BAR mapping with enabled write combining (wc) */ #define RTE_PCI_DRV_WC_ACTIVATE 0x0002 +/** Device already probed can be probed again to check for new ports. */ +#define RTE_PCI_DRV_PROBE_AGAIN 0x0004 /** Device driver supports link state interrupt */ #define RTE_PCI_DRV_INTR_LSC 0x0008 /** Device driver supports device removal interrupt */ @@ -219,6 +223,8 @@ void rte_pci_unregister(struct rte_pci_driver *driver); * The length of the data buffer. * @param offset * The offset into PCI config space + * @return + * Number of bytes read on success, negative on error. */ int rte_pci_read_config(const struct rte_pci_device *device, void *buf, size_t len, off_t offset); diff --git a/drivers/bus/vdev/Makefile b/drivers/bus/vdev/Makefile index bd0bb895..803b8ea7 100644 --- a/drivers/bus/vdev/Makefile +++ b/drivers/bus/vdev/Makefile @@ -16,11 +16,12 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API EXPORT_MAP := rte_bus_vdev_version.map # library version -LIBABIVER := 1 +LIBABIVER := 2 SRCS-y += vdev.c +SRCS-y += vdev_params.c -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_kvargs # # Export include files diff --git a/drivers/bus/vdev/meson.build b/drivers/bus/vdev/meson.build index 2ee648b4..803785f1 100644 --- a/drivers/bus/vdev/meson.build +++ b/drivers/bus/vdev/meson.build @@ -1,7 +1,12 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2017 Intel Corporation -sources = files('vdev.c') +version = 2 + +sources = files('vdev.c', + 'vdev_params.c') install_headers('rte_bus_vdev.h') allow_experimental_apis = true + +deps += ['kvargs'] diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c index 6139dd55..9c66bdc7 100644 --- a/drivers/bus/vdev/vdev.c +++ b/drivers/bus/vdev/vdev.c @@ -23,6 +23,7 @@ #include "rte_bus_vdev.h" #include "vdev_logs.h" +#include "vdev_private.h" #define VDEV_MP_KEY "bus_vdev_mp" @@ -40,7 +41,7 @@ static struct vdev_device_list vdev_device_list = static rte_spinlock_recursive_t vdev_device_list_lock = RTE_SPINLOCK_RECURSIVE_INITIALIZER; -struct vdev_driver_list vdev_driver_list = +static struct vdev_driver_list vdev_driver_list = TAILQ_HEAD_INITIALIZER(vdev_driver_list); struct vdev_custom_scan { @@ -149,10 +150,9 @@ vdev_probe_all_drivers(struct rte_vdev_device *dev) if (vdev_parse(name, &driver)) return -1; - dev->device.driver = &driver->driver; ret = driver->probe(dev); - if (ret) - dev->device.driver = NULL; + if (ret == 0) + dev->device.driver = &driver->driver; return ret; } @@ -202,7 +202,9 @@ alloc_devargs(const char *name, const char *args) } static int -insert_vdev(const char *name, const char *args, struct rte_vdev_device **p_dev) +insert_vdev(const char *name, const char *args, + struct rte_vdev_device **p_dev, + bool init) { struct rte_vdev_device *dev; struct rte_devargs *devargs; @@ -221,17 +223,24 @@ insert_vdev(const char *name, const char *args, struct rte_vdev_device **p_dev) goto fail; } + dev->device.bus = &rte_vdev_bus; dev->device.devargs = devargs; dev->device.numa_node = SOCKET_ID_ANY; dev->device.name = devargs->name; if (find_vdev(name)) { + /* + * A vdev is expected to have only one port. + * So there is no reason to try probing again, + * even with new arguments. + */ ret = -EEXIST; goto fail; } TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); - rte_devargs_insert(devargs); + if (init) + rte_devargs_insert(devargs); if (p_dev) *p_dev = dev; @@ -248,20 +257,18 @@ int rte_vdev_init(const char *name, const char *args) { struct rte_vdev_device *dev; - struct rte_devargs *devargs; int ret; rte_spinlock_recursive_lock(&vdev_device_list_lock); - ret = insert_vdev(name, args, &dev); + ret = insert_vdev(name, args, &dev, true); if (ret == 0) { ret = vdev_probe_all_drivers(dev); if (ret) { if (ret > 0) VDEV_LOG(ERR, "no driver found for %s", name); /* If fails, remove it from vdev list */ - devargs = dev->device.devargs; TAILQ_REMOVE(&vdev_device_list, dev, next); - rte_devargs_remove(devargs->bus->name, devargs->name); + rte_devargs_remove(dev->device.devargs); free(dev); } } @@ -289,7 +296,6 @@ int rte_vdev_uninit(const char *name) { struct rte_vdev_device *dev; - struct rte_devargs *devargs; int ret; if (name == NULL) @@ -308,8 +314,7 @@ rte_vdev_uninit(const char *name) goto unlock; TAILQ_REMOVE(&vdev_device_list, dev, next); - devargs = dev->device.devargs; - rte_devargs_remove(devargs->bus->name, devargs->name); + rte_devargs_remove(dev->device.devargs); free(dev); unlock: @@ -346,6 +351,7 @@ vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; const char *devname; int num; + int ret; strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); mp_resp.len_param = sizeof(*ou); @@ -380,7 +386,10 @@ vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) break; case VDEV_SCAN_ONE: VDEV_LOG(INFO, "receive vdev, %s", in->name); - if (insert_vdev(in->name, NULL, NULL) < 0) + ret = insert_vdev(in->name, NULL, NULL, false); + if (ret == -EEXIST) + VDEV_LOG(DEBUG, "device already exist, %s", in->name); + else if (ret < 0) VDEV_LOG(ERR, "failed to add vdev, %s", in->name); break; default: @@ -419,6 +428,7 @@ vdev_scan(void) mp_rep = &mp_reply.msgs[0]; resp = (struct vdev_param *)mp_rep->param; VDEV_LOG(INFO, "Received %d vdevs", resp->num); + free(mp_reply.msgs); } else VDEV_LOG(ERR, "Failed to request vdev from primary"); @@ -455,6 +465,7 @@ vdev_scan(void) continue; } + dev->device.bus = &rte_vdev_bus; dev->device.devargs = devargs; dev->device.numa_node = SOCKET_ID_ANY; dev->device.name = devargs->name; @@ -480,7 +491,7 @@ vdev_probe(void) * we call each driver probe. */ - if (dev->device.driver) + if (rte_dev_is_probed(&dev->device)) continue; if (vdev_probe_all_drivers(dev)) { @@ -493,9 +504,9 @@ vdev_probe(void) return ret; } -static struct rte_device * -vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, - const void *data) +struct rte_device * +rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) { const struct rte_vdev_device *vstart; struct rte_vdev_device *dev; @@ -532,10 +543,11 @@ vdev_unplug(struct rte_device *dev) static struct rte_bus rte_vdev_bus = { .scan = vdev_scan, .probe = vdev_probe, - .find_device = vdev_find_device, + .find_device = rte_vdev_find_device, .plug = vdev_plug, .unplug = vdev_unplug, .parse = vdev_parse, + .dev_iterate = rte_vdev_dev_iterate, }; RTE_REGISTER_BUS(vdev, rte_vdev_bus); diff --git a/drivers/bus/vdev/vdev_params.c b/drivers/bus/vdev/vdev_params.c new file mode 100644 index 00000000..6f74704d --- /dev/null +++ b/drivers/bus/vdev/vdev_params.c @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 Gaëtan Rivet + */ + +#include + +#include +#include +#include +#include + +#include "vdev_logs.h" +#include "vdev_private.h" + +enum vdev_params { + RTE_VDEV_PARAM_NAME, + RTE_VDEV_PARAM_MAX, +}; + +static const char * const vdev_params_keys[] = { + [RTE_VDEV_PARAM_NAME] = "name", + [RTE_VDEV_PARAM_MAX] = NULL, +}; + +static int +vdev_dev_match(const struct rte_device *dev, + const void *_kvlist) +{ + int ret; + const struct rte_kvargs *kvlist = _kvlist; + char *name; + + /* cannot pass const dev->name to rte_kvargs_process() */ + name = strdup(dev->name); + if (name == NULL) + return -1; + ret = rte_kvargs_process(kvlist, + vdev_params_keys[RTE_VDEV_PARAM_NAME], + rte_kvargs_strcmp, name); + free(name); + if (ret != 0) + return -1; + + return 0; +} + +void * +rte_vdev_dev_iterate(const void *start, + const char *str, + const struct rte_dev_iterator *it __rte_unused) +{ + struct rte_kvargs *kvargs = NULL; + struct rte_device *dev; + + if (str != NULL) { + kvargs = rte_kvargs_parse(str, vdev_params_keys); + if (kvargs == NULL) { + VDEV_LOG(ERR, "cannot parse argument list\n"); + rte_errno = EINVAL; + return NULL; + } + } + dev = rte_vdev_find_device(start, vdev_dev_match, kvargs); + rte_kvargs_free(kvargs); + return dev; +} diff --git a/drivers/bus/vdev/vdev_private.h b/drivers/bus/vdev/vdev_private.h new file mode 100644 index 00000000..ba6dc48f --- /dev/null +++ b/drivers/bus/vdev/vdev_private.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 Gaëtan Rivet + */ + +#ifndef _VDEV_PRIVATE_H_ +#define _VDEV_PRIVATE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct rte_device * +rte_vdev_find_device(const struct rte_device *start, + rte_dev_cmp_t cmp, + const void *data); + +void * +rte_vdev_dev_iterate(const void *start, + const char *str, + const struct rte_dev_iterator *it); + +#ifdef __cplusplus +} +#endif + +#endif /* _VDEV_PRIVATE_H_ */ diff --git a/drivers/bus/vmbus/Makefile b/drivers/bus/vmbus/Makefile index deee9dd1..e54c557c 100644 --- a/drivers/bus/vmbus/Makefile +++ b/drivers/bus/vmbus/Makefile @@ -3,7 +3,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_bus_vmbus.a -LIBABIVER := 1 +LIBABIVER := 2 EXPORT_MAP := rte_bus_vmbus_version.map CFLAGS += -I$(SRCDIR) diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c index 52d6a3c0..a4755a38 100644 --- a/drivers/bus/vmbus/linux/vmbus_bus.c +++ b/drivers/bus/vmbus/linux/vmbus_bus.c @@ -229,6 +229,7 @@ vmbus_scan_one(const char *name) if (dev == NULL) return -1; + dev->device.bus = &rte_vmbus_bus.bus; dev->device.name = strdup(name); if (!dev->device.name) goto error; @@ -276,6 +277,8 @@ vmbus_scan_one(const char *name) dev->device.numa_node = SOCKET_ID_ANY; } + dev->device.devargs = vmbus_devargs_lookup(dev); + /* device is valid, add in list (sorted) */ VMBUS_LOG(DEBUG, "Adding vmbus device %s", name); diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index 18daabec..0e4d058e 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -1,5 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause +version = 2 + allow_experimental_apis = true install_headers('rte_bus_vmbus.h','rte_vmbus_reg.h') diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h index 9964fc42..211127dd 100644 --- a/drivers/bus/vmbus/private.h +++ b/drivers/bus/vmbus/private.h @@ -10,11 +10,14 @@ #include #include #include +#include #ifndef PAGE_SIZE #define PAGE_SIZE 4096 #endif +extern struct rte_vmbus_bus rte_vmbus_bus; + extern int vmbus_logtype_bus; #define VMBUS_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, vmbus_logtype_bus, "%s(): " fmt "\n", \ @@ -66,6 +69,9 @@ struct vmbus_channel { #define VMBUS_MAX_CHANNELS 64 +struct rte_devargs * +vmbus_devargs_lookup(struct rte_vmbus_device *dev); + int vmbus_chan_create(const struct rte_vmbus_device *device, uint16_t relid, uint16_t subid, uint8_t monitor_id, struct vmbus_channel **new_chan); diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h index 4a2c1f6f..2839fef5 100644 --- a/drivers/bus/vmbus/rte_bus_vmbus.h +++ b/drivers/bus/vmbus/rte_bus_vmbus.h @@ -364,6 +364,21 @@ void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read) */ uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan); +/** + * Set the host monitor latency hint + * + * @param dev + * VMBUS device + * @param chan + * Pointer to vmbus_channel structure. + * @param latency + * Approximate wait period between hypervisor examinations of + * the trigger page (in nanoseconds). + */ +void rte_vmbus_set_latency(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + uint32_t latency); + /** * Register a VMBUS driver. * diff --git a/drivers/bus/vmbus/rte_bus_vmbus_version.map b/drivers/bus/vmbus/rte_bus_vmbus_version.map index dabb9203..ae231ad3 100644 --- a/drivers/bus/vmbus/rte_bus_vmbus_version.map +++ b/drivers/bus/vmbus/rte_bus_vmbus_version.map @@ -27,3 +27,10 @@ DPDK_18.08 { local: *; }; + +DPDK_18.11 { + global: + + rte_vmbus_set_latency; + +} DPDK_18.08; diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c index cc5f3e83..bd14c066 100644 --- a/drivers/bus/vmbus/vmbus_channel.c +++ b/drivers/bus/vmbus/vmbus_channel.c @@ -59,6 +59,32 @@ vmbus_set_event(const struct rte_vmbus_device *dev, vmbus_set_monitor(dev, chan->monitor_id); } +/* + * Set the wait between when hypervisor examines the trigger. + */ +void +rte_vmbus_set_latency(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + uint32_t latency) +{ + uint32_t trig_idx = chan->monitor_id / VMBUS_MONTRIG_LEN; + uint32_t trig_offs = chan->monitor_id % VMBUS_MONTRIG_LEN; + + if (latency >= UINT16_MAX * 100) { + VMBUS_LOG(ERR, "invalid latency value %u", latency); + return; + } + + if (trig_idx >= VMBUS_MONTRIGS_MAX) { + VMBUS_LOG(ERR, "invalid monitor trigger %u", + trig_idx); + return; + } + + /* Host value is expressed in 100 nanosecond units */ + dev->monitor_page->lat[trig_idx][trig_offs] = latency / 100; +} + /* * Notify host that there are data pending on our TX bufring. * diff --git a/drivers/bus/vmbus/vmbus_common.c b/drivers/bus/vmbus/vmbus_common.c index c7165ad5..48a219f7 100644 --- a/drivers/bus/vmbus/vmbus_common.c +++ b/drivers/bus/vmbus/vmbus_common.c @@ -85,7 +85,6 @@ vmbus_match(const struct rte_vmbus_driver *dr, return false; } - /* * If device ID match, call the devinit() function of the driver. */ @@ -112,7 +111,6 @@ vmbus_probe_one_driver(struct rte_vmbus_driver *dr, /* reference driver structure */ dev->driver = dr; - dev->device.driver = &dr->driver; if (dev->device.numa_node < 0) { VMBUS_LOG(WARNING, " Invalid NUMA socket, default to 0"); @@ -125,6 +123,8 @@ vmbus_probe_one_driver(struct rte_vmbus_driver *dr, if (ret) { dev->driver = NULL; rte_vmbus_unmap_device(dev); + } else { + dev->device.driver = &dr->driver; } return ret; @@ -143,7 +143,7 @@ vmbus_probe_all_drivers(struct rte_vmbus_device *dev) int rc; /* Check if a driver is already loaded */ - if (dev->driver != NULL) { + if (rte_dev_is_probed(&dev->device)) { VMBUS_LOG(DEBUG, "VMBUS driver already loaded"); return 0; } @@ -204,6 +204,27 @@ vmbus_parse(const char *name, void *addr) return ret; } +/* + * scan for matching device args on command line + * example: + * -w 'vmbus:635a7ae3-091e-4410-ad59-667c4f8c04c3,latency=20' + */ +struct rte_devargs * +vmbus_devargs_lookup(struct rte_vmbus_device *dev) +{ + struct rte_devargs *devargs; + rte_uuid_t addr; + + RTE_EAL_DEVARGS_FOREACH("vmbus", devargs) { + vmbus_parse(devargs->name, &addr); + + if (rte_uuid_compare(dev->device_id, addr) == 0) + return devargs; + } + return NULL; + +} + /* register vmbus driver */ void rte_vmbus_register(struct rte_vmbus_driver *driver) -- cgit 1.2.3-korg