diff options
Diffstat (limited to 'drivers/bus')
76 files changed, 5911 insertions, 905 deletions
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile index 7ef25939..cea3b55e 100644 --- a/drivers/bus/Makefile +++ b/drivers/bus/Makefile @@ -4,8 +4,12 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += dpaa +ifeq ($(CONFIG_RTE_EAL_VFIO),y) DIRS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += fslmc +endif +DIRS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga DIRS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci DIRS-$(CONFIG_RTE_LIBRTE_VDEV_BUS) += vdev +DIRS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/drivers/bus/dpaa/base/fman/fman.c b/drivers/bus/dpaa/base/fman/fman.c index bda62e01..bdb70042 100644 --- a/drivers/bus/dpaa/base/fman/fman.c +++ b/drivers/bus/dpaa/base/fman/fman.c @@ -300,7 +300,7 @@ fman_if_init(const struct device_node *dpa_node) _errno = fman_get_mac_index(regs_addr_host, &__if->__if.mac_idx); if (_errno) { - FMAN_ERR(-EINVAL, "Invalid register address: %lu", + FMAN_ERR(-EINVAL, "Invalid register address: %" PRIx64, regs_addr_host); goto err; } @@ -442,6 +442,7 @@ fman_if_init(const struct device_node *dpa_node) if (!pool_node) { FMAN_ERR(-ENXIO, "%s: bad fsl,bman-buffer-pools\n", dname); + free(bpool); goto err; } pname = pool_node->full_name; @@ -449,6 +450,7 @@ fman_if_init(const struct device_node *dpa_node) prop = of_get_property(pool_node, "fsl,bpid", &proplen); if (!prop) { FMAN_ERR(-EINVAL, "%s: no fsl,bpid\n", pname); + free(bpool); goto err; } assert(proplen == sizeof(*prop)); @@ -502,7 +504,7 @@ fman_if_init(const struct device_node *dpa_node) /* Parsing of the network interface is complete, add it to the list */ DPAA_BUS_LOG(DEBUG, "Found %s, Tx Channel = %x, FMAN = %x," - "Port ID = %x\n", + "Port ID = %x", dname, __if->__if.tx_channel_id, __if->__if.fman_idx, __if->__if.mac_idx); diff --git a/drivers/bus/dpaa/base/fman/fman_hw.c b/drivers/bus/dpaa/base/fman/fman_hw.c index 0148b98e..4ebbc3d3 100644 --- a/drivers/bus/dpaa/base/fman/fman_hw.c +++ b/drivers/bus/dpaa/base/fman/fman_hw.c @@ -16,6 +16,9 @@ #include <fsl_fman_crc64.h> #include <fsl_bman.h> +#define FMAN_SP_SG_DISABLE 0x80000000 +#define FMAN_SP_EXT_BUF_MARG_START_SHIFT 16 + /* Instantiate the global variable that the inline CRC64 implementation (in * <fsl_fman.h>) depends on. */ @@ -422,20 +425,16 @@ fman_if_set_fc_quanta(struct fman_if *fm_if, u16 pause_quanta) int fman_if_get_fdoff(struct fman_if *fm_if) { - u32 fmbm_ricp; + u32 fmbm_rebm; int fdoff; - int iceof_mask = 0x001f0000; - int icsz_mask = 0x0000001f; struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); assert(fman_ccsr_map_fd != -1); - fmbm_ricp = - in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ricp); - /*iceof + icsz*/ - fdoff = ((fmbm_ricp & iceof_mask) >> 16) * 16 + - (fmbm_ricp & icsz_mask) * 16; + fmbm_rebm = in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm); + + fdoff = (fmbm_rebm >> FMAN_SP_EXT_BUF_MARG_START_SHIFT) & 0x1ff; return fdoff; } @@ -502,12 +501,16 @@ fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset) { struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); unsigned int *fmbm_rebm; + int val = 0; + int fmbm_mask = 0x01ff0000; + + val = fd_offset << FMAN_SP_EXT_BUF_MARG_START_SHIFT; assert(fman_ccsr_map_fd != -1); fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm; - out_be32(fmbm_rebm, in_be32(fmbm_rebm) | (fd_offset << 16)); + out_be32(fmbm_rebm, (in_be32(fmbm_rebm) & ~fmbm_mask) | val); } void @@ -536,6 +539,47 @@ fman_if_get_maxfrm(struct fman_if *fm_if) return (in_be32(reg_maxfrm) | 0x0000FFFF); } +/* MSB in fmbm_rebm register + * 0 - If BMI cannot store the frame in a single buffer it may select a buffer + * of smaller size and store the frame in scatter gather (S/G) buffers + * 1 - Scatter gather format is not enabled for frame storage. If BMI cannot + * store the frame in a single buffer, the frame is discarded. + */ + +int +fman_if_get_sg_enable(struct fman_if *fm_if) +{ + u32 fmbm_rebm; + + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + fmbm_rebm = in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm); + + return (fmbm_rebm & FMAN_SP_SG_DISABLE) ? 0 : 1; +} + +void +fman_if_set_sg(struct fman_if *fm_if, int enable) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *fmbm_rebm; + int val; + int fmbm_mask = FMAN_SP_SG_DISABLE; + + if (enable) + val = 0; + else + val = FMAN_SP_SG_DISABLE; + + assert(fman_ccsr_map_fd != -1); + + fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm; + + out_be32(fmbm_rebm, (in_be32(fmbm_rebm) & ~fmbm_mask) | val); +} + void fman_if_set_dnia(struct fman_if *fm_if, uint32_t nia) { diff --git a/drivers/bus/dpaa/base/fman/netcfg_layer.c b/drivers/bus/dpaa/base/fman/netcfg_layer.c index 3e956ce1..031c6f1a 100644 --- a/drivers/bus/dpaa/base/fman/netcfg_layer.c +++ b/drivers/bus/dpaa/base/fman/netcfg_layer.c @@ -18,11 +18,6 @@ #include <rte_dpaa_logs.h> #include <netcfg.h> -/* Structure contains information about all the interfaces given by user - * on command line. - */ -struct netcfg_interface *netcfg_interface; - /* This data structure contaings all configurations information * related to usages of DPA devices. */ diff --git a/drivers/bus/dpaa/base/fman/of.c b/drivers/bus/dpaa/base/fman/of.c index 1b2dbe26..a7f3174e 100644 --- a/drivers/bus/dpaa/base/fman/of.c +++ b/drivers/bus/dpaa/base/fman/of.c @@ -182,6 +182,11 @@ linear_dir(struct dt_dir *d) DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s", d->node.node.full_name); d->lphandle = f; + } else if (!strcmp(f->node.node.name, "phandle")) { + if (d->lphandle) + DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s", + d->node.node.full_name); + d->lphandle = f; } else if (!strcmp(f->node.node.name, "#address-cells")) { if (d->a_cells) DPAA_BUS_LOG(DEBUG, "Duplicate a_cells in %s", @@ -541,3 +546,42 @@ of_device_is_compatible(const struct device_node *dev_node, return true; return false; } + +static const void *of_get_mac_addr(const struct device_node *np, + const char *name) +{ + return of_get_property(np, name, NULL); +} + +/** + * Search the device tree for the best MAC address to use. 'mac-address' is + * checked first, because that is supposed to contain to "most recent" MAC + * address. If that isn't set, then 'local-mac-address' is checked next, + * because that is the default address. If that isn't set, then the obsolete + * 'address' is checked, just in case we're using an old device tree. + * + * Note that the 'address' property is supposed to contain a virtual address of + * the register set, but some DTS files have redefined that property to be the + * MAC address. + * + * All-zero MAC addresses are rejected, because those could be properties that + * exist in the device tree, but were not set by U-Boot. For example, the + * DTS could define 'mac-address' and 'local-mac-address', with zero MAC + * addresses. Some older U-Boots only initialized 'local-mac-address'. In + * this case, the real MAC is in 'local-mac-address', and 'mac-address' exists + * but is all zeros. + */ +const void *of_get_mac_address(const struct device_node *np) +{ + const void *addr; + + addr = of_get_mac_addr(np, "mac-address"); + if (addr) + return addr; + + addr = of_get_mac_addr(np, "local-mac-address"); + if (addr) + return addr; + + return of_get_mac_addr(np, "address"); +} diff --git a/drivers/bus/dpaa/base/qbman/bman_driver.c b/drivers/bus/dpaa/base/qbman/bman_driver.c index a1ef3921..b14b5905 100644 --- a/drivers/bus/dpaa/base/qbman/bman_driver.c +++ b/drivers/bus/dpaa/base/qbman/bman_driver.c @@ -15,9 +15,9 @@ /* * Global variables of the max portal/pool number this bman version supported */ -u16 bman_ip_rev; +static u16 bman_ip_rev; u16 bman_pool_max; -void *bman_ccsr_map; +static void *bman_ccsr_map; /*****************/ /* Portal driver */ @@ -161,7 +161,7 @@ int bman_init_ccsr(const struct device_node *node) PROT_WRITE, MAP_SHARED, ccsr_map_fd, phys_addr); if (bman_ccsr_map == MAP_FAILED) { pr_err("Can not map BMan CCSR base Bman: " - "0x%x Phys: 0x%lx size 0x%lx", + "0x%x Phys: 0x%" PRIx64 " size 0x%" PRIu64, *bman_addr, phys_addr, regs_size); return -EINVAL; } diff --git a/drivers/bus/dpaa/base/qbman/qman.c b/drivers/bus/dpaa/base/qbman/qman.c index 2b97671b..7c17027f 100644 --- a/drivers/bus/dpaa/base/qbman/qman.c +++ b/drivers/bus/dpaa/base/qbman/qman.c @@ -314,9 +314,9 @@ loop: if (!msg) return 0; } - if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { + if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { /* We aren't draining anything but FQRNIs */ - pr_err("Found verb 0x%x in MR\n", msg->verb); + pr_err("Found verb 0x%x in MR\n", msg->ern.verb); return -1; } qm_mr_next(p); @@ -483,7 +483,7 @@ static inline void qm_mr_pvb_update(struct qm_portal *portal) /* when accessing 'verb', use __raw_readb() to ensure that compiler * inlining doesn't try to optimise out "excess reads". */ - if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) { + if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) { mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); if (!mr->pi) mr->vbit ^= QM_MR_VERB_VBIT; @@ -625,7 +625,7 @@ fail_eqcr: #define MAX_GLOBAL_PORTALS 8 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS]; -rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS]; +static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS]; static struct qman_portal * qman_alloc_global_portal(void) @@ -832,7 +832,7 @@ mr_loop: goto mr_done; swapped_msg = *msg; hw_fd_to_cpu(&swapped_msg.ern.fd); - verb = msg->verb & QM_MR_VERB_TYPE_MASK; + verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK; /* The message is a software ERN iff the 0x20 bit is set */ if (verb & 0x20) { switch (verb) { @@ -1058,7 +1058,7 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit, struct qm_portal *portal = &p->p; register struct qm_dqrr *dqrr = &portal->dqrr; struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE]; - struct qman_fq *fq[QM_DQRR_SIZE]; + struct qman_fq *fq; unsigned int limit = 0, rx_number = 0; uint32_t consume = 0; @@ -1087,18 +1087,18 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit, shadow[rx_number]->fd.opaque = be32_to_cpu(dq[rx_number]->fd.opaque); #else - shadow = dq; + shadow[rx_number] = dq[rx_number]; #endif /* SDQCR: context_b points to the FQ */ #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP - fq[rx_number] = qman_fq_lookup_table[be32_to_cpu( - dq[rx_number]->contextB)]; + fq = qman_fq_lookup_table[be32_to_cpu(dq[rx_number]->contextB)]; #else - fq[rx_number] = (void *)(uintptr_t)be32_to_cpu(dq->contextB); + fq = (void *)be32_to_cpu(dq[rx_number]->contextB); #endif - fq[rx_number]->cb.dqrr_prepare(shadow[rx_number], - &bufs[rx_number]); + if (fq->cb.dqrr_prepare) + fq->cb.dqrr_prepare(shadow[rx_number], + &bufs[rx_number]); consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number]))); rx_number++; @@ -1106,7 +1106,7 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit, } while (++limit < poll_limit); if (rx_number) - fq[0]->cb.dqrr_dpdk_pull_cb(fq, shadow, bufs, rx_number); + fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number); /* Consume all the DQRR enries together */ qm_out(DQRR_DCAP, (1 << 8) | consume); @@ -1665,7 +1665,7 @@ int qman_retire_fq(struct qman_fq *fq, u32 *flags) */ struct qm_mr_entry msg; - msg.verb = QM_MR_VERB_FQRNI; + msg.ern.verb = QM_MR_VERB_FQRNI; msg.fq.fqs = mcr->alterfq.fqs; msg.fq.fqid = fq->fqid; #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP @@ -2002,13 +2002,13 @@ int qman_query_congestion(struct qm_mcr_querycongestion *congestion) return 0; } -int qman_set_vdq(struct qman_fq *fq, u16 num) +int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags) { struct qman_portal *p = get_affine_portal(); uint32_t vdqcr; int ret = -EBUSY; - vdqcr = QM_VDQCR_EXACT; + vdqcr = vdqcr_flags; vdqcr |= QM_VDQCR_NUMFRAMES_SET(num); if ((fq->state != qman_fq_state_parked) && @@ -2642,7 +2642,7 @@ int qman_shutdown_fq(u32 fqid) qm_mr_pvb_update(low_p); msg = qm_mr_current(low_p); while (msg) { - if ((msg->verb & + if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) == QM_MR_VERB_FQRN) found_fqrn = 1; @@ -2710,7 +2710,7 @@ int qman_shutdown_fq(u32 fqid) qm_mr_pvb_update(low_p); msg = qm_mr_current(low_p); while (msg) { - if ((msg->verb & QM_MR_VERB_TYPE_MASK) == + if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) == QM_MR_VERB_FQRL) orl_empty = 1; qm_mr_next(low_p); diff --git a/drivers/bus/dpaa/base/qbman/qman_driver.c b/drivers/bus/dpaa/base/qbman/qman_driver.c index 7cfa8ee4..f6ecd6b2 100644 --- a/drivers/bus/dpaa/base/qbman/qman_driver.c +++ b/drivers/bus/dpaa/base/qbman/qman_driver.c @@ -20,9 +20,9 @@ u16 qm_channel_caam = QMAN_CHANNEL_CAAM; u16 qm_channel_pme = QMAN_CHANNEL_PME; /* Ccsr map address to access ccsrbased register */ -void *qman_ccsr_map; +static void *qman_ccsr_map; /* The qman clock frequency */ -u32 qman_clk; +static u32 qman_clk; static __thread int qmfd = -1; static __thread struct qm_portal_config qpcfg; @@ -160,6 +160,7 @@ struct qman_portal *fsl_qman_portal_create(void) &cpuset); if (ret) { error(0, ret, "pthread_getaffinity_np()"); + kfree(q_pcfg); return NULL; } @@ -168,12 +169,14 @@ struct qman_portal *fsl_qman_portal_create(void) if (CPU_ISSET(loop, &cpuset)) { if (q_pcfg->cpu != -1) { pr_err("Thread is not affine to 1 cpu\n"); + kfree(q_pcfg); return NULL; } q_pcfg->cpu = loop; } if (q_pcfg->cpu == -1) { pr_err("Bug in getaffinity handling!\n"); + kfree(q_pcfg); return NULL; } @@ -183,6 +186,7 @@ struct qman_portal *fsl_qman_portal_create(void) ret = process_portal_map(&q_map); if (ret) { error(0, ret, "process_portal_map()"); + kfree(q_pcfg); return NULL; } q_pcfg->channel = q_map.channel; @@ -217,6 +221,7 @@ err2: close(q_fd); err1: process_portal_unmap(&q_map.addr); + kfree(q_pcfg); return NULL; } @@ -246,7 +251,6 @@ int fsl_qman_portal_destroy(struct qman_portal *qp) int qman_global_init(void) { const struct device_node *dt_node; - int ret = 0; size_t lenp; const u32 *chanid; static int ccsr_map_fd; @@ -352,9 +356,7 @@ int qman_global_init(void) qman_clk = be32_to_cpu(*clk); #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP - ret = qman_setup_fq_lookup_table(CONFIG_FSL_QMAN_FQ_LOOKUP_MAX); - if (ret) - return ret; + return qman_setup_fq_lookup_table(CONFIG_FSL_QMAN_FQ_LOOKUP_MAX); #endif return 0; } diff --git a/drivers/bus/dpaa/base/qbman/qman_priv.h b/drivers/bus/dpaa/base/qbman/qman_priv.h index 9e4471e6..02f6301f 100644 --- a/drivers/bus/dpaa/base/qbman/qman_priv.h +++ b/drivers/bus/dpaa/base/qbman/qman_priv.h @@ -139,7 +139,6 @@ struct qm_portal_config { #define QMAN_REV31 0x0301 #define QMAN_REV32 0x0302 extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */ -extern u32 qman_clk; int qm_set_wpm(int wpm); int qm_get_wpm(int *wpm); diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c index f2bb3b15..16fabd1b 100644 --- a/drivers/bus/dpaa/dpaa_bus.c +++ b/drivers/bus/dpaa/dpaa_bus.c @@ -19,7 +19,6 @@ #include <rte_interrupts.h> #include <rte_log.h> #include <rte_debug.h> -#include <rte_pci.h> #include <rte_atomic.h> #include <rte_branch_prediction.h> #include <rte_memory.h> @@ -51,10 +50,12 @@ struct rte_dpaa_bus rte_dpaa_bus; struct netcfg_info *dpaa_netcfg; /* define a variable to hold the portal_key, once created.*/ -pthread_key_t dpaa_portal_key; +static pthread_key_t dpaa_portal_key; unsigned int dpaa_svr_family; +#define FSL_DPAA_BUS_NAME dpaa_bus + RTE_DEFINE_PER_LCORE(bool, dpaa_io); RTE_DEFINE_PER_LCORE(struct dpaa_portal_dqrr, held_bufs); @@ -130,6 +131,22 @@ dpaa_sec_available(void) static void dpaa_clean_device_list(void); +static struct rte_devargs * +dpaa_devargs_lookup(struct rte_dpaa_device *dev) +{ + struct rte_devargs *devargs; + char dev_name[32]; + + RTE_EAL_DEVARGS_FOREACH("dpaa_bus", devargs) { + devargs->bus->parse(devargs->name, &dev_name); + if (strcmp(dev_name, dev->device.name) == 0) { + DPAA_BUS_INFO("**Devargs matched %s", dev_name); + return devargs; + } + } + return NULL; +} + static int dpaa_create_device_list(void) { @@ -161,8 +178,9 @@ dpaa_create_device_list(void) memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN); sprintf(dev->name, "fm%d-mac%d", (fman_intf->fman_idx + 1), fman_intf->mac_idx); - DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name); + DPAA_BUS_LOG(INFO, "%s netdev added", dev->name); dev->device.name = dev->name; + dev->device.devargs = dpaa_devargs_lookup(dev); dpaa_add_to_device_list(dev); } @@ -198,7 +216,9 @@ dpaa_create_device_list(void) */ memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN); sprintf(dev->name, "dpaa-sec%d", i); - DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name); + DPAA_BUS_LOG(INFO, "%s cryptodev added", dev->name); + dev->device.name = dev->name; + dev->device.devargs = dpaa_devargs_lookup(dev); dpaa_add_to_device_list(dev); } @@ -235,7 +255,7 @@ int rte_dpaa_portal_init(void *arg) BUS_INIT_FUNC_TRACE(); - if ((uint64_t)arg == 1 || cpu == LCORE_ID_ANY) + if ((size_t)arg == 1 || cpu == LCORE_ID_ANY) cpu = rte_get_master_lcore(); /* if the core id is not supported */ else @@ -309,9 +329,15 @@ rte_dpaa_portal_fq_init(void *arg, struct qman_fq *fq) /* Affine above created portal with channel*/ u32 sdqcr; struct qman_portal *qp; + int ret; - if (unlikely(!RTE_PER_LCORE(dpaa_io))) - rte_dpaa_portal_init(arg); + if (unlikely(!RTE_PER_LCORE(dpaa_io))) { + ret = rte_dpaa_portal_init(arg); + if (ret < 0) { + DPAA_BUS_LOG(ERR, "portal initialization failure"); + return ret; + } + } /* Initialise qman specific portals */ qp = fsl_qman_portal_create(); @@ -352,6 +378,51 @@ dpaa_portal_finish(void *arg) RTE_PER_LCORE(dpaa_io) = false; } +static int +rte_dpaa_bus_parse(const char *name, void *out_name) +{ + int i, j; + int max_fman = 2, max_macs = 16; + char *sep = strchr(name, ':'); + + if (strncmp(name, RTE_STR(FSL_DPAA_BUS_NAME), + strlen(RTE_STR(FSL_DPAA_BUS_NAME)))) { + return -EINVAL; + } + + if (!sep) { + DPAA_BUS_ERR("Incorrect device name observed"); + return -EINVAL; + } + + sep = (char *) (sep + 1); + + for (i = 0; i < max_fman; i++) { + for (j = 0; j < max_macs; j++) { + char fm_name[16]; + snprintf(fm_name, 16, "fm%d-mac%d", i, j); + if (strcmp(fm_name, sep) == 0) { + if (out_name) + strcpy(out_name, sep); + return 0; + } + } + } + + for (i = 0; i < RTE_LIBRTE_DPAA_MAX_CRYPTODEV; i++) { + char sec_name[16]; + + snprintf(sec_name, 16, "dpaa-sec%d", i); + if (strcmp(sec_name, sep) == 0) { + if (out_name) + strcpy(out_name, sep); + return 0; + } + } + + return -EINVAL; +} + #define DPAA_DEV_PATH1 "/sys/devices/platform/soc/soc:fsl,dpaa" #define DPAA_DEV_PATH2 "/sys/devices/platform/fsl,dpaa" @@ -390,14 +461,11 @@ rte_dpaa_bus_scan(void) return 0; } - DPAA_BUS_LOG(DEBUG, "Bus: Address of netcfg=%p, Ethports=%d", - dpaa_netcfg, dpaa_netcfg->num_ethports); - #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER dump_netcfg(dpaa_netcfg); #endif - DPAA_BUS_LOG(DEBUG, "Number of devices = %d\n", + DPAA_BUS_LOG(DEBUG, "Number of ethernet devices = %d", dpaa_netcfg->num_ethports); ret = dpaa_create_device_list(); if (ret) { @@ -415,9 +483,6 @@ rte_dpaa_bus_scan(void) return ret; } - DPAA_BUS_LOG(DEBUG, "dpaa_portal_key=%u, ret=%d\n", - (unsigned int)dpaa_portal_key, ret); - return 0; } @@ -453,22 +518,15 @@ static int rte_dpaa_device_match(struct rte_dpaa_driver *drv, struct rte_dpaa_device *dev) { - int ret = -1; - - BUS_INIT_FUNC_TRACE(); - if (!drv || !dev) { DPAA_BUS_DEBUG("Invalid drv or dev received."); - return ret; + return -1; } - if (drv->drv_type == dev->device_type) { - DPAA_BUS_INFO("Device: %s matches for driver: %s", - dev->name, drv->driver.name); - ret = 0; /* Found a match */ - } + if (drv->drv_type == dev->device_type) + return 0; - return ret; + return -1; } static int @@ -479,8 +537,14 @@ rte_dpaa_bus_probe(void) struct rte_dpaa_driver *drv; FILE *svr_file = NULL; unsigned int svr_ver; + int probe_all = rte_dpaa_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST; - BUS_INIT_FUNC_TRACE(); + svr_file = fopen(DPAA_SOC_ID_FILE, "r"); + if (svr_file) { + if (fscanf(svr_file, "svr:%x", &svr_ver) > 0) + dpaa_svr_family = svr_ver & SVR_MASK; + fclose(svr_file); + } /* For each registered driver, and device, call the driver->probe */ TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) { @@ -489,13 +553,19 @@ rte_dpaa_bus_probe(void) if (ret) continue; - if (!drv->probe) + if (!drv->probe || + (dev->device.devargs && + dev->device.devargs->policy == RTE_DEV_BLACKLISTED)) continue; - ret = drv->probe(drv, dev); - if (ret) - DPAA_BUS_ERR("Unable to probe.\n"); - + if (probe_all || + (dev->device.devargs && + dev->device.devargs->policy == + RTE_DEV_WHITELISTED)) { + ret = drv->probe(drv, dev); + if (ret) + DPAA_BUS_ERR("Unable to probe.\n"); + } break; } } @@ -506,13 +576,6 @@ rte_dpaa_bus_probe(void) if (!TAILQ_EMPTY(&rte_dpaa_bus.device_list)) rte_mbuf_set_platform_mempool_ops(DPAA_MEMPOOL_OPS_NAME); - svr_file = fopen(DPAA_SOC_ID_FILE, "r"); - if (svr_file) { - if (fscanf(svr_file, "svr:%x", &svr_ver) > 0) - dpaa_svr_family = svr_ver & SVR_MASK; - fclose(svr_file); - } - return 0; } @@ -552,6 +615,7 @@ struct rte_dpaa_bus rte_dpaa_bus = { .bus = { .scan = rte_dpaa_bus_scan, .probe = rte_dpaa_bus_probe, + .parse = rte_dpaa_bus_parse, .find_device = rte_dpaa_find_device, .get_iommu_class = rte_dpaa_get_iommu_class, }, @@ -562,9 +626,7 @@ struct rte_dpaa_bus rte_dpaa_bus = { RTE_REGISTER_BUS(FSL_DPAA_BUS_NAME, rte_dpaa_bus.bus); -RTE_INIT(dpaa_init_log); -static void -dpaa_init_log(void) +RTE_INIT(dpaa_init_log) { dpaa_logtype_bus = rte_log_register("bus.dpaa"); if (dpaa_logtype_bus >= 0) @@ -574,11 +636,11 @@ dpaa_init_log(void) if (dpaa_logtype_mempool >= 0) rte_log_set_level(dpaa_logtype_mempool, RTE_LOG_NOTICE); - dpaa_logtype_pmd = rte_log_register("pmd.dpaa"); + dpaa_logtype_pmd = rte_log_register("pmd.net.dpaa"); if (dpaa_logtype_pmd >= 0) rte_log_set_level(dpaa_logtype_pmd, RTE_LOG_NOTICE); - dpaa_logtype_eventdev = rte_log_register("eventdev.dpaa"); + dpaa_logtype_eventdev = rte_log_register("pmd.event.dpaa"); if (dpaa_logtype_eventdev >= 0) rte_log_set_level(dpaa_logtype_eventdev, RTE_LOG_NOTICE); } diff --git a/drivers/bus/dpaa/include/compat.h b/drivers/bus/dpaa/include/compat.h index 53707bb4..92241d23 100644 --- a/drivers/bus/dpaa/include/compat.h +++ b/drivers/bus/dpaa/include/compat.h @@ -39,6 +39,7 @@ #include <rte_spinlock.h> #include <rte_common.h> #include <rte_debug.h> +#include <rte_cycles.h> /* The following definitions are primarily to allow the single-source driver * interfaces to be included by arbitrary program code. Ie. for interfaces that @@ -47,9 +48,15 @@ */ /* Required compiler attributes */ +#ifndef __maybe_unused #define __maybe_unused __rte_unused +#endif +#ifndef __always_unused #define __always_unused __rte_unused +#endif +#ifndef __packed #define __packed __rte_packed +#endif #define noinline __attribute__((noinline)) #define L1_CACHE_BYTES 64 @@ -127,13 +134,15 @@ static inline void out_be32(volatile void *__p, u32 val) *p = rte_cpu_to_be_32(val); } +#define hwsync() rte_rmb() +#define lwsync() rte_wmb() + #define dcbt_ro(p) __builtin_prefetch(p, 0) #define dcbt_rw(p) __builtin_prefetch(p, 1) +#if defined(RTE_ARCH_ARM64) #define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); } #define dcbz_64(p) dcbz(p) -#define hwsync() rte_rmb() -#define lwsync() rte_wmb() #define dcbf(p) { asm volatile("dc cvac, %0" : : "r"(p) : "memory"); } #define dcbf_64(p) dcbf(p) #define dccivac(p) { asm volatile("dc civac, %0" : : "r"(p) : "memory"); } @@ -144,9 +153,27 @@ static inline void out_be32(volatile void *__p, u32 val) asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); \ } while (0) +#elif defined(RTE_ARCH_ARM) +#define dcbz(p) memset((p), 0, 32) +#define dcbz_64(p) memset((p), 0, 64) +#define dcbf(p) RTE_SET_USED(p) +#define dcbf_64(p) dcbf(p) +#define dccivac(p) RTE_SET_USED(p) +#define dcbit_ro(p) RTE_SET_USED(p) + +#else +#define dcbz(p) RTE_SET_USED(p) +#define dcbz_64(p) dcbz(p) +#define dcbf(p) RTE_SET_USED(p) +#define dcbf_64(p) dcbf(p) +#define dccivac(p) RTE_SET_USED(p) +#define dcbit_ro(p) RTE_SET_USED(p) +#endif + #define barrier() { asm volatile ("" : : : "memory"); } #define cpu_relax barrier +#if defined(RTE_ARCH_ARM64) static inline uint64_t mfatb(void) { uint64_t ret, ret_new, timeout = 200; @@ -160,6 +187,11 @@ static inline uint64_t mfatb(void) DPAA_BUG_ON(!timeout && (ret != ret_new)); return ret * 64; } +#else + +#define mfatb rte_rdtsc + +#endif /* Spin for a few cycles without bothering the bus */ static inline void cpu_spin(int cycles) diff --git a/drivers/bus/dpaa/include/fsl_fman.h b/drivers/bus/dpaa/include/fsl_fman.h index c0ef1bff..1d1ce867 100644 --- a/drivers/bus/dpaa/include/fsl_fman.h +++ b/drivers/bus/dpaa/include/fsl_fman.h @@ -108,6 +108,12 @@ int fman_if_get_fdoff(struct fman_if *fm_if); /* Set interface fd->offset value */ void fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset); +/* Get interface SG enable status value */ +int fman_if_get_sg_enable(struct fman_if *fm_if); + +/* Set interface SG support mode */ +void fman_if_set_sg(struct fman_if *fm_if, int enable); + /* Get interface Max Frame length (MTU) */ uint16_t fman_if_get_maxfrm(struct fman_if *fm_if); diff --git a/drivers/bus/dpaa/include/fsl_qman.h b/drivers/bus/dpaa/include/fsl_qman.h index e9793f30..b18cf037 100644 --- a/drivers/bus/dpaa/include/fsl_qman.h +++ b/drivers/bus/dpaa/include/fsl_qman.h @@ -284,20 +284,20 @@ static inline dma_addr_t qm_sg_addr(const struct qm_sg_entry *sg) } while (0) /* See 1.5.8.1: "Enqueue Command" */ -struct qm_eqcr_entry { +struct __rte_aligned(8) qm_eqcr_entry { u8 __dont_write_directly__verb; u8 dca; u16 seqnum; u32 orp; /* 24-bit */ u32 fqid; /* 24-bit */ u32 tag; - struct qm_fd fd; + struct qm_fd fd; /* this has alignment 8 */ u8 __reserved3[32]; } __packed; /* "Frame Dequeue Response" */ -struct qm_dqrr_entry { +struct __rte_aligned(8) qm_dqrr_entry { u8 verb; u8 stat; u16 seqnum; /* 15-bit */ @@ -305,7 +305,7 @@ struct qm_dqrr_entry { u8 __reserved2[3]; u32 fqid; /* 24-bit */ u32 contextB; - struct qm_fd fd; + struct qm_fd fd; /* this has alignment 8 */ u8 __reserved4[32]; }; @@ -323,18 +323,19 @@ struct qm_dqrr_entry { /* "ERN Message Response" */ /* "FQ State Change Notification" */ struct qm_mr_entry { - u8 verb; union { struct { + u8 verb; u8 dca; u16 seqnum; u8 rc; /* Rejection Code */ u32 orp:24; u32 fqid; /* 24-bit */ u32 tag; - struct qm_fd fd; - } __packed ern; + struct qm_fd fd; /* this has alignment 8 */ + } __packed __rte_aligned(8) ern; struct { + u8 verb; #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */ u8 __reserved1:4; @@ -349,18 +350,19 @@ struct qm_mr_entry { u32 __reserved3:24; u32 fqid; /* 24-bit */ u32 tag; - struct qm_fd fd; - } __packed dcern; + struct qm_fd fd; /* this has alignment 8 */ + } __packed __rte_aligned(8) dcern; struct { + u8 verb; u8 fqs; /* Frame Queue Status */ u8 __reserved1[6]; u32 fqid; /* 24-bit */ u32 contextB; u8 __reserved2[16]; - } __packed fq; /* FQRN/FQRNI/FQRL/FQPN */ + } __packed __rte_aligned(8) fq; /* FQRN/FQRNI/FQRL/FQPN */ }; u8 __reserved2[32]; -} __packed; +} __packed __rte_aligned(8); #define QM_MR_VERB_VBIT 0x80 /* * ERNs originating from direct-connect portals ("dcern") use 0x20 as a verb @@ -1330,10 +1332,11 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit, * qman_set_vdq - Issue a volatile dequeue command * @fq: Frame Queue on which the volatile dequeue command is issued * @num: Number of Frames requested for volatile dequeue + * @vdqcr_flags: QM_VDQCR_EXACT flag to for VDQCR command * * This function will issue a volatile dequeue command to the QMAN. */ -int qman_set_vdq(struct qman_fq *fq, u16 num); +int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags); /** * qman_dequeue - Get the DQRR entry after volatile dequeue command diff --git a/drivers/bus/dpaa/include/of.h b/drivers/bus/dpaa/include/of.h index 151be5a3..7ea7608f 100644 --- a/drivers/bus/dpaa/include/of.h +++ b/drivers/bus/dpaa/include/of.h @@ -109,6 +109,8 @@ const struct device_node *of_get_parent(const struct device_node *dev_node); const struct device_node *of_get_next_child(const struct device_node *dev_node, const struct device_node *prev); +const void *of_get_mac_address(const struct device_node *np); + #define for_each_child_node(parent, child) \ for (child = of_get_next_child(parent, NULL); child != NULL; \ child = of_get_next_child(parent, child)) diff --git a/drivers/bus/dpaa/meson.build b/drivers/bus/dpaa/meson.build new file mode 100644 index 00000000..d10b62c0 --- /dev/null +++ b/drivers/bus/dpaa/meson.build @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018 NXP + +if host_machine.system() != 'linux' + build = false +endif + +deps += ['eventdev'] +sources = files('base/fman/fman.c', + 'base/fman/fman_hw.c', + 'base/fman/netcfg_layer.c', + 'base/fman/of.c', + 'base/qbman/bman.c', + 'base/qbman/bman_driver.c', + 'base/qbman/dpaa_alloc.c', + 'base/qbman/dpaa_sys.c', + 'base/qbman/process.c', + 'base/qbman/qman.c', + 'base/qbman/qman_driver.c', + 'dpaa_bus.c') + +allow_experimental_apis = true + +if cc.has_argument('-Wno-cast-qual') + cflags += '-Wno-cast-qual' +endif + +includes += include_directories('include', 'base/qbman') +cflags += ['-D_GNU_SOURCE'] diff --git a/drivers/bus/dpaa/rte_bus_dpaa_version.map b/drivers/bus/dpaa/rte_bus_dpaa_version.map index 8d902854..7d6d6243 100644 --- a/drivers/bus/dpaa/rte_bus_dpaa_version.map +++ b/drivers/bus/dpaa/rte_bus_dpaa_version.map @@ -92,3 +92,13 @@ DPDK_18.02 { local: *; } DPDK_17.11; + +DPDK_18.08 { + global: + + fman_if_get_sg_enable; + fman_if_set_sg; + of_get_mac_address; + + local: *; +} DPDK_18.02; diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h index 718701b1..15dc6a4a 100644 --- a/drivers/bus/dpaa/rte_dpaa_bus.h +++ b/drivers/bus/dpaa/rte_dpaa_bus.h @@ -15,8 +15,6 @@ #include <of.h> #include <netcfg.h> -#define FSL_DPAA_BUS_NAME "FSL_DPAA_BUS" - #define DPAA_MEMPOOL_OPS_NAME "dpaa" #define DEV_TO_DPAA_DEVICE(ptr) \ @@ -95,20 +93,35 @@ struct dpaa_portal { uint64_t tid;/**< Parent Thread id for this portal */ }; -/* TODO - this is costly, need to write a fast coversion routine */ +/* Various structures representing contiguous memory maps */ +struct dpaa_memseg { + TAILQ_ENTRY(dpaa_memseg) next; + char *vaddr; + rte_iova_t iova; + size_t len; +}; + +TAILQ_HEAD(dpaa_memseg_list, dpaa_memseg); +extern struct dpaa_memseg_list rte_dpaa_memsegs; + +/* Either iterate over the list of internal memseg references or fallback to + * EAL memseg based iova2virt. + */ static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr) { - const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); - int i; - - for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr != NULL; i++) { - if (paddr >= memseg[i].iova && paddr < - memseg[i].iova + memseg[i].len) - return (uint8_t *)(memseg[i].addr) + - (paddr - memseg[i].iova); + struct dpaa_memseg *ms; + + /* Check if the address is already part of the memseg list internally + * maintained by the dpaa driver. + */ + TAILQ_FOREACH(ms, &rte_dpaa_memsegs, next) { + if (paddr >= ms->iova && paddr < + ms->iova + ms->len) + return RTE_PTR_ADD(ms->vaddr, (uintptr_t)(paddr - ms->iova)); } - return NULL; + /* If not, Fallback to full memseg list searching */ + return rte_mem_iova2virt(paddr); } /** @@ -151,8 +164,7 @@ void dpaa_portal_finish(void *arg); /** Helper for DPAA device registration from driver (eth, crypto) instance */ #define RTE_PMD_REGISTER_DPAA(nm, dpaa_drv) \ -RTE_INIT(dpaainitfn_ ##nm); \ -static void dpaainitfn_ ##nm(void) \ +RTE_INIT(dpaainitfn_ ##nm) \ {\ (dpaa_drv).driver.name = RTE_STR(nm);\ rte_dpaa_driver_register(&dpaa_drv); \ diff --git a/drivers/bus/dpaa/rte_dpaa_logs.h b/drivers/bus/dpaa/rte_dpaa_logs.h index 0fd70bbf..e4143543 100644 --- a/drivers/bus/dpaa/rte_dpaa_logs.h +++ b/drivers/bus/dpaa/rte_dpaa_logs.h @@ -15,10 +15,7 @@ extern int dpaa_logtype_pmd; extern int dpaa_logtype_eventdev; #define DPAA_BUS_LOG(level, fmt, args...) \ - rte_log(RTE_LOG_ ## level, dpaa_logtype_bus, "%s(): " fmt "\n", \ - __func__, ##args) - -#define BUS_INIT_FUNC_TRACE() DPAA_BUS_LOG(DEBUG, " >>") + rte_log(RTE_LOG_ ## level, dpaa_logtype_bus, "dpaa: " fmt "\n", ##args) #ifdef RTE_LIBRTE_DPAA_DEBUG_BUS #define DPAA_BUS_HWWARN(cond, fmt, args...) \ @@ -31,7 +28,11 @@ extern int dpaa_logtype_eventdev; #endif #define DPAA_BUS_DEBUG(fmt, args...) \ - DPAA_BUS_LOG(DEBUG, fmt, ## args) + rte_log(RTE_LOG_DEBUG, dpaa_logtype_bus, "dpaa: %s(): " fmt "\n", \ + __func__, ##args) + +#define BUS_INIT_FUNC_TRACE() DPAA_BUS_DEBUG(" >>") + #define DPAA_BUS_INFO(fmt, args...) \ DPAA_BUS_LOG(INFO, fmt, ## args) #define DPAA_BUS_ERR(fmt, args...) \ diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile index de237f0f..515d0f53 100644 --- a/drivers/bus/fslmc/Makefile +++ b/drivers/bus/fslmc/Makefile @@ -9,23 +9,13 @@ include $(RTE_SDK)/mk/rte.vars.mk # LIB = librte_bus_fslmc.a -ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_PMD),y) -CONFIG_RTE_LIBRTE_FSLMC_BUS = $(CONFIG_RTE_LIBRTE_DPAA2_PMD) -endif - CFLAGS += -DALLOW_EXPERIMENTAL_API -ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y) -CFLAGS += -O0 -g -CFLAGS += "-Wno-error" -else CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -endif CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include -CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev @@ -42,11 +32,12 @@ SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ mc/dpmng.c \ - mc/dpbp.c \ - mc/dpio.c \ - mc/mc_sys.c \ + mc/dpbp.c \ + mc/dpio.c \ + mc/mc_sys.c \ mc/dpcon.c \ - mc/dpci.c + mc/dpci.c \ + mc/dpdmai.c SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpio.c SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpbp.c diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c index 5ee0beb8..d2900edc 100644 --- a/drivers/bus/fslmc/fslmc_bus.c +++ b/drivers/bus/fslmc/fslmc_bus.c @@ -18,11 +18,12 @@ #include <rte_fslmc.h> #include <fslmc_vfio.h> +#include "fslmc_logs.h" -#define FSLMC_BUS_LOG(level, fmt, args...) \ - RTE_LOG(level, EAL, fmt "\n", ##args) +int dpaa2_logtype_bus; #define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups" +#define FSLMC_BUS_NAME fslmc struct rte_fslmc_bus rte_fslmc_bus; uint8_t dpaa2_virt_mode; @@ -93,6 +94,41 @@ insert_in_device_list(struct rte_dpaa2_device *newdev) TAILQ_INSERT_TAIL(&rte_fslmc_bus.device_list, newdev, next); } +static struct rte_devargs * +fslmc_devargs_lookup(struct rte_dpaa2_device *dev) +{ + struct rte_devargs *devargs; + char dev_name[32]; + + RTE_EAL_DEVARGS_FOREACH("fslmc", devargs) { + devargs->bus->parse(devargs->name, &dev_name); + if (strcmp(dev_name, dev->device.name) == 0) { + DPAA2_BUS_INFO("**Devargs matched %s", dev_name); + return devargs; + } + } + return NULL; +} + +static void +dump_device_list(void) +{ + struct rte_dpaa2_device *dev; + uint32_t global_log_level; + int local_log_level; + + /* Only if the log level has been set to Debugging, print list */ + global_log_level = rte_log_get_global_level(); + local_log_level = rte_log_get_level(dpaa2_logtype_bus); + if (global_log_level == RTE_LOG_DEBUG || + local_log_level == RTE_LOG_DEBUG) { + DPAA2_BUS_LOG(DEBUG, "List of devices scanned on bus:"); + TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { + DPAA2_BUS_LOG(DEBUG, "\t\t%s", dev->device.name); + } + } +} + static int scan_one_fslmc_device(char *dev_name) { @@ -109,7 +145,7 @@ scan_one_fslmc_device(char *dev_name) /* Creating a temporary copy to perform cut-parse over string */ dup_dev_name = strdup(dev_name); if (!dup_dev_name) { - FSLMC_BUS_LOG(ERR, "Out of memory."); + DPAA2_BUS_ERR("Unable to allocate device name memory"); return -ENOMEM; } @@ -120,7 +156,7 @@ scan_one_fslmc_device(char *dev_name) */ dev = calloc(1, sizeof(struct rte_dpaa2_device)); if (!dev) { - FSLMC_BUS_LOG(ERR, "Out of memory."); + DPAA2_BUS_ERR("Unable to allocate device object"); free(dup_dev_name); return -ENOMEM; } @@ -128,7 +164,7 @@ scan_one_fslmc_device(char *dev_name) /* Parse the device name and ID */ t_ptr = strtok(dup_dev_name, "."); if (!t_ptr) { - FSLMC_BUS_LOG(ERR, "Incorrect device string observed."); + DPAA2_BUS_ERR("Incorrect device name observed"); goto cleanup; } if (!strncmp("dpni", t_ptr, 4)) @@ -145,6 +181,8 @@ scan_one_fslmc_device(char *dev_name) dev->dev_type = DPAA2_CI; else if (!strncmp("dpmcp", t_ptr, 5)) dev->dev_type = DPAA2_MPORTAL; + else if (!strncmp("dpdmai", t_ptr, 6)) + dev->dev_type = DPAA2_QDMA; else dev->dev_type = DPAA2_UNKNOWN; @@ -153,17 +191,17 @@ scan_one_fslmc_device(char *dev_name) t_ptr = strtok(NULL, "."); if (!t_ptr) { - FSLMC_BUS_LOG(ERR, "Incorrect device string observed (%s).", - t_ptr); + DPAA2_BUS_ERR("Incorrect device string observed (%s)", t_ptr); goto cleanup; } sscanf(t_ptr, "%hu", &dev->object_id); dev->device.name = strdup(dev_name); if (!dev->device.name) { - FSLMC_BUS_LOG(ERR, "Out of memory."); + DPAA2_BUS_ERR("Unable to clone device name. Out of memory"); goto cleanup; } + dev->device.devargs = fslmc_devargs_lookup(dev); /* Add device in the fslmc device list */ insert_in_device_list(dev); @@ -182,6 +220,54 @@ cleanup: } static int +rte_fslmc_parse(const char *name, void *addr) +{ + uint16_t dev_id; + char *t_ptr; + char *sep = strchr(name, ':'); + + if (strncmp(name, RTE_STR(FSLMC_BUS_NAME), + strlen(RTE_STR(FSLMC_BUS_NAME)))) { + return -EINVAL; + } + + if (!sep) { + DPAA2_BUS_ERR("Incorrect device name observed"); + return -EINVAL; + } + + t_ptr = (char *)(sep + 1); + + if (strncmp("dpni", t_ptr, 4) && + strncmp("dpseci", t_ptr, 6) && + strncmp("dpcon", t_ptr, 5) && + strncmp("dpbp", t_ptr, 4) && + strncmp("dpio", t_ptr, 4) && + strncmp("dpci", t_ptr, 4) && + strncmp("dpmcp", t_ptr, 5) && + strncmp("dpdmai", t_ptr, 6)) { + DPAA2_BUS_ERR("Unknown or unsupported device"); + return -EINVAL; + } + + t_ptr = strchr(name, '.'); + if (!t_ptr) { + DPAA2_BUS_ERR("Incorrect device string observed (%s)", t_ptr); + return -EINVAL; + } + + t_ptr = (char *)(t_ptr + 1); + if (sscanf(t_ptr, "%hu", &dev_id) <= 0) { + DPAA2_BUS_ERR("Incorrect device string observed (%s)", t_ptr); + return -EINVAL; + } + + if (addr) + strcpy(addr, (char *)(sep + 1)); + return 0; +} + +static int rte_fslmc_scan(void) { int ret; @@ -193,8 +279,7 @@ rte_fslmc_scan(void) int groupid; if (process_once) { - FSLMC_BUS_LOG(DEBUG, - "Fslmc bus already scanned. Not rescanning"); + DPAA2_BUS_DEBUG("Fslmc bus already scanned. Not rescanning"); return 0; } process_once = 1; @@ -208,7 +293,7 @@ rte_fslmc_scan(void) groupid); dir = opendir(fslmc_dirpath); if (!dir) { - FSLMC_BUS_LOG(ERR, "Unable to open VFIO group dir."); + DPAA2_BUS_ERR("Unable to open VFIO group directory"); goto scan_fail; } @@ -224,9 +309,12 @@ rte_fslmc_scan(void) device_count += 1; } - FSLMC_BUS_LOG(INFO, "fslmc: Bus scan completed"); - closedir(dir); + + DPAA2_BUS_INFO("FSLMC Bus scan completed"); + /* If debugging is enabled, device list is dumped to log output */ + dump_device_list(); + return 0; scan_fail_cleanup: @@ -235,7 +323,7 @@ scan_fail_cleanup: /* Remove all devices in the list */ cleanup_fslmc_device_list(); scan_fail: - FSLMC_BUS_LOG(DEBUG, "FSLMC Bus Not Available. Skipping."); + DPAA2_BUS_DEBUG("FSLMC Bus Not Available. Skipping"); /* Irrespective of failure, scan only return success */ return 0; } @@ -254,6 +342,8 @@ static int rte_fslmc_probe(void) { int ret = 0; + int probe_all; + struct rte_dpaa2_device *dev; struct rte_dpaa2_driver *drv; @@ -262,16 +352,29 @@ rte_fslmc_probe(void) ret = fslmc_vfio_setup_group(); if (ret) { - FSLMC_BUS_LOG(ERR, "Unable to setup VFIO %d", ret); + DPAA2_BUS_ERR("Unable to setup VFIO %d", ret); + return 0; + } + + /* Map existing segments as well as, in case of hotpluggable memory, + * install callback handler. + */ + ret = rte_fslmc_vfio_dmamap(); + if (ret) { + DPAA2_BUS_ERR("Unable to DMA map existing VAs: (%d)", ret); + /* Not continuing ahead */ + DPAA2_BUS_ERR("FSLMC VFIO Mapping failed"); return 0; } ret = fslmc_vfio_process_group(); if (ret) { - FSLMC_BUS_LOG(ERR, "Unable to setup devices %d", ret); + DPAA2_BUS_ERR("Unable to setup devices %d", ret); return 0; } + probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST; + TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { ret = rte_fslmc_match(drv, dev); @@ -281,9 +384,21 @@ rte_fslmc_probe(void) if (!drv->probe) continue; - ret = drv->probe(drv, dev); - if (ret) - FSLMC_BUS_LOG(ERR, "Unable to probe.\n"); + if (dev->device.devargs && + dev->device.devargs->policy == RTE_DEV_BLACKLISTED) { + DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping", + dev->device.name); + continue; + } + + if (probe_all || + (dev->device.devargs && + dev->device.devargs->policy == + RTE_DEV_WHITELISTED)) { + ret = drv->probe(drv, dev); + if (ret) + DPAA2_BUS_ERR("Unable to probe"); + } break; } } @@ -298,16 +413,19 @@ static struct rte_device * rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, const void *data) { + const struct rte_dpaa2_device *dstart; struct rte_dpaa2_device *dev; - TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { - if (start && &dev->device == start) { - start = NULL; /* starting point found */ - continue; - } - + if (start != NULL) { + dstart = RTE_DEV_TO_FSLMC_CONST(start); + dev = TAILQ_NEXT(dstart, next); + } else { + dev = TAILQ_FIRST(&rte_fslmc_bus.device_list); + } + while (dev != NULL) { if (cmp(&dev->device, data) == 0) return &dev->device; + dev = TAILQ_NEXT(dev, next); } return NULL; @@ -390,6 +508,7 @@ struct rte_fslmc_bus rte_fslmc_bus = { .bus = { .scan = rte_fslmc_scan, .probe = rte_fslmc_probe, + .parse = rte_fslmc_parse, .find_device = rte_fslmc_find_device, .get_iommu_class = rte_dpaa2_get_iommu_class, }, @@ -398,4 +517,12 @@ struct rte_fslmc_bus rte_fslmc_bus = { .device_count = {0}, }; -RTE_REGISTER_BUS(fslmc, rte_fslmc_bus.bus); +RTE_REGISTER_BUS(FSLMC_BUS_NAME, rte_fslmc_bus.bus); + +RTE_INIT(fslmc_init_log) +{ + /* Bus level logs */ + dpaa2_logtype_bus = rte_log_register("bus.fslmc"); + if (dpaa2_logtype_bus >= 0) + rte_log_set_level(dpaa2_logtype_bus, RTE_LOG_NOTICE); +} diff --git a/drivers/bus/fslmc/fslmc_logs.h b/drivers/bus/fslmc/fslmc_logs.h index d87b6386..dd74cb7d 100644 --- a/drivers/bus/fslmc/fslmc_logs.h +++ b/drivers/bus/fslmc/fslmc_logs.h @@ -7,44 +7,35 @@ #ifndef _FSLMC_LOGS_H_ #define _FSLMC_LOGS_H_ -#define PMD_INIT_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ##args) - -#ifdef RTE_LIBRTE_DPAA2_DEBUG_INIT -#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") -#else -#define PMD_INIT_FUNC_TRACE() do { } while (0) -#endif - -#ifdef RTE_LIBRTE_DPAA2_DEBUG_RX -#define PMD_RX_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) -#else -#define PMD_RX_LOG(level, fmt, args...) do { } while (0) -#endif - -#ifdef RTE_LIBRTE_DPAA2_DEBUG_TX -#define PMD_TX_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) -#else -#define PMD_TX_LOG(level, fmt, args...) do { } while (0) -#endif - -#ifdef RTE_LIBRTE_DPAA2_DEBUG_TX_FREE -#define PMD_TX_FREE_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) -#else -#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while (0) -#endif - -#ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER -#define PMD_DRV_LOG_RAW(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) -#else -#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0) -#endif - -#define PMD_DRV_LOG(level, fmt, args...) \ - PMD_DRV_LOG_RAW(level, fmt "\n", ## args) +extern int dpaa2_logtype_bus; + +#define DPAA2_BUS_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, dpaa2_logtype_bus, "fslmc: " fmt "\n", \ + ##args) + +/* Debug logs are with Function names */ +#define DPAA2_BUS_DEBUG(fmt, args...) \ + rte_log(RTE_LOG_DEBUG, dpaa2_logtype_bus, "fslmc: %s(): " fmt "\n", \ + __func__, ##args) + +#define BUS_INIT_FUNC_TRACE() DPAA2_BUS_DEBUG(" >>") + +#define DPAA2_BUS_INFO(fmt, args...) \ + DPAA2_BUS_LOG(INFO, fmt, ## args) +#define DPAA2_BUS_ERR(fmt, args...) \ + DPAA2_BUS_LOG(ERR, fmt, ## args) +#define DPAA2_BUS_WARN(fmt, args...) \ + DPAA2_BUS_LOG(WARNING, fmt, ## args) + +/* DP Logs, toggled out at compile time if level lower than current level */ +#define DPAA2_BUS_DP_LOG(level, fmt, args...) \ + RTE_LOG_DP(level, PMD, fmt, ## args) + +#define DPAA2_BUS_DP_DEBUG(fmt, args...) \ + DPAA2_BUS_DP_LOG(DEBUG, fmt, ## args) +#define DPAA2_BUS_DP_INFO(fmt, args...) \ + DPAA2_BUS_DP_LOG(INFO, fmt, ## args) +#define DPAA2_BUS_DP_WARN(fmt, args...) \ + DPAA2_BUS_DP_LOG(WARNING, fmt, ## args) #endif /* _FSLMC_LOGS_H_ */ diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c index 1241295b..4c2cd2a8 100644 --- a/drivers/bus/fslmc/fslmc_vfio.c +++ b/drivers/bus/fslmc/fslmc_vfio.c @@ -30,17 +30,16 @@ #include <rte_kvargs.h> #include <rte_dev.h> #include <rte_bus.h> +#include <rte_eal_memconfig.h> #include "rte_fslmc.h" #include "fslmc_vfio.h" +#include "fslmc_logs.h" #include <mc/fsl_dpmng.h> #include "portal/dpaa2_hw_pvt.h" #include "portal/dpaa2_hw_dpio.h" -#define FSLMC_VFIO_LOG(level, fmt, args...) \ - RTE_LOG(level, EAL, fmt "\n", ##args) - /** Pathname of FSL-MC devices directory. */ #define SYSFS_FSL_MC_DEVICES "/sys/bus/fsl-mc/devices" @@ -53,7 +52,6 @@ static int container_device_fd; static char *g_container; static uint32_t *msi_intr_vaddr; void *(*rte_mcp_ptr_list); -static int is_dma_done; static struct rte_dpaa2_object_list dpaa2_obj_list = TAILQ_HEAD_INITIALIZER(dpaa2_obj_list); @@ -76,33 +74,32 @@ fslmc_get_container_group(int *groupid) if (!g_container) { container = getenv("DPRC"); if (container == NULL) { - RTE_LOG(WARNING, EAL, "DPAA2: DPRC not available\n"); + DPAA2_BUS_DEBUG("DPAA2: DPRC not available"); return -EINVAL; } if (strlen(container) >= FSLMC_CONTAINER_MAX_LEN) { - FSLMC_VFIO_LOG(ERR, "Invalid container name: %s\n", - container); + DPAA2_BUS_ERR("Invalid container name: %s", container); return -1; } g_container = strdup(container); if (!g_container) { - FSLMC_VFIO_LOG(ERR, "Out of memory."); + DPAA2_BUS_ERR("Mem alloc failure; Container name"); return -ENOMEM; } } /* get group number */ - ret = vfio_get_group_no(SYSFS_FSL_MC_DEVICES, g_container, groupid); + ret = rte_vfio_get_group_num(SYSFS_FSL_MC_DEVICES, + g_container, groupid); if (ret <= 0) { - FSLMC_VFIO_LOG(ERR, "Unable to find %s IOMMU group", - g_container); + DPAA2_BUS_ERR("Unable to find %s IOMMU group", g_container); return -1; } - FSLMC_VFIO_LOG(DEBUG, "Container: %s has VFIO iommu group id = %d", - g_container, *groupid); + DPAA2_BUS_DEBUG("Container: %s has VFIO iommu group id = %d", + g_container, *groupid); return 0; } @@ -113,14 +110,14 @@ vfio_connect_container(void) int fd, ret; if (vfio_container.used) { - FSLMC_VFIO_LOG(DEBUG, "No container available."); + DPAA2_BUS_DEBUG("No container available"); return -1; } /* Try connecting to vfio container if already created */ if (!ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER, &vfio_container.fd)) { - FSLMC_VFIO_LOG(INFO, + DPAA2_BUS_DEBUG( "Container pre-exists with FD[0x%x] for this group", vfio_container.fd); vfio_group.container = &vfio_container; @@ -128,9 +125,9 @@ vfio_connect_container(void) } /* Opens main vfio file descriptor which represents the "container" */ - fd = vfio_get_container_fd(); + fd = rte_vfio_get_container_fd(); if (fd < 0) { - FSLMC_VFIO_LOG(ERR, "Failed to open VFIO container"); + DPAA2_BUS_ERR("Failed to open VFIO container"); return -errno; } @@ -139,19 +136,19 @@ vfio_connect_container(void) /* Connect group to container */ ret = ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER, &fd); if (ret) { - FSLMC_VFIO_LOG(ERR, "Failed to setup group container"); + DPAA2_BUS_ERR("Failed to setup group container"); close(fd); return -errno; } ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); if (ret) { - FSLMC_VFIO_LOG(ERR, "Failed to setup VFIO iommu"); + DPAA2_BUS_ERR("Failed to setup VFIO iommu"); close(fd); return -errno; } } else { - FSLMC_VFIO_LOG(ERR, "No supported IOMMU available"); + DPAA2_BUS_ERR("No supported IOMMU available"); close(fd); return -EINVAL; } @@ -179,7 +176,7 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group) vaddr = (unsigned long *)mmap(NULL, 0x1000, PROT_WRITE | PROT_READ, MAP_SHARED, container_device_fd, 0x6030000); if (vaddr == MAP_FAILED) { - FSLMC_VFIO_LOG(ERR, "Unable to map region (errno = %d)", errno); + DPAA2_BUS_ERR("Unable to map region (errno = %d)", errno); return -errno; } @@ -189,88 +186,195 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group) if (ret == 0) return 0; - FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA fails (errno = %d)", errno); + DPAA2_BUS_ERR("Unable to map DMA address (errno = %d)", errno); return -errno; } -int rte_fslmc_vfio_dmamap(void) +static int fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len); +static int fslmc_unmap_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len); + +static void +fslmc_memevent_cb(enum rte_mem_event type, const void *addr, size_t len, + void *arg __rte_unused) { + struct rte_memseg_list *msl; + struct rte_memseg *ms; + size_t cur_len = 0, map_len = 0; + uint64_t virt_addr; + rte_iova_t iova_addr; int ret; + + msl = rte_mem_virt2memseg_list(addr); + + while (cur_len < len) { + const void *va = RTE_PTR_ADD(addr, cur_len); + + ms = rte_mem_virt2memseg(va, msl); + iova_addr = ms->iova; + virt_addr = ms->addr_64; + map_len = ms->len; + + DPAA2_BUS_DEBUG("Request for %s, va=%p, " + "virt_addr=0x%" PRIx64 ", " + "iova=0x%" PRIx64 ", map_len=%zu", + type == RTE_MEM_EVENT_ALLOC ? + "alloc" : "dealloc", + va, virt_addr, iova_addr, map_len); + + if (type == RTE_MEM_EVENT_ALLOC) + ret = fslmc_map_dma(virt_addr, iova_addr, map_len); + else + ret = fslmc_unmap_dma(virt_addr, iova_addr, map_len); + + if (ret != 0) { + DPAA2_BUS_ERR("DMA Mapping/Unmapping failed. " + "Map=%d, addr=%p, len=%zu, err:(%d)", + type, va, map_len, ret); + return; + } + + cur_len += map_len; + } + + if (type == RTE_MEM_EVENT_ALLOC) + DPAA2_BUS_DEBUG("Total Mapped: addr=%p, len=%zu", + addr, len); + else + DPAA2_BUS_DEBUG("Total Unmapped: addr=%p, len=%zu", + addr, len); +} + +static int +fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr __rte_unused, size_t len) +{ struct fslmc_vfio_group *group; struct vfio_iommu_type1_dma_map dma_map = { .argsz = sizeof(struct vfio_iommu_type1_dma_map), .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, }; + int ret; - int i; - const struct rte_memseg *memseg; + dma_map.size = len; + dma_map.vaddr = vaddr; - if (is_dma_done) - return 0; +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + dma_map.iova = iovaddr; +#else + dma_map.iova = dma_map.vaddr; +#endif - memseg = rte_eal_get_physmem_layout(); - if (memseg == NULL) { - FSLMC_VFIO_LOG(ERR, "Cannot get physical layout."); - return -ENODEV; + /* SET DMA MAP for IOMMU */ + group = &vfio_group; + + if (!group->container) { + DPAA2_BUS_ERR("Container is not connected "); + return -1; } - for (i = 0; i < RTE_MAX_MEMSEG; i++) { - if (memseg[i].addr == NULL && memseg[i].len == 0) { - FSLMC_VFIO_LOG(DEBUG, "Total %d segments found.", i); - break; - } + DPAA2_BUS_DEBUG("--> Map address: 0x%"PRIx64", size: %"PRIu64"", + (uint64_t)dma_map.vaddr, (uint64_t)dma_map.size); + ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &dma_map); + if (ret) { + DPAA2_BUS_ERR("VFIO_IOMMU_MAP_DMA API(errno = %d)", + errno); + return -1; + } - dma_map.size = memseg[i].len; - dma_map.vaddr = memseg[i].addr_64; -#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA - if (rte_eal_iova_mode() == RTE_IOVA_VA) - dma_map.iova = dma_map.vaddr; - else - dma_map.iova = memseg[i].iova; -#else - dma_map.iova = dma_map.vaddr; -#endif + return 0; +} - /* SET DMA MAP for IOMMU */ - group = &vfio_group; +static int +fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len) +{ + struct fslmc_vfio_group *group; + struct vfio_iommu_type1_dma_unmap dma_unmap = { + .argsz = sizeof(struct vfio_iommu_type1_dma_unmap), + .flags = 0, + }; + int ret; - if (!group->container) { - FSLMC_VFIO_LOG(ERR, "Container is not connected "); - return -1; - } + dma_unmap.size = len; + dma_unmap.iova = vaddr; - FSLMC_VFIO_LOG(DEBUG, "-->Initial SHM Virtual ADDR %llX", - dma_map.vaddr); - FSLMC_VFIO_LOG(DEBUG, "-----> DMA size 0x%llX", dma_map.size); - ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, - &dma_map); - if (ret) { - FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA API(errno = %d)", - errno); - return ret; - } + /* SET DMA MAP for IOMMU */ + group = &vfio_group; + + if (!group->container) { + DPAA2_BUS_ERR("Container is not connected "); + return -1; } - /* Verifying that at least single segment is available */ - if (i <= 0) { - FSLMC_VFIO_LOG(ERR, "No Segments found for VFIO Mapping"); + DPAA2_BUS_DEBUG("--> Unmap address: 0x%"PRIx64", size: %"PRIu64"", + (uint64_t)dma_unmap.iova, (uint64_t)dma_unmap.size); + ret = ioctl(group->container->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap); + if (ret) { + DPAA2_BUS_ERR("VFIO_IOMMU_UNMAP_DMA API(errno = %d)", + errno); return -1; } + return 0; +} + +static int +fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused, + const struct rte_memseg *ms, void *arg) +{ + int *n_segs = arg; + int ret; + + ret = fslmc_map_dma(ms->addr_64, ms->iova, ms->len); + if (ret) + DPAA2_BUS_ERR("Unable to VFIO map (addr=%p, len=%zu)", + ms->addr, ms->len); + else + (*n_segs)++; + + return ret; +} + +int rte_fslmc_vfio_dmamap(void) +{ + int i = 0, ret; + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + rte_rwlock_t *mem_lock = &mcfg->memory_hotplug_lock; + + /* Lock before parsing and registering callback to memory subsystem */ + rte_rwlock_read_lock(mem_lock); + + if (rte_memseg_walk(fslmc_dmamap_seg, &i) < 0) { + rte_rwlock_read_unlock(mem_lock); + return -1; + } + + ret = rte_mem_event_callback_register("fslmc_memevent_clb", + fslmc_memevent_cb, NULL); + if (ret && rte_errno == ENOTSUP) + DPAA2_BUS_DEBUG("Memory event callbacks not supported"); + else if (ret) + DPAA2_BUS_DEBUG("Unable to install memory handler"); + else + DPAA2_BUS_DEBUG("Installed memory callback handler"); + + DPAA2_BUS_DEBUG("Total %d segments found.", i); + /* TODO - This is a W.A. as VFIO currently does not add the mapping of * the interrupt region to SMMU. This should be removed once the * support is added in the Kernel. */ - vfio_map_irq_region(group); + vfio_map_irq_region(&vfio_group); - is_dma_done = 1; + /* Existing segments have been mapped and memory callback for hotplug + * has been installed. + */ + rte_rwlock_read_unlock(mem_lock); return 0; } static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj) { - int64_t v_addr = (int64_t)MAP_FAILED; + intptr_t v_addr = (intptr_t)MAP_FAILED; int32_t ret, mc_fd; struct vfio_device_info d_info = { .argsz = sizeof(d_info) }; @@ -279,29 +383,26 @@ static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj) /* getting the mcp object's fd*/ mc_fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, mcp_obj); if (mc_fd < 0) { - FSLMC_VFIO_LOG(ERR, "error in VFIO get dev %s fd from group %d", - mcp_obj, group->fd); + DPAA2_BUS_ERR("Error in VFIO get dev %s fd from group %d", + mcp_obj, group->fd); return v_addr; } /* getting device info*/ ret = ioctl(mc_fd, VFIO_DEVICE_GET_INFO, &d_info); if (ret < 0) { - FSLMC_VFIO_LOG(ERR, "error in VFIO getting DEVICE_INFO"); + DPAA2_BUS_ERR("Error in VFIO getting DEVICE_INFO"); goto MC_FAILURE; } /* getting device region info*/ ret = ioctl(mc_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info); if (ret < 0) { - FSLMC_VFIO_LOG(ERR, "error in VFIO getting REGION_INFO"); + DPAA2_BUS_ERR("Error in VFIO getting REGION_INFO"); goto MC_FAILURE; } - FSLMC_VFIO_LOG(DEBUG, "region offset = %llx , region size = %llx", - reg_info.offset, reg_info.size); - - v_addr = (uint64_t)mmap(NULL, reg_info.size, + v_addr = (size_t)mmap(NULL, reg_info.size, PROT_WRITE | PROT_READ, MAP_SHARED, mc_fd, reg_info.offset); @@ -334,8 +435,8 @@ int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index) ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set); if (ret) { - RTE_LOG(ERR, EAL, "Error:dpaa2 SET IRQs fd=%d, err = %d(%s)\n", - intr_handle->fd, errno, strerror(errno)); + DPAA2_BUS_ERR("Error:dpaa2 SET IRQs fd=%d, err = %d(%s)", + intr_handle->fd, errno, strerror(errno)); return ret; } @@ -359,8 +460,8 @@ int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index) ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set); if (ret) - RTE_LOG(ERR, EAL, - "Error disabling dpaa2 interrupts for fd %d\n", + DPAA2_BUS_ERR( + "Error disabling dpaa2 interrupts for fd %d", intr_handle->fd); return ret; @@ -383,9 +484,8 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle, ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); if (ret < 0) { - FSLMC_VFIO_LOG(ERR, - "cannot get IRQ(%d) info, error %i (%s)", - i, errno, strerror(errno)); + DPAA2_BUS_ERR("Cannot get IRQ(%d) info, error %i (%s)", + i, errno, strerror(errno)); return -1; } @@ -399,9 +499,8 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle, /* set up an eventfd for interrupts */ fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); if (fd < 0) { - FSLMC_VFIO_LOG(ERR, - "cannot set up eventfd, error %i (%s)\n", - errno, strerror(errno)); + DPAA2_BUS_ERR("Cannot set up eventfd, error %i (%s)", + errno, strerror(errno)); return -1; } @@ -430,13 +529,14 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) dev_fd = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD, dev->device.name); if (dev_fd <= 0) { - FSLMC_VFIO_LOG(ERR, "Unable to obtain device FD for device:%s", - dev->device.name); + DPAA2_BUS_ERR("Unable to obtain device FD for device:%s", + dev->device.name); return -1; } if (ioctl(dev_fd, VFIO_DEVICE_GET_INFO, &device_info)) { - FSLMC_VFIO_LOG(ERR, "DPAA2 VFIO_DEVICE_GET_INFO fail"); + DPAA2_BUS_ERR("Unable to obtain information for device:%s", + dev->device.name); return -1; } @@ -461,61 +561,74 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) break; } - FSLMC_VFIO_LOG(DEBUG, "Device (%s) abstracted from VFIO", - dev->device.name); + DPAA2_BUS_LOG(DEBUG, "Device (%s) abstracted from VFIO", + dev->device.name); return 0; } static int fslmc_process_mcp(struct rte_dpaa2_device *dev) { - int64_t v_addr; - char *dev_name; + int ret; + intptr_t v_addr; + char *dev_name = NULL; struct fsl_mc_io dpmng = {0}; struct mc_version mc_ver_info = {0}; rte_mcp_ptr_list = malloc(sizeof(void *) * 1); if (!rte_mcp_ptr_list) { - FSLMC_VFIO_LOG(ERR, "Out of memory"); - return -ENOMEM; + DPAA2_BUS_ERR("Unable to allocate MC portal memory"); + ret = -ENOMEM; + goto cleanup; } dev_name = strdup(dev->device.name); if (!dev_name) { - FSLMC_VFIO_LOG(ERR, "Out of memory."); - free(rte_mcp_ptr_list); - rte_mcp_ptr_list = NULL; - return -ENOMEM; + DPAA2_BUS_ERR("Unable to allocate MC device name memory"); + ret = -ENOMEM; + goto cleanup; } v_addr = vfio_map_mcp_obj(&vfio_group, dev_name); - if (v_addr == (int64_t)MAP_FAILED) { - FSLMC_VFIO_LOG(ERR, "Error mapping region (errno = %d)", - errno); - free(rte_mcp_ptr_list); - rte_mcp_ptr_list = NULL; - return -1; + if (v_addr == (intptr_t)MAP_FAILED) { + DPAA2_BUS_ERR("Error mapping region (errno = %d)", errno); + ret = -1; + goto cleanup; } /* check the MC version compatibility */ dpmng.regs = (void *)v_addr; - if (mc_get_version(&dpmng, CMD_PRI_LOW, &mc_ver_info)) - RTE_LOG(WARNING, PMD, "\tmc_get_version failed\n"); + if (mc_get_version(&dpmng, CMD_PRI_LOW, &mc_ver_info)) { + DPAA2_BUS_ERR("Unable to obtain MC version"); + ret = -1; + goto cleanup; + } if ((mc_ver_info.major != MC_VER_MAJOR) || (mc_ver_info.minor < MC_VER_MINOR)) { - RTE_LOG(ERR, PMD, "DPAA2 MC version not compatible!" - " Expected %d.%d.x, Detected %d.%d.%d\n", - MC_VER_MAJOR, MC_VER_MINOR, - mc_ver_info.major, mc_ver_info.minor, - mc_ver_info.revision); - free(rte_mcp_ptr_list); - rte_mcp_ptr_list = NULL; - return -1; + DPAA2_BUS_ERR("DPAA2 MC version not compatible!" + " Expected %d.%d.x, Detected %d.%d.%d", + MC_VER_MAJOR, MC_VER_MINOR, + mc_ver_info.major, mc_ver_info.minor, + mc_ver_info.revision); + ret = -1; + goto cleanup; } rte_mcp_ptr_list[0] = (void *)v_addr; + free(dev_name); return 0; + +cleanup: + if (dev_name) + free(dev_name); + + if (rte_mcp_ptr_list) { + free(rte_mcp_ptr_list); + rte_mcp_ptr_list = NULL; + } + + return ret; } int @@ -530,7 +643,7 @@ fslmc_vfio_process_group(void) if (dev->dev_type == DPAA2_MPORTAL) { ret = fslmc_process_mcp(dev); if (ret) { - FSLMC_VFIO_LOG(DEBUG, "Unable to map Portal."); + DPAA2_BUS_ERR("Unable to map MC Portal"); return -1; } if (!found_mportal) @@ -547,23 +660,19 @@ fslmc_vfio_process_group(void) /* Cannot continue if there is not even a single mportal */ if (!found_mportal) { - FSLMC_VFIO_LOG(DEBUG, - "No MC Portal device found. Not continuing."); + DPAA2_BUS_ERR("No MC Portal device found. Not continuing"); return -1; } TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) { - if (!dev) - break; - switch (dev->dev_type) { case DPAA2_ETH: case DPAA2_CRYPTO: + case DPAA2_QDMA: ret = fslmc_process_iodevices(dev); if (ret) { - FSLMC_VFIO_LOG(DEBUG, - "Dev (%s) init failed.", - dev->device.name); + DPAA2_BUS_DEBUG("Dev (%s) init failed", + dev->device.name); return ret; } break; @@ -576,9 +685,8 @@ fslmc_vfio_process_group(void) */ ret = fslmc_process_iodevices(dev); if (ret) { - FSLMC_VFIO_LOG(DEBUG, - "Dev (%s) init failed.", - dev->device.name); + DPAA2_BUS_DEBUG("Dev (%s) init failed", + dev->device.name); return -1; } @@ -592,8 +700,8 @@ fslmc_vfio_process_group(void) case DPAA2_UNKNOWN: default: /* Unknown - ignore */ - FSLMC_VFIO_LOG(DEBUG, "Found unknown device (%s).", - dev->device.name); + DPAA2_BUS_DEBUG("Found unknown device (%s)", + dev->device.name); TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next); free(dev); dev = NULL; @@ -622,12 +730,12 @@ fslmc_vfio_setup_group(void) * processing. */ if (vfio_group.groupid == groupid) { - FSLMC_VFIO_LOG(ERR, "groupid already exists %d", groupid); + DPAA2_BUS_ERR("groupid already exists %d", groupid); return 0; } /* Get the actual group fd */ - ret = vfio_get_group_fd(groupid); + ret = rte_vfio_get_group_fd(groupid); if (ret < 0) return ret; vfio_group.fd = ret; @@ -635,14 +743,14 @@ fslmc_vfio_setup_group(void) /* Check group viability */ ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_STATUS, &status); if (ret) { - FSLMC_VFIO_LOG(ERR, "VFIO error getting group status"); + DPAA2_BUS_ERR("VFIO error getting group status"); close(vfio_group.fd); rte_vfio_clear_group(vfio_group.fd); return ret; } if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { - FSLMC_VFIO_LOG(ERR, "VFIO group not viable"); + DPAA2_BUS_ERR("VFIO group not viable"); close(vfio_group.fd); rte_vfio_clear_group(vfio_group.fd); return -EPERM; @@ -655,7 +763,7 @@ fslmc_vfio_setup_group(void) /* Now connect this IOMMU group to given container */ ret = vfio_connect_container(); if (ret) { - FSLMC_VFIO_LOG(ERR, + DPAA2_BUS_ERR( "Error connecting container with groupid %d", groupid); close(vfio_group.fd); @@ -667,15 +775,15 @@ fslmc_vfio_setup_group(void) /* Get Device information */ ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD, g_container); if (ret < 0) { - FSLMC_VFIO_LOG(ERR, "Error getting device %s fd from group %d", - g_container, vfio_group.groupid); + DPAA2_BUS_ERR("Error getting device %s fd from group %d", + g_container, vfio_group.groupid); close(vfio_group.fd); rte_vfio_clear_group(vfio_group.fd); return ret; } container_device_fd = ret; - FSLMC_VFIO_LOG(DEBUG, "VFIO Container FD is [0x%X]", - container_device_fd); + DPAA2_BUS_DEBUG("VFIO Container FD is [0x%X]", + container_device_fd); return 0; } diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h index e8fb3445..9e2c4fee 100644 --- a/drivers/bus/fslmc/fslmc_vfio.h +++ b/drivers/bus/fslmc/fslmc_vfio.h @@ -10,8 +10,6 @@ #include <rte_vfio.h> -#include "eal_vfio.h" - #define DPAA2_MC_DPNI_DEVID 7 #define DPAA2_MC_DPSECI_DEVID 3 #define DPAA2_MC_DPCON_DEVID 5 diff --git a/drivers/bus/fslmc/mc/dpdmai.c b/drivers/bus/fslmc/mc/dpdmai.c new file mode 100644 index 00000000..528889df --- /dev/null +++ b/drivers/bus/fslmc/mc/dpdmai.c @@ -0,0 +1,429 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 NXP + */ + +#include <fsl_mc_sys.h> +#include <fsl_mc_cmd.h> +#include <fsl_dpdmai.h> +#include <fsl_dpdmai_cmd.h> + +/** + * dpdmai_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpdmai_id: DPDMAI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpdmai_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpdmai_id, + uint16_t *token) +{ + struct dpdmai_cmd_open *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN, + cmd_flags, + 0); + cmd_params = (struct dpdmai_cmd_open *)cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpdmai_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_create() - Create the DPDMAI object + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id + * + * Create the DPDMAI object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpdmai_cfg *cfg, + uint32_t *obj_id) +{ + struct dpdmai_cmd_create *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE, + cmd_flags, + dprc_token); + cmd_params = (struct dpdmai_cmd_create *)cmd.params; + cmd_params->priorities[0] = cfg->priorities[0]; + cmd_params->priorities[1] = cfg->priorities[1]; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; error code otherwise. + */ +int dpdmai_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) +{ + struct dpdmai_cmd_destroy *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, + cmd_flags, + dprc_token); + cmd_params = (struct dpdmai_cmd_destroy *)cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(object_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_is_enabled() - Check if the DPDMAI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct dpdmai_rsp_is_enabled *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_IS_ENABLED, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpdmai_rsp_is_enabled *)cmd.params; + *en = dpdmai_get_field(rsp_params->en, ENABLE); + + return 0; +} + +/** + * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_get_attributes() - Retrieve DPDMAI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpdmai_attr *attr) +{ + struct dpdmai_rsp_get_attr *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpdmai_rsp_get_attr *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->num_of_priorities = rsp_params->num_of_priorities; + + return 0; +} + +/** + * dpdmai_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @priority: Select the queue relative to number of + * priorities configured at DPDMAI creation; use + * DPDMAI_ALL_QUEUES to configure all Rx queues + * identically. + * @cfg: Rx queue configuration + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + const struct dpdmai_rx_queue_cfg *cfg) +{ + struct dpdmai_cmd_set_rx_queue *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpdmai_cmd_set_rx_queue *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->priority = priority; + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); + cmd_params->options = cpu_to_le32(cfg->options); + dpdmai_set_field(cmd_params->dest_type, + DEST_TYPE, + cfg->dest_cfg.dest_type); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_get_rx_queue() - Retrieve Rx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @priority: Select the queue relative to number of + * priorities configured at DPDMAI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpdmai_rx_queue_attr *attr) +{ + struct dpdmai_cmd_get_queue *cmd_params; + struct dpdmai_rsp_get_rx_queue *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params; + cmd_params->priority = priority; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpdmai_rsp_get_rx_queue *)cmd.params; + attr->user_ctx = le64_to_cpu(rsp_params->user_ctx); + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + attr->dest_cfg.priority = le32_to_cpu(rsp_params->dest_priority); + attr->dest_cfg.dest_type = dpdmai_get_field(rsp_params->dest_type, + DEST_TYPE); + + return 0; +} + +/** + * dpdmai_get_tx_queue() - Retrieve Tx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @priority: Select the queue relative to number of + * priorities configured at DPDMAI creation + * @attr: Returned Tx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpdmai_tx_queue_attr *attr) +{ + struct dpdmai_cmd_get_queue *cmd_params; + struct dpdmai_rsp_get_tx_queue *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params; + cmd_params->priority = priority; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params; + attr->fqid = le32_to_cpu(rsp_params->fqid); + + return 0; +} diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai.h b/drivers/bus/fslmc/mc/fsl_dpdmai.h new file mode 100644 index 00000000..03e46ec1 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpdmai.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 NXP + */ + +#ifndef __FSL_DPDMAI_H +#define __FSL_DPDMAI_H + +struct fsl_mc_io; + +/* Data Path DMA Interface API + * Contains initialization APIs and runtime control APIs for DPDMAI + */ + +/* General DPDMAI macros */ + +/** + * Maximum number of Tx/Rx priorities per DPDMAI object + */ +#define DPDMAI_PRIO_NUM 2 + +/** + * All queues considered; see dpdmai_set_rx_queue() + */ +#define DPDMAI_ALL_QUEUES (uint8_t)(-1) + +int dpdmai_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpdmai_id, + uint16_t *token); + +int dpdmai_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpdmai_cfg - Structure representing DPDMAI configuration + * @priorities: Priorities for the DMA hardware processing; valid priorities are + * configured with values 1-8; the entry following last valid entry + * should be configured with 0 + */ +struct dpdmai_cfg { + uint8_t priorities[DPDMAI_PRIO_NUM]; +}; + +int dpdmai_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpdmai_cfg *cfg, + uint32_t *obj_id); + +int dpdmai_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); + +int dpdmai_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpdmai_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpdmai_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpdmai_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpdmai_attr - Structure representing DPDMAI attributes + * @id: DPDMAI object ID + * @num_of_priorities: number of priorities + */ +struct dpdmai_attr { + int id; + uint8_t num_of_priorities; +}; + +int dpdmai_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpdmai_attr *attr); + +/** + * enum dpdmai_dest - DPDMAI destination types + * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode + * and does not generate FQDAN notifications; user is expected to dequeue + * from the queue based on polling or other user-defined method + * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected to dequeue + * from the queue only after notification is received + * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified DPCON object; + * user is expected to dequeue from the DPCON channel + */ +enum dpdmai_dest { + DPDMAI_DEST_NONE = 0, + DPDMAI_DEST_DPIO = 1, + DPDMAI_DEST_DPCON = 2 +}; + +/** + * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that + * channel; not relevant for 'DPDMAI_DEST_NONE' option + */ +struct dpdmai_dest_cfg { + enum dpdmai_dest dest_type; + int dest_id; + uint8_t priority; +}; + +/* DPDMAI queue modification options */ + +/** + * Select to modify the user's context associated with the queue + */ +#define DPDMAI_QUEUE_OPT_USER_CTX 0x00000001 + +/** + * Select to modify the queue's destination + */ +#define DPDMAI_QUEUE_OPT_DEST 0x00000002 + +/** + * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPDMAI_QUEUE_OPT_<X>' flags + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; + * valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options' + * @dest_cfg: Queue destination parameters; + * valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options' + */ +struct dpdmai_rx_queue_cfg { + uint32_t options; + uint64_t user_ctx; + struct dpdmai_dest_cfg dest_cfg; + +}; + +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + const struct dpdmai_rx_queue_cfg *cfg); + +/** + * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations + */ +struct dpdmai_rx_queue_attr { + uint64_t user_ctx; + struct dpdmai_dest_cfg dest_cfg; + uint32_t fqid; +}; + +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpdmai_rx_queue_attr *attr); + +/** + * struct dpdmai_tx_queue_attr - Structure representing attributes of Tx queues + * @fqid: Virtual FQID to be used for sending frames to DMA hardware + */ + +struct dpdmai_tx_queue_attr { + uint32_t fqid; +}; + +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpdmai_tx_queue_attr *attr); + +#endif /* __FSL_DPDMAI_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h new file mode 100644 index 00000000..618e19ea --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 NXP + */ + +#ifndef _FSL_DPDMAI_CMD_H +#define _FSL_DPDMAI_CMD_H + +/* DPDMAI Version */ +#define DPDMAI_VER_MAJOR 3 +#define DPDMAI_VER_MINOR 2 + +/* Command versioning */ +#define DPDMAI_CMD_BASE_VERSION 1 +#define DPDMAI_CMD_ID_OFFSET 4 + +#define DPDMAI_CMD(id) ((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) + +/* Command IDs */ +#define DPDMAI_CMDID_CLOSE DPDMAI_CMD(0x800) +#define DPDMAI_CMDID_OPEN DPDMAI_CMD(0x80E) +#define DPDMAI_CMDID_CREATE DPDMAI_CMD(0x90E) +#define DPDMAI_CMDID_DESTROY DPDMAI_CMD(0x98E) +#define DPDMAI_CMDID_GET_API_VERSION DPDMAI_CMD(0xa0E) + +#define DPDMAI_CMDID_ENABLE DPDMAI_CMD(0x002) +#define DPDMAI_CMDID_DISABLE DPDMAI_CMD(0x003) +#define DPDMAI_CMDID_GET_ATTR DPDMAI_CMD(0x004) +#define DPDMAI_CMDID_RESET DPDMAI_CMD(0x005) +#define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMD(0x006) + +#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMD(0x1A0) +#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMD(0x1A1) +#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMD(0x1A2) + +/* Macros for accessing command fields smaller than 1byte */ +#define DPDMAI_MASK(field) \ + GENMASK(DPDMAI_##field##_SHIFT + DPDMAI_##field##_SIZE - 1, \ + DPDMAI_##field##_SHIFT) +#define dpdmai_set_field(var, field, val) \ + ((var) |= (((val) << DPDMAI_##field##_SHIFT) & DPDMAI_MASK(field))) +#define dpdmai_get_field(var, field) \ + (((var) & DPDMAI_MASK(field)) >> DPDMAI_##field##_SHIFT) + +#pragma pack(push, 1) +struct dpdmai_cmd_open { + uint32_t dpdmai_id; +}; + +struct dpdmai_cmd_create { + uint8_t pad; + uint8_t priorities[2]; +}; + +struct dpdmai_cmd_destroy { + uint32_t dpdmai_id; +}; + +#define DPDMAI_ENABLE_SHIFT 0 +#define DPDMAI_ENABLE_SIZE 1 + +struct dpdmai_rsp_is_enabled { + /* only the LSB bit */ + uint8_t en; +}; + +struct dpdmai_rsp_get_attr { + uint32_t id; + uint8_t num_of_priorities; +}; + +#define DPDMAI_DEST_TYPE_SHIFT 0 +#define DPDMAI_DEST_TYPE_SIZE 4 + +struct dpdmai_cmd_set_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t priority; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad; + uint64_t user_ctx; + uint32_t options; +}; + +struct dpdmai_cmd_get_queue { + uint8_t pad[5]; + uint8_t priority; +}; + +struct dpdmai_rsp_get_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t pad1; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad2; + uint64_t user_ctx; + uint32_t fqid; +}; + +struct dpdmai_rsp_get_tx_queue { + uint64_t pad; + uint32_t fqid; +}; + +#pragma pack(pop) +#endif /* _FSL_DPDMAI_CMD_H */ diff --git a/drivers/bus/fslmc/mc/fsl_mc_cmd.h b/drivers/bus/fslmc/mc/fsl_mc_cmd.h index a3c3e798..ac919610 100644 --- a/drivers/bus/fslmc/mc/fsl_mc_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_mc_cmd.h @@ -27,7 +27,7 @@ #define le32_to_cpu rte_le_to_cpu_32 #define le16_to_cpu rte_le_to_cpu_16 -#define BITS_PER_LONG 64 +#define BITS_PER_LONG (__SIZEOF_LONG__ * 8) #define GENMASK(h, l) \ (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) diff --git a/drivers/bus/fslmc/meson.build b/drivers/bus/fslmc/meson.build new file mode 100644 index 00000000..22a56a6f --- /dev/null +++ b/drivers/bus/fslmc/meson.build @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018 NXP + +if host_machine.system() != 'linux' + build = false +endif + +deps += ['eventdev', 'kvargs'] +sources = files('fslmc_bus.c', + 'fslmc_vfio.c', + 'mc/dpbp.c', + 'mc/dpci.c', + 'mc/dpcon.c', + 'mc/dpdmai.c', + 'mc/dpio.c', + 'mc/dpmng.c', + 'mc/mc_sys.c', + 'portal/dpaa2_hw_dpbp.c', + 'portal/dpaa2_hw_dpci.c', + 'portal/dpaa2_hw_dpio.c', + 'qbman/qbman_portal.c', + 'qbman/qbman_debug.c') + +allow_experimental_apis = true + +includes += include_directories('mc', 'qbman/include', 'portal') +cflags += ['-D_GNU_SOURCE'] diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c index f1f14e29..39c5adf9 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c @@ -44,7 +44,7 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused, /* Allocate DPAA2 dpbp handle */ dpbp_node = rte_malloc(NULL, sizeof(struct dpaa2_dpbp_dev), 0); if (!dpbp_node) { - PMD_INIT_LOG(ERR, "Memory allocation failed for DPBP Device"); + DPAA2_BUS_ERR("Memory allocation failed for DPBP Device"); return -1; } @@ -53,8 +53,8 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused, ret = dpbp_open(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_id, &dpbp_node->token); if (ret) { - PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d", - ret); + DPAA2_BUS_ERR("Unable to open buffer pool object: err(%d)", + ret); rte_free(dpbp_node); return -1; } @@ -62,8 +62,8 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused, /* Clean the device first */ ret = dpbp_reset(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token); if (ret) { - PMD_INIT_LOG(ERR, "Failure cleaning dpbp device with" - " error code %d\n", ret); + DPAA2_BUS_ERR("Unable to reset buffer pool device. err(%d)", + ret); dpbp_close(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token); rte_free(dpbp_node); return -1; @@ -74,8 +74,6 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused, TAILQ_INSERT_TAIL(&dpbp_dev_list, dpbp_node, next); - RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpbp.%d]\n", dpbp_id); - if (!register_once) { rte_mbuf_set_platform_mempool_ops(DPAA2_MEMPOOL_OPS_NAME); register_once = 1; diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c index fb28e497..5ad0374d 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c @@ -39,13 +39,14 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, struct dpci_attr attr; struct dpci_rx_queue_cfg rx_queue_cfg; struct dpci_rx_queue_attr rx_attr; + struct dpci_tx_queue_attr tx_attr; int ret, i; /* Allocate DPAA2 dpci handle */ dpci_node = rte_malloc(NULL, sizeof(struct dpaa2_dpci_dev), 0); if (!dpci_node) { - PMD_INIT_LOG(ERR, "Memory allocation failed for DPCI Device"); - return -1; + DPAA2_BUS_ERR("Memory allocation failed for DPCI Device"); + return -ENOMEM; } /* Open the dpci object */ @@ -53,43 +54,57 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, ret = dpci_open(&dpci_node->dpci, CMD_PRI_LOW, dpci_id, &dpci_node->token); if (ret) { - PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d", - ret); - rte_free(dpci_node); - return -1; + DPAA2_BUS_ERR("Resource alloc failure with err code: %d", ret); + goto err; } /* Get the device attributes */ ret = dpci_get_attributes(&dpci_node->dpci, CMD_PRI_LOW, dpci_node->token, &attr); if (ret != 0) { - PMD_INIT_LOG(ERR, "Reading device failed with err code: %d", - ret); - rte_free(dpci_node); - return -1; + DPAA2_BUS_ERR("Reading device failed with err code: %d", ret); + goto err; } - /* Set up the Rx Queue */ - memset(&rx_queue_cfg, 0, sizeof(struct dpci_rx_queue_cfg)); - ret = dpci_set_rx_queue(&dpci_node->dpci, - CMD_PRI_LOW, - dpci_node->token, - 0, &rx_queue_cfg); - if (ret) { - PMD_INIT_LOG(ERR, "Setting Rx queue failed with err code: %d", - ret); - rte_free(dpci_node); - return -1; + for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) { + struct dpaa2_queue *rxq; + + memset(&rx_queue_cfg, 0, sizeof(struct dpci_rx_queue_cfg)); + ret = dpci_set_rx_queue(&dpci_node->dpci, + CMD_PRI_LOW, + dpci_node->token, + i, &rx_queue_cfg); + if (ret) { + DPAA2_BUS_ERR("Setting Rx queue failed with err code: %d", + ret); + goto err; + } + + /* Allocate DQ storage for the DPCI Rx queues */ + rxq = &(dpci_node->rx_queue[i]); + rxq->q_storage = rte_malloc("dq_storage", + sizeof(struct queue_storage_info_t), + RTE_CACHE_LINE_SIZE); + if (!rxq->q_storage) { + DPAA2_BUS_ERR("q_storage allocation failed\n"); + ret = -ENOMEM; + goto err; + } + + memset(rxq->q_storage, 0, sizeof(struct queue_storage_info_t)); + ret = dpaa2_alloc_dq_storage(rxq->q_storage); + if (ret) { + DPAA2_BUS_ERR("dpaa2_alloc_dq_storage failed\n"); + goto err; + } } /* Enable the device */ ret = dpci_enable(&dpci_node->dpci, CMD_PRI_LOW, dpci_node->token); if (ret != 0) { - PMD_INIT_LOG(ERR, "Enabling device failed with err code: %d", - ret); - rte_free(dpci_node); - return -1; + DPAA2_BUS_ERR("Enabling device failed with err code: %d", ret); + goto err; } for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) { @@ -99,14 +114,22 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, dpci_node->token, i, &rx_attr); if (ret != 0) { - PMD_INIT_LOG(ERR, - "Reading device failed with err code: %d", - ret); - rte_free(dpci_node); - return -1; + DPAA2_BUS_ERR("Rx queue fetch failed with err code: %d", + ret); + goto err; } + dpci_node->rx_queue[i].fqid = rx_attr.fqid; - dpci_node->queue[i].fqid = rx_attr.fqid; + ret = dpci_get_tx_queue(&dpci_node->dpci, + CMD_PRI_LOW, + dpci_node->token, i, + &tx_attr); + if (ret != 0) { + DPAA2_BUS_ERR("Reading device failed with err code: %d", + ret); + goto err; + } + dpci_node->tx_queue[i].fqid = tx_attr.fqid; } dpci_node->dpci_id = dpci_id; @@ -114,9 +137,20 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, TAILQ_INSERT_TAIL(&dpci_dev_list, dpci_node, next); - RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpci.%d]\n", dpci_id); - return 0; + +err: + for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) { + struct dpaa2_queue *rxq = &(dpci_node->rx_queue[i]); + + if (rxq->q_storage) { + dpaa2_free_dq_storage(rxq->q_storage); + rte_free(rxq->q_storage); + } + } + rte_free(dpci_node); + + return ret; } struct dpaa2_dpci_dev *rte_dpaa2_alloc_dpci_dev(void) diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c index eefde155..99f70be1 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c @@ -101,7 +101,7 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id) snprintf(string, STRING_LEN, "dpio.%d", dpio_id); file = fopen("/proc/interrupts", "r"); if (!file) { - PMD_DRV_LOG(WARNING, "Failed to open /proc/interrupts file\n"); + DPAA2_BUS_WARN("Failed to open /proc/interrupts file"); return; } while (getline(&temp, &len, file) != -1) { @@ -112,8 +112,8 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id) } if (!token) { - PMD_DRV_LOG(WARNING, "Failed to get interrupt id for dpio.%d\n", - dpio_id); + DPAA2_BUS_WARN("Failed to get interrupt id for dpio.%d", + dpio_id); if (temp) free(temp); fclose(file); @@ -125,10 +125,10 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id) cpu_mask, token); ret = system(command); if (ret < 0) - PMD_DRV_LOG(WARNING, - "Failed to affine interrupts on respective core\n"); + DPAA2_BUS_WARN( + "Failed to affine interrupts on respective core"); else - PMD_DRV_LOG(WARNING, " %s command is executed\n", command); + DPAA2_BUS_DEBUG(" %s command is executed", command); free(temp); fclose(file); @@ -143,7 +143,7 @@ static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev) dpio_epoll_fd = epoll_create(1); ret = rte_dpaa2_intr_enable(&dpio_dev->intr_handle, 0); if (ret) { - PMD_DRV_LOG(ERR, "Interrupt registeration failed\n"); + DPAA2_BUS_ERR("Interrupt registeration failed"); return -1; } @@ -166,7 +166,7 @@ static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev) ret = epoll_ctl(dpio_epoll_fd, EPOLL_CTL_ADD, eventfd, &epoll_ev); if (ret < 0) { - PMD_DRV_LOG(ERR, "epoll_ctl failed\n"); + DPAA2_BUS_ERR("epoll_ctl failed"); return -1; } dpio_dev->epoll_fd = dpio_epoll_fd; @@ -185,28 +185,27 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io)); if (!dpio_dev->dpio) { - PMD_INIT_LOG(ERR, "Memory allocation failure\n"); + DPAA2_BUS_ERR("Memory allocation failure"); return -1; } - PMD_DRV_LOG(DEBUG, "Allocated DPIO Portal[%p]", dpio_dev->dpio); dpio_dev->dpio->regs = dpio_dev->mc_portal; if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id, &dpio_dev->token)) { - PMD_INIT_LOG(ERR, "Failed to allocate IO space\n"); + DPAA2_BUS_ERR("Failed to allocate IO space"); free(dpio_dev->dpio); return -1; } if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { - PMD_INIT_LOG(ERR, "Failed to reset dpio\n"); + DPAA2_BUS_ERR("Failed to reset dpio"); dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); free(dpio_dev->dpio); return -1; } if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { - PMD_INIT_LOG(ERR, "Failed to Enable dpio\n"); + DPAA2_BUS_ERR("Failed to Enable dpio"); dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); free(dpio_dev->dpio); return -1; @@ -214,7 +213,7 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token, &attr)) { - PMD_INIT_LOG(ERR, "DPIO Get attribute failed\n"); + DPAA2_BUS_ERR("DPIO Get attribute failed"); dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); free(dpio_dev->dpio); @@ -231,7 +230,7 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) dpio_dev->sw_portal = qbman_swp_init(&p_des); if (dpio_dev->sw_portal == NULL) { - PMD_DRV_LOG(ERR, " QBMan SW Portal Init failed\n"); + DPAA2_BUS_ERR("QBMan SW Portal Init failed"); dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); free(dpio_dev->dpio); return -1; @@ -249,7 +248,7 @@ dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id) if (cpu_id < 0) { cpu_id = rte_get_master_lcore(); if (cpu_id < 0) { - RTE_LOG(ERR, PMD, "\tGetting CPU Index failed\n"); + DPAA2_BUS_ERR("Getting CPU Index failed"); return -1; } } @@ -258,19 +257,19 @@ dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id) */ sdest = dpaa2_core_cluster_sdest(cpu_id); - PMD_DRV_LOG(DEBUG, "Portal= %d CPU= %u SDEST= %d", - dpio_dev->index, cpu_id, sdest); + DPAA2_BUS_DEBUG("Portal= %d CPU= %u SDEST= %d", + dpio_dev->index, cpu_id, sdest); ret = dpio_set_stashing_destination(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token, sdest); if (ret) { - PMD_DRV_LOG(ERR, "%d ERROR in SDEST\n", ret); + DPAA2_BUS_ERR("%d ERROR in SDEST", ret); return -1; } #ifdef RTE_LIBRTE_PMD_DPAA2_EVENTDEV if (dpaa2_dpio_intr_init(dpio_dev)) { - PMD_DRV_LOG(ERR, "Interrupt registration failed for dpio\n"); + DPAA2_BUS_ERR("Interrupt registration failed for dpio"); return -1; } #endif @@ -291,12 +290,12 @@ struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id) if (!dpio_dev) return NULL; - PMD_DRV_LOG(DEBUG, "New Portal=0x%x (%d) affined thread - %lu", - dpio_dev, dpio_dev->index, syscall(SYS_gettid)); + DPAA2_BUS_DEBUG("New Portal %p (%d) affined thread - %lu", + dpio_dev, dpio_dev->index, syscall(SYS_gettid)); ret = dpaa2_configure_stashing(dpio_dev, cpu_id); if (ret) - PMD_DRV_LOG(ERR, "dpaa2_configure_stashing failed"); + DPAA2_BUS_ERR("dpaa2_configure_stashing failed"); return dpio_dev; } @@ -314,8 +313,9 @@ dpaa2_affine_qbman_swp(void) return -1; if (dpaa2_io_portal[lcore_id].dpio_dev) { - PMD_DRV_LOG(INFO, "DPAA Portal=0x%x (%d) is being shared" - " between thread %lu and current %lu", + DPAA2_BUS_DP_INFO("DPAA Portal=%p (%d) is being shared" + " between thread %" PRIu64 " and current " + "%" PRIu64 "\n", dpaa2_io_portal[lcore_id].dpio_dev, dpaa2_io_portal[lcore_id].dpio_dev->index, dpaa2_io_portal[lcore_id].net_tid, @@ -326,7 +326,8 @@ dpaa2_affine_qbman_swp(void) [lcore_id].dpio_dev->ref_count); dpaa2_io_portal[lcore_id].net_tid = tid; - PMD_DRV_LOG(DEBUG, "Old Portal=0x%x (%d) affined thread - %lu", + DPAA2_BUS_DP_DEBUG("Old Portal=%p (%d) affined thread - " + "%" PRIu64 "\n", dpaa2_io_portal[lcore_id].dpio_dev, dpaa2_io_portal[lcore_id].dpio_dev->index, tid); @@ -348,7 +349,7 @@ dpaa2_affine_qbman_swp(void) } int -dpaa2_affine_qbman_swp_sec(void) +dpaa2_affine_qbman_ethrx_swp(void) { unsigned int lcore_id = rte_lcore_id(); uint64_t tid = syscall(SYS_gettid); @@ -359,32 +360,36 @@ dpaa2_affine_qbman_swp_sec(void) else if (lcore_id >= RTE_MAX_LCORE) return -1; - if (dpaa2_io_portal[lcore_id].sec_dpio_dev) { - PMD_DRV_LOG(INFO, "DPAA Portal=0x%x (%d) is being shared" - " between thread %lu and current %lu", - dpaa2_io_portal[lcore_id].sec_dpio_dev, - dpaa2_io_portal[lcore_id].sec_dpio_dev->index, - dpaa2_io_portal[lcore_id].sec_tid, - tid); - RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev - = dpaa2_io_portal[lcore_id].sec_dpio_dev; + if (dpaa2_io_portal[lcore_id].ethrx_dpio_dev) { + DPAA2_BUS_DP_INFO( + "DPAA Portal=%p (%d) is being shared between thread" + " %" PRIu64 " and current %" PRIu64 "\n", + dpaa2_io_portal[lcore_id].ethrx_dpio_dev, + dpaa2_io_portal[lcore_id].ethrx_dpio_dev->index, + dpaa2_io_portal[lcore_id].sec_tid, + tid); + RTE_PER_LCORE(_dpaa2_io).ethrx_dpio_dev + = dpaa2_io_portal[lcore_id].ethrx_dpio_dev; rte_atomic16_inc(&dpaa2_io_portal - [lcore_id].sec_dpio_dev->ref_count); + [lcore_id].ethrx_dpio_dev->ref_count); dpaa2_io_portal[lcore_id].sec_tid = tid; - PMD_DRV_LOG(DEBUG, "Old Portal=0x%x (%d) affined thread - %lu", - dpaa2_io_portal[lcore_id].sec_dpio_dev, - dpaa2_io_portal[lcore_id].sec_dpio_dev->index, - tid); + DPAA2_BUS_DP_DEBUG( + "Old Portal=%p (%d) affined thread" + " - %" PRIu64 "\n", + dpaa2_io_portal[lcore_id].ethrx_dpio_dev, + dpaa2_io_portal[lcore_id].ethrx_dpio_dev->index, + tid); return 0; } /* Populate the dpaa2_io_portal structure */ - dpaa2_io_portal[lcore_id].sec_dpio_dev = dpaa2_get_qbman_swp(lcore_id); + dpaa2_io_portal[lcore_id].ethrx_dpio_dev = + dpaa2_get_qbman_swp(lcore_id); - if (dpaa2_io_portal[lcore_id].sec_dpio_dev) { - RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev - = dpaa2_io_portal[lcore_id].sec_dpio_dev; + if (dpaa2_io_portal[lcore_id].ethrx_dpio_dev) { + RTE_PER_LCORE(_dpaa2_io).ethrx_dpio_dev + = dpaa2_io_portal[lcore_id].ethrx_dpio_dev; dpaa2_io_portal[lcore_id].sec_tid = tid; return 0; } else { @@ -401,15 +406,14 @@ dpaa2_create_dpio_device(int vdev_fd, struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)}; if (obj_info->num_regions < NUM_DPIO_REGIONS) { - PMD_INIT_LOG(ERR, "ERROR, Not sufficient number " - "of DPIO regions.\n"); + DPAA2_BUS_ERR("Not sufficient number of DPIO regions"); return -1; } dpio_dev = rte_malloc(NULL, sizeof(struct dpaa2_dpio_dev), RTE_CACHE_LINE_SIZE); if (!dpio_dev) { - PMD_INIT_LOG(ERR, "Memory allocation failed for DPIO Device\n"); + DPAA2_BUS_ERR("Memory allocation failed for DPIO Device"); return -1; } @@ -421,31 +425,31 @@ dpaa2_create_dpio_device(int vdev_fd, reg_info.index = 0; if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { - PMD_INIT_LOG(ERR, "vfio: error getting region info\n"); + DPAA2_BUS_ERR("vfio: error getting region info"); rte_free(dpio_dev); return -1; } dpio_dev->ce_size = reg_info.size; - dpio_dev->qbman_portal_ce_paddr = (uint64_t)mmap(NULL, reg_info.size, + dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size, PROT_WRITE | PROT_READ, MAP_SHARED, vdev_fd, reg_info.offset); reg_info.index = 1; if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { - PMD_INIT_LOG(ERR, "vfio: error getting region info\n"); + DPAA2_BUS_ERR("vfio: error getting region info"); rte_free(dpio_dev); return -1; } dpio_dev->ci_size = reg_info.size; - dpio_dev->qbman_portal_ci_paddr = (uint64_t)mmap(NULL, reg_info.size, + dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size, PROT_WRITE | PROT_READ, MAP_SHARED, vdev_fd, reg_info.offset); if (configure_dpio_qbman_swp(dpio_dev)) { - PMD_INIT_LOG(ERR, - "Fail to configure the dpio qbman portal for %d\n", + DPAA2_BUS_ERR( + "Fail to configure the dpio qbman portal for %d", dpio_dev->hw_id); rte_free(dpio_dev); return -1; @@ -455,8 +459,8 @@ dpaa2_create_dpio_device(int vdev_fd, dpio_dev->index = io_space_count; if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) { - PMD_INIT_LOG(ERR, "Fail to setup interrupt for %d\n", - dpio_dev->hw_id); + DPAA2_BUS_ERR("Fail to setup interrupt for %d", + dpio_dev->hw_id); rte_free(dpio_dev); } @@ -466,21 +470,20 @@ dpaa2_create_dpio_device(int vdev_fd, if (mc_get_soc_version(dpio_dev->dpio, CMD_PRI_LOW, &mc_plat_info)) { - PMD_INIT_LOG(ERR, "\tmc_get_soc_version failed\n"); + DPAA2_BUS_ERR("Unable to get SoC version information"); } else if ((mc_plat_info.svr & 0xffff0000) == SVR_LS1080A) { dpaa2_core_cluster_base = 0x02; dpaa2_cluster_sz = 4; - PMD_INIT_LOG(DEBUG, "\tLS108x (A53) Platform Detected"); + DPAA2_BUS_DEBUG("LS108x (A53) Platform Detected"); } else if ((mc_plat_info.svr & 0xffff0000) == SVR_LX2160A) { dpaa2_core_cluster_base = 0x00; dpaa2_cluster_sz = 2; - PMD_INIT_LOG(DEBUG, "\tLX2160 Platform Detected"); + DPAA2_BUS_DEBUG("LX2160 Platform Detected"); } dpaa2_svr_family = (mc_plat_info.svr & 0xffff0000); } TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next); - RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpio.%d]\n", object_id); return 0; } diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h index c0bd8782..d593eea7 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h @@ -13,7 +13,7 @@ struct dpaa2_io_portal_t { struct dpaa2_dpio_dev *dpio_dev; - struct dpaa2_dpio_dev *sec_dpio_dev; + struct dpaa2_dpio_dev *ethrx_dpio_dev; uint64_t net_tid; uint64_t sec_tid; void *eventdev; @@ -25,8 +25,8 @@ RTE_DECLARE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io); #define DPAA2_PER_LCORE_DPIO RTE_PER_LCORE(_dpaa2_io).dpio_dev #define DPAA2_PER_LCORE_PORTAL DPAA2_PER_LCORE_DPIO->sw_portal -#define DPAA2_PER_LCORE_SEC_DPIO RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev -#define DPAA2_PER_LCORE_SEC_PORTAL DPAA2_PER_LCORE_SEC_DPIO->sw_portal +#define DPAA2_PER_LCORE_ETHRX_DPIO RTE_PER_LCORE(_dpaa2_io).ethrx_dpio_dev +#define DPAA2_PER_LCORE_ETHRX_PORTAL DPAA2_PER_LCORE_ETHRX_DPIO->sw_portal /* Variable to store DPAA2 platform type */ extern uint32_t dpaa2_svr_family; @@ -39,7 +39,7 @@ struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id); int dpaa2_affine_qbman_swp(void); /* Affine additional DPIO portal to current crypto processing thread */ -int dpaa2_affine_qbman_swp_sec(void); +int dpaa2_affine_qbman_ethrx_swp(void); /* allocate memory for FQ - dq storage */ int diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h index d421dbf0..82075936 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h @@ -142,7 +142,8 @@ struct dpaa2_dpci_dev { uint16_t token; rte_atomic16_t in_use; uint32_t dpci_id; /*HW ID for DPCI object */ - struct dpaa2_queue queue[DPAA2_DPCI_MAX_QUEUES]; + struct dpaa2_queue rx_queue[DPAA2_DPCI_MAX_QUEUES]; + struct dpaa2_queue tx_queue[DPAA2_DPCI_MAX_QUEUES]; }; /*! Global MCP list */ @@ -174,7 +175,7 @@ enum qbman_fd_format { }; /*Macros to define operations on FD*/ #define DPAA2_SET_FD_ADDR(fd, addr) do { \ - (fd)->simple.addr_lo = lower_32_bits((uint64_t)(addr)); \ + (fd)->simple.addr_lo = lower_32_bits((size_t)(addr)); \ (fd)->simple.addr_hi = upper_32_bits((uint64_t)(addr)); \ } while (0) #define DPAA2_SET_FD_LEN(fd, length) ((fd)->simple.len = length) @@ -188,50 +189,55 @@ enum qbman_fd_format { ((fd)->simple.frc = (0x80000000 | (len))) #define DPAA2_GET_FD_FRC_PARSE_SUM(fd) \ ((uint16_t)(((fd)->simple.frc & 0xffff0000) >> 16)) -#define DPAA2_SET_FD_FRC(fd, frc) ((fd)->simple.frc = frc) +#define DPAA2_SET_FD_FRC(fd, _frc) ((fd)->simple.frc = _frc) #define DPAA2_RESET_FD_CTRL(fd) ((fd)->simple.ctrl = 0) #define DPAA2_SET_FD_ASAL(fd, asal) ((fd)->simple.ctrl |= (asal << 16)) #define DPAA2_SET_FD_FLC(fd, addr) do { \ - (fd)->simple.flc_lo = lower_32_bits((uint64_t)(addr)); \ + (fd)->simple.flc_lo = lower_32_bits((size_t)(addr)); \ (fd)->simple.flc_hi = upper_32_bits((uint64_t)(addr)); \ } while (0) #define DPAA2_SET_FLE_INTERNAL_JD(fle, len) ((fle)->frc = (0x80000000 | (len))) #define DPAA2_GET_FLE_ADDR(fle) \ - (uint64_t)((((uint64_t)((fle)->addr_hi)) << 32) + (fle)->addr_lo) + (size_t)((((uint64_t)((fle)->addr_hi)) << 32) + (fle)->addr_lo) #define DPAA2_SET_FLE_ADDR(fle, addr) do { \ - (fle)->addr_lo = lower_32_bits((uint64_t)addr); \ - (fle)->addr_hi = upper_32_bits((uint64_t)addr); \ + (fle)->addr_lo = lower_32_bits((size_t)addr); \ + (fle)->addr_hi = upper_32_bits((uint64_t)addr); \ } while (0) #define DPAA2_GET_FLE_CTXT(fle) \ - (uint64_t)((((uint64_t)((fle)->reserved[1])) << 32) + \ - (fle)->reserved[0]) + ((((uint64_t)((fle)->reserved[1])) << 32) + (fle)->reserved[0]) #define DPAA2_FLE_SAVE_CTXT(fle, addr) do { \ - (fle)->reserved[0] = lower_32_bits((uint64_t)addr); \ - (fle)->reserved[1] = upper_32_bits((uint64_t)addr); \ + (fle)->reserved[0] = lower_32_bits((size_t)addr); \ + (fle)->reserved[1] = upper_32_bits((uint64_t)addr); \ } while (0) #define DPAA2_SET_FLE_OFFSET(fle, offset) \ ((fle)->fin_bpid_offset |= (uint32_t)(offset) << 16) -#define DPAA2_SET_FLE_BPID(fle, bpid) ((fle)->fin_bpid_offset |= (uint64_t)bpid) +#define DPAA2_SET_FLE_LEN(fle, len) ((fle)->length = len) +#define DPAA2_SET_FLE_BPID(fle, bpid) ((fle)->fin_bpid_offset |= (size_t)bpid) #define DPAA2_GET_FLE_BPID(fle) ((fle)->fin_bpid_offset & 0x000000ff) -#define DPAA2_SET_FLE_FIN(fle) ((fle)->fin_bpid_offset |= (uint64_t)1 << 31) +#define DPAA2_SET_FLE_FIN(fle) ((fle)->fin_bpid_offset |= 1 << 31) #define DPAA2_SET_FLE_IVP(fle) (((fle)->fin_bpid_offset |= 0x00004000)) +#define DPAA2_SET_FLE_BMT(fle) (((fle)->fin_bpid_offset |= 0x00008000)) #define DPAA2_SET_FD_COMPOUND_FMT(fd) \ ((fd)->simple.bpid_offset |= (uint32_t)1 << 28) #define DPAA2_GET_FD_ADDR(fd) \ -((uint64_t)((((uint64_t)((fd)->simple.addr_hi)) << 32) + (fd)->simple.addr_lo)) +(((((uint64_t)((fd)->simple.addr_hi)) << 32) + (fd)->simple.addr_lo)) #define DPAA2_GET_FD_LEN(fd) ((fd)->simple.len) #define DPAA2_GET_FD_BPID(fd) (((fd)->simple.bpid_offset & 0x00003FFF)) #define DPAA2_GET_FD_IVP(fd) (((fd)->simple.bpid_offset & 0x00004000) >> 14) #define DPAA2_GET_FD_OFFSET(fd) (((fd)->simple.bpid_offset & 0x0FFF0000) >> 16) +#define DPAA2_GET_FD_FRC(fd) ((fd)->simple.frc) +#define DPAA2_GET_FD_FLC(fd) \ + (((uint64_t)((fd)->simple.flc_hi) << 32) + (fd)->simple.flc_lo) +#define DPAA2_GET_FD_ERR(fd) ((fd)->simple.bpid_offset & 0x000000FF) #define DPAA2_GET_FLE_OFFSET(fle) (((fle)->fin_bpid_offset & 0x0FFF0000) >> 16) #define DPAA2_SET_FLE_SG_EXT(fle) ((fle)->fin_bpid_offset |= (uint64_t)1 << 29) #define DPAA2_IS_SET_FLE_SG_EXT(fle) \ (((fle)->fin_bpid_offset & ((uint64_t)1 << 29)) ? 1 : 0) #define DPAA2_INLINE_MBUF_FROM_BUF(buf, meta_data_size) \ - ((struct rte_mbuf *)((uint64_t)(buf) - (meta_data_size))) + ((struct rte_mbuf *)((size_t)(buf) - (meta_data_size))) #define DPAA2_ASAL_VAL (DPAA2_MBUF_HW_ANNOTATION / 64) @@ -255,47 +261,53 @@ enum qbman_fd_format { */ #define DPAA2_EQ_RESP_ALWAYS 1 +/* Various structures representing contiguous memory maps */ +struct dpaa2_memseg { + TAILQ_ENTRY(dpaa2_memseg) next; + char *vaddr; + rte_iova_t iova; + size_t len; +}; + +TAILQ_HEAD(dpaa2_memseg_list, dpaa2_memseg); +extern struct dpaa2_memseg_list rte_dpaa2_memsegs; + #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA extern uint8_t dpaa2_virt_mode; static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused)); /* todo - this is costly, need to write a fast coversion routine */ static void *dpaa2_mem_ptov(phys_addr_t paddr) { - const struct rte_memseg *memseg; - int i; + struct dpaa2_memseg *ms; if (dpaa2_virt_mode) - return (void *)paddr; - - memseg = rte_eal_get_physmem_layout(); - - for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { - if (paddr >= memseg[i].iova && - (char *)paddr < (char *)memseg[i].iova + memseg[i].len) - return (void *)(memseg[i].addr_64 - + (paddr - memseg[i].iova)); + return (void *)(size_t)paddr; + + /* Check if the address is already part of the memseg list internally + * maintained by the dpaa2 driver. + */ + TAILQ_FOREACH(ms, &rte_dpaa2_memsegs, next) { + if (paddr >= ms->iova && paddr < + ms->iova + ms->len) + return RTE_PTR_ADD(ms->vaddr, (uintptr_t)(paddr - ms->iova)); } - return NULL; + + /* If not, Fallback to full memseg list searching */ + return rte_mem_iova2virt(paddr); } static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused)); static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) { const struct rte_memseg *memseg; - int i; if (dpaa2_virt_mode) return vaddr; - memseg = rte_eal_get_physmem_layout(); - - for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { - if (vaddr >= memseg[i].addr_64 && - vaddr < memseg[i].addr_64 + memseg[i].len) - return memseg[i].iova - + (vaddr - memseg[i].addr_64); - } - return (phys_addr_t)(NULL); + memseg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr, NULL); + if (memseg) + return memseg->phys_addr + RTE_PTR_DIFF(vaddr, memseg->addr); + return (size_t)NULL; } /** @@ -306,28 +318,26 @@ static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) */ #define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_iova) -#define DPAA2_OP_VADDR_TO_IOVA(op) (op->phys_addr) /** * macro to convert Virtual address to IOVA */ -#define DPAA2_VADDR_TO_IOVA(_vaddr) dpaa2_mem_vtop((uint64_t)(_vaddr)) +#define DPAA2_VADDR_TO_IOVA(_vaddr) dpaa2_mem_vtop((size_t)(_vaddr)) /** * macro to convert IOVA to Virtual address */ -#define DPAA2_IOVA_TO_VADDR(_iova) dpaa2_mem_ptov((phys_addr_t)(_iova)) +#define DPAA2_IOVA_TO_VADDR(_iova) dpaa2_mem_ptov((size_t)(_iova)) /** * macro to convert modify the memory containing IOVA to Virtual address */ #define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type) \ - {_mem = (_type)(dpaa2_mem_ptov((phys_addr_t)(_mem))); } + {_mem = (_type)(dpaa2_mem_ptov((size_t)(_mem))); } #else /* RTE_LIBRTE_DPAA2_USE_PHYS_IOVA */ #define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_addr) -#define DPAA2_OP_VADDR_TO_IOVA(op) (op) #define DPAA2_VADDR_TO_IOVA(_vaddr) (_vaddr) #define DPAA2_IOVA_TO_VADDR(_iova) (_iova) #define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type) diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h index 96269ed4..bb60a98f 100644 --- a/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h +++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h @@ -6,8 +6,6 @@ #ifndef _FSL_QBMAN_BASE_H #define _FSL_QBMAN_BASE_H -typedef uint64_t dma_addr_t; - /** * DOC: QBMan basic structures * diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c index e2217335..07145005 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.c +++ b/drivers/bus/fslmc/qbman/qbman_portal.c @@ -122,8 +122,7 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) p->vdq.valid_bit = QB_VALID_BIT; p->dqrr.next_idx = 0; p->dqrr.valid_bit = QB_VALID_BIT; - qman_version = p->desc.qman_version; - if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) { + if ((p->desc.qman_version & 0xFFFF0000) < QMAN_REV_4100) { p->dqrr.dqrr_size = 4; p->dqrr.reset_bug = 1; } else { @@ -553,10 +552,9 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, /* Flush all the cacheline without load/store in between */ eqcr_pi = s->eqcr.pi; - addr_cena = (uint64_t)s->sys.addr_cena; + addr_cena = (size_t)s->sys.addr_cena; for (i = 0; i < num_enqueued; i++) { - dcbf((uint64_t *)(addr_cena + - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); eqcr_pi++; eqcr_pi &= 0xF; } @@ -620,10 +618,9 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, /* Flush all the cacheline without load/store in between */ eqcr_pi = s->eqcr.pi; - addr_cena = (uint64_t)s->sys.addr_cena; + addr_cena = (size_t)s->sys.addr_cena; for (i = 0; i < num_enqueued; i++) { - dcbf((uint64_t *)(addr_cena + - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); eqcr_pi++; eqcr_pi &= 0xF; } @@ -690,7 +687,7 @@ void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, dma_addr_t storage_phys, int stash) { - d->pull.rsp_addr_virt = (uint64_t)storage; + d->pull.rsp_addr_virt = (size_t)storage; if (!storage) { d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT); @@ -749,7 +746,7 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) } d->pull.tok = s->sys.idx + 1; - s->vdq.storage = (void *)d->pull.rsp_addr_virt; + s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt; p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR); memcpy(&p[1], &cl[1], 12); diff --git a/drivers/bus/fslmc/qbman/qbman_portal.h b/drivers/bus/fslmc/qbman/qbman_portal.h index 8bff0b4f..dbea22a1 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.h +++ b/drivers/bus/fslmc/qbman/qbman_portal.h @@ -7,7 +7,6 @@ #include "qbman_sys.h" #include <fsl_qbman_portal.h> -uint32_t qman_version; #define QMAN_REV_4000 0x04000000 #define QMAN_REV_4100 0x04010000 #define QMAN_REV_4101 0x04010001 diff --git a/drivers/bus/fslmc/qbman/qbman_sys.h b/drivers/bus/fslmc/qbman/qbman_sys.h index 846788ea..2bd33ea5 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys.h +++ b/drivers/bus/fslmc/qbman/qbman_sys.h @@ -20,6 +20,9 @@ #include "qbman_sys_decl.h" +#define CENA_WRITE_ENABLE 0 +#define CINH_WRITE_ENABLE 1 + /* Debugging assists */ static inline void __hexdump(unsigned long start, unsigned long end, unsigned long p, size_t sz, const unsigned char *c) @@ -178,7 +181,11 @@ static inline void *qbman_cena_write_start_wo_shadow(struct qbman_swp_sys *s, s->addr_cena, s->idx, offset); #endif QBMAN_BUG_ON(offset & 63); +#ifdef RTE_ARCH_64 return (s->addr_cena + offset); +#else + return (s->addr_cinh + offset); +#endif } static inline void qbman_cena_write_complete(struct qbman_swp_sys *s, @@ -191,11 +198,19 @@ static inline void qbman_cena_write_complete(struct qbman_swp_sys *s, s->addr_cena, s->idx, offset, shadow); hexdump(cmd, 64); #endif +#ifdef RTE_ARCH_64 for (loop = 15; loop >= 1; loop--) __raw_writel(shadow[loop], s->addr_cena + offset + loop * 4); lwsync(); __raw_writel(shadow[0], s->addr_cena + offset); +#else + for (loop = 15; loop >= 1; loop--) + __raw_writel(shadow[loop], s->addr_cinh + + offset + loop * 4); + lwsync(); + __raw_writel(shadow[0], s->addr_cinh + offset); +#endif dcbf(s->addr_cena + offset); } @@ -224,9 +239,15 @@ static inline void *qbman_cena_read(struct qbman_swp_sys *s, uint32_t offset) s->addr_cena, s->idx, offset, shadow); #endif +#ifdef RTE_ARCH_64 for (loop = 0; loop < 16; loop++) shadow[loop] = __raw_readl(s->addr_cena + offset + loop * 4); +#else + for (loop = 0; loop < 16; loop++) + shadow[loop] = __raw_readl(s->addr_cinh + offset + + loop * 4); +#endif #ifdef QBMAN_CENA_TRACE hexdump(shadow, 64); #endif @@ -313,6 +334,11 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, uint8_t dqrr_size) { uint32_t reg; +#ifdef RTE_ARCH_64 + uint8_t wn = CENA_WRITE_ENABLE; +#else + uint8_t wn = CINH_WRITE_ENABLE; +#endif s->addr_cena = d->cena_bar; s->addr_cinh = d->cinh_bar; @@ -333,10 +359,10 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, QBMAN_BUG_ON(reg); #endif if (s->eqcr_mode == qman_eqcr_vb_array) - reg = qbman_set_swp_cfg(dqrr_size, 0, 0, 3, 2, 3, 1, 1, 1, 1, + reg = qbman_set_swp_cfg(dqrr_size, wn, 0, 3, 2, 3, 1, 1, 1, 1, 1, 1); else - reg = qbman_set_swp_cfg(dqrr_size, 0, 1, 3, 2, 2, 1, 1, 1, 1, + reg = qbman_set_swp_cfg(dqrr_size, wn, 1, 3, 2, 2, 1, 1, 1, 1, 1, 1); qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg); reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); diff --git a/drivers/bus/fslmc/qbman/qbman_sys_decl.h b/drivers/bus/fslmc/qbman/qbman_sys_decl.h index f82bb18c..fa6977fe 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys_decl.h +++ b/drivers/bus/fslmc/qbman/qbman_sys_decl.h @@ -15,6 +15,7 @@ /****************/ /* arch assists */ /****************/ +#if defined(RTE_ARCH_ARM64) #define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); } #define lwsync() { asm volatile("dmb st" : : : "memory"); } #define dcbf(p) { asm volatile("dc cvac, %0" : : "r"(p) : "memory"); } @@ -28,3 +29,25 @@ static inline void prefetch_for_store(void *p) { asm volatile("prfm pstl1keep, [%0, #0]" : : "r" (p)); } +#elif defined(RTE_ARCH_ARM) +#define dcbz(p) memset(p, 0, 64) +#define lwsync() { asm volatile("dmb st" : : : "memory"); } +#define dcbf(p) RTE_SET_USED(p) +#define dccivac(p) RTE_SET_USED(p) +#define prefetch_for_load(p) { asm volatile ("pld [%0]" : : "r" (p)); } +#define prefetch_for_store(p) { asm volatile ("pld [%0]" : : "r" (p)); } + +#else +#define dcbz(p) RTE_SET_USED(p) +#define lwsync() +#define dcbf(p) RTE_SET_USED(p) +#define dccivac(p) RTE_SET_USED(p) +static inline void prefetch_for_load(void *p) +{ + RTE_SET_USED(p); +} +static inline void prefetch_for_store(void *p) +{ + RTE_SET_USED(p); +} +#endif diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map index b7db0741..fe45a113 100644 --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map @@ -2,7 +2,6 @@ DPDK_17.05 { global: dpaa2_affine_qbman_swp; - dpaa2_affine_qbman_swp_sec; dpaa2_alloc_dpbp_dev; dpaa2_alloc_dq_storage; dpaa2_free_dpbp_dev; @@ -101,3 +100,19 @@ DPDK_18.02 { rte_fslmc_get_device_count; } DPDK_17.11; + +DPDK_18.05 { + global: + + dpaa2_affine_qbman_ethrx_swp; + dpdmai_close; + dpdmai_disable; + dpdmai_enable; + dpdmai_get_attributes; + dpdmai_get_rx_queue; + dpdmai_get_tx_queue; + dpdmai_open; + dpdmai_set_rx_queue; + rte_dpaa2_free_dpci_dev; + +} DPDK_18.02; diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h index 69d0fec7..cea5b78f 100644 --- a/drivers/bus/fslmc/rte_fslmc.h +++ b/drivers/bus/fslmc/rte_fslmc.h @@ -31,6 +31,7 @@ extern "C" { #include <rte_dev.h> #include <rte_bus.h> #include <rte_tailq.h> +#include <rte_devargs.h> #include <fslmc_vfio.h> @@ -49,6 +50,9 @@ struct rte_dpaa2_driver; TAILQ_HEAD(rte_fslmc_device_list, rte_dpaa2_device); TAILQ_HEAD(rte_fslmc_driver_list, rte_dpaa2_driver); +#define RTE_DEV_TO_FSLMC_CONST(ptr) \ + container_of(ptr, const struct rte_dpaa2_device, device) + extern struct rte_fslmc_bus rte_fslmc_bus; enum rte_dpaa2_dev_type { @@ -61,6 +65,7 @@ enum rte_dpaa2_dev_type { DPAA2_IO, /**< DPIO type device */ DPAA2_CI, /**< DPCI type device */ DPAA2_MPORTAL, /**< DPMCP type device */ + DPAA2_QDMA, /**< DPDMAI type device */ /* Unknown device placeholder */ DPAA2_UNKNOWN, DPAA2_DEVTYPE_MAX, @@ -91,6 +96,7 @@ struct rte_dpaa2_device { union { struct rte_eth_dev *eth_dev; /**< ethernet device */ struct rte_cryptodev *cryptodev; /**< Crypto Device */ + struct rte_rawdev *rawdev; /**< Raw Device */ }; enum rte_dpaa2_dev_type dev_type; /**< Device Type */ uint16_t object_id; /**< DPAA2 Object ID */ @@ -167,8 +173,7 @@ void rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver); /** Helper for DPAA2 device registration from driver (eth, crypto) instance */ #define RTE_PMD_REGISTER_DPAA2(nm, dpaa2_drv) \ -RTE_INIT(dpaa2initfn_ ##nm); \ -static void dpaa2initfn_ ##nm(void) \ +RTE_INIT(dpaa2initfn_ ##nm) \ {\ (dpaa2_drv).driver.name = RTE_STR(nm);\ rte_fslmc_driver_register(&dpaa2_drv); \ @@ -197,8 +202,7 @@ uint32_t rte_fslmc_get_device_count(enum rte_dpaa2_dev_type device_type); /** Helper for DPAA2 object registration */ #define RTE_PMD_REGISTER_DPAA2_OBJECT(nm, dpaa2_obj) \ -RTE_INIT(dpaa2objinitfn_ ##nm); \ -static void dpaa2objinitfn_ ##nm(void) \ +RTE_INIT(dpaa2objinitfn_ ##nm) \ {\ (dpaa2_obj).name = RTE_STR(nm);\ rte_fslmc_object_register(&dpaa2_obj); \ diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile new file mode 100644 index 00000000..3ff3bdb8 --- /dev/null +++ b/drivers/bus/ifpga/Makefile @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_bus_ifpga.a + +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal +LDLIBS += -lrte_rawdev +LDLIBS += -lrte_kvargs + +# versioning export map +EXPORT_MAP := rte_bus_ifpga_version.map + +# library version +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_bus.c +SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_common.c + +# +# Export include files +# +SYMLINK-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)-include += rte_bus_ifpga.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c new file mode 100644 index 00000000..b324872e --- /dev/null +++ b/drivers/bus/ifpga/ifpga_bus.c @@ -0,0 +1,467 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation + */ + +#include <string.h> +#include <inttypes.h> +#include <stdint.h> +#include <stdlib.h> +#include <stdio.h> +#include <sys/queue.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <unistd.h> +#include <fcntl.h> + +#include <rte_errno.h> +#include <rte_bus.h> +#include <rte_per_lcore.h> +#include <rte_memory.h> +#include <rte_memzone.h> +#include <rte_eal.h> +#include <rte_common.h> +#include <rte_devargs.h> +#include <rte_kvargs.h> +#include <rte_alarm.h> +#include <rte_string_fns.h> + +#include "rte_rawdev.h" +#include "rte_rawdev_pmd.h" +#include "rte_bus_ifpga.h" +#include "ifpga_logs.h" +#include "ifpga_common.h" + +int ifpga_bus_logtype; + +/* Forward declaration to access Intel FPGA bus + * on which iFPGA devices are connected + */ +static struct rte_bus rte_ifpga_bus; + +static struct ifpga_afu_dev_list ifpga_afu_dev_list = + TAILQ_HEAD_INITIALIZER(ifpga_afu_dev_list); +static struct ifpga_afu_drv_list ifpga_afu_drv_list = + TAILQ_HEAD_INITIALIZER(ifpga_afu_drv_list); + + +/* register a ifpga bus based driver */ +void rte_ifpga_driver_register(struct rte_afu_driver *driver) +{ + RTE_VERIFY(driver); + + TAILQ_INSERT_TAIL(&ifpga_afu_drv_list, driver, next); +} + +/* un-register a fpga bus based driver */ +void rte_ifpga_driver_unregister(struct rte_afu_driver *driver) +{ + TAILQ_REMOVE(&ifpga_afu_drv_list, driver, next); +} + +static struct rte_afu_device * +ifpga_find_afu_dev(const struct rte_rawdev *rdev, + const struct rte_afu_id *afu_id) +{ + struct rte_afu_device *afu_dev = NULL; + + TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) { + if (afu_dev && + afu_dev->rawdev == rdev && + !ifpga_afu_id_cmp(&afu_dev->id, afu_id)) + return afu_dev; + } + return NULL; +} + +static const char * const valid_args[] = { +#define IFPGA_ARG_NAME "ifpga" + IFPGA_ARG_NAME, +#define IFPGA_ARG_PORT "port" + IFPGA_ARG_PORT, +#define IFPGA_AFU_BTS "afu_bts" + IFPGA_AFU_BTS, + NULL +}; + +/* + * Scan the content of the FPGA bus, and the devices in the devices + * list + */ +static struct rte_afu_device * +ifpga_scan_one(struct rte_rawdev *rawdev, + struct rte_devargs *devargs) +{ + struct rte_kvargs *kvlist = NULL; + struct rte_afu_device *afu_dev = NULL; + struct rte_afu_pr_conf afu_pr_conf; + int ret = 0; + char *path = NULL; + + memset(&afu_pr_conf, 0, sizeof(struct rte_afu_pr_conf)); + + kvlist = rte_kvargs_parse(devargs->args, valid_args); + if (!kvlist) { + IFPGA_BUS_ERR("error when parsing param"); + goto end; + } + + if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) { + if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT, + &rte_ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) { + IFPGA_BUS_ERR("error to parse %s", + IFPGA_ARG_PORT); + goto end; + } + } else { + IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus", + IFPGA_ARG_PORT); + goto end; + } + + if (rte_kvargs_count(kvlist, IFPGA_AFU_BTS) == 1) { + if (rte_kvargs_process(kvlist, IFPGA_AFU_BTS, + &rte_ifpga_get_string_arg, &path) < 0) { + IFPGA_BUS_ERR("Failed to parse %s", + IFPGA_AFU_BTS); + goto end; + } + } else { + IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus", + IFPGA_AFU_BTS); + goto end; + } + + afu_pr_conf.afu_id.uuid.uuid_low = 0; + afu_pr_conf.afu_id.uuid.uuid_high = 0; + afu_pr_conf.pr_enable = path?1:0; + + if (ifpga_find_afu_dev(rawdev, &afu_pr_conf.afu_id)) + goto end; + + afu_dev = calloc(1, sizeof(*afu_dev)); + if (!afu_dev) + goto end; + + afu_dev->device.devargs = devargs; + afu_dev->device.numa_node = SOCKET_ID_ANY; + afu_dev->device.name = devargs->name; + afu_dev->rawdev = rawdev; + afu_dev->id.uuid.uuid_low = 0; + afu_dev->id.uuid.uuid_high = 0; + afu_dev->id.port = afu_pr_conf.afu_id.port; + + if (rawdev->dev_ops && rawdev->dev_ops->dev_info_get) + rawdev->dev_ops->dev_info_get(rawdev, afu_dev); + + if (rawdev->dev_ops && + rawdev->dev_ops->dev_start && + rawdev->dev_ops->dev_start(rawdev)) + goto end; + + strlcpy(afu_pr_conf.bs_path, path, sizeof(afu_pr_conf.bs_path)); + if (rawdev->dev_ops && + rawdev->dev_ops->firmware_load && + rawdev->dev_ops->firmware_load(rawdev, + &afu_pr_conf)){ + IFPGA_BUS_ERR("firmware load error %d\n", ret); + goto end; + } + afu_dev->id.uuid.uuid_low = afu_pr_conf.afu_id.uuid.uuid_low; + afu_dev->id.uuid.uuid_high = afu_pr_conf.afu_id.uuid.uuid_high; + + rte_kvargs_free(kvlist); + free(path); + return afu_dev; + +end: + if (kvlist) + rte_kvargs_free(kvlist); + if (path) + free(path); + if (afu_dev) + free(afu_dev); + + return NULL; +} + +/* + * Scan the content of the FPGA bus, and the devices in the devices + * list + */ +static int +ifpga_scan(void) +{ + struct rte_devargs *devargs; + struct rte_kvargs *kvlist = NULL; + struct rte_rawdev *rawdev = NULL; + char *name = NULL; + char name1[RTE_RAWDEV_NAME_MAX_LEN]; + struct rte_afu_device *afu_dev = NULL; + + /* for FPGA devices we scan the devargs_list populated via cmdline */ + RTE_EAL_DEVARGS_FOREACH(IFPGA_ARG_NAME, devargs) { + if (devargs->bus != &rte_ifpga_bus) + continue; + + kvlist = rte_kvargs_parse(devargs->args, valid_args); + if (!kvlist) { + IFPGA_BUS_ERR("error when parsing param"); + goto end; + } + + if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) { + if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME, + &rte_ifpga_get_string_arg, &name) < 0) { + IFPGA_BUS_ERR("error to parse %s", + IFPGA_ARG_NAME); + goto end; + } + } else { + IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus", + IFPGA_ARG_NAME); + goto end; + } + + memset(name1, 0, sizeof(name1)); + snprintf(name1, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name); + + rawdev = rte_rawdev_pmd_get_named_dev(name1); + if (!rawdev) + goto end; + + afu_dev = ifpga_scan_one(rawdev, devargs); + if (afu_dev != NULL) + TAILQ_INSERT_TAIL(&ifpga_afu_dev_list, afu_dev, next); + } + +end: + if (kvlist) + rte_kvargs_free(kvlist); + if (name) + free(name); + + return 0; +} + +/* + * Match the AFU Driver and AFU Device using the ID Table + */ +static int +rte_afu_match(const struct rte_afu_driver *afu_drv, + const struct rte_afu_device *afu_dev) +{ + const struct rte_afu_uuid *id_table; + + for (id_table = afu_drv->id_table; + ((id_table->uuid_low != 0) && (id_table->uuid_high != 0)); + id_table++) { + /* check if device's identifiers match the driver's ones */ + if ((id_table->uuid_low != afu_dev->id.uuid.uuid_low) || + id_table->uuid_high != + afu_dev->id.uuid.uuid_high) + continue; + + return 1; + } + + return 0; +} + +static int +ifpga_probe_one_driver(struct rte_afu_driver *drv, + struct rte_afu_device *afu_dev) +{ + int ret; + + if (!rte_afu_match(drv, afu_dev)) + /* Match of device and driver failed */ + return 1; + + /* reference driver structure */ + afu_dev->driver = drv; + afu_dev->device.driver = &drv->driver; + + /* call the driver probe() function */ + ret = drv->probe(afu_dev); + if (ret) { + afu_dev->driver = NULL; + afu_dev->device.driver = NULL; + } + + return ret; +} + +static int +ifpga_probe_all_drivers(struct rte_afu_device *afu_dev) +{ + struct rte_afu_driver *drv = NULL; + int ret = 0; + + if (afu_dev == NULL) + return -1; + + /* Check if a driver is already loaded */ + if (afu_dev->driver != NULL) + return 0; + + TAILQ_FOREACH(drv, &ifpga_afu_drv_list, next) { + if (ifpga_probe_one_driver(drv, afu_dev)) { + ret = -1; + break; + } + } + return ret; +} + +/* + * Scan the content of the Intel FPGA bus, and call the probe() function for + * all registered drivers that have a matching entry in its id_table + * for discovered devices. + */ +static int +ifpga_probe(void) +{ + struct rte_afu_device *afu_dev = NULL; + int ret = 0; + + TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) { + if (afu_dev->device.driver) + continue; + + ret = ifpga_probe_all_drivers(afu_dev); + if (ret < 0) + IFPGA_BUS_ERR("failed to initialize %s device\n", + rte_ifpga_device_name(afu_dev)); + } + + return ret; +} + +static int +ifpga_plug(struct rte_device *dev) +{ + return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev)); +} + +static int +ifpga_remove_driver(struct rte_afu_device *afu_dev) +{ + const char *name; + const struct rte_afu_driver *driver; + + name = rte_ifpga_device_name(afu_dev); + if (!afu_dev->device.driver) { + IFPGA_BUS_DEBUG("no driver attach to device %s\n", name); + return 1; + } + + driver = RTE_DRV_TO_AFU_CONST(afu_dev->device.driver); + return driver->remove(afu_dev); +} + +static int +ifpga_unplug(struct rte_device *dev) +{ + struct rte_afu_device *afu_dev = NULL; + struct rte_devargs *devargs = NULL; + int ret; + + if (dev == NULL) + return -EINVAL; + + afu_dev = RTE_DEV_TO_AFU(dev); + if (!afu_dev) + return -ENOENT; + + devargs = dev->devargs; + + ret = ifpga_remove_driver(afu_dev); + if (ret) + return ret; + + TAILQ_REMOVE(&ifpga_afu_dev_list, afu_dev, next); + + rte_devargs_remove(devargs->bus->name, devargs->name); + free(afu_dev); + return 0; + +} + +static struct rte_device * +ifpga_find_device(const struct rte_device *start, + rte_dev_cmp_t cmp, const void *data) +{ + struct rte_afu_device *afu_dev; + + TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) { + if (start && &afu_dev->device == start) { + start = NULL; + continue; + } + if (cmp(&afu_dev->device, data) == 0) + return &afu_dev->device; + } + + return NULL; +} +static int +ifpga_parse(const char *name, void *addr) +{ + int *out = addr; + struct rte_rawdev *rawdev = NULL; + char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN]; + char *c1 = NULL; + char *c2 = NULL; + int port = IFPGA_BUS_DEV_PORT_MAX; + char str_port[8]; + int str_port_len = 0; + int ret; + + memset(str_port, 0, 8); + c1 = strchr(name, '|'); + if (c1 != NULL) { + str_port_len = c1 - name; + c2 = c1 + 1; + } + + if (str_port_len < 8 && + str_port_len > 0) { + memcpy(str_port, name, str_port_len); + ret = sscanf(str_port, "%d", &port); + if (ret == -1) + return 0; + } + + memset(rawdev_name, 0, sizeof(rawdev_name)); + snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", c2); + rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name); + + if ((port < IFPGA_BUS_DEV_PORT_MAX) && + rawdev && + (addr != NULL)) + *out = port; + + if ((port < IFPGA_BUS_DEV_PORT_MAX) && + rawdev) + return 0; + else + return 1; +} + +static struct rte_bus rte_ifpga_bus = { + .scan = ifpga_scan, + .probe = ifpga_probe, + .find_device = ifpga_find_device, + .plug = ifpga_plug, + .unplug = ifpga_unplug, + .parse = ifpga_parse, +}; + +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus); + +RTE_INIT(ifpga_init_log) +{ + ifpga_bus_logtype = rte_log_register("bus.ifpga"); + if (ifpga_bus_logtype >= 0) + rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE); +} diff --git a/drivers/bus/ifpga/ifpga_common.c b/drivers/bus/ifpga/ifpga_common.c new file mode 100644 index 00000000..78e2eaee --- /dev/null +++ b/drivers/bus/ifpga/ifpga_common.c @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation + */ + +#include <string.h> +#include <inttypes.h> +#include <stdint.h> +#include <stdlib.h> +#include <stdio.h> +#include <sys/queue.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <unistd.h> +#include <fcntl.h> + +#include <rte_errno.h> +#include <rte_bus.h> +#include <rte_per_lcore.h> +#include <rte_memory.h> +#include <rte_memzone.h> +#include <rte_eal.h> +#include <rte_common.h> + +#include <rte_devargs.h> +#include <rte_kvargs.h> +#include <rte_alarm.h> + +#include "rte_bus_ifpga.h" +#include "ifpga_logs.h" +#include "ifpga_common.h" + +int rte_ifpga_get_string_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + if (!value || !extra_args) + return -EINVAL; + + *(char **)extra_args = strdup(value); + + if (!*(char **)extra_args) + return -ENOMEM; + + return 0; +} +int rte_ifpga_get_integer32_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + if (!value || !extra_args) + return -EINVAL; + + *(int *)extra_args = strtoull(value, NULL, 0); + + return 0; +} +int ifpga_get_integer64_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + if (!value || !extra_args) + return -EINVAL; + + *(uint64_t *)extra_args = strtoull(value, NULL, 0); + + return 0; +} +int ifpga_get_unsigned_long(const char *str, int base) +{ + unsigned long num; + char *end = NULL; + + errno = 0; + + num = strtoul(str, &end, base); + if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0)) + return -1; + + return num; +} + +int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, + const struct rte_afu_id *afu_id1) +{ + if ((afu_id0->uuid.uuid_low == afu_id1->uuid.uuid_low) && + (afu_id0->uuid.uuid_high == afu_id1->uuid.uuid_high) && + (afu_id0->port == afu_id1->port)) { + return 0; + } else + return 1; +} diff --git a/drivers/bus/ifpga/ifpga_common.h b/drivers/bus/ifpga/ifpga_common.h new file mode 100644 index 00000000..f9254b9d --- /dev/null +++ b/drivers/bus/ifpga/ifpga_common.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation + */ + +#ifndef _IFPGA_COMMON_H_ +#define _IFPGA_COMMON_H_ + +int rte_ifpga_get_string_arg(const char *key __rte_unused, + const char *value, void *extra_args); +int rte_ifpga_get_integer32_arg(const char *key __rte_unused, + const char *value, void *extra_args); +int ifpga_get_integer64_arg(const char *key __rte_unused, + const char *value, void *extra_args); +int ifpga_get_unsigned_long(const char *str, int base); +int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, + const struct rte_afu_id *afu_id1); + +#endif /* _IFPGA_COMMON_H_ */ diff --git a/drivers/bus/ifpga/ifpga_logs.h b/drivers/bus/ifpga/ifpga_logs.h new file mode 100644 index 00000000..873e0a4f --- /dev/null +++ b/drivers/bus/ifpga/ifpga_logs.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation + */ + +#ifndef _IFPGA_LOGS_H_ +#define _IFPGA_LOGS_H_ + +#include <rte_log.h> + +extern int ifpga_bus_logtype; + +#define IFPGA_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \ + __func__, ##args) + +#define IFPGA_BUS_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \ + __func__, ##args) + +#define IFPGA_BUS_FUNC_TRACE() IFPGA_BUS_LOG(DEBUG, ">>") + +#define IFPGA_BUS_DEBUG(fmt, args...) \ + IFPGA_BUS_LOG(DEBUG, fmt, ## args) +#define IFPGA_BUS_INFO(fmt, args...) \ + IFPGA_BUS_LOG(INFO, fmt, ## args) +#define IFPGA_BUS_ERR(fmt, args...) \ + IFPGA_BUS_LOG(ERR, fmt, ## args) +#define IFPGA_BUS_WARN(fmt, args...) \ + IFPGA_BUS_LOG(WARNING, fmt, ## args) + +#endif /* _IFPGA_BUS_LOGS_H_ */ diff --git a/drivers/bus/ifpga/meson.build b/drivers/bus/ifpga/meson.build new file mode 100644 index 00000000..c9b08c86 --- /dev/null +++ b/drivers/bus/ifpga/meson.build @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2018 Intel Corporation + +deps += ['pci', 'kvargs', 'rawdev'] +install_headers('rte_bus_ifpga.h') +sources = files('ifpga_common.c', 'ifpga_bus.c') + +allow_experimental_apis = true diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h new file mode 100644 index 00000000..51d5ae0d --- /dev/null +++ b/drivers/bus/ifpga/rte_bus_ifpga.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation + */ + +#ifndef _RTE_BUS_IFPGA_H_ +#define _RTE_BUS_IFPGA_H_ + +/** + * @file + * + * RTE Intel FPGA Bus Interface + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_bus.h> +#include <rte_pci.h> + +/** Name of Intel FPGA Bus */ +#define IFPGA_BUS_NAME ifpga + +/* Forward declarations */ +struct rte_afu_device; +struct rte_afu_driver; + +/** Double linked list of Intel FPGA AFU device. */ +TAILQ_HEAD(ifpga_afu_dev_list, rte_afu_device); +/** Double linked list of Intel FPGA AFU device drivers. */ +TAILQ_HEAD(ifpga_afu_drv_list, rte_afu_driver); + +#define IFPGA_BUS_BITSTREAM_PATH_MAX_LEN 256 + +struct rte_afu_uuid { + uint64_t uuid_low; + uint64_t uuid_high; +} __attribute__ ((packed)); + +#define IFPGA_BUS_DEV_PORT_MAX 4 + +/** + * A structure describing an ID for a AFU driver. Each driver provides a + * table of these IDs for each device that it supports. + */ +struct rte_afu_id { + struct rte_afu_uuid uuid; + int port; /**< port number */ +} __attribute__ ((packed)); + +/** + * A structure PR (Partial Reconfiguration) configuration AFU driver. + */ + +struct rte_afu_pr_conf { + struct rte_afu_id afu_id; + int pr_enable; + char bs_path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN]; +}; + +#define AFU_PRI_STR_SIZE (PCI_PRI_STR_SIZE + 8) + +/** + * A structure describing a AFU device. + */ +struct rte_afu_device { + TAILQ_ENTRY(rte_afu_device) next; /**< Next in device list. */ + struct rte_device device; /**< Inherit core device */ + struct rte_rawdev *rawdev; /**< Point Rawdev */ + struct rte_afu_id id; /**< AFU id within FPGA. */ + uint32_t num_region; /**< number of regions found */ + struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE]; + /**< AFU Memory Resource */ + struct rte_intr_handle intr_handle; /**< Interrupt handle */ + struct rte_afu_driver *driver; /**< Associated driver */ + char path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN]; +} __attribute__ ((packed)); + +/** + * @internal + * Helper macro for drivers that need to convert to struct rte_afu_device. + */ +#define RTE_DEV_TO_AFU(ptr) \ + container_of(ptr, struct rte_afu_device, device) + +#define RTE_DRV_TO_AFU_CONST(ptr) \ + container_of(ptr, const struct rte_afu_driver, driver) + +/** + * Initialization function for the driver called during FPGA BUS probing. + */ +typedef int (afu_probe_t)(struct rte_afu_device *); + +/** + * Uninitialization function for the driver called during hotplugging. + */ +typedef int (afu_remove_t)(struct rte_afu_device *); + +/** + * A structure describing a AFU device. + */ +struct rte_afu_driver { + TAILQ_ENTRY(rte_afu_driver) next; /**< Next afu driver. */ + struct rte_driver driver; /**< Inherit core driver. */ + afu_probe_t *probe; /**< Device Probe function. */ + afu_remove_t *remove; /**< Device Remove function. */ + const struct rte_afu_uuid *id_table; /**< AFU uuid within FPGA. */ +}; + +static inline const char * +rte_ifpga_device_name(const struct rte_afu_device *afu) +{ + if (afu && afu->device.name) + return afu->device.name; + return NULL; +} + +/** + * Register a ifpga afu device driver. + * + * @param driver + * A pointer to a rte_afu_driver structure describing the driver + * to be registered. + */ +void rte_ifpga_driver_register(struct rte_afu_driver *driver); + +/** + * Unregister a ifpga afu device driver. + * + * @param driver + * A pointer to a rte_afu_driver structure describing the driver + * to be unregistered. + */ +void rte_ifpga_driver_unregister(struct rte_afu_driver *driver); + +#define RTE_PMD_REGISTER_AFU(nm, afudrv)\ +static const char *afudrvinit_ ## nm ## _alias;\ +RTE_INIT(afudrvinitfn_ ##afudrv)\ +{\ + (afudrv).driver.name = RTE_STR(nm);\ + (afudrv).driver.alias = afudrvinit_ ## nm ## _alias;\ + rte_ifpga_driver_register(&afudrv);\ +} \ +RTE_PMD_EXPORT_NAME(nm, __COUNTER__) + +#define RTE_PMD_REGISTER_AFU_ALIAS(nm, alias)\ +static const char *afudrvinit_ ## nm ## _alias = RTE_STR(alias) + +#endif /* _RTE_BUS_IFPGA_H_ */ diff --git a/drivers/bus/ifpga/rte_bus_ifpga_version.map b/drivers/bus/ifpga/rte_bus_ifpga_version.map new file mode 100644 index 00000000..a0279797 --- /dev/null +++ b/drivers/bus/ifpga/rte_bus_ifpga_version.map @@ -0,0 +1,10 @@ +DPDK_18.05 { + global: + + rte_ifpga_get_integer32_arg; + rte_ifpga_get_string_arg; + rte_ifpga_driver_register; + rte_ifpga_driver_unregister; + + local: *; +}; diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build index c6af500e..80de2d91 100644 --- a/drivers/bus/meson.build +++ b/drivers/bus/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2017 Intel Corporation -drivers = ['pci', 'vdev'] +drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus'] std_deps = ['eal'] config_flag_fmt = 'RTE_LIBRTE_@0@_BUS' driver_name_fmt = 'rte_bus_@0@' diff --git a/drivers/bus/pci/Makefile b/drivers/bus/pci/Makefile index f3df1c4c..cf373068 100644 --- a/drivers/bus/pci/Makefile +++ b/drivers/bus/pci/Makefile @@ -1,33 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2017 6WIND S.A. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 6WIND S.A. include $(RTE_SDK)/mk/rte.vars.mk @@ -37,6 +9,7 @@ EXPORT_MAP := rte_bus_pci_version.map CFLAGS := -I$(SRCDIR) $(CFLAGS) CFLAGS += -O3 $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),) SYSTEM := linux @@ -49,6 +22,9 @@ CFLAGS += -I$(RTE_SDK)/drivers/bus/pci/$(SYSTEM) CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal +# memseg walk is not part of stable API yet +CFLAGS += -DALLOW_EXPERIMENTAL_API + LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev -lrte_pci diff --git a/drivers/bus/pci/bsd/Makefile b/drivers/bus/pci/bsd/Makefile index 4450913e..c1b54c05 100644 --- a/drivers/bus/pci/bsd/Makefile +++ b/drivers/bus/pci/bsd/Makefile @@ -1,32 +1,4 @@ -# BSD LICENSE -# -# Copyright(c) 2017 6WIND S.A. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 6WIND S.A. SRCS += pci.c diff --git a/drivers/bus/pci/linux/Makefile b/drivers/bus/pci/linux/Makefile index 77c5f970..96ea1d54 100644 --- a/drivers/bus/pci/linux/Makefile +++ b/drivers/bus/pci/linux/Makefile @@ -1,33 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2017 6WIND S.A. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 6WIND S.A. SRCS += pci.c SRCS += pci_uio.c diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index abde6411..04648ac9 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -15,7 +15,6 @@ #include <rte_memcpy.h> #include <rte_vfio.h> -#include "eal_private.h" #include "eal_filesystem.h" #include "private.h" @@ -33,7 +32,8 @@ extern struct rte_pci_bus rte_pci_bus; static int -pci_get_kernel_driver_by_path(const char *filename, char *dri_name) +pci_get_kernel_driver_by_path(const char *filename, char *dri_name, + size_t len) { int count; char path[PATH_MAX]; @@ -54,7 +54,7 @@ pci_get_kernel_driver_by_path(const char *filename, char *dri_name) name = strrchr(path, '/'); if (name) { - strncpy(dri_name, name + 1, strlen(name + 1) + 1); + strlcpy(dri_name, name + 1, len); return 0; } @@ -116,24 +116,28 @@ rte_pci_unmap_device(struct rte_pci_device *dev) } } -void * -pci_find_max_end_va(void) +static int +find_max_end_va(const struct rte_memseg_list *msl, void *arg) { - const struct rte_memseg *seg = rte_eal_get_physmem_layout(); - const struct rte_memseg *last = seg; - unsigned i = 0; + size_t sz = msl->memseg_arr.len * msl->page_sz; + void *end_va = RTE_PTR_ADD(msl->base_va, sz); + void **max_va = arg; - for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) { - if (seg->addr == NULL) - break; + if (*max_va < end_va) + *max_va = end_va; + return 0; +} - if (seg->addr > last->addr) - last = seg; +void * +pci_find_max_end_va(void) +{ + void *va = NULL; - } - return RTE_PTR_ADD(last->addr, last->len); + rte_memseg_list_walk(find_max_end_va, &va); + return va; } + /* parse one line of the "resource" sysfs file (note that the 'line' * string is modified) */ @@ -310,7 +314,7 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr) /* parse driver */ snprintf(filename, sizeof(filename), "%s/driver", dirname); - ret = pci_get_kernel_driver_by_path(filename, driver); + ret = pci_get_kernel_driver_by_path(filename, driver, sizeof(driver)); if (ret < 0) { RTE_LOG(ERR, EAL, "Fail to get kernel driver\n"); free(dev); diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c index d423e4bb..a7c14421 100644 --- a/drivers/bus/pci/linux/pci_uio.c +++ b/drivers/bus/pci/linux/pci_uio.c @@ -282,22 +282,19 @@ int pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, struct mapped_pci_resource *uio_res, int map_idx) { - int fd; + int fd = -1; char devname[PATH_MAX]; void *mapaddr; struct rte_pci_addr *loc; struct pci_map *maps; + int wc_activate = 0; + + if (dev->driver != NULL) + wc_activate = dev->driver->drv_flags & RTE_PCI_DRV_WC_ACTIVATE; loc = &dev->addr; maps = uio_res->maps; - /* update devname for mmap */ - snprintf(devname, sizeof(devname), - "%s/" PCI_PRI_FMT "/resource%d", - rte_pci_get_sysfs_path(), - loc->domain, loc->bus, loc->devid, - loc->function, res_idx); - /* allocate memory to keep path */ maps[map_idx].path = rte_malloc(NULL, strlen(devname) + 1, 0); if (maps[map_idx].path == NULL) { @@ -309,11 +306,37 @@ pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, /* * open resource file, to mmap it */ - fd = open(devname, O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + if (wc_activate) { + /* update devname for mmap */ + snprintf(devname, sizeof(devname), + "%s/" PCI_PRI_FMT "/resource%d_wc", + rte_pci_get_sysfs_path(), + loc->domain, loc->bus, loc->devid, + loc->function, res_idx); + + if (access(devname, R_OK|W_OK) != -1) { + fd = open(devname, O_RDWR); + if (fd < 0) + RTE_LOG(INFO, EAL, "%s cannot be mapped. " + "Fall-back to non prefetchable mode.\n", + devname); + } + } + + if (!wc_activate || fd < 0) { + snprintf(devname, sizeof(devname), + "%s/" PCI_PRI_FMT "/resource%d", + rte_pci_get_sysfs_path(), + loc->domain, loc->bus, loc->devid, + loc->function, res_idx); + + /* then try to map resource file */ + fd = open(devname, O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", devname, strerror(errno)); - goto error; + goto error; + } } /* try mapping somewhere close to the end of hugepages */ diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c index aeeaa9ed..686386d6 100644 --- a/drivers/bus/pci/linux/pci_vfio.c +++ b/drivers/bus/pci/linux/pci_vfio.c @@ -584,6 +584,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev) dev->mem_resource[i].addr = maps[i].addr; } + /* we need save vfio_dev_fd, so it can be used during release */ + dev->intr_handle.vfio_dev_fd = vfio_dev_fd; + return 0; err_vfio_dev_fd: close(vfio_dev_fd); @@ -603,22 +606,58 @@ pci_vfio_map_resource(struct rte_pci_device *dev) return pci_vfio_map_resource_secondary(dev); } -int -pci_vfio_unmap_resource(struct rte_pci_device *dev) +static struct mapped_pci_resource * +find_and_unmap_vfio_resource(struct mapped_pci_res_list *vfio_res_list, + struct rte_pci_device *dev, + const char *pci_addr) +{ + struct mapped_pci_resource *vfio_res = NULL; + struct pci_map *maps; + int i; + + /* Get vfio_res */ + TAILQ_FOREACH(vfio_res, vfio_res_list, next) { + if (rte_pci_addr_cmp(&vfio_res->pci_addr, &dev->addr)) + continue; + break; + } + + if (vfio_res == NULL) + return vfio_res; + + RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n", + pci_addr); + + maps = vfio_res->maps; + for (i = 0; i < (int) vfio_res->nb_maps; i++) { + + /* + * We do not need to be aware of MSI-X table BAR mappings as + * when mapping. Just using current maps array is enough + */ + if (maps[i].addr) { + RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n", + pci_addr, maps[i].addr); + pci_unmap_resource(maps[i].addr, maps[i].size); + } + } + + return vfio_res; +} + +static int +pci_vfio_unmap_resource_primary(struct rte_pci_device *dev) { char pci_addr[PATH_MAX] = {0}; struct rte_pci_addr *loc = &dev->addr; - int i, ret; struct mapped_pci_resource *vfio_res = NULL; struct mapped_pci_res_list *vfio_res_list; - - struct pci_map *maps; + int ret; /* store PCI address string */ snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, loc->domain, loc->bus, loc->devid, loc->function); - if (close(dev->intr_handle.fd) < 0) { RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n", pci_addr); @@ -639,13 +678,10 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev) return ret; } - vfio_res_list = RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); - /* Get vfio_res */ - TAILQ_FOREACH(vfio_res, vfio_res_list, next) { - if (memcmp(&vfio_res->pci_addr, &dev->addr, sizeof(dev->addr))) - continue; - break; - } + vfio_res_list = + RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); + vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr); + /* if we haven't found our tailq entry, something's wrong */ if (vfio_res == NULL) { RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n", @@ -653,30 +689,56 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev) return -1; } - /* unmap BARs */ - maps = vfio_res->maps; + TAILQ_REMOVE(vfio_res_list, vfio_res, next); - RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n", - pci_addr); - for (i = 0; i < (int) vfio_res->nb_maps; i++) { + return 0; +} - /* - * We do not need to be aware of MSI-X table BAR mappings as - * when mapping. Just using current maps array is enough - */ - if (maps[i].addr) { - RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n", - pci_addr, maps[i].addr); - pci_unmap_resource(maps[i].addr, maps[i].size); - } +static int +pci_vfio_unmap_resource_secondary(struct rte_pci_device *dev) +{ + char pci_addr[PATH_MAX] = {0}; + struct rte_pci_addr *loc = &dev->addr; + struct mapped_pci_resource *vfio_res = NULL; + struct mapped_pci_res_list *vfio_res_list; + int ret; + + /* store PCI address string */ + snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, + loc->domain, loc->bus, loc->devid, loc->function); + + ret = rte_vfio_release_device(rte_pci_get_sysfs_path(), pci_addr, + dev->intr_handle.vfio_dev_fd); + if (ret < 0) { + RTE_LOG(ERR, EAL, + "%s(): cannot release device\n", __func__); + return ret; } - TAILQ_REMOVE(vfio_res_list, vfio_res, next); + vfio_res_list = + RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); + vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr); + + /* if we haven't found our tailq entry, something's wrong */ + if (vfio_res == NULL) { + RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n", + pci_addr); + return -1; + } return 0; } int +pci_vfio_unmap_resource(struct rte_pci_device *dev) +{ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + return pci_vfio_unmap_resource_primary(dev); + else + return pci_vfio_unmap_resource_secondary(dev); +} + +int pci_vfio_ioport_map(struct rte_pci_device *dev, int bar, struct rte_pci_ioport *p) { diff --git a/drivers/bus/pci/meson.build b/drivers/bus/pci/meson.build index 12756a4d..72939e59 100644 --- a/drivers/bus/pci/meson.build +++ b/drivers/bus/pci/meson.build @@ -14,3 +14,6 @@ else sources += files('bsd/pci.c') includes += include_directories('bsd') endif + +# memseg walk is not part of stable API yet +allow_experimental_apis = true diff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c index 2a00f365..7736b3f9 100644 --- a/drivers/bus/pci/pci_common.c +++ b/drivers/bus/pci/pci_common.c @@ -26,6 +26,7 @@ #include "private.h" + extern struct rte_pci_bus rte_pci_bus; #define SYSFS_PCI_DEVICES "/sys/bus/pci/devices" @@ -45,12 +46,8 @@ static struct rte_devargs *pci_devargs_lookup(struct rte_pci_device *dev) { struct rte_devargs *devargs; struct rte_pci_addr addr; - struct rte_bus *pbus; - pbus = rte_bus_find_by_name("pci"); - TAILQ_FOREACH(devargs, &devargs_list, next) { - if (devargs->bus != pbus) - continue; + RTE_EAL_DEVARGS_FOREACH("pci", devargs) { devargs->bus->parse(devargs->name, &addr); if (!rte_pci_addr_cmp(&dev->addr, &addr)) return devargs; @@ -73,7 +70,7 @@ pci_name_set(struct rte_pci_device *dev) */ if (devargs != NULL) /* If an rte_devargs exists, the generic rte_device uses the - * given name as its namea + * given name as its name. */ dev->device.name = dev->device.devargs->name; else @@ -159,17 +156,24 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, RTE_LOG(INFO, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id, dev->id.device_id, dr->driver.name); + /* + * reference driver structure + * This needs to be before rte_pci_map_device(), as it enables to use + * driver flags for adjusting configuration. + */ + dev->driver = dr; + dev->device.driver = &dr->driver; + if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) { /* map resources for devices that use igb_uio */ ret = rte_pci_map_device(dev); - if (ret != 0) + if (ret != 0) { + dev->driver = NULL; + dev->device.driver = NULL; return ret; + } } - /* reference driver structure */ - dev->driver = dr; - dev->device.driver = &dr->driver; - /* call the driver probe() function */ ret = dr->probe(dr, dev); if (ret) { @@ -259,81 +263,6 @@ pci_probe_all_drivers(struct rte_pci_device *dev) } /* - * Find the pci device specified by pci address, then invoke probe function of - * the driver of the device. - */ -int -rte_pci_probe_one(const struct rte_pci_addr *addr) -{ - struct rte_pci_device *dev = NULL; - - int ret = 0; - - if (addr == NULL) - return -1; - - /* update current pci device in global list, kernel bindings might have - * changed since last time we looked at it. - */ - if (pci_update_device(addr) < 0) - goto err_return; - - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (rte_pci_addr_cmp(&dev->addr, addr)) - continue; - - ret = pci_probe_all_drivers(dev); - if (ret) - goto err_return; - return 0; - } - return -1; - -err_return: - RTE_LOG(WARNING, EAL, - "Requested device " PCI_PRI_FMT " cannot be used\n", - addr->domain, addr->bus, addr->devid, addr->function); - return -1; -} - -/* - * Detach device specified by its pci address. - */ -int -rte_pci_detach(const struct rte_pci_addr *addr) -{ - struct rte_pci_device *dev = NULL; - int ret = 0; - - if (addr == NULL) - return -1; - - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (rte_pci_addr_cmp(&dev->addr, addr)) - continue; - - ret = rte_pci_detach_dev(dev); - if (ret < 0) - /* negative value is an error */ - goto err_return; - if (ret > 0) - /* positive value means driver doesn't support it */ - continue; - - rte_pci_remove_device(dev); - free(dev); - return 0; - } - return -1; - -err_return: - RTE_LOG(WARNING, EAL, "Requested device " PCI_PRI_FMT - " cannot be used\n", dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - return -1; -} - -/* * Scan the content of the PCI bus, and call the probe() function for * all registered drivers that have a matching entry in its id_table * for discovered devices. @@ -449,7 +378,7 @@ rte_pci_insert_device(struct rte_pci_device *exist_pci_dev, } /* Remove a device from PCI bus */ -void +static void rte_pci_remove_device(struct rte_pci_device *pci_dev) { TAILQ_REMOVE(&rte_pci_bus.device_list, pci_dev, next); @@ -459,17 +388,20 @@ static struct rte_device * pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, const void *data) { - struct rte_pci_device *dev; + const struct rte_pci_device *pstart; + struct rte_pci_device *pdev; - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (start && &dev->device == start) { - start = NULL; /* starting point found */ - continue; - } - if (cmp(&dev->device, data) == 0) - return &dev->device; + if (start != NULL) { + pstart = RTE_DEV_TO_PCI_CONST(start); + pdev = TAILQ_NEXT(pstart, next); + } else { + pdev = TAILQ_FIRST(&rte_pci_bus.device_list); + } + while (pdev != NULL) { + if (cmp(&pdev->device, data) == 0) + return &pdev->device; + pdev = TAILQ_NEXT(pdev, next); } - return NULL; } diff --git a/drivers/bus/pci/private.h b/drivers/bus/pci/private.h index 88fa587e..8ddd03e1 100644 --- a/drivers/bus/pci/private.h +++ b/drivers/bus/pci/private.h @@ -33,36 +33,6 @@ rte_pci_probe(void); int rte_pci_scan(void); /** - * Probe the single PCI device. - * - * Scan the content of the PCI bus, and find the pci device specified by pci - * address, then call the probe() function for registered driver that has a - * matching entry in its id_table for discovered device. - * - * @param addr - * The PCI Bus-Device-Function address to probe. - * @return - * - 0 on success. - * - Negative on error. - */ -int rte_pci_probe_one(const struct rte_pci_addr *addr); - -/** - * Close the single PCI device. - * - * Scan the content of the PCI bus, and find the pci device specified by pci - * address, then call the remove() function for registered driver that has a - * matching entry in its id_table for discovered device. - * - * @param addr - * The PCI Bus-Device-Function address to close. - * @return - * - 0 on success. - * - Negative on error. - */ -int rte_pci_detach(const struct rte_pci_addr *addr); - -/** * Find the name of a PCI device. */ void @@ -94,16 +64,6 @@ void rte_pci_insert_device(struct rte_pci_device *exist_pci_dev, struct rte_pci_device *new_pci_dev); /** - * Remove a PCI device from the PCI Bus. This sets to NULL the bus references - * in the PCI device object as well as the generic device object. - * - * @param pci_device - * PCI device to be removed from PCI Bus - * @return void - */ -void rte_pci_remove_device(struct rte_pci_device *pci_device); - -/** * Update a pci device object by asking the kernel for the latest information. * * This function is private to EAL. @@ -117,16 +77,6 @@ void rte_pci_remove_device(struct rte_pci_device *pci_device); int pci_update_device(const struct rte_pci_addr *addr); /** - * Unbind kernel driver for this device - * - * This function is private to EAL. - * - * @return - * 0 on success, negative on error - */ -int pci_unbind_kernel_driver(struct rte_pci_device *dev); - -/** * Map the PCI resource of a PCI device in virtual memory * * This function is private to EAL. diff --git a/drivers/bus/pci/rte_bus_pci.h b/drivers/bus/pci/rte_bus_pci.h index 357afb91..0d1955ff 100644 --- a/drivers/bus/pci/rte_bus_pci.h +++ b/drivers/bus/pci/rte_bus_pci.h @@ -74,6 +74,9 @@ struct rte_pci_device { */ #define RTE_DEV_TO_PCI(ptr) container_of(ptr, struct rte_pci_device, device) +#define RTE_DEV_TO_PCI_CONST(ptr) \ + container_of(ptr, const struct rte_pci_device, device) + #define RTE_ETH_DEV_TO_PCI(eth_dev) RTE_DEV_TO_PCI((eth_dev)->device) /** Any PCI device identifier (vendor, device, ...) */ @@ -132,6 +135,8 @@ struct rte_pci_bus { /** Device needs PCI BAR mapping (done with either IGB_UIO or VFIO) */ #define RTE_PCI_DRV_NEED_MAPPING 0x0001 +/** Device needs PCI BAR mapping with enabled write combining (wc) */ +#define RTE_PCI_DRV_WC_ACTIVATE 0x0002 /** Device driver supports link state interrupt */ #define RTE_PCI_DRV_INTR_LSC 0x0008 /** Device driver supports device removal interrupt */ @@ -186,8 +191,7 @@ void rte_pci_register(struct rte_pci_driver *driver); /** Helper for PCI device registration from driver (eth, crypto) instance */ #define RTE_PMD_REGISTER_PCI(nm, pci_drv) \ -RTE_INIT(pciinitfn_ ##nm); \ -static void pciinitfn_ ##nm(void) \ +RTE_INIT(pciinitfn_ ##nm) \ {\ (pci_drv).driver.name = RTE_STR(nm);\ rte_pci_register(&pci_drv); \ diff --git a/drivers/bus/vdev/Makefile b/drivers/bus/vdev/Makefile index 24d424a3..bd0bb895 100644 --- a/drivers/bus/vdev/Makefile +++ b/drivers/bus/vdev/Makefile @@ -10,6 +10,7 @@ LIB = librte_bus_vdev.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API # versioning export map EXPORT_MAP := rte_bus_vdev_version.map diff --git a/drivers/bus/vdev/meson.build b/drivers/bus/vdev/meson.build index 03cb87eb..2ee648b4 100644 --- a/drivers/bus/vdev/meson.build +++ b/drivers/bus/vdev/meson.build @@ -3,3 +3,5 @@ sources = files('vdev.c') install_headers('rte_bus_vdev.h') + +allow_experimental_apis = true diff --git a/drivers/bus/vdev/rte_bus_vdev.h b/drivers/bus/vdev/rte_bus_vdev.h index f9d8a238..9ae3eaae 100644 --- a/drivers/bus/vdev/rte_bus_vdev.h +++ b/drivers/bus/vdev/rte_bus_vdev.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016 RehiveTech. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of RehiveTech nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016 RehiveTech. All rights reserved. */ #ifndef RTE_VDEV_H @@ -53,6 +25,9 @@ struct rte_vdev_device { #define RTE_DEV_TO_VDEV(ptr) \ container_of(ptr, struct rte_vdev_device, device) +#define RTE_DEV_TO_VDEV_CONST(ptr) \ + container_of(ptr, const struct rte_vdev_device, device) + static inline const char * rte_vdev_device_name(const struct rte_vdev_device *dev) { @@ -111,9 +86,8 @@ void rte_vdev_register(struct rte_vdev_driver *driver); void rte_vdev_unregister(struct rte_vdev_driver *driver); #define RTE_PMD_REGISTER_VDEV(nm, vdrv)\ -RTE_INIT(vdrvinitfn_ ##vdrv);\ static const char *vdrvinit_ ## nm ## _alias;\ -static void vdrvinitfn_ ##vdrv(void)\ +RTE_INIT(vdrvinitfn_ ##vdrv)\ {\ (vdrv).driver.name = RTE_STR(nm);\ (vdrv).driver.alias = vdrvinit_ ## nm ## _alias;\ diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c index e4bc7246..6139dd55 100644 --- a/drivers/bus/vdev/vdev.c +++ b/drivers/bus/vdev/vdev.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016 RehiveTech. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of RehiveTech nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016 RehiveTech. All rights reserved. */ #include <string.h> @@ -46,11 +18,14 @@ #include <rte_memory.h> #include <rte_tailq.h> #include <rte_spinlock.h> +#include <rte_string_fns.h> #include <rte_errno.h> #include "rte_bus_vdev.h" #include "vdev_logs.h" +#define VDEV_MP_KEY "bus_vdev_mp" + int vdev_logtype_bus; /* Forward declare to access virtual bus name */ @@ -61,6 +36,10 @@ TAILQ_HEAD(vdev_device_list, rte_vdev_device); static struct vdev_device_list vdev_device_list = TAILQ_HEAD_INITIALIZER(vdev_device_list); +/* The lock needs to be recursive because a vdev can manage another vdev. */ +static rte_spinlock_recursive_t vdev_device_list_lock = + RTE_SPINLOCK_RECURSIVE_INITIALIZER; + struct vdev_driver_list vdev_driver_list = TAILQ_HEAD_INITIALIZER(vdev_driver_list); @@ -165,7 +144,7 @@ vdev_probe_all_drivers(struct rte_vdev_device *dev) name = rte_vdev_device_name(dev); - VDEV_LOG(DEBUG, "Search driver %s to probe device %s\n", name, + VDEV_LOG(DEBUG, "Search driver %s to probe device %s", name, rte_vdev_device_name(dev)); if (vdev_parse(name, &driver)) @@ -177,6 +156,7 @@ vdev_probe_all_drivers(struct rte_vdev_device *dev) return ret; } +/* The caller shall be responsible for thread-safe */ static struct rte_vdev_device * find_vdev(const char *name) { @@ -188,7 +168,7 @@ find_vdev(const char *name) TAILQ_FOREACH(dev, &vdev_device_list, next) { const char *devname = rte_vdev_device_name(dev); - if (!strncmp(devname, name, strlen(name))) + if (!strcmp(devname, name)) return dev; } @@ -221,8 +201,8 @@ alloc_devargs(const char *name, const char *args) return devargs; } -int -rte_vdev_init(const char *name, const char *args) +static int +insert_vdev(const char *name, const char *args, struct rte_vdev_device **p_dev) { struct rte_vdev_device *dev; struct rte_devargs *devargs; @@ -231,10 +211,6 @@ rte_vdev_init(const char *name, const char *args) if (name == NULL) return -EINVAL; - dev = find_vdev(name); - if (dev) - return -EEXIST; - devargs = alloc_devargs(name, args); if (!devargs) return -ENOMEM; @@ -249,18 +225,18 @@ rte_vdev_init(const char *name, const char *args) dev->device.numa_node = SOCKET_ID_ANY; dev->device.name = devargs->name; - ret = vdev_probe_all_drivers(dev); - if (ret) { - if (ret > 0) - VDEV_LOG(ERR, "no driver found for %s\n", name); + if (find_vdev(name)) { + ret = -EEXIST; goto fail; } - TAILQ_INSERT_TAIL(&devargs_list, devargs, next); - TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); - return 0; + rte_devargs_insert(devargs); + + if (p_dev) + *p_dev = dev; + return 0; fail: free(devargs->args); free(devargs); @@ -268,6 +244,31 @@ fail: return ret; } +int +rte_vdev_init(const char *name, const char *args) +{ + struct rte_vdev_device *dev; + struct rte_devargs *devargs; + int ret; + + rte_spinlock_recursive_lock(&vdev_device_list_lock); + ret = insert_vdev(name, args, &dev); + if (ret == 0) { + ret = vdev_probe_all_drivers(dev); + if (ret) { + if (ret > 0) + VDEV_LOG(ERR, "no driver found for %s", name); + /* If fails, remove it from vdev list */ + devargs = dev->device.devargs; + TAILQ_REMOVE(&vdev_device_list, dev, next); + rte_devargs_remove(devargs->bus->name, devargs->name); + free(dev); + } + } + rte_spinlock_recursive_unlock(&vdev_device_list_lock); + return ret; +} + static int vdev_remove_driver(struct rte_vdev_device *dev) { @@ -275,7 +276,7 @@ vdev_remove_driver(struct rte_vdev_device *dev) const struct rte_vdev_driver *driver; if (!dev->device.driver) { - VDEV_LOG(DEBUG, "no driver attach to device %s\n", name); + VDEV_LOG(DEBUG, "no driver attach to device %s", name); return 1; } @@ -294,23 +295,98 @@ rte_vdev_uninit(const char *name) if (name == NULL) return -EINVAL; - dev = find_vdev(name); - if (!dev) - return -ENOENT; + rte_spinlock_recursive_lock(&vdev_device_list_lock); - devargs = dev->device.devargs; + dev = find_vdev(name); + if (!dev) { + ret = -ENOENT; + goto unlock; + } ret = vdev_remove_driver(dev); if (ret) - return ret; + goto unlock; TAILQ_REMOVE(&vdev_device_list, dev, next); + devargs = dev->device.devargs; + rte_devargs_remove(devargs->bus->name, devargs->name); + free(dev); - TAILQ_REMOVE(&devargs_list, devargs, next); +unlock: + rte_spinlock_recursive_unlock(&vdev_device_list_lock); + return ret; +} + +struct vdev_param { +#define VDEV_SCAN_REQ 1 +#define VDEV_SCAN_ONE 2 +#define VDEV_SCAN_REP 3 + int type; + int num; + char name[RTE_DEV_NAME_MAX_LEN]; +}; + +static int vdev_plug(struct rte_device *dev); + +/** + * This function works as the action for both primary and secondary process + * for static vdev discovery when a secondary process is booting. + * + * step 1, secondary process sends a sync request to ask for vdev in primary; + * step 2, primary process receives the request, and send vdevs one by one; + * step 3, primary process sends back reply, which indicates how many vdevs + * are sent. + */ +static int +vdev_action(const struct rte_mp_msg *mp_msg, const void *peer) +{ + struct rte_vdev_device *dev; + struct rte_mp_msg mp_resp; + struct vdev_param *ou = (struct vdev_param *)&mp_resp.param; + const struct vdev_param *in = (const struct vdev_param *)mp_msg->param; + const char *devname; + int num; + + strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name)); + mp_resp.len_param = sizeof(*ou); + mp_resp.num_fds = 0; + + switch (in->type) { + case VDEV_SCAN_REQ: + ou->type = VDEV_SCAN_ONE; + ou->num = 1; + num = 0; + + rte_spinlock_recursive_lock(&vdev_device_list_lock); + TAILQ_FOREACH(dev, &vdev_device_list, next) { + devname = rte_vdev_device_name(dev); + if (strlen(devname) == 0) { + VDEV_LOG(INFO, "vdev with no name is not sent"); + continue; + } + VDEV_LOG(INFO, "send vdev, %s", devname); + strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN); + if (rte_mp_sendmsg(&mp_resp) < 0) + VDEV_LOG(ERR, "send vdev, %s, failed, %s", + devname, strerror(rte_errno)); + num++; + } + rte_spinlock_recursive_unlock(&vdev_device_list_lock); + + ou->type = VDEV_SCAN_REP; + ou->num = num; + if (rte_mp_reply(&mp_resp, peer) < 0) + VDEV_LOG(ERR, "Failed to reply a scan request"); + break; + case VDEV_SCAN_ONE: + VDEV_LOG(INFO, "receive vdev, %s", in->name); + if (insert_vdev(in->name, NULL, NULL) < 0) + VDEV_LOG(ERR, "failed to add vdev, %s", in->name); + break; + default: + VDEV_LOG(ERR, "vdev cannot recognize this message"); + } - free(devargs->args); - free(devargs); - free(dev); return 0; } @@ -321,13 +397,41 @@ vdev_scan(void) struct rte_devargs *devargs; struct vdev_custom_scan *custom_scan; + if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 && + rte_errno != EEXIST) { + VDEV_LOG(ERR, "Failed to add vdev mp action"); + return -1; + } + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + struct rte_mp_msg mp_req, *mp_rep; + struct rte_mp_reply mp_reply; + struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; + struct vdev_param *req = (struct vdev_param *)mp_req.param; + struct vdev_param *resp; + + strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name)); + mp_req.len_param = sizeof(*req); + mp_req.num_fds = 0; + req->type = VDEV_SCAN_REQ; + if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && + mp_reply.nb_received == 1) { + mp_rep = &mp_reply.msgs[0]; + resp = (struct vdev_param *)mp_rep->param; + VDEV_LOG(INFO, "Received %d vdevs", resp->num); + } else + VDEV_LOG(ERR, "Failed to request vdev from primary"); + + /* Fall through to allow private vdevs in secondary process */ + } + /* call custom scan callbacks if any */ rte_spinlock_lock(&vdev_custom_scan_lock); TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) { if (custom_scan->callback != NULL) /* * the callback should update devargs list - * by calling rte_eal_devargs_insert() with + * by calling rte_devargs_insert() with * devargs.bus = rte_bus_find_by_name("vdev"); * devargs.type = RTE_DEVTYPE_VIRTUAL; * devargs.policy = RTE_DEV_WHITELISTED; @@ -337,24 +441,27 @@ vdev_scan(void) rte_spinlock_unlock(&vdev_custom_scan_lock); /* for virtual devices we scan the devargs_list populated via cmdline */ - TAILQ_FOREACH(devargs, &devargs_list, next) { - - if (devargs->bus != &rte_vdev_bus) - continue; - - dev = find_vdev(devargs->name); - if (dev) - continue; + RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { dev = calloc(1, sizeof(*dev)); if (!dev) return -1; + rte_spinlock_recursive_lock(&vdev_device_list_lock); + + if (find_vdev(devargs->name)) { + rte_spinlock_recursive_unlock(&vdev_device_list_lock); + free(dev); + continue; + } + dev->device.devargs = devargs; dev->device.numa_node = SOCKET_ID_ANY; dev->device.name = devargs->name; TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); + + rte_spinlock_recursive_unlock(&vdev_device_list_lock); } return 0; @@ -368,12 +475,16 @@ vdev_probe(void) /* call the init function for each virtual device */ TAILQ_FOREACH(dev, &vdev_device_list, next) { + /* we don't use the vdev lock here, as it's only used in DPDK + * initialization; and we don't want to hold such a lock when + * we call each driver probe. + */ if (dev->device.driver) continue; if (vdev_probe_all_drivers(dev)) { - VDEV_LOG(ERR, "failed to initialize %s device\n", + VDEV_LOG(ERR, "failed to initialize %s device", rte_vdev_device_name(dev)); ret = -1; } @@ -386,17 +497,24 @@ static struct rte_device * vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, const void *data) { + const struct rte_vdev_device *vstart; struct rte_vdev_device *dev; - TAILQ_FOREACH(dev, &vdev_device_list, next) { - if (start && &dev->device == start) { - start = NULL; - continue; - } + rte_spinlock_recursive_lock(&vdev_device_list_lock); + if (start != NULL) { + vstart = RTE_DEV_TO_VDEV_CONST(start); + dev = TAILQ_NEXT(vstart, next); + } else { + dev = TAILQ_FIRST(&vdev_device_list); + } + while (dev != NULL) { if (cmp(&dev->device, data) == 0) - return &dev->device; + break; + dev = TAILQ_NEXT(dev, next); } - return NULL; + rte_spinlock_recursive_unlock(&vdev_device_list_lock); + + return dev ? &dev->device : NULL; } static int diff --git a/drivers/bus/vmbus/Makefile b/drivers/bus/vmbus/Makefile new file mode 100644 index 00000000..deee9dd1 --- /dev/null +++ b/drivers/bus/vmbus/Makefile @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause + +include $(RTE_SDK)/mk/rte.vars.mk + +LIB = librte_bus_vmbus.a +LIBABIVER := 1 +EXPORT_MAP := rte_bus_vmbus_version.map + +CFLAGS += -I$(SRCDIR) +CFLAGS += -O3 $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API + +ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),) +SYSTEM := linux +endif +ifneq ($(CONFIG_RTE_EXEC_ENV_BSDAPP),) +$(error "VMBUS not implemented for BSD yet") +endif + +CFLAGS += -I$(RTE_SDK)/drivers/bus/vmbus/$(SYSTEM) +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal + +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev + +include $(RTE_SDK)/drivers/bus/vmbus/$(SYSTEM)/Makefile +SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) := $(addprefix $(SYSTEM)/,$(SRCS)) +SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_common.c +SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_channel.c vmbus_bufring.c +SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_common_uio.c + +SYMLINK-$(CONFIG_RTE_LIBRTE_VMBUS)-include += rte_bus_vmbus.h +SYMLINK-$(CONFIG_RTE_LIBRTE_VMBUS)-include += rte_vmbus_reg.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/bus/vmbus/linux/Makefile b/drivers/bus/vmbus/linux/Makefile new file mode 100644 index 00000000..ef0d30b2 --- /dev/null +++ b/drivers/bus/vmbus/linux/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: BSD-3-Clause + +SRCS += vmbus_bus.c vmbus_uio.c diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c new file mode 100644 index 00000000..52d6a3c0 --- /dev/null +++ b/drivers/bus/vmbus/linux/vmbus_bus.c @@ -0,0 +1,355 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <string.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> + +#include <rte_eal.h> +#include <rte_uuid.h> +#include <rte_tailq.h> +#include <rte_log.h> +#include <rte_devargs.h> +#include <rte_memory.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> + +#include "eal_filesystem.h" +#include "private.h" + +/** Pathname of VMBUS devices directory. */ +#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" + +extern struct rte_vmbus_bus rte_vmbus_bus; + +/* Read sysfs file to get UUID */ +static int +parse_sysfs_uuid(const char *filename, rte_uuid_t uu) +{ + char buf[BUFSIZ]; + char *cp, *in = buf; + FILE *f; + + f = fopen(filename, "r"); + if (f == NULL) { + VMBUS_LOG(ERR, "cannot open sysfs value %s: %s", + filename, strerror(errno)); + return -1; + } + + if (fgets(buf, sizeof(buf), f) == NULL) { + VMBUS_LOG(ERR, "cannot read sysfs value %s", + filename); + fclose(f); + return -1; + } + fclose(f); + + cp = strchr(buf, '\n'); + if (cp) + *cp = '\0'; + + /* strip { } notation */ + if (buf[0] == '{') { + in = buf + 1; + cp = strchr(in, '}'); + if (cp) + *cp = '\0'; + } + + if (rte_uuid_parse(in, uu) < 0) { + VMBUS_LOG(ERR, "%s %s not a valid UUID", + filename, buf); + return -1; + } + + return 0; +} + +static int +get_sysfs_string(const char *filename, char *buf, size_t buflen) +{ + char *cp; + FILE *f; + + f = fopen(filename, "r"); + if (f == NULL) { + VMBUS_LOG(ERR, "cannot open sysfs value %s:%s", + filename, strerror(errno)); + return -1; + } + + if (fgets(buf, buflen, f) == NULL) { + VMBUS_LOG(ERR, "cannot read sysfs value %s", + filename); + fclose(f); + return -1; + } + fclose(f); + + /* remove trailing newline */ + cp = memchr(buf, '\n', buflen); + if (cp) + *cp = '\0'; + + return 0; +} + +static int +vmbus_get_uio_dev(const struct rte_vmbus_device *dev, + char *dstbuf, size_t buflen) +{ + char dirname[PATH_MAX]; + unsigned int uio_num; + struct dirent *e; + DIR *dir; + + /* Assume recent kernel where uio is in uio/uioX */ + snprintf(dirname, sizeof(dirname), + SYSFS_VMBUS_DEVICES "/%s/uio", dev->device.name); + + dir = opendir(dirname); + if (dir == NULL) + return -1; /* Not a UIO device */ + + /* take the first file starting with "uio" */ + while ((e = readdir(dir)) != NULL) { + const int prefix_len = 3; + char *endptr; + + if (strncmp(e->d_name, "uio", prefix_len) != 0) + continue; + + /* try uio%d */ + errno = 0; + uio_num = strtoull(e->d_name + prefix_len, &endptr, 10); + if (errno == 0 && endptr != (e->d_name + prefix_len)) { + snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); + break; + } + } + closedir(dir); + + if (e == NULL) + return -1; + + return uio_num; +} + +/* Check map names with kernel names */ +static const char *map_names[VMBUS_MAX_RESOURCE] = { + [HV_TXRX_RING_MAP] = "txrx_rings", + [HV_INT_PAGE_MAP] = "int_page", + [HV_MON_PAGE_MAP] = "monitor_page", + [HV_RECV_BUF_MAP] = "recv:", + [HV_SEND_BUF_MAP] = "send:", +}; + + +/* map the resources of a vmbus device in virtual memory */ +int +rte_vmbus_map_device(struct rte_vmbus_device *dev) +{ + char uioname[PATH_MAX], filename[PATH_MAX]; + char dirname[PATH_MAX], mapname[64]; + int i; + + dev->uio_num = vmbus_get_uio_dev(dev, uioname, sizeof(uioname)); + if (dev->uio_num < 0) { + VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped"); + return 1; + } + + /* Extract resource value */ + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { + struct rte_mem_resource *res = &dev->resource[i]; + unsigned long len, gpad = 0; + char *cp; + + snprintf(dirname, sizeof(dirname), + "%s/maps/map%d", uioname, i); + + snprintf(filename, sizeof(filename), + "%s/name", dirname); + + if (get_sysfs_string(filename, mapname, sizeof(mapname)) < 0) { + VMBUS_LOG(ERR, "could not read %s", filename); + return -1; + } + + if (strncmp(map_names[i], mapname, strlen(map_names[i])) != 0) { + VMBUS_LOG(ERR, + "unexpected resource %s (expected %s)", + mapname, map_names[i]); + return -1; + } + + snprintf(filename, sizeof(filename), + "%s/size", dirname); + if (eal_parse_sysfs_value(filename, &len) < 0) { + VMBUS_LOG(ERR, + "could not read %s", filename); + return -1; + } + res->len = len; + + /* both send and receive buffers have gpad in name */ + cp = memchr(mapname, ':', sizeof(mapname)); + if (cp) + gpad = strtoul(cp+1, NULL, 0); + + /* put the GPAD value in physical address */ + res->phys_addr = gpad; + } + + return vmbus_uio_map_resource(dev); +} + +void +rte_vmbus_unmap_device(struct rte_vmbus_device *dev) +{ + vmbus_uio_unmap_resource(dev); +} + +/* Scan one vmbus sysfs entry, and fill the devices list from it. */ +static int +vmbus_scan_one(const char *name) +{ + struct rte_vmbus_device *dev, *dev2; + char filename[PATH_MAX]; + char dirname[PATH_MAX]; + unsigned long tmp; + + dev = calloc(1, sizeof(*dev)); + if (dev == NULL) + return -1; + + dev->device.name = strdup(name); + if (!dev->device.name) + goto error; + + /* sysfs base directory + * /sys/bus/vmbus/devices/7a08391f-f5a0-4ac0-9802-d13fd964f8df + * or on older kernel + * /sys/bus/vmbus/devices/vmbus_1 + */ + snprintf(dirname, sizeof(dirname), "%s/%s", + SYSFS_VMBUS_DEVICES, name); + + /* get device id */ + snprintf(filename, sizeof(filename), "%s/device_id", dirname); + if (parse_sysfs_uuid(filename, dev->device_id) < 0) + goto error; + + /* get device class */ + snprintf(filename, sizeof(filename), "%s/class_id", dirname); + if (parse_sysfs_uuid(filename, dev->class_id) < 0) + goto error; + + /* get relid */ + snprintf(filename, sizeof(filename), "%s/id", dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) + goto error; + dev->relid = tmp; + + /* get monitor id */ + snprintf(filename, sizeof(filename), "%s/monitor_id", dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) + goto error; + dev->monitor_id = tmp; + + /* get numa node (if present) */ + snprintf(filename, sizeof(filename), "%s/numa_node", + dirname); + + if (access(filename, R_OK) == 0) { + if (eal_parse_sysfs_value(filename, &tmp) < 0) + goto error; + dev->device.numa_node = tmp; + } else { + /* if no NUMA support, set default to 0 */ + dev->device.numa_node = SOCKET_ID_ANY; + } + + /* device is valid, add in list (sorted) */ + VMBUS_LOG(DEBUG, "Adding vmbus device %s", name); + + TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) { + int ret; + + ret = rte_uuid_compare(dev->device_id, dev2->device_id); + if (ret > 0) + continue; + + if (ret < 0) { + vmbus_insert_device(dev2, dev); + } else { /* already registered */ + VMBUS_LOG(NOTICE, + "%s already registered", name); + free(dev); + } + return 0; + } + + vmbus_add_device(dev); + return 0; +error: + VMBUS_LOG(DEBUG, "failed"); + + free(dev); + return -1; +} + +/* + * Scan the content of the vmbus, and the devices in the devices list + */ +int +rte_vmbus_scan(void) +{ + struct dirent *e; + DIR *dir; + + dir = opendir(SYSFS_VMBUS_DEVICES); + if (dir == NULL) { + if (errno == ENOENT) + return 0; + + VMBUS_LOG(ERR, "opendir %s failed: %s", + SYSFS_VMBUS_DEVICES, strerror(errno)); + return -1; + } + + while ((e = readdir(dir)) != NULL) { + if (e->d_name[0] == '.') + continue; + + if (vmbus_scan_one(e->d_name) < 0) + goto error; + } + closedir(dir); + return 0; + +error: + closedir(dir); + return -1; +} + +void rte_vmbus_irq_mask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 1); +} + +void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 0); +} + +int rte_vmbus_irq_read(struct rte_vmbus_device *device) +{ + return vmbus_uio_irq_read(device); +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c new file mode 100644 index 00000000..856c6d66 --- /dev/null +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -0,0 +1,398 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <dirent.h> +#include <inttypes.h> +#include <sys/stat.h> +#include <sys/mman.h> + +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_memory.h> +#include <rte_eal_memconfig.h> +#include <rte_common.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> +#include <rte_string_fns.h> + +#include "private.h" + +/** Pathname of VMBUS devices directory. */ +#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" + +static void *vmbus_map_addr; + +/* Control interrupts */ +void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) +{ + if (write(dev->intr_handle.fd, &onoff, sizeof(onoff)) < 0) { + VMBUS_LOG(ERR, "cannot write to %d:%s", + dev->intr_handle.fd, strerror(errno)); + } +} + +int vmbus_uio_irq_read(struct rte_vmbus_device *dev) +{ + int32_t count; + int cc; + + cc = read(dev->intr_handle.fd, &count, sizeof(count)); + if (cc < (int)sizeof(count)) { + if (cc < 0) { + VMBUS_LOG(ERR, "IRQ read failed %s", + strerror(errno)); + return -errno; + } + VMBUS_LOG(ERR, "can't read IRQ count"); + return -EINVAL; + } + + return count; +} + +void +vmbus_uio_free_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource *uio_res) +{ + rte_free(uio_res); + + if (dev->intr_handle.uio_cfg_fd >= 0) { + close(dev->intr_handle.uio_cfg_fd); + dev->intr_handle.uio_cfg_fd = -1; + } + + if (dev->intr_handle.fd >= 0) { + close(dev->intr_handle.fd); + dev->intr_handle.fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + } +} + +int +vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource **uio_res) +{ + char devname[PATH_MAX]; /* contains the /dev/uioX */ + + /* save fd if in primary process */ + snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); + dev->intr_handle.fd = open(devname, O_RDWR); + if (dev->intr_handle.fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + devname, strerror(errno)); + goto error; + } + dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX; + + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { + VMBUS_LOG(ERR, "cannot store uio mmap details"); + goto error; + } + + strlcpy((*uio_res)->path, devname, PATH_MAX); + rte_uuid_copy((*uio_res)->id, dev->device_id); + + return 0; + +error: + vmbus_uio_free_resource(dev, *uio_res); + return -1; +} + +static int +find_max_end_va(const struct rte_memseg_list *msl, void *arg) +{ + size_t sz = msl->memseg_arr.len * msl->page_sz; + void *end_va = RTE_PTR_ADD(msl->base_va, sz); + void **max_va = arg; + + if (*max_va < end_va) + *max_va = end_va; + return 0; +} + +/* + * TODO: this should be part of memseg api. + * code is duplicated from PCI. + */ +static void * +vmbus_find_max_end_va(void) +{ + void *va = NULL; + + rte_memseg_list_walk(find_max_end_va, &va); + return va; +} + +int +vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, + struct mapped_vmbus_resource *uio_res, + int flags) +{ + size_t size = dev->resource[idx].len; + struct vmbus_map *maps = uio_res->maps; + void *mapaddr; + off_t offset; + int fd; + + /* devname for mmap */ + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + /* try mapping somewhere close to the end of hugepages */ + if (vmbus_map_addr == NULL) + vmbus_map_addr = vmbus_find_max_end_va(); + + /* offset is special in uio it indicates which resource */ + offset = idx * PAGE_SIZE; + + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); + close(fd); + + if (mapaddr == MAP_FAILED) + return -1; + + dev->resource[idx].addr = mapaddr; + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); + + /* Record result of sucessful mapping for use by secondary */ + maps[idx].addr = mapaddr; + maps[idx].size = size; + + return 0; +} + +static int vmbus_uio_map_primary(struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + struct mapped_vmbus_resource *uio_res; + + uio_res = vmbus_uio_find_resource(chan->device); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources!"); + return -ENOMEM; + } + + if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { + VMBUS_LOG(ERR, "VMBUS: only %u resources found!", + uio_res->nb_maps); + return -EINVAL; + } + + *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; + *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; + return 0; +} + +static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + char ring_path[PATH_MAX]; + size_t file_size; + struct stat sb; + int fd; + + snprintf(ring_path, sizeof(ring_path), + "%s/%s/channels/%u/ring", + SYSFS_VMBUS_DEVICES, dev->device.name, + chan->relid); + + fd = open(ring_path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + ring_path, strerror(errno)); + return -errno; + } + + if (fstat(fd, &sb) < 0) { + VMBUS_LOG(ERR, "Cannot state %s: %s", + ring_path, strerror(errno)); + close(fd); + return -errno; + } + file_size = sb.st_size; + + if (file_size == 0 || (file_size & (PAGE_SIZE - 1))) { + VMBUS_LOG(ERR, "incorrect size %s: %zu", + ring_path, file_size); + + close(fd); + return -EINVAL; + } + + *ring_size = file_size / 2; + *ring_buf = vmbus_map_resource(vmbus_map_addr, fd, + 0, sb.st_size, 0); + close(fd); + + if (ring_buf == MAP_FAILED) + return -EIO; + + vmbus_map_addr = RTE_PTR_ADD(ring_buf, file_size); + return 0; +} + +int vmbus_uio_map_rings(struct vmbus_channel *chan) +{ + const struct rte_vmbus_device *dev = chan->device; + uint32_t ring_size; + void *ring_buf; + int ret; + + /* Primary channel */ + if (chan->subchannel_id == 0) + ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); + else + ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size); + + if (ret) + return ret; + + vmbus_br_setup(&chan->txbr, ring_buf, ring_size); + vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); + return 0; +} + +static int vmbus_uio_sysfs_read(const char *dir, const char *name, + unsigned long *val, unsigned long max_range) +{ + char path[PATH_MAX]; + FILE *f; + int ret; + + snprintf(path, sizeof(path), "%s/%s", dir, name); + f = fopen(path, "r"); + if (!f) { + VMBUS_LOG(ERR, "can't open %s:%s", + path, strerror(errno)); + return -errno; + } + + if (fscanf(f, "%lu", val) != 1) + ret = -EIO; + else if (*val > max_range) + ret = -ERANGE; + else + ret = 0; + fclose(f); + + return ret; +} + +static bool vmbus_uio_ring_present(const struct rte_vmbus_device *dev, + uint32_t relid) +{ + char ring_path[PATH_MAX]; + + /* Check if kernel has subchannel sysfs files */ + snprintf(ring_path, sizeof(ring_path), + "%s/%s/channels/%u/ring", + SYSFS_VMBUS_DEVICES, dev->device.name, relid); + + return access(ring_path, R_OK|W_OK) == 0; +} + +bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan) +{ + return vmbus_uio_ring_present(dev, chan->relid); +} + +static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, + unsigned long id) +{ + const struct vmbus_channel *c; + + STAILQ_FOREACH(c, &primary->subchannel_list, next) { + if (c->relid == id) + return false; + } + return true; +} + +int vmbus_uio_get_subchan(struct vmbus_channel *primary, + struct vmbus_channel **subchan) +{ + const struct rte_vmbus_device *dev = primary->device; + char chan_path[PATH_MAX], subchan_path[PATH_MAX]; + struct dirent *ent; + DIR *chan_dir; + + snprintf(chan_path, sizeof(chan_path), + "%s/%s/channels", + SYSFS_VMBUS_DEVICES, dev->device.name); + + chan_dir = opendir(chan_path); + if (!chan_dir) { + VMBUS_LOG(ERR, "cannot open %s: %s", + chan_path, strerror(errno)); + return -errno; + } + + while ((ent = readdir(chan_dir))) { + unsigned long relid, subid, monid; + char *endp; + int err; + + if (ent->d_name[0] == '.') + continue; + + errno = 0; + relid = strtoul(ent->d_name, &endp, 0); + if (*endp || errno != 0 || relid > UINT16_MAX) { + VMBUS_LOG(NOTICE, "not a valid channel relid: %s", + ent->d_name); + continue; + } + + snprintf(subchan_path, sizeof(subchan_path), "%s/%lu", + chan_path, relid); + err = vmbus_uio_sysfs_read(subchan_path, "subchannel_id", + &subid, UINT16_MAX); + if (err) { + VMBUS_LOG(NOTICE, "invalid subchannel id %lu", + subid); + closedir(chan_dir); + return err; + } + + if (subid == 0) + continue; /* skip primary channel */ + + if (!vmbus_isnew_subchannel(primary, relid)) + continue; + + if (!vmbus_uio_ring_present(dev, relid)) + continue; /* Ring may not be ready yet */ + + err = vmbus_uio_sysfs_read(subchan_path, "monitor_id", + &monid, UINT8_MAX); + if (err) { + VMBUS_LOG(NOTICE, "invalid monitor id %lu", + monid); + return err; + } + + err = vmbus_chan_create(dev, relid, subid, monid, subchan); + if (err) { + VMBUS_LOG(NOTICE, "subchannel setup failed"); + return err; + } + break; + } + closedir(chan_dir); + + return (ent == NULL) ? -ENOENT : 0; +} diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build new file mode 100644 index 00000000..18daabec --- /dev/null +++ b/drivers/bus/vmbus/meson.build @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: BSD-3-Clause + +allow_experimental_apis = true + +install_headers('rte_bus_vmbus.h','rte_vmbus_reg.h') + +sources = files('vmbus_common.c', + 'vmbus_channel.c', + 'vmbus_bufring.c', + 'vmbus_common_uio.c') + +if host_machine.system() == 'linux' + sources += files('linux/vmbus_bus.c', + 'linux/vmbus_uio.c') + includes += include_directories('linux') +else + build = false +endif diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h new file mode 100644 index 00000000..9964fc42 --- /dev/null +++ b/drivers/bus/vmbus/private.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#ifndef _VMBUS_PRIVATE_H_ +#define _VMBUS_PRIVATE_H_ + +#include <stdbool.h> +#include <sys/uio.h> +#include <rte_log.h> +#include <rte_vmbus_reg.h> + +#ifndef PAGE_SIZE +#define PAGE_SIZE 4096 +#endif + +extern int vmbus_logtype_bus; +#define VMBUS_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, vmbus_logtype_bus, "%s(): " fmt "\n", \ + __func__, ##args) + +struct vmbus_br { + struct vmbus_bufring *vbr; + uint32_t dsize; + uint32_t windex; /* next available location */ +}; + +#define UIO_NAME_MAX 64 + +struct vmbus_map { + void *addr; /* user mmap of resource */ + uint64_t size; /* length */ +}; + +/* + * For multi-process we need to reproduce all vmbus mappings in secondary + * processes, so save them in a tailq. + */ +struct mapped_vmbus_resource { + TAILQ_ENTRY(mapped_vmbus_resource) next; + + rte_uuid_t id; + int nb_maps; + struct vmbus_map maps[VMBUS_MAX_RESOURCE]; + char path[PATH_MAX]; +}; + +TAILQ_HEAD(mapped_vmbus_res_list, mapped_vmbus_resource); + +#define HV_MON_TRIG_LEN 32 +#define HV_MON_TRIG_MAX 4 + +struct vmbus_channel { + STAILQ_HEAD(, vmbus_channel) subchannel_list; + STAILQ_ENTRY(vmbus_channel) next; + const struct rte_vmbus_device *device; + + struct vmbus_br rxbr; + struct vmbus_br txbr; + + uint16_t relid; + uint16_t subchannel_id; + uint8_t monitor_id; +}; + +#define VMBUS_MAX_CHANNELS 64 + +int vmbus_chan_create(const struct rte_vmbus_device *device, + uint16_t relid, uint16_t subid, uint8_t monitor_id, + struct vmbus_channel **new_chan); + +void vmbus_add_device(struct rte_vmbus_device *vmbus_dev); +void vmbus_insert_device(struct rte_vmbus_device *exist_vmbus_dev, + struct rte_vmbus_device *new_vmbus_dev); +void vmbus_remove_device(struct rte_vmbus_device *vmbus_device); + +void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff); +int vmbus_uio_irq_read(struct rte_vmbus_device *dev); + +int vmbus_uio_map_resource(struct rte_vmbus_device *dev); +void vmbus_uio_unmap_resource(struct rte_vmbus_device *dev); + +int vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource **uio_res); +void vmbus_uio_free_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource *uio_res); + +struct mapped_vmbus_resource * +vmbus_uio_find_resource(const struct rte_vmbus_device *dev); +int vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int res_idx, + struct mapped_vmbus_resource *uio_res, + int flags); + +void *vmbus_map_resource(void *requested_addr, int fd, off_t offset, + size_t size, int additional_flags); +void vmbus_unmap_resource(void *requested_addr, size_t size); + +bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan); +int vmbus_uio_get_subchan(struct vmbus_channel *primary, + struct vmbus_channel **subchan); +int vmbus_uio_map_rings(struct vmbus_channel *chan); + +void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); + +/* Amount of space available for write */ +static inline uint32_t +vmbus_br_availwrite(const struct vmbus_br *br, uint32_t windex) +{ + uint32_t rindex = br->vbr->rindex; + + if (windex >= rindex) + return br->dsize - (windex - rindex); + else + return rindex - windex; +} + +static inline uint32_t +vmbus_br_availread(const struct vmbus_br *br) +{ + return br->dsize - vmbus_br_availwrite(br, br->vbr->windex); +} + +int vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen, + bool *need_sig); + +int vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen); + +int vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t hlen); + +#endif /* _VMBUS_PRIVATE_H_ */ diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h new file mode 100644 index 00000000..4a2c1f6f --- /dev/null +++ b/drivers/bus/vmbus/rte_bus_vmbus.h @@ -0,0 +1,407 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#ifndef _VMBUS_H_ +#define _VMBUS_H_ + +/** + * @file + * + * VMBUS Interface + */ +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <limits.h> +#include <stdbool.h> +#include <errno.h> +#include <sys/queue.h> +#include <stdint.h> +#include <inttypes.h> + +#include <rte_compat.h> +#include <rte_uuid.h> +#include <rte_debug.h> +#include <rte_interrupts.h> +#include <rte_dev.h> +#include <rte_vmbus_reg.h> + +/* Forward declarations */ +struct rte_vmbus_device; +struct rte_vmbus_driver; +struct rte_vmbus_bus; +struct vmbus_channel; +struct vmbus_mon_page; + +TAILQ_HEAD(rte_vmbus_device_list, rte_vmbus_device); +TAILQ_HEAD(rte_vmbus_driver_list, rte_vmbus_driver); + +/* VMBus iterators */ +#define FOREACH_DEVICE_ON_VMBUS(p) \ + TAILQ_FOREACH(p, &(rte_vmbus_bus.device_list), next) + +#define FOREACH_DRIVER_ON_VMBUS(p) \ + TAILQ_FOREACH(p, &(rte_vmbus_bus.driver_list), next) + +/** Maximum number of VMBUS resources. */ +enum hv_uio_map { + HV_TXRX_RING_MAP = 0, + HV_INT_PAGE_MAP, + HV_MON_PAGE_MAP, + HV_RECV_BUF_MAP, + HV_SEND_BUF_MAP +}; +#define VMBUS_MAX_RESOURCE 5 + +/** + * A structure describing a VMBUS device. + */ +struct rte_vmbus_device { + TAILQ_ENTRY(rte_vmbus_device) next; /**< Next probed VMBUS device */ + const struct rte_vmbus_driver *driver; /**< Associated driver */ + struct rte_device device; /**< Inherit core device */ + rte_uuid_t device_id; /**< VMBUS device id */ + rte_uuid_t class_id; /**< VMBUS device type */ + uint32_t relid; /**< id for primary */ + uint8_t monitor_id; /**< monitor page */ + int uio_num; /**< UIO device number */ + uint32_t *int_page; /**< VMBUS interrupt page */ + struct vmbus_channel *primary; /**< VMBUS primary channel */ + struct vmbus_mon_page *monitor_page; /**< VMBUS monitor page */ + + struct rte_intr_handle intr_handle; /**< Interrupt handle */ + struct rte_mem_resource resource[VMBUS_MAX_RESOURCE]; +}; + +/** + * Initialization function for the driver called during VMBUS probing. + */ +typedef int (vmbus_probe_t)(struct rte_vmbus_driver *, + struct rte_vmbus_device *); + +/** + * Initialization function for the driver called during hot plugging. + */ +typedef int (vmbus_remove_t)(struct rte_vmbus_device *); + +/** + * A structure describing a VMBUS driver. + */ +struct rte_vmbus_driver { + TAILQ_ENTRY(rte_vmbus_driver) next; /**< Next in list. */ + struct rte_driver driver; + struct rte_vmbus_bus *bus; /**< VM bus reference. */ + vmbus_probe_t *probe; /**< Device Probe function. */ + vmbus_remove_t *remove; /**< Device Remove function. */ + + const rte_uuid_t *id_table; /**< ID table. */ +}; + + +/** + * Structure describing the VM bus + */ +struct rte_vmbus_bus { + struct rte_bus bus; /**< Inherit the generic class */ + struct rte_vmbus_device_list device_list; /**< List of devices */ + struct rte_vmbus_driver_list driver_list; /**< List of drivers */ +}; + +/** + * Scan the content of the VMBUS bus, and the devices in the devices + * list + * + * @return + * 0 on success, negative on error + */ +int rte_vmbus_scan(void); + +/** + * Probe the VMBUS bus + * + * @return + * - 0 on success. + * - !0 on error. + */ +int rte_vmbus_probe(void); + +/** + * Map the VMBUS device resources in user space virtual memory address + * + * @param dev + * A pointer to a rte_vmbus_device structure describing the device + * to use + * + * @return + * 0 on success, negative on error and positive if no driver + * is found for the device. + */ +int rte_vmbus_map_device(struct rte_vmbus_device *dev); + +/** + * Unmap this device + * + * @param dev + * A pointer to a rte_vmbus_device structure describing the device + * to use + */ +void rte_vmbus_unmap_device(struct rte_vmbus_device *dev); + +/** + * Get connection to primary VMBUS channel + * + * @param device + * A pointer to a rte_vmbus_device structure describing the device + * @param chan + * A pointer to a VMBUS channel pointer that will be filled. + * @return + * - 0 Success; channel opened. + * - -ENOMEM: Not enough memory available. + * - -EINVAL: Regions could not be mapped. + */ +int rte_vmbus_chan_open(struct rte_vmbus_device *device, + struct vmbus_channel **chan); + +/** + * Free connection to VMBUS channel + * + * @param chan + * VMBUS channel + */ +void rte_vmbus_chan_close(struct vmbus_channel *chan); + +/** + * Gets the maximum number of channels supported on device + * + * @param device + * A pointer to a rte_vmbus_device structure describing the device + * @return + * Number of channels available. + */ +int rte_vmbus_max_channels(const struct rte_vmbus_device *device); + +/** + * Get a connection to new secondary vmbus channel + * + * @param primary + * A pointer to primary VMBUS channel + * @param chan + * A pointer to a secondary VMBUS channel pointer that will be filled. + * @return + * - 0 Success; channel opened. + * - -ENOMEM: Not enough memory available. + * - -EINVAL: Regions could not be mapped. + */ +int rte_vmbus_subchan_open(struct vmbus_channel *primary, + struct vmbus_channel **new_chan); + +/** + * Disable IRQ for device + * + * @param device + * VMBUS device + */ +void rte_vmbus_irq_mask(struct rte_vmbus_device *device); + +/** + * Enable IRQ for device + * + * @param device + * VMBUS device + */ +void rte_vmbus_irq_unmask(struct rte_vmbus_device *device); + +/** + * Read (and wait) for IRQ + * + * @param device + * VMBUS device + */ +int rte_vmbus_irq_read(struct rte_vmbus_device *device); + +/** + * Test if channel is empty + * + * @param channel + * Pointer to vmbus_channel structure. + * @return + * Return true if no data present in incoming ring. + */ +bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel); + +/** + * Send the specified buffer on the given channel + * + * @param channel + * Pointer to vmbus_channel structure. + * @param type + * Type of packet that is being send e.g. negotiate, time + * packet etc. + * @param data + * Pointer to the buffer to send + * @param dlen + * Number of bytes of data to send + * @param xact + * Identifier of the request + * @param flags + * Message type inband, rxbuf, gpa + * @param need_sig + * Is host signal tx is required (optional) + * + * Sends data in buffer directly to hyper-v via the vmbus + */ +int rte_vmbus_chan_send(struct vmbus_channel *channel, uint16_t type, + void *data, uint32_t dlen, + uint64_t xact, uint32_t flags, bool *need_sig); + +/** + * Explicitly signal host that data is available + * + * @param + * Pointer to vmbus_channel structure. + * + * Used when batching multiple sends and only signaling host + * after the last send. + */ +void rte_vmbus_chan_signal_tx(const struct vmbus_channel *channel); + +/* Structure for scatter/gather I/O */ +struct iova_list { + rte_iova_t addr; + uint32_t len; +}; +#define MAX_PAGE_BUFFER_COUNT 32 + +/** + * Send a scattered buffer on the given channel + * + * @param channel + * Pointer to vmbus_channel structure. + * @param type + * Type of packet that is being send e.g. negotiate, time + * packet etc. + * @param gpa + * Array of buffers to send + * @param gpacnt + * Number of elements in iov + * @param data + * Pointer to the buffer additional data to send + * @param dlen + * Maximum size of what the the buffer will hold + * @param xact + * Identifier of the request + * @param flags + * Message type inband, rxbuf, gpa + * @param need_sig + * Is host signal tx is required (optional) + * + * Sends data in buffer directly to hyper-v via the vmbus + */ +int rte_vmbus_chan_send_sglist(struct vmbus_channel *channel, + struct vmbus_gpa gpa[], uint32_t gpacnt, + void *data, uint32_t dlen, + uint64_t xact, bool *need_sig); +/** + * Receive response to request on the given channel + * skips the channel header. + * + * @param channel + * Pointer to vmbus_channel structure. + * @param data + * Pointer to the buffer you want to receive the data into. + * @param len + * Pointer to size of receive buffer (in/out) + * @param + * Pointer to received transaction_id + * @return + * On success, returns 0 + * On failure, returns negative errno. + */ +int rte_vmbus_chan_recv(struct vmbus_channel *chan, + void *data, uint32_t *len, + uint64_t *request_id); + +/** + * Receive response to request on the given channel + * includes the channel header. + * + * @param channel + * Pointer to vmbus_channel structure. + * @param data + * Pointer to the buffer you want to receive the data into. + * @param len + * Pointer to size of receive buffer (in/out) + * @return + * On success, returns number of bytes read. + * On failure, returns negative errno. + */ +int rte_vmbus_chan_recv_raw(struct vmbus_channel *chan, + void *data, uint32_t *len); + +/** + * Notify host of bytes read (after recv_raw) + * Signals host if required. + * + * @param channel + * Pointer to vmbus_channel structure. + * @param bytes_read + * Number of bytes read since last signal + */ +void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read); + +/** + * Determine sub channel index of the given channel + * + * @param channel + * Pointer to vmbus_channel structure. + * @return + * Sub channel index (0 for primary) + */ +uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan); + +/** + * Register a VMBUS driver. + * + * @param driver + * A pointer to a rte_vmbus_driver structure describing the driver + * to be registered. + */ +void rte_vmbus_register(struct rte_vmbus_driver *driver); + +/** + * For debug dump contents of ring buffer. + * + * @param channel + * Pointer to vmbus_channel structure. + */ +void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan); + +/** + * Unregister a VMBUS driver. + * + * @param driver + * A pointer to a rte_vmbus_driver structure describing the driver + * to be unregistered. + */ +void rte_vmbus_unregister(struct rte_vmbus_driver *driver); + +/** Helper for VMBUS device registration from driver instance */ +#define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ + RTE_INIT(vmbusinitfn_ ##nm); \ + static void vmbusinitfn_ ##nm(void) \ + { \ + (vmbus_drv).driver.name = RTE_STR(nm); \ + rte_vmbus_register(&vmbus_drv); \ + } \ + RTE_PMD_EXPORT_NAME(nm, __COUNTER__) + +#ifdef __cplusplus +} +#endif + +#endif /* _VMBUS_H_ */ diff --git a/drivers/bus/vmbus/rte_bus_vmbus_version.map b/drivers/bus/vmbus/rte_bus_vmbus_version.map new file mode 100644 index 00000000..dabb9203 --- /dev/null +++ b/drivers/bus/vmbus/rte_bus_vmbus_version.map @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +DPDK_18.08 { + global: + + rte_vmbus_chan_close; + rte_vmbus_chan_open; + rte_vmbus_chan_recv; + rte_vmbus_chan_recv_raw; + rte_vmbus_chan_rx_empty; + rte_vmbus_chan_send; + rte_vmbus_chan_send_sglist; + rte_vmbus_chan_signal_read; + rte_vmbus_chan_signal_tx; + rte_vmbus_irq_mask; + rte_vmbus_irq_read; + rte_vmbus_irq_unmask; + rte_vmbus_map_device; + rte_vmbus_max_channels; + rte_vmbus_probe; + rte_vmbus_register; + rte_vmbus_scan; + rte_vmbus_sub_channel_index; + rte_vmbus_subchan_open; + rte_vmbus_unmap_device; + rte_vmbus_unregister; + + local: *; +}; diff --git a/drivers/bus/vmbus/rte_vmbus_reg.h b/drivers/bus/vmbus/rte_vmbus_reg.h new file mode 100644 index 00000000..f5a0693d --- /dev/null +++ b/drivers/bus/vmbus/rte_vmbus_reg.h @@ -0,0 +1,344 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#ifndef _VMBUS_REG_H_ +#define _VMBUS_REG_H_ + +/* + * Hyper-V SynIC message format. + */ +#define VMBUS_MSG_DSIZE_MAX 240 +#define VMBUS_MSG_SIZE 256 + +struct vmbus_message { + uint32_t type; /* HYPERV_MSGTYPE_ */ + uint8_t dsize; /* data size */ + uint8_t flags; /* VMBUS_MSGFLAG_ */ + uint16_t rsvd; + uint64_t id; + uint8_t data[VMBUS_MSG_DSIZE_MAX]; +} __rte_packed; + +#define VMBUS_MSGFLAG_PENDING 0x01 + +/* + * Hyper-V Monitor Notification Facility + */ + +struct vmbus_mon_trig { + uint32_t pending; + uint32_t armed; +} __rte_packed; + +#define VMBUS_MONTRIGS_MAX 4 +#define VMBUS_MONTRIG_LEN 32 + +/* + * Hyper-V Monitor Notification Facility + */ +struct hyperv_mon_param { + uint32_t connid; + uint16_t evtflag_ofs; + uint16_t rsvd; +} __rte_packed; + +struct vmbus_mon_page { + uint32_t state; + uint32_t rsvd1; + + struct vmbus_mon_trig trigs[VMBUS_MONTRIGS_MAX]; + uint8_t rsvd2[536]; + + uint16_t lat[VMBUS_MONTRIGS_MAX][VMBUS_MONTRIG_LEN]; + uint8_t rsvd3[256]; + + struct hyperv_mon_param + param[VMBUS_MONTRIGS_MAX][VMBUS_MONTRIG_LEN]; + uint8_t rsvd4[1984]; +} __rte_packed; + +/* + * Buffer ring + */ + +struct vmbus_bufring { + volatile uint32_t windex; + volatile uint32_t rindex; + + /* + * Interrupt mask {0,1} + * + * For TX bufring, host set this to 1, when it is processing + * the TX bufring, so that we can safely skip the TX event + * notification to host. + * + * For RX bufring, once this is set to 1 by us, host will not + * further dispatch interrupts to us, even if there are data + * pending on the RX bufring. This effectively disables the + * interrupt of the channel to which this RX bufring is attached. + */ + volatile uint32_t imask; + + /* + * Win8 uses some of the reserved bits to implement + * interrupt driven flow management. On the send side + * we can request that the receiver interrupt the sender + * when the ring transitions from being full to being able + * to handle a message of size "pending_send_sz". + * + * Add necessary state for this enhancement. + */ + volatile uint32_t pending_send; + uint32_t reserved1[12]; + + union { + struct { + uint32_t feat_pending_send_sz:1; + }; + uint32_t value; + } feature_bits; + + /* Pad it to PAGE_SIZE so that data starts on page boundary */ + uint8_t reserved2[4028]; + + /* + * Ring data starts here + RingDataStartOffset + * !!! DO NOT place any fields below this !!! + */ + uint8_t data[0]; +} __rte_packed; + +/* + * Channel packets + */ + +/* Channel packet flags */ +#define VMBUS_CHANPKT_TYPE_INBAND 0x0006 +#define VMBUS_CHANPKT_TYPE_RXBUF 0x0007 +#define VMBUS_CHANPKT_TYPE_GPA 0x0009 +#define VMBUS_CHANPKT_TYPE_COMP 0x000b + +#define VMBUS_CHANPKT_FLAG_NONE 0 +#define VMBUS_CHANPKT_FLAG_RC 0x0001 /* report completion */ + +#define VMBUS_CHANPKT_SIZE_SHIFT 3 +#define VMBUS_CHANPKT_SIZE_ALIGN (1 << VMBUS_CHANPKT_SIZE_SHIFT) +#define VMBUS_CHANPKT_HLEN_MIN \ + (sizeof(struct vmbus_chanpkt_hdr) >> VMBUS_CHANPKT_SIZE_SHIFT) + +static inline uint32_t +vmbus_chanpkt_getlen(uint16_t pktlen) +{ + return (uint32_t)pktlen << VMBUS_CHANPKT_SIZE_SHIFT; +} + +/* + * GPA stuffs. + */ +struct vmbus_gpa_range { + uint32_t len; + uint32_t ofs; + uint64_t page[0]; +} __rte_packed; + +/* This is actually vmbus_gpa_range.gpa_page[1] */ +struct vmbus_gpa { + uint32_t len; + uint32_t ofs; + uint64_t page; +} __rte_packed; + +struct vmbus_chanpkt_hdr { + uint16_t type; /* VMBUS_CHANPKT_TYPE_ */ + uint16_t hlen; /* header len, in 8 bytes */ + uint16_t tlen; /* total len, in 8 bytes */ + uint16_t flags; /* VMBUS_CHANPKT_FLAG_ */ + uint64_t xactid; +} __rte_packed; + +static inline uint32_t +vmbus_chanpkt_datalen(const struct vmbus_chanpkt_hdr *pkt) +{ + return vmbus_chanpkt_getlen(pkt->tlen) + - vmbus_chanpkt_getlen(pkt->hlen); +} + +struct vmbus_chanpkt { + struct vmbus_chanpkt_hdr hdr; +} __rte_packed; + +struct vmbus_rxbuf_desc { + uint32_t len; + uint32_t ofs; +} __rte_packed; + +struct vmbus_chanpkt_rxbuf { + struct vmbus_chanpkt_hdr hdr; + uint16_t rxbuf_id; + uint16_t rsvd; + uint32_t rxbuf_cnt; + struct vmbus_rxbuf_desc rxbuf[]; +} __rte_packed; + +struct vmbus_chanpkt_sglist { + struct vmbus_chanpkt_hdr hdr; + uint32_t rsvd; + uint32_t gpa_cnt; + struct vmbus_gpa gpa[]; +} __rte_packed; + +/* + * Channel messages + * - Embedded in vmbus_message.msg_data, e.g. response and notification. + * - Embedded in hypercall_postmsg_in.hc_data, e.g. request. + */ + +#define VMBUS_CHANMSG_TYPE_CHOFFER 1 /* NOTE */ +#define VMBUS_CHANMSG_TYPE_CHRESCIND 2 /* NOTE */ +#define VMBUS_CHANMSG_TYPE_CHREQUEST 3 /* REQ */ +#define VMBUS_CHANMSG_TYPE_CHOFFER_DONE 4 /* NOTE */ +#define VMBUS_CHANMSG_TYPE_CHOPEN 5 /* REQ */ +#define VMBUS_CHANMSG_TYPE_CHOPEN_RESP 6 /* RESP */ +#define VMBUS_CHANMSG_TYPE_CHCLOSE 7 /* REQ */ +#define VMBUS_CHANMSG_TYPE_GPADL_CONN 8 /* REQ */ +#define VMBUS_CHANMSG_TYPE_GPADL_SUBCONN 9 /* REQ */ +#define VMBUS_CHANMSG_TYPE_GPADL_CONNRESP 10 /* RESP */ +#define VMBUS_CHANMSG_TYPE_GPADL_DISCONN 11 /* REQ */ +#define VMBUS_CHANMSG_TYPE_GPADL_DISCONNRESP 12 /* RESP */ +#define VMBUS_CHANMSG_TYPE_CHFREE 13 /* REQ */ +#define VMBUS_CHANMSG_TYPE_CONNECT 14 /* REQ */ +#define VMBUS_CHANMSG_TYPE_CONNECT_RESP 15 /* RESP */ +#define VMBUS_CHANMSG_TYPE_DISCONNECT 16 /* REQ */ +#define VMBUS_CHANMSG_TYPE_MAX 22 + +struct vmbus_chanmsg_hdr { + uint32_t type; /* VMBUS_CHANMSG_TYPE_ */ + uint32_t rsvd; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CONNECT */ +struct vmbus_chanmsg_connect { + struct vmbus_chanmsg_hdr hdr; + uint32_t ver; + uint32_t rsvd; + uint64_t evtflags; + uint64_t mnf1; + uint64_t mnf2; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CONNECT_RESP */ +struct vmbus_chanmsg_connect_resp { + struct vmbus_chanmsg_hdr hdr; + uint8_t done; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CHREQUEST */ +struct vmbus_chanmsg_chrequest { + struct vmbus_chanmsg_hdr hdr; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_DISCONNECT */ +struct vmbus_chanmsg_disconnect { + struct vmbus_chanmsg_hdr hdr; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CHOPEN */ +struct vmbus_chanmsg_chopen { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; + uint32_t openid; + uint32_t gpadl; + uint32_t vcpuid; + uint32_t txbr_pgcnt; +#define VMBUS_CHANMSG_CHOPEN_UDATA_SIZE 120 + uint8_t udata[VMBUS_CHANMSG_CHOPEN_UDATA_SIZE]; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CHOPEN_RESP */ +struct vmbus_chanmsg_chopen_resp { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; + uint32_t openid; + uint32_t status; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_GPADL_CONN */ +struct vmbus_chanmsg_gpadl_conn { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; + uint32_t gpadl; + uint16_t range_len; + uint16_t range_cnt; + struct vmbus_gpa_range range; +} __rte_packed; + +#define VMBUS_CHANMSG_GPADL_CONN_PGMAX 26 + +/* VMBUS_CHANMSG_TYPE_GPADL_SUBCONN */ +struct vmbus_chanmsg_gpadl_subconn { + struct vmbus_chanmsg_hdr hdr; + uint32_t msgno; + uint32_t gpadl; + uint64_t gpa_page[]; +} __rte_packed; + +#define VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX 28 + +/* VMBUS_CHANMSG_TYPE_GPADL_CONNRESP */ +struct vmbus_chanmsg_gpadl_connresp { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; + uint32_t gpadl; + uint32_t status; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CHCLOSE */ +struct vmbus_chanmsg_chclose { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_GPADL_DISCONN */ +struct vmbus_chanmsg_gpadl_disconn { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; + uint32_t gpadl; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CHFREE */ +struct vmbus_chanmsg_chfree { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CHRESCIND */ +struct vmbus_chanmsg_chrescind { + struct vmbus_chanmsg_hdr hdr; + uint32_t chanid; +} __rte_packed; + +/* VMBUS_CHANMSG_TYPE_CHOFFER */ +struct vmbus_chanmsg_choffer { + struct vmbus_chanmsg_hdr hdr; + rte_uuid_t chtype; + rte_uuid_t chinst; + uint64_t chlat; /* unit: 100ns */ + uint32_t chrev; + uint32_t svrctx_sz; + uint16_t chflags; + uint16_t mmio_sz; /* unit: MB */ + uint8_t udata[120]; + uint16_t subidx; + uint16_t rsvd; + uint32_t chanid; + uint8_t montrig; + uint8_t flags1; /* VMBUS_CHOFFER_FLAG1_ */ + uint16_t flags2; + uint32_t connid; +} __rte_packed; + +#define VMBUS_CHOFFER_FLAG1_HASMNF 0x01 + +#endif /* !_VMBUS_REG_H_ */ diff --git a/drivers/bus/vmbus/vmbus_bufring.c b/drivers/bus/vmbus/vmbus_bufring.c new file mode 100644 index 00000000..c8800160 --- /dev/null +++ b/drivers/bus/vmbus/vmbus_bufring.c @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2009-2012,2016 Microsoft Corp. + * Copyright (c) 2012 NetApp Inc. + * Copyright (c) 2012 Citrix Inc. + * All rights reserved. + */ + +#include <unistd.h> +#include <stdint.h> +#include <stdbool.h> +#include <string.h> +#include <sys/uio.h> + +#include <rte_eal.h> +#include <rte_tailq.h> +#include <rte_log.h> +#include <rte_malloc.h> +#include <rte_bus.h> +#include <rte_atomic.h> +#include <rte_memory.h> +#include <rte_pause.h> +#include <rte_bus_vmbus.h> + +#include "private.h" + +/* Increase bufring index by inc with wraparound */ +static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz) +{ + idx += inc; + if (idx >= sz) + idx -= sz; + + return idx; +} + +void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen) +{ + br->vbr = buf; + br->windex = br->vbr->windex; + br->dsize = blen - sizeof(struct vmbus_bufring); +} + +/* + * When we write to the ring buffer, check if the host needs to be + * signaled. + * + * The contract: + * - The host guarantees that while it is draining the TX bufring, + * it will set the br_imask to indicate it does not need to be + * interrupted when new data are added. + * - The host guarantees that it will completely drain the TX bufring + * before exiting the read loop. Further, once the TX bufring is + * empty, it will clear the br_imask and re-check to see if new + * data have arrived. + */ +static inline bool +vmbus_txbr_need_signal(const struct vmbus_br *tbr, uint32_t old_windex) +{ + rte_smp_mb(); + if (tbr->vbr->imask) + return false; + + rte_smp_rmb(); + + /* + * This is the only case we need to signal when the + * ring transitions from being empty to non-empty. + */ + return old_windex == tbr->vbr->rindex; +} + +static inline uint32_t +vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex, + const void *src0, uint32_t cplen) +{ + uint8_t *br_data = tbr->vbr->data; + uint32_t br_dsize = tbr->dsize; + const uint8_t *src = src0; + + /* XXX use double mapping like Linux kernel? */ + if (cplen > br_dsize - windex) { + uint32_t fraglen = br_dsize - windex; + + /* Wrap-around detected */ + memcpy(br_data + windex, src, fraglen); + memcpy(br_data, src + fraglen, cplen - fraglen); + } else { + memcpy(br_data + windex, src, cplen); + } + + return vmbus_br_idxinc(windex, cplen, br_dsize); +} + +/* + * Write scattered channel packet to TX bufring. + * + * The offset of this channel packet is written as a 64bits value + * immediately after this channel packet. + * + * The write goes through three stages: + * 1. Reserve space in ring buffer for the new data. + * Writer atomically moves priv_write_index. + * 2. Copy the new data into the ring. + * 3. Update the tail of the ring (visible to host) that indicates + * next read location. Writer updates write_index + */ +int +vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen, + bool *need_sig) +{ + struct vmbus_bufring *vbr = tbr->vbr; + uint32_t ring_size = tbr->dsize; + uint32_t old_windex, next_windex, windex, total; + uint64_t save_windex; + int i; + + total = 0; + for (i = 0; i < iovlen; i++) + total += iov[i].iov_len; + total += sizeof(save_windex); + + /* Reserve space in ring */ + do { + uint32_t avail; + + /* Get current free location */ + old_windex = tbr->windex; + + /* Prevent compiler reordering this with calculation */ + rte_compiler_barrier(); + + avail = vmbus_br_availwrite(tbr, old_windex); + + /* If not enough space in ring, then tell caller. */ + if (avail <= total) + return -EAGAIN; + + next_windex = vmbus_br_idxinc(old_windex, total, ring_size); + + /* Atomic update of next write_index for other threads */ + } while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex)); + + /* Space from old..new is now reserved */ + windex = old_windex; + for (i = 0; i < iovlen; i++) { + windex = vmbus_txbr_copyto(tbr, windex, + iov[i].iov_base, iov[i].iov_len); + } + + /* Set the offset of the current channel packet. */ + save_windex = ((uint64_t)old_windex) << 32; + windex = vmbus_txbr_copyto(tbr, windex, &save_windex, + sizeof(save_windex)); + + /* The region reserved should match region used */ + RTE_ASSERT(windex == next_windex); + + /* Ensure that data is available before updating host index */ + rte_smp_wmb(); + + /* Checkin for our reservation. wait for our turn to update host */ + while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex)) + rte_pause(); + + /* If host had read all data before this, then need to signal */ + *need_sig |= vmbus_txbr_need_signal(tbr, old_windex); + return 0; +} + +static inline uint32_t +vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex, + void *dst0, size_t cplen) +{ + const uint8_t *br_data = rbr->vbr->data; + uint32_t br_dsize = rbr->dsize; + uint8_t *dst = dst0; + + if (cplen > br_dsize - rindex) { + uint32_t fraglen = br_dsize - rindex; + + /* Wrap-around detected. */ + memcpy(dst, br_data + rindex, fraglen); + memcpy(dst + fraglen, br_data, cplen - fraglen); + } else { + memcpy(dst, br_data + rindex, cplen); + } + + return vmbus_br_idxinc(rindex, cplen, br_dsize); +} + +/* Copy data from receive ring but don't change index */ +int +vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen) +{ + uint32_t avail; + + /* + * The requested data and the 64bits channel packet + * offset should be there at least. + */ + avail = vmbus_br_availread(rbr); + if (avail < dlen + sizeof(uint64_t)) + return -EAGAIN; + + vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen); + return 0; +} + +/* + * Copy data from receive ring and change index + * NOTE: + * We assume (dlen + skip) == sizeof(channel packet). + */ +int +vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip) +{ + struct vmbus_bufring *vbr = rbr->vbr; + uint32_t br_dsize = rbr->dsize; + uint32_t rindex; + + if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t)) + return -EAGAIN; + + /* Record where host was when we started read (for debug) */ + rbr->windex = rbr->vbr->windex; + + /* + * Copy channel packet from RX bufring. + */ + rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize); + rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen); + + /* + * Discard this channel packet's 64bits offset, which is useless to us. + */ + rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize); + + /* Update the read index _after_ the channel packet is fetched. */ + rte_compiler_barrier(); + + vbr->rindex = rindex; + + return 0; +} diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c new file mode 100644 index 00000000..cc5f3e83 --- /dev/null +++ b/drivers/bus/vmbus/vmbus_channel.c @@ -0,0 +1,405 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <unistd.h> +#include <stdint.h> +#include <string.h> +#include <sys/uio.h> + +#include <rte_eal.h> +#include <rte_tailq.h> +#include <rte_log.h> +#include <rte_malloc.h> +#include <rte_bus.h> +#include <rte_atomic.h> +#include <rte_memory.h> +#include <rte_bus_vmbus.h> + +#include "private.h" + +static inline void +vmbus_sync_set_bit(volatile uint32_t *addr, uint32_t mask) +{ + /* Use GCC builtin which atomic does atomic OR operation */ + __sync_or_and_fetch(addr, mask); +} + +static inline void +vmbus_send_interrupt(const struct rte_vmbus_device *dev, uint32_t relid) +{ + uint32_t *int_addr; + uint32_t int_mask; + + int_addr = dev->int_page + relid / 32; + int_mask = 1u << (relid % 32); + + vmbus_sync_set_bit(int_addr, int_mask); +} + +static inline void +vmbus_set_monitor(const struct rte_vmbus_device *dev, uint32_t monitor_id) +{ + uint32_t *monitor_addr, monitor_mask; + unsigned int trigger_index; + + trigger_index = monitor_id / HV_MON_TRIG_LEN; + monitor_mask = 1u << (monitor_id % HV_MON_TRIG_LEN); + + monitor_addr = &dev->monitor_page->trigs[trigger_index].pending; + vmbus_sync_set_bit(monitor_addr, monitor_mask); +} + +static void +vmbus_set_event(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan) +{ + vmbus_send_interrupt(dev, chan->relid); + vmbus_set_monitor(dev, chan->monitor_id); +} + +/* + * Notify host that there are data pending on our TX bufring. + * + * Since this in userspace, rely on the monitor page. + * Can't do a hypercall from userspace. + */ +void +rte_vmbus_chan_signal_tx(const struct vmbus_channel *chan) +{ + const struct rte_vmbus_device *dev = chan->device; + const struct vmbus_br *tbr = &chan->txbr; + + /* Make sure all updates are done before signaling host */ + rte_smp_wmb(); + + /* If host is ignoring interrupts? */ + if (tbr->vbr->imask) + return; + + vmbus_set_event(dev, chan); +} + + +/* Do a simple send directly using transmit ring. */ +int rte_vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, + void *data, uint32_t dlen, + uint64_t xactid, uint32_t flags, bool *need_sig) +{ + struct vmbus_chanpkt pkt; + unsigned int pktlen, pad_pktlen; + const uint32_t hlen = sizeof(pkt); + bool send_evt = false; + uint64_t pad = 0; + struct iovec iov[3]; + int error; + + pktlen = hlen + dlen; + pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t)); + + pkt.hdr.type = type; + pkt.hdr.flags = flags; + pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT; + pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT; + pkt.hdr.xactid = xactid; + + iov[0].iov_base = &pkt; + iov[0].iov_len = hlen; + iov[1].iov_base = data; + iov[1].iov_len = dlen; + iov[2].iov_base = &pad; + iov[2].iov_len = pad_pktlen - pktlen; + + error = vmbus_txbr_write(&chan->txbr, iov, 3, &send_evt); + + /* + * caller sets need_sig to non-NULL if it will handle + * signaling if required later. + * if need_sig is NULL, signal now if needed. + */ + if (need_sig) + *need_sig |= send_evt; + else if (error == 0 && send_evt) + rte_vmbus_chan_signal_tx(chan); + return error; +} + +/* Do a scatter/gather send where the descriptor points to data. */ +int rte_vmbus_chan_send_sglist(struct vmbus_channel *chan, + struct vmbus_gpa sg[], uint32_t sglen, + void *data, uint32_t dlen, + uint64_t xactid, bool *need_sig) +{ + struct vmbus_chanpkt_sglist pkt; + unsigned int pktlen, pad_pktlen, hlen; + bool send_evt = false; + struct iovec iov[4]; + uint64_t pad = 0; + int error; + + hlen = offsetof(struct vmbus_chanpkt_sglist, gpa[sglen]); + pktlen = hlen + dlen; + pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t)); + + pkt.hdr.type = VMBUS_CHANPKT_TYPE_GPA; + pkt.hdr.flags = VMBUS_CHANPKT_FLAG_RC; + pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT; + pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT; + pkt.hdr.xactid = xactid; + pkt.rsvd = 0; + pkt.gpa_cnt = sglen; + + iov[0].iov_base = &pkt; + iov[0].iov_len = sizeof(pkt); + iov[1].iov_base = sg; + iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; + iov[2].iov_base = data; + iov[2].iov_len = dlen; + iov[3].iov_base = &pad; + iov[3].iov_len = pad_pktlen - pktlen; + + error = vmbus_txbr_write(&chan->txbr, iov, 4, &send_evt); + + /* if caller is batching, just propagate the status */ + if (need_sig) + *need_sig |= send_evt; + else if (error == 0 && send_evt) + rte_vmbus_chan_signal_tx(chan); + return error; +} + +bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel) +{ + const struct vmbus_br *br = &channel->rxbr; + + return br->vbr->rindex == br->vbr->windex; +} + +/* Signal host after reading N bytes */ +void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read) +{ + struct vmbus_br *rbr = &chan->rxbr; + uint32_t write_sz, pending_sz; + + /* No need for signaling on older versions */ + if (!rbr->vbr->feature_bits.feat_pending_send_sz) + return; + + /* Make sure reading of pending happens after new read index */ + rte_mb(); + + pending_sz = rbr->vbr->pending_send; + if (!pending_sz) + return; + + rte_smp_rmb(); + write_sz = vmbus_br_availwrite(rbr, rbr->vbr->windex); + + /* If there was space before then host was not blocked */ + if (write_sz - bytes_read > pending_sz) + return; + + /* If pending write will not fit */ + if (write_sz <= pending_sz) + return; + + vmbus_set_event(chan->device, chan); +} + +int rte_vmbus_chan_recv(struct vmbus_channel *chan, void *data, uint32_t *len, + uint64_t *request_id) +{ + struct vmbus_chanpkt_hdr pkt; + uint32_t dlen, hlen, bufferlen = *len; + int error; + + *len = 0; + + error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt)); + if (error) + return error; + + if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) { + VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen); + /* XXX this channel is dead actually. */ + return -EIO; + } + + if (unlikely(pkt.hlen > pkt.tlen)) { + VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u", + pkt.hlen, pkt.tlen); + return -EIO; + } + + /* Length are in quad words */ + hlen = pkt.hlen << VMBUS_CHANPKT_SIZE_SHIFT; + dlen = (pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT) - hlen; + *len = dlen; + + /* If caller buffer is not large enough */ + if (unlikely(dlen > bufferlen)) + return -ENOBUFS; + + if (request_id) + *request_id = pkt.xactid; + + /* Read data and skip packet header */ + error = vmbus_rxbr_read(&chan->rxbr, data, dlen, hlen); + if (error) + return error; + + rte_vmbus_chan_signal_read(chan, dlen + hlen + sizeof(uint64_t)); + return 0; +} + +/* TODO: replace this with inplace ring buffer (no copy) */ +int rte_vmbus_chan_recv_raw(struct vmbus_channel *chan, + void *data, uint32_t *len) +{ + struct vmbus_chanpkt_hdr pkt; + uint32_t dlen, bufferlen = *len; + int error; + + error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt)); + if (error) + return error; + + if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) { + VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen); + /* XXX this channel is dead actually. */ + return -EIO; + } + + if (unlikely(pkt.hlen > pkt.tlen)) { + VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u", + pkt.hlen, pkt.tlen); + return -EIO; + } + + /* Length are in quad words */ + dlen = pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT; + *len = dlen; + + /* If caller buffer is not large enough */ + if (unlikely(dlen > bufferlen)) + return -ENOBUFS; + + /* Read data and skip packet header */ + error = vmbus_rxbr_read(&chan->rxbr, data, dlen, 0); + if (error) + return error; + + /* Return the number of bytes read */ + return dlen + sizeof(uint64_t); +} + +int vmbus_chan_create(const struct rte_vmbus_device *device, + uint16_t relid, uint16_t subid, uint8_t monitor_id, + struct vmbus_channel **new_chan) +{ + struct vmbus_channel *chan; + int err; + + chan = rte_zmalloc_socket("VMBUS", sizeof(*chan), RTE_CACHE_LINE_SIZE, + device->device.numa_node); + if (!chan) + return -ENOMEM; + + STAILQ_INIT(&chan->subchannel_list); + chan->device = device; + chan->subchannel_id = subid; + chan->relid = relid; + chan->monitor_id = monitor_id; + *new_chan = chan; + + err = vmbus_uio_map_rings(chan); + if (err) { + rte_free(chan); + return err; + } + + return 0; +} + +/* Setup the primary channel */ +int rte_vmbus_chan_open(struct rte_vmbus_device *device, + struct vmbus_channel **new_chan) +{ + int err; + + err = vmbus_chan_create(device, device->relid, 0, + device->monitor_id, new_chan); + if (!err) + device->primary = *new_chan; + + return err; +} + +int rte_vmbus_max_channels(const struct rte_vmbus_device *device) +{ + if (vmbus_uio_subchannels_supported(device, device->primary)) + return VMBUS_MAX_CHANNELS; + else + return 1; +} + +/* Setup secondary channel */ +int rte_vmbus_subchan_open(struct vmbus_channel *primary, + struct vmbus_channel **new_chan) +{ + struct vmbus_channel *chan; + int err; + + err = vmbus_uio_get_subchan(primary, &chan); + if (err) + return err; + + STAILQ_INSERT_TAIL(&primary->subchannel_list, chan, next); + *new_chan = chan; + return 0; +} + +uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan) +{ + return chan->subchannel_id; +} + +void rte_vmbus_chan_close(struct vmbus_channel *chan) +{ + const struct rte_vmbus_device *device = chan->device; + struct vmbus_channel *primary = device->primary; + + if (chan != primary) + STAILQ_REMOVE(&primary->subchannel_list, chan, + vmbus_channel, next); + + rte_free(chan); +} + +static void vmbus_dump_ring(FILE *f, const char *id, const struct vmbus_br *br) +{ + const struct vmbus_bufring *vbr = br->vbr; + struct vmbus_chanpkt_hdr pkt; + + fprintf(f, "%s windex=%u rindex=%u mask=%u pending=%u feature=%#x\n", + id, vbr->windex, vbr->rindex, vbr->imask, + vbr->pending_send, vbr->feature_bits.value); + fprintf(f, " size=%u avail write=%u read=%u\n", + br->dsize, vmbus_br_availwrite(br, vbr->windex), + vmbus_br_availread(br)); + + if (vmbus_rxbr_peek(br, &pkt, sizeof(pkt)) == 0) + fprintf(f, " pkt type %#x len %u flags %#x xactid %#"PRIx64"\n", + pkt.type, + pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT, + pkt.flags, pkt.xactid); +} + +void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan) +{ + fprintf(f, "channel[%u] relid=%u monitor=%u\n", + chan->subchannel_id, chan->relid, chan->monitor_id); + vmbus_dump_ring(f, "rxbr", &chan->rxbr); + vmbus_dump_ring(f, "txbr", &chan->txbr); +} diff --git a/drivers/bus/vmbus/vmbus_common.c b/drivers/bus/vmbus/vmbus_common.c new file mode 100644 index 00000000..c7165ad5 --- /dev/null +++ b/drivers/bus/vmbus/vmbus_common.c @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <string.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <sys/queue.h> +#include <sys/mman.h> + +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_eal.h> +#include <rte_tailq.h> +#include <rte_devargs.h> +#include <rte_malloc.h> +#include <rte_errno.h> +#include <rte_memory.h> +#include <rte_bus_vmbus.h> + +#include "private.h" + +int vmbus_logtype_bus; +extern struct rte_vmbus_bus rte_vmbus_bus; + +/* map a particular resource from a file */ +void * +vmbus_map_resource(void *requested_addr, int fd, off_t offset, size_t size, + int flags) +{ + void *mapaddr; + + /* Map the memory resource of device */ + mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE, + MAP_SHARED | flags, fd, offset); + if (mapaddr == MAP_FAILED) { + VMBUS_LOG(ERR, + "mmap(%d, %p, %zu, %ld) failed: %s", + fd, requested_addr, size, (long)offset, + strerror(errno)); + } + return mapaddr; +} + +/* unmap a particular resource */ +void +vmbus_unmap_resource(void *requested_addr, size_t size) +{ + if (requested_addr == NULL) + return; + + /* Unmap the VMBUS memory resource of device */ + if (munmap(requested_addr, size)) { + VMBUS_LOG(ERR, "munmap(%p, 0x%lx) failed: %s", + requested_addr, (unsigned long)size, + strerror(errno)); + } else + VMBUS_LOG(DEBUG, " VMBUS memory unmapped at %p", + requested_addr); +} + +/** + * Match the VMBUS driver and device using UUID table + * + * @param drv + * VMBUS driver from which ID table would be extracted + * @param pci_dev + * VMBUS device to match against the driver + * @return + * true for successful match + * false for unsuccessful match + */ +static bool +vmbus_match(const struct rte_vmbus_driver *dr, + const struct rte_vmbus_device *dev) +{ + const rte_uuid_t *id_table; + + for (id_table = dr->id_table; !rte_uuid_is_null(*id_table); ++id_table) { + if (rte_uuid_compare(*id_table, dev->class_id) == 0) + return true; + } + + return false; +} + +/* + * If device ID match, call the devinit() function of the driver. + */ +static int +vmbus_probe_one_driver(struct rte_vmbus_driver *dr, + struct rte_vmbus_device *dev) +{ + char guid[RTE_UUID_STRLEN]; + int ret; + + if (!vmbus_match(dr, dev)) + return 1; /* not supported */ + + rte_uuid_unparse(dev->device_id, guid, sizeof(guid)); + VMBUS_LOG(INFO, "VMBUS device %s on NUMA socket %i", + guid, dev->device.numa_node); + + /* TODO add blacklisted */ + + /* map resources for device */ + ret = rte_vmbus_map_device(dev); + if (ret != 0) + return ret; + + /* reference driver structure */ + dev->driver = dr; + dev->device.driver = &dr->driver; + + if (dev->device.numa_node < 0) { + VMBUS_LOG(WARNING, " Invalid NUMA socket, default to 0"); + dev->device.numa_node = 0; + } + + /* call the driver probe() function */ + VMBUS_LOG(INFO, " probe driver: %s", dr->driver.name); + ret = dr->probe(dr, dev); + if (ret) { + dev->driver = NULL; + rte_vmbus_unmap_device(dev); + } + + return ret; +} + +/* + * IF device class GUID mathces, call the probe function of + * registere drivers for the vmbus device. + * Return -1 if initialization failed, + * and 1 if no driver found for this device. + */ +static int +vmbus_probe_all_drivers(struct rte_vmbus_device *dev) +{ + struct rte_vmbus_driver *dr; + int rc; + + /* Check if a driver is already loaded */ + if (dev->driver != NULL) { + VMBUS_LOG(DEBUG, "VMBUS driver already loaded"); + return 0; + } + + FOREACH_DRIVER_ON_VMBUS(dr) { + rc = vmbus_probe_one_driver(dr, dev); + if (rc < 0) /* negative is an error */ + return -1; + + if (rc > 0) /* positive driver doesn't support it */ + continue; + + return 0; + } + return 1; +} + +/* + * Scan the vmbus, and call the devinit() function for + * all registered drivers that have a matching entry in its id_table + * for discovered devices. + */ +int +rte_vmbus_probe(void) +{ + struct rte_vmbus_device *dev; + size_t probed = 0, failed = 0; + char ubuf[RTE_UUID_STRLEN]; + + FOREACH_DEVICE_ON_VMBUS(dev) { + probed++; + + rte_uuid_unparse(dev->device_id, ubuf, sizeof(ubuf)); + + /* TODO: add whitelist/blacklist */ + + if (vmbus_probe_all_drivers(dev) < 0) { + VMBUS_LOG(NOTICE, + "Requested device %s cannot be used", ubuf); + rte_errno = errno; + failed++; + } + } + + return (probed && probed == failed) ? -1 : 0; +} + +static int +vmbus_parse(const char *name, void *addr) +{ + rte_uuid_t guid; + int ret; + + ret = rte_uuid_parse(name, guid); + if (ret == 0 && addr) + memcpy(addr, &guid, sizeof(guid)); + + return ret; +} + +/* register vmbus driver */ +void +rte_vmbus_register(struct rte_vmbus_driver *driver) +{ + VMBUS_LOG(DEBUG, + "Registered driver %s", driver->driver.name); + + TAILQ_INSERT_TAIL(&rte_vmbus_bus.driver_list, driver, next); + driver->bus = &rte_vmbus_bus; +} + +/* unregister vmbus driver */ +void +rte_vmbus_unregister(struct rte_vmbus_driver *driver) +{ + TAILQ_REMOVE(&rte_vmbus_bus.driver_list, driver, next); + driver->bus = NULL; +} + +/* Add a device to VMBUS bus */ +void +vmbus_add_device(struct rte_vmbus_device *vmbus_dev) +{ + TAILQ_INSERT_TAIL(&rte_vmbus_bus.device_list, vmbus_dev, next); +} + +/* Insert a device into a predefined position in VMBUS bus */ +void +vmbus_insert_device(struct rte_vmbus_device *exist_vmbus_dev, + struct rte_vmbus_device *new_vmbus_dev) +{ + TAILQ_INSERT_BEFORE(exist_vmbus_dev, new_vmbus_dev, next); +} + +/* Remove a device from VMBUS bus */ +void +vmbus_remove_device(struct rte_vmbus_device *vmbus_dev) +{ + TAILQ_REMOVE(&rte_vmbus_bus.device_list, vmbus_dev, next); +} + +/* VMBUS doesn't support hotplug */ +static struct rte_device * +vmbus_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_vmbus_device *dev; + + FOREACH_DEVICE_ON_VMBUS(dev) { + if (start && &dev->device == start) { + start = NULL; + continue; + } + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + + return NULL; +} + + +struct rte_vmbus_bus rte_vmbus_bus = { + .bus = { + .scan = rte_vmbus_scan, + .probe = rte_vmbus_probe, + .find_device = vmbus_find_device, + .parse = vmbus_parse, + }, + .device_list = TAILQ_HEAD_INITIALIZER(rte_vmbus_bus.device_list), + .driver_list = TAILQ_HEAD_INITIALIZER(rte_vmbus_bus.driver_list), +}; + +RTE_REGISTER_BUS(vmbus, rte_vmbus_bus.bus); + +RTE_INIT(vmbus_init_log) +{ + vmbus_logtype_bus = rte_log_register("bus.vmbus"); + if (vmbus_logtype_bus >= 0) + rte_log_set_level(vmbus_logtype_bus, RTE_LOG_NOTICE); +} diff --git a/drivers/bus/vmbus/vmbus_common_uio.c b/drivers/bus/vmbus/vmbus_common_uio.c new file mode 100644 index 00000000..5ddd36ab --- /dev/null +++ b/drivers/bus/vmbus/vmbus_common_uio.c @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <fcntl.h> +#include <string.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/mman.h> + +#include <rte_eal.h> +#include <rte_tailq.h> +#include <rte_log.h> +#include <rte_malloc.h> +#include <rte_bus.h> +#include <rte_bus_vmbus.h> + +#include "private.h" + +static struct rte_tailq_elem vmbus_tailq = { + .name = "VMBUS_RESOURCE_LIST", +}; +EAL_REGISTER_TAILQ(vmbus_tailq) + +static int +vmbus_uio_map_secondary(struct rte_vmbus_device *dev) +{ + int fd, i; + struct mapped_vmbus_resource *uio_res; + struct mapped_vmbus_res_list *uio_res_list + = RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list); + + TAILQ_FOREACH(uio_res, uio_res_list, next) { + + /* skip this element if it doesn't match our UUID */ + if (rte_uuid_compare(uio_res->id, dev->device_id) != 0) + continue; + + /* open /dev/uioX */ + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + for (i = 0; i != uio_res->nb_maps; i++) { + void *mapaddr; + + mapaddr = vmbus_map_resource(uio_res->maps[i].addr, + fd, 0, + uio_res->maps[i].size, 0); + + if (mapaddr == uio_res->maps[i].addr) + continue; + + VMBUS_LOG(ERR, + "Cannot mmap device resource file %s to address: %p", + uio_res->path, uio_res->maps[i].addr); + + if (mapaddr != MAP_FAILED) + /* unmap addr wrongly mapped */ + vmbus_unmap_resource(mapaddr, + (size_t)uio_res->maps[i].size); + + /* unmap addrs correctly mapped */ + while (--i >= 0) + vmbus_unmap_resource(uio_res->maps[i].addr, + (size_t)uio_res->maps[i].size); + + close(fd); + return -1; + } + + /* fd is not needed in slave process, close it */ + close(fd); + return 0; + } + + VMBUS_LOG(ERR, "Cannot find resource for device"); + return 1; +} + +static int +vmbus_uio_map_primary(struct rte_vmbus_device *dev) +{ + int i, ret; + struct mapped_vmbus_resource *uio_res = NULL; + struct mapped_vmbus_res_list *uio_res_list = + RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list); + + /* allocate uio resource */ + ret = vmbus_uio_alloc_resource(dev, &uio_res); + if (ret) + return ret; + + /* Map the resources */ + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { + /* skip empty BAR */ + if (dev->resource[i].len == 0) + continue; + + ret = vmbus_uio_map_resource_by_index(dev, i, uio_res, 0); + if (ret) + goto error; + } + + uio_res->nb_maps = i; + + TAILQ_INSERT_TAIL(uio_res_list, uio_res, next); + + return 0; +error: + while (--i >= 0) { + vmbus_unmap_resource(uio_res->maps[i].addr, + (size_t)uio_res->maps[i].size); + } + vmbus_uio_free_resource(dev, uio_res); + return -1; +} + + +struct mapped_vmbus_resource * +vmbus_uio_find_resource(const struct rte_vmbus_device *dev) +{ + struct mapped_vmbus_resource *uio_res; + struct mapped_vmbus_res_list *uio_res_list = + RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list); + + if (dev == NULL) + return NULL; + + TAILQ_FOREACH(uio_res, uio_res_list, next) { + /* skip this element if it doesn't match our VMBUS address */ + if (rte_uuid_compare(uio_res->id, dev->device_id) == 0) + return uio_res; + } + return NULL; +} + +/* map the VMBUS resource of a VMBUS device in virtual memory */ +int +vmbus_uio_map_resource(struct rte_vmbus_device *dev) +{ + struct mapped_vmbus_resource *uio_res; + int ret; + + /* TODO: handle rescind */ + dev->intr_handle.fd = -1; + dev->intr_handle.uio_cfg_fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + + /* secondary processes - use already recorded details */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + ret = vmbus_uio_map_secondary(dev); + else + ret = vmbus_uio_map_primary(dev); + + if (ret != 0) + return ret; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources!"); + return -EIO; + } + + if (uio_res->nb_maps <= HV_MON_PAGE_MAP) { + VMBUS_LOG(ERR, "VMBUS: only %u resources found!", + uio_res->nb_maps); + return -EINVAL; + } + + dev->int_page = (uint32_t *)((char *)uio_res->maps[HV_INT_PAGE_MAP].addr + + (PAGE_SIZE >> 1)); + dev->monitor_page = uio_res->maps[HV_MON_PAGE_MAP].addr; + return 0; +} + +static void +vmbus_uio_unmap(struct mapped_vmbus_resource *uio_res) +{ + int i; + + if (uio_res == NULL) + return; + + for (i = 0; i != uio_res->nb_maps; i++) { + vmbus_unmap_resource(uio_res->maps[i].addr, + (size_t)uio_res->maps[i].size); + } +} + +/* unmap the VMBUS resource of a VMBUS device in virtual memory */ +void +vmbus_uio_unmap_resource(struct rte_vmbus_device *dev) +{ + struct mapped_vmbus_resource *uio_res; + struct mapped_vmbus_res_list *uio_res_list = + RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list); + + if (dev == NULL) + return; + + /* find an entry for the device */ + uio_res = vmbus_uio_find_resource(dev); + if (uio_res == NULL) + return; + + /* secondary processes - just free maps */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return vmbus_uio_unmap(uio_res); + + TAILQ_REMOVE(uio_res_list, uio_res, next); + + /* unmap all resources */ + vmbus_uio_unmap(uio_res); + + /* free uio resource */ + rte_free(uio_res); + + /* close fd if in primary process */ + close(dev->intr_handle.fd); + if (dev->intr_handle.uio_cfg_fd >= 0) { + close(dev->intr_handle.uio_cfg_fd); + dev->intr_handle.uio_cfg_fd = -1; + } + + dev->intr_handle.fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; +} |