aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/bus
diff options
context:
space:
mode:
authorLuca Boccassi <luca.boccassi@gmail.com>2018-08-14 18:52:30 +0100
committerLuca Boccassi <luca.boccassi@gmail.com>2018-08-14 18:53:17 +0100
commitb63264c8342e6a1b6971c79550d2af2024b6a4de (patch)
tree83114aac64286fe616506c0b3dfaec2ab86ef835 /drivers/bus
parentca33590b6af032bff57d9cc70455660466a654b2 (diff)
New upstream version 18.08upstream/18.08
Change-Id: I32fdf5e5016556d9c0a6d88ddaf1fc468961790a Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>
Diffstat (limited to 'drivers/bus')
-rw-r--r--drivers/bus/Makefile4
-rw-r--r--drivers/bus/dpaa/base/fman/fman.c6
-rw-r--r--drivers/bus/dpaa/base/fman/fman_hw.c62
-rw-r--r--drivers/bus/dpaa/base/fman/netcfg_layer.c5
-rw-r--r--drivers/bus/dpaa/base/fman/of.c44
-rw-r--r--drivers/bus/dpaa/base/qbman/bman_driver.c6
-rw-r--r--drivers/bus/dpaa/base/qbman/qman.c36
-rw-r--r--drivers/bus/dpaa/base/qbman/qman_driver.c14
-rw-r--r--drivers/bus/dpaa/base/qbman/qman_priv.h1
-rw-r--r--drivers/bus/dpaa/dpaa_bus.c148
-rw-r--r--drivers/bus/dpaa/include/compat.h36
-rw-r--r--drivers/bus/dpaa/include/fsl_fman.h6
-rw-r--r--drivers/bus/dpaa/include/fsl_qman.h27
-rw-r--r--drivers/bus/dpaa/include/of.h2
-rw-r--r--drivers/bus/dpaa/meson.build29
-rw-r--r--drivers/bus/dpaa/rte_bus_dpaa_version.map10
-rw-r--r--drivers/bus/dpaa/rte_dpaa_bus.h40
-rw-r--r--drivers/bus/dpaa/rte_dpaa_logs.h11
-rw-r--r--drivers/bus/fslmc/Makefile19
-rw-r--r--drivers/bus/fslmc/fslmc_bus.c179
-rw-r--r--drivers/bus/fslmc/fslmc_logs.h69
-rw-r--r--drivers/bus/fslmc/fslmc_vfio.c384
-rw-r--r--drivers/bus/fslmc/fslmc_vfio.h2
-rw-r--r--drivers/bus/fslmc/mc/dpdmai.c429
-rw-r--r--drivers/bus/fslmc/mc/fsl_dpdmai.h189
-rw-r--r--drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h107
-rw-r--r--drivers/bus/fslmc/mc/fsl_mc_cmd.h2
-rw-r--r--drivers/bus/fslmc/meson.build27
-rw-r--r--drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c12
-rw-r--r--drivers/bus/fslmc/portal/dpaa2_hw_dpci.c100
-rw-r--r--drivers/bus/fslmc/portal/dpaa2_hw_dpio.c123
-rw-r--r--drivers/bus/fslmc/portal/dpaa2_hw_dpio.h8
-rw-r--r--drivers/bus/fslmc/portal/dpaa2_hw_pvt.h94
-rw-r--r--drivers/bus/fslmc/qbman/include/fsl_qbman_base.h2
-rw-r--r--drivers/bus/fslmc/qbman/qbman_portal.c17
-rw-r--r--drivers/bus/fslmc/qbman/qbman_portal.h1
-rw-r--r--drivers/bus/fslmc/qbman/qbman_sys.h30
-rw-r--r--drivers/bus/fslmc/qbman/qbman_sys_decl.h23
-rw-r--r--drivers/bus/fslmc/rte_bus_fslmc_version.map17
-rw-r--r--drivers/bus/fslmc/rte_fslmc.h12
-rw-r--r--drivers/bus/ifpga/Makefile32
-rw-r--r--drivers/bus/ifpga/ifpga_bus.c467
-rw-r--r--drivers/bus/ifpga/ifpga_common.c88
-rw-r--r--drivers/bus/ifpga/ifpga_common.h18
-rw-r--r--drivers/bus/ifpga/ifpga_logs.h31
-rw-r--r--drivers/bus/ifpga/meson.build8
-rw-r--r--drivers/bus/ifpga/rte_bus_ifpga.h149
-rw-r--r--drivers/bus/ifpga/rte_bus_ifpga_version.map10
-rw-r--r--drivers/bus/meson.build2
-rw-r--r--drivers/bus/pci/Makefile36
-rw-r--r--drivers/bus/pci/bsd/Makefile32
-rw-r--r--drivers/bus/pci/linux/Makefile32
-rw-r--r--drivers/bus/pci/linux/pci.c36
-rw-r--r--drivers/bus/pci/linux/pci_uio.c47
-rw-r--r--drivers/bus/pci/linux/pci_vfio.c118
-rw-r--r--drivers/bus/pci/meson.build3
-rw-r--r--drivers/bus/pci/pci_common.c124
-rw-r--r--drivers/bus/pci/private.h50
-rw-r--r--drivers/bus/pci/rte_bus_pci.h8
-rw-r--r--drivers/bus/vdev/Makefile1
-rw-r--r--drivers/bus/vdev/meson.build2
-rw-r--r--drivers/bus/vdev/rte_bus_vdev.h38
-rw-r--r--drivers/bus/vdev/vdev.c262
-rw-r--r--drivers/bus/vmbus/Makefile36
-rw-r--r--drivers/bus/vmbus/linux/Makefile3
-rw-r--r--drivers/bus/vmbus/linux/vmbus_bus.c355
-rw-r--r--drivers/bus/vmbus/linux/vmbus_uio.c398
-rw-r--r--drivers/bus/vmbus/meson.build18
-rw-r--r--drivers/bus/vmbus/private.h132
-rw-r--r--drivers/bus/vmbus/rte_bus_vmbus.h407
-rw-r--r--drivers/bus/vmbus/rte_bus_vmbus_version.map29
-rw-r--r--drivers/bus/vmbus/rte_vmbus_reg.h344
-rw-r--r--drivers/bus/vmbus/vmbus_bufring.c244
-rw-r--r--drivers/bus/vmbus/vmbus_channel.c405
-rw-r--r--drivers/bus/vmbus/vmbus_common.c286
-rw-r--r--drivers/bus/vmbus/vmbus_common_uio.c232
76 files changed, 5911 insertions, 905 deletions
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index 7ef25939..cea3b55e 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -4,8 +4,12 @@
include $(RTE_SDK)/mk/rte.vars.mk
DIRS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += dpaa
+ifeq ($(CONFIG_RTE_EAL_VFIO),y)
DIRS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += fslmc
+endif
+DIRS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga
DIRS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci
DIRS-$(CONFIG_RTE_LIBRTE_VDEV_BUS) += vdev
+DIRS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus
include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/bus/dpaa/base/fman/fman.c b/drivers/bus/dpaa/base/fman/fman.c
index bda62e01..bdb70042 100644
--- a/drivers/bus/dpaa/base/fman/fman.c
+++ b/drivers/bus/dpaa/base/fman/fman.c
@@ -300,7 +300,7 @@ fman_if_init(const struct device_node *dpa_node)
_errno = fman_get_mac_index(regs_addr_host, &__if->__if.mac_idx);
if (_errno) {
- FMAN_ERR(-EINVAL, "Invalid register address: %lu",
+ FMAN_ERR(-EINVAL, "Invalid register address: %" PRIx64,
regs_addr_host);
goto err;
}
@@ -442,6 +442,7 @@ fman_if_init(const struct device_node *dpa_node)
if (!pool_node) {
FMAN_ERR(-ENXIO, "%s: bad fsl,bman-buffer-pools\n",
dname);
+ free(bpool);
goto err;
}
pname = pool_node->full_name;
@@ -449,6 +450,7 @@ fman_if_init(const struct device_node *dpa_node)
prop = of_get_property(pool_node, "fsl,bpid", &proplen);
if (!prop) {
FMAN_ERR(-EINVAL, "%s: no fsl,bpid\n", pname);
+ free(bpool);
goto err;
}
assert(proplen == sizeof(*prop));
@@ -502,7 +504,7 @@ fman_if_init(const struct device_node *dpa_node)
/* Parsing of the network interface is complete, add it to the list */
DPAA_BUS_LOG(DEBUG, "Found %s, Tx Channel = %x, FMAN = %x,"
- "Port ID = %x\n",
+ "Port ID = %x",
dname, __if->__if.tx_channel_id, __if->__if.fman_idx,
__if->__if.mac_idx);
diff --git a/drivers/bus/dpaa/base/fman/fman_hw.c b/drivers/bus/dpaa/base/fman/fman_hw.c
index 0148b98e..4ebbc3d3 100644
--- a/drivers/bus/dpaa/base/fman/fman_hw.c
+++ b/drivers/bus/dpaa/base/fman/fman_hw.c
@@ -16,6 +16,9 @@
#include <fsl_fman_crc64.h>
#include <fsl_bman.h>
+#define FMAN_SP_SG_DISABLE 0x80000000
+#define FMAN_SP_EXT_BUF_MARG_START_SHIFT 16
+
/* Instantiate the global variable that the inline CRC64 implementation (in
* <fsl_fman.h>) depends on.
*/
@@ -422,20 +425,16 @@ fman_if_set_fc_quanta(struct fman_if *fm_if, u16 pause_quanta)
int
fman_if_get_fdoff(struct fman_if *fm_if)
{
- u32 fmbm_ricp;
+ u32 fmbm_rebm;
int fdoff;
- int iceof_mask = 0x001f0000;
- int icsz_mask = 0x0000001f;
struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
assert(fman_ccsr_map_fd != -1);
- fmbm_ricp =
- in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ricp);
- /*iceof + icsz*/
- fdoff = ((fmbm_ricp & iceof_mask) >> 16) * 16 +
- (fmbm_ricp & icsz_mask) * 16;
+ fmbm_rebm = in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm);
+
+ fdoff = (fmbm_rebm >> FMAN_SP_EXT_BUF_MARG_START_SHIFT) & 0x1ff;
return fdoff;
}
@@ -502,12 +501,16 @@ fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset)
{
struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
unsigned int *fmbm_rebm;
+ int val = 0;
+ int fmbm_mask = 0x01ff0000;
+
+ val = fd_offset << FMAN_SP_EXT_BUF_MARG_START_SHIFT;
assert(fman_ccsr_map_fd != -1);
fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm;
- out_be32(fmbm_rebm, in_be32(fmbm_rebm) | (fd_offset << 16));
+ out_be32(fmbm_rebm, (in_be32(fmbm_rebm) & ~fmbm_mask) | val);
}
void
@@ -536,6 +539,47 @@ fman_if_get_maxfrm(struct fman_if *fm_if)
return (in_be32(reg_maxfrm) | 0x0000FFFF);
}
+/* MSB in fmbm_rebm register
+ * 0 - If BMI cannot store the frame in a single buffer it may select a buffer
+ * of smaller size and store the frame in scatter gather (S/G) buffers
+ * 1 - Scatter gather format is not enabled for frame storage. If BMI cannot
+ * store the frame in a single buffer, the frame is discarded.
+ */
+
+int
+fman_if_get_sg_enable(struct fman_if *fm_if)
+{
+ u32 fmbm_rebm;
+
+ struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
+
+ assert(fman_ccsr_map_fd != -1);
+
+ fmbm_rebm = in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm);
+
+ return (fmbm_rebm & FMAN_SP_SG_DISABLE) ? 0 : 1;
+}
+
+void
+fman_if_set_sg(struct fman_if *fm_if, int enable)
+{
+ struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
+ unsigned int *fmbm_rebm;
+ int val;
+ int fmbm_mask = FMAN_SP_SG_DISABLE;
+
+ if (enable)
+ val = 0;
+ else
+ val = FMAN_SP_SG_DISABLE;
+
+ assert(fman_ccsr_map_fd != -1);
+
+ fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm;
+
+ out_be32(fmbm_rebm, (in_be32(fmbm_rebm) & ~fmbm_mask) | val);
+}
+
void
fman_if_set_dnia(struct fman_if *fm_if, uint32_t nia)
{
diff --git a/drivers/bus/dpaa/base/fman/netcfg_layer.c b/drivers/bus/dpaa/base/fman/netcfg_layer.c
index 3e956ce1..031c6f1a 100644
--- a/drivers/bus/dpaa/base/fman/netcfg_layer.c
+++ b/drivers/bus/dpaa/base/fman/netcfg_layer.c
@@ -18,11 +18,6 @@
#include <rte_dpaa_logs.h>
#include <netcfg.h>
-/* Structure contains information about all the interfaces given by user
- * on command line.
- */
-struct netcfg_interface *netcfg_interface;
-
/* This data structure contaings all configurations information
* related to usages of DPA devices.
*/
diff --git a/drivers/bus/dpaa/base/fman/of.c b/drivers/bus/dpaa/base/fman/of.c
index 1b2dbe26..a7f3174e 100644
--- a/drivers/bus/dpaa/base/fman/of.c
+++ b/drivers/bus/dpaa/base/fman/of.c
@@ -182,6 +182,11 @@ linear_dir(struct dt_dir *d)
DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s",
d->node.node.full_name);
d->lphandle = f;
+ } else if (!strcmp(f->node.node.name, "phandle")) {
+ if (d->lphandle)
+ DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s",
+ d->node.node.full_name);
+ d->lphandle = f;
} else if (!strcmp(f->node.node.name, "#address-cells")) {
if (d->a_cells)
DPAA_BUS_LOG(DEBUG, "Duplicate a_cells in %s",
@@ -541,3 +546,42 @@ of_device_is_compatible(const struct device_node *dev_node,
return true;
return false;
}
+
+static const void *of_get_mac_addr(const struct device_node *np,
+ const char *name)
+{
+ return of_get_property(np, name, NULL);
+}
+
+/**
+ * Search the device tree for the best MAC address to use. 'mac-address' is
+ * checked first, because that is supposed to contain to "most recent" MAC
+ * address. If that isn't set, then 'local-mac-address' is checked next,
+ * because that is the default address. If that isn't set, then the obsolete
+ * 'address' is checked, just in case we're using an old device tree.
+ *
+ * Note that the 'address' property is supposed to contain a virtual address of
+ * the register set, but some DTS files have redefined that property to be the
+ * MAC address.
+ *
+ * All-zero MAC addresses are rejected, because those could be properties that
+ * exist in the device tree, but were not set by U-Boot. For example, the
+ * DTS could define 'mac-address' and 'local-mac-address', with zero MAC
+ * addresses. Some older U-Boots only initialized 'local-mac-address'. In
+ * this case, the real MAC is in 'local-mac-address', and 'mac-address' exists
+ * but is all zeros.
+ */
+const void *of_get_mac_address(const struct device_node *np)
+{
+ const void *addr;
+
+ addr = of_get_mac_addr(np, "mac-address");
+ if (addr)
+ return addr;
+
+ addr = of_get_mac_addr(np, "local-mac-address");
+ if (addr)
+ return addr;
+
+ return of_get_mac_addr(np, "address");
+}
diff --git a/drivers/bus/dpaa/base/qbman/bman_driver.c b/drivers/bus/dpaa/base/qbman/bman_driver.c
index a1ef3921..b14b5905 100644
--- a/drivers/bus/dpaa/base/qbman/bman_driver.c
+++ b/drivers/bus/dpaa/base/qbman/bman_driver.c
@@ -15,9 +15,9 @@
/*
* Global variables of the max portal/pool number this bman version supported
*/
-u16 bman_ip_rev;
+static u16 bman_ip_rev;
u16 bman_pool_max;
-void *bman_ccsr_map;
+static void *bman_ccsr_map;
/*****************/
/* Portal driver */
@@ -161,7 +161,7 @@ int bman_init_ccsr(const struct device_node *node)
PROT_WRITE, MAP_SHARED, ccsr_map_fd, phys_addr);
if (bman_ccsr_map == MAP_FAILED) {
pr_err("Can not map BMan CCSR base Bman: "
- "0x%x Phys: 0x%lx size 0x%lx",
+ "0x%x Phys: 0x%" PRIx64 " size 0x%" PRIu64,
*bman_addr, phys_addr, regs_size);
return -EINVAL;
}
diff --git a/drivers/bus/dpaa/base/qbman/qman.c b/drivers/bus/dpaa/base/qbman/qman.c
index 2b97671b..7c17027f 100644
--- a/drivers/bus/dpaa/base/qbman/qman.c
+++ b/drivers/bus/dpaa/base/qbman/qman.c
@@ -314,9 +314,9 @@ loop:
if (!msg)
return 0;
}
- if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
+ if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
/* We aren't draining anything but FQRNIs */
- pr_err("Found verb 0x%x in MR\n", msg->verb);
+ pr_err("Found verb 0x%x in MR\n", msg->ern.verb);
return -1;
}
qm_mr_next(p);
@@ -483,7 +483,7 @@ static inline void qm_mr_pvb_update(struct qm_portal *portal)
/* when accessing 'verb', use __raw_readb() to ensure that compiler
* inlining doesn't try to optimise out "excess reads".
*/
- if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
+ if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) {
mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
if (!mr->pi)
mr->vbit ^= QM_MR_VERB_VBIT;
@@ -625,7 +625,7 @@ fail_eqcr:
#define MAX_GLOBAL_PORTALS 8
static struct qman_portal global_portals[MAX_GLOBAL_PORTALS];
-rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS];
+static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS];
static struct qman_portal *
qman_alloc_global_portal(void)
@@ -832,7 +832,7 @@ mr_loop:
goto mr_done;
swapped_msg = *msg;
hw_fd_to_cpu(&swapped_msg.ern.fd);
- verb = msg->verb & QM_MR_VERB_TYPE_MASK;
+ verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK;
/* The message is a software ERN iff the 0x20 bit is set */
if (verb & 0x20) {
switch (verb) {
@@ -1058,7 +1058,7 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
struct qm_portal *portal = &p->p;
register struct qm_dqrr *dqrr = &portal->dqrr;
struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE];
- struct qman_fq *fq[QM_DQRR_SIZE];
+ struct qman_fq *fq;
unsigned int limit = 0, rx_number = 0;
uint32_t consume = 0;
@@ -1087,18 +1087,18 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
shadow[rx_number]->fd.opaque =
be32_to_cpu(dq[rx_number]->fd.opaque);
#else
- shadow = dq;
+ shadow[rx_number] = dq[rx_number];
#endif
/* SDQCR: context_b points to the FQ */
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
- fq[rx_number] = qman_fq_lookup_table[be32_to_cpu(
- dq[rx_number]->contextB)];
+ fq = qman_fq_lookup_table[be32_to_cpu(dq[rx_number]->contextB)];
#else
- fq[rx_number] = (void *)(uintptr_t)be32_to_cpu(dq->contextB);
+ fq = (void *)be32_to_cpu(dq[rx_number]->contextB);
#endif
- fq[rx_number]->cb.dqrr_prepare(shadow[rx_number],
- &bufs[rx_number]);
+ if (fq->cb.dqrr_prepare)
+ fq->cb.dqrr_prepare(shadow[rx_number],
+ &bufs[rx_number]);
consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number])));
rx_number++;
@@ -1106,7 +1106,7 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
} while (++limit < poll_limit);
if (rx_number)
- fq[0]->cb.dqrr_dpdk_pull_cb(fq, shadow, bufs, rx_number);
+ fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number);
/* Consume all the DQRR enries together */
qm_out(DQRR_DCAP, (1 << 8) | consume);
@@ -1665,7 +1665,7 @@ int qman_retire_fq(struct qman_fq *fq, u32 *flags)
*/
struct qm_mr_entry msg;
- msg.verb = QM_MR_VERB_FQRNI;
+ msg.ern.verb = QM_MR_VERB_FQRNI;
msg.fq.fqs = mcr->alterfq.fqs;
msg.fq.fqid = fq->fqid;
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
@@ -2002,13 +2002,13 @@ int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
return 0;
}
-int qman_set_vdq(struct qman_fq *fq, u16 num)
+int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags)
{
struct qman_portal *p = get_affine_portal();
uint32_t vdqcr;
int ret = -EBUSY;
- vdqcr = QM_VDQCR_EXACT;
+ vdqcr = vdqcr_flags;
vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
if ((fq->state != qman_fq_state_parked) &&
@@ -2642,7 +2642,7 @@ int qman_shutdown_fq(u32 fqid)
qm_mr_pvb_update(low_p);
msg = qm_mr_current(low_p);
while (msg) {
- if ((msg->verb &
+ if ((msg->ern.verb &
QM_MR_VERB_TYPE_MASK)
== QM_MR_VERB_FQRN)
found_fqrn = 1;
@@ -2710,7 +2710,7 @@ int qman_shutdown_fq(u32 fqid)
qm_mr_pvb_update(low_p);
msg = qm_mr_current(low_p);
while (msg) {
- if ((msg->verb & QM_MR_VERB_TYPE_MASK) ==
+ if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) ==
QM_MR_VERB_FQRL)
orl_empty = 1;
qm_mr_next(low_p);
diff --git a/drivers/bus/dpaa/base/qbman/qman_driver.c b/drivers/bus/dpaa/base/qbman/qman_driver.c
index 7cfa8ee4..f6ecd6b2 100644
--- a/drivers/bus/dpaa/base/qbman/qman_driver.c
+++ b/drivers/bus/dpaa/base/qbman/qman_driver.c
@@ -20,9 +20,9 @@ u16 qm_channel_caam = QMAN_CHANNEL_CAAM;
u16 qm_channel_pme = QMAN_CHANNEL_PME;
/* Ccsr map address to access ccsrbased register */
-void *qman_ccsr_map;
+static void *qman_ccsr_map;
/* The qman clock frequency */
-u32 qman_clk;
+static u32 qman_clk;
static __thread int qmfd = -1;
static __thread struct qm_portal_config qpcfg;
@@ -160,6 +160,7 @@ struct qman_portal *fsl_qman_portal_create(void)
&cpuset);
if (ret) {
error(0, ret, "pthread_getaffinity_np()");
+ kfree(q_pcfg);
return NULL;
}
@@ -168,12 +169,14 @@ struct qman_portal *fsl_qman_portal_create(void)
if (CPU_ISSET(loop, &cpuset)) {
if (q_pcfg->cpu != -1) {
pr_err("Thread is not affine to 1 cpu\n");
+ kfree(q_pcfg);
return NULL;
}
q_pcfg->cpu = loop;
}
if (q_pcfg->cpu == -1) {
pr_err("Bug in getaffinity handling!\n");
+ kfree(q_pcfg);
return NULL;
}
@@ -183,6 +186,7 @@ struct qman_portal *fsl_qman_portal_create(void)
ret = process_portal_map(&q_map);
if (ret) {
error(0, ret, "process_portal_map()");
+ kfree(q_pcfg);
return NULL;
}
q_pcfg->channel = q_map.channel;
@@ -217,6 +221,7 @@ err2:
close(q_fd);
err1:
process_portal_unmap(&q_map.addr);
+ kfree(q_pcfg);
return NULL;
}
@@ -246,7 +251,6 @@ int fsl_qman_portal_destroy(struct qman_portal *qp)
int qman_global_init(void)
{
const struct device_node *dt_node;
- int ret = 0;
size_t lenp;
const u32 *chanid;
static int ccsr_map_fd;
@@ -352,9 +356,7 @@ int qman_global_init(void)
qman_clk = be32_to_cpu(*clk);
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
- ret = qman_setup_fq_lookup_table(CONFIG_FSL_QMAN_FQ_LOOKUP_MAX);
- if (ret)
- return ret;
+ return qman_setup_fq_lookup_table(CONFIG_FSL_QMAN_FQ_LOOKUP_MAX);
#endif
return 0;
}
diff --git a/drivers/bus/dpaa/base/qbman/qman_priv.h b/drivers/bus/dpaa/base/qbman/qman_priv.h
index 9e4471e6..02f6301f 100644
--- a/drivers/bus/dpaa/base/qbman/qman_priv.h
+++ b/drivers/bus/dpaa/base/qbman/qman_priv.h
@@ -139,7 +139,6 @@ struct qm_portal_config {
#define QMAN_REV31 0x0301
#define QMAN_REV32 0x0302
extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */
-extern u32 qman_clk;
int qm_set_wpm(int wpm);
int qm_get_wpm(int *wpm);
diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c
index f2bb3b15..16fabd1b 100644
--- a/drivers/bus/dpaa/dpaa_bus.c
+++ b/drivers/bus/dpaa/dpaa_bus.c
@@ -19,7 +19,6 @@
#include <rte_interrupts.h>
#include <rte_log.h>
#include <rte_debug.h>
-#include <rte_pci.h>
#include <rte_atomic.h>
#include <rte_branch_prediction.h>
#include <rte_memory.h>
@@ -51,10 +50,12 @@ struct rte_dpaa_bus rte_dpaa_bus;
struct netcfg_info *dpaa_netcfg;
/* define a variable to hold the portal_key, once created.*/
-pthread_key_t dpaa_portal_key;
+static pthread_key_t dpaa_portal_key;
unsigned int dpaa_svr_family;
+#define FSL_DPAA_BUS_NAME dpaa_bus
+
RTE_DEFINE_PER_LCORE(bool, dpaa_io);
RTE_DEFINE_PER_LCORE(struct dpaa_portal_dqrr, held_bufs);
@@ -130,6 +131,22 @@ dpaa_sec_available(void)
static void dpaa_clean_device_list(void);
+static struct rte_devargs *
+dpaa_devargs_lookup(struct rte_dpaa_device *dev)
+{
+ struct rte_devargs *devargs;
+ char dev_name[32];
+
+ RTE_EAL_DEVARGS_FOREACH("dpaa_bus", devargs) {
+ devargs->bus->parse(devargs->name, &dev_name);
+ if (strcmp(dev_name, dev->device.name) == 0) {
+ DPAA_BUS_INFO("**Devargs matched %s", dev_name);
+ return devargs;
+ }
+ }
+ return NULL;
+}
+
static int
dpaa_create_device_list(void)
{
@@ -161,8 +178,9 @@ dpaa_create_device_list(void)
memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN);
sprintf(dev->name, "fm%d-mac%d", (fman_intf->fman_idx + 1),
fman_intf->mac_idx);
- DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name);
+ DPAA_BUS_LOG(INFO, "%s netdev added", dev->name);
dev->device.name = dev->name;
+ dev->device.devargs = dpaa_devargs_lookup(dev);
dpaa_add_to_device_list(dev);
}
@@ -198,7 +216,9 @@ dpaa_create_device_list(void)
*/
memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN);
sprintf(dev->name, "dpaa-sec%d", i);
- DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name);
+ DPAA_BUS_LOG(INFO, "%s cryptodev added", dev->name);
+ dev->device.name = dev->name;
+ dev->device.devargs = dpaa_devargs_lookup(dev);
dpaa_add_to_device_list(dev);
}
@@ -235,7 +255,7 @@ int rte_dpaa_portal_init(void *arg)
BUS_INIT_FUNC_TRACE();
- if ((uint64_t)arg == 1 || cpu == LCORE_ID_ANY)
+ if ((size_t)arg == 1 || cpu == LCORE_ID_ANY)
cpu = rte_get_master_lcore();
/* if the core id is not supported */
else
@@ -309,9 +329,15 @@ rte_dpaa_portal_fq_init(void *arg, struct qman_fq *fq)
/* Affine above created portal with channel*/
u32 sdqcr;
struct qman_portal *qp;
+ int ret;
- if (unlikely(!RTE_PER_LCORE(dpaa_io)))
- rte_dpaa_portal_init(arg);
+ if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
+ ret = rte_dpaa_portal_init(arg);
+ if (ret < 0) {
+ DPAA_BUS_LOG(ERR, "portal initialization failure");
+ return ret;
+ }
+ }
/* Initialise qman specific portals */
qp = fsl_qman_portal_create();
@@ -352,6 +378,51 @@ dpaa_portal_finish(void *arg)
RTE_PER_LCORE(dpaa_io) = false;
}
+static int
+rte_dpaa_bus_parse(const char *name, void *out_name)
+{
+ int i, j;
+ int max_fman = 2, max_macs = 16;
+ char *sep = strchr(name, ':');
+
+ if (strncmp(name, RTE_STR(FSL_DPAA_BUS_NAME),
+ strlen(RTE_STR(FSL_DPAA_BUS_NAME)))) {
+ return -EINVAL;
+ }
+
+ if (!sep) {
+ DPAA_BUS_ERR("Incorrect device name observed");
+ return -EINVAL;
+ }
+
+ sep = (char *) (sep + 1);
+
+ for (i = 0; i < max_fman; i++) {
+ for (j = 0; j < max_macs; j++) {
+ char fm_name[16];
+ snprintf(fm_name, 16, "fm%d-mac%d", i, j);
+ if (strcmp(fm_name, sep) == 0) {
+ if (out_name)
+ strcpy(out_name, sep);
+ return 0;
+ }
+ }
+ }
+
+ for (i = 0; i < RTE_LIBRTE_DPAA_MAX_CRYPTODEV; i++) {
+ char sec_name[16];
+
+ snprintf(sec_name, 16, "dpaa-sec%d", i);
+ if (strcmp(sec_name, sep) == 0) {
+ if (out_name)
+ strcpy(out_name, sep);
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
#define DPAA_DEV_PATH1 "/sys/devices/platform/soc/soc:fsl,dpaa"
#define DPAA_DEV_PATH2 "/sys/devices/platform/fsl,dpaa"
@@ -390,14 +461,11 @@ rte_dpaa_bus_scan(void)
return 0;
}
- DPAA_BUS_LOG(DEBUG, "Bus: Address of netcfg=%p, Ethports=%d",
- dpaa_netcfg, dpaa_netcfg->num_ethports);
-
#ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
dump_netcfg(dpaa_netcfg);
#endif
- DPAA_BUS_LOG(DEBUG, "Number of devices = %d\n",
+ DPAA_BUS_LOG(DEBUG, "Number of ethernet devices = %d",
dpaa_netcfg->num_ethports);
ret = dpaa_create_device_list();
if (ret) {
@@ -415,9 +483,6 @@ rte_dpaa_bus_scan(void)
return ret;
}
- DPAA_BUS_LOG(DEBUG, "dpaa_portal_key=%u, ret=%d\n",
- (unsigned int)dpaa_portal_key, ret);
-
return 0;
}
@@ -453,22 +518,15 @@ static int
rte_dpaa_device_match(struct rte_dpaa_driver *drv,
struct rte_dpaa_device *dev)
{
- int ret = -1;
-
- BUS_INIT_FUNC_TRACE();
-
if (!drv || !dev) {
DPAA_BUS_DEBUG("Invalid drv or dev received.");
- return ret;
+ return -1;
}
- if (drv->drv_type == dev->device_type) {
- DPAA_BUS_INFO("Device: %s matches for driver: %s",
- dev->name, drv->driver.name);
- ret = 0; /* Found a match */
- }
+ if (drv->drv_type == dev->device_type)
+ return 0;
- return ret;
+ return -1;
}
static int
@@ -479,8 +537,14 @@ rte_dpaa_bus_probe(void)
struct rte_dpaa_driver *drv;
FILE *svr_file = NULL;
unsigned int svr_ver;
+ int probe_all = rte_dpaa_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST;
- BUS_INIT_FUNC_TRACE();
+ svr_file = fopen(DPAA_SOC_ID_FILE, "r");
+ if (svr_file) {
+ if (fscanf(svr_file, "svr:%x", &svr_ver) > 0)
+ dpaa_svr_family = svr_ver & SVR_MASK;
+ fclose(svr_file);
+ }
/* For each registered driver, and device, call the driver->probe */
TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) {
@@ -489,13 +553,19 @@ rte_dpaa_bus_probe(void)
if (ret)
continue;
- if (!drv->probe)
+ if (!drv->probe ||
+ (dev->device.devargs &&
+ dev->device.devargs->policy == RTE_DEV_BLACKLISTED))
continue;
- ret = drv->probe(drv, dev);
- if (ret)
- DPAA_BUS_ERR("Unable to probe.\n");
-
+ if (probe_all ||
+ (dev->device.devargs &&
+ dev->device.devargs->policy ==
+ RTE_DEV_WHITELISTED)) {
+ ret = drv->probe(drv, dev);
+ if (ret)
+ DPAA_BUS_ERR("Unable to probe.\n");
+ }
break;
}
}
@@ -506,13 +576,6 @@ rte_dpaa_bus_probe(void)
if (!TAILQ_EMPTY(&rte_dpaa_bus.device_list))
rte_mbuf_set_platform_mempool_ops(DPAA_MEMPOOL_OPS_NAME);
- svr_file = fopen(DPAA_SOC_ID_FILE, "r");
- if (svr_file) {
- if (fscanf(svr_file, "svr:%x", &svr_ver) > 0)
- dpaa_svr_family = svr_ver & SVR_MASK;
- fclose(svr_file);
- }
-
return 0;
}
@@ -552,6 +615,7 @@ struct rte_dpaa_bus rte_dpaa_bus = {
.bus = {
.scan = rte_dpaa_bus_scan,
.probe = rte_dpaa_bus_probe,
+ .parse = rte_dpaa_bus_parse,
.find_device = rte_dpaa_find_device,
.get_iommu_class = rte_dpaa_get_iommu_class,
},
@@ -562,9 +626,7 @@ struct rte_dpaa_bus rte_dpaa_bus = {
RTE_REGISTER_BUS(FSL_DPAA_BUS_NAME, rte_dpaa_bus.bus);
-RTE_INIT(dpaa_init_log);
-static void
-dpaa_init_log(void)
+RTE_INIT(dpaa_init_log)
{
dpaa_logtype_bus = rte_log_register("bus.dpaa");
if (dpaa_logtype_bus >= 0)
@@ -574,11 +636,11 @@ dpaa_init_log(void)
if (dpaa_logtype_mempool >= 0)
rte_log_set_level(dpaa_logtype_mempool, RTE_LOG_NOTICE);
- dpaa_logtype_pmd = rte_log_register("pmd.dpaa");
+ dpaa_logtype_pmd = rte_log_register("pmd.net.dpaa");
if (dpaa_logtype_pmd >= 0)
rte_log_set_level(dpaa_logtype_pmd, RTE_LOG_NOTICE);
- dpaa_logtype_eventdev = rte_log_register("eventdev.dpaa");
+ dpaa_logtype_eventdev = rte_log_register("pmd.event.dpaa");
if (dpaa_logtype_eventdev >= 0)
rte_log_set_level(dpaa_logtype_eventdev, RTE_LOG_NOTICE);
}
diff --git a/drivers/bus/dpaa/include/compat.h b/drivers/bus/dpaa/include/compat.h
index 53707bb4..92241d23 100644
--- a/drivers/bus/dpaa/include/compat.h
+++ b/drivers/bus/dpaa/include/compat.h
@@ -39,6 +39,7 @@
#include <rte_spinlock.h>
#include <rte_common.h>
#include <rte_debug.h>
+#include <rte_cycles.h>
/* The following definitions are primarily to allow the single-source driver
* interfaces to be included by arbitrary program code. Ie. for interfaces that
@@ -47,9 +48,15 @@
*/
/* Required compiler attributes */
+#ifndef __maybe_unused
#define __maybe_unused __rte_unused
+#endif
+#ifndef __always_unused
#define __always_unused __rte_unused
+#endif
+#ifndef __packed
#define __packed __rte_packed
+#endif
#define noinline __attribute__((noinline))
#define L1_CACHE_BYTES 64
@@ -127,13 +134,15 @@ static inline void out_be32(volatile void *__p, u32 val)
*p = rte_cpu_to_be_32(val);
}
+#define hwsync() rte_rmb()
+#define lwsync() rte_wmb()
+
#define dcbt_ro(p) __builtin_prefetch(p, 0)
#define dcbt_rw(p) __builtin_prefetch(p, 1)
+#if defined(RTE_ARCH_ARM64)
#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); }
#define dcbz_64(p) dcbz(p)
-#define hwsync() rte_rmb()
-#define lwsync() rte_wmb()
#define dcbf(p) { asm volatile("dc cvac, %0" : : "r"(p) : "memory"); }
#define dcbf_64(p) dcbf(p)
#define dccivac(p) { asm volatile("dc civac, %0" : : "r"(p) : "memory"); }
@@ -144,9 +153,27 @@ static inline void out_be32(volatile void *__p, u32 val)
asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); \
} while (0)
+#elif defined(RTE_ARCH_ARM)
+#define dcbz(p) memset((p), 0, 32)
+#define dcbz_64(p) memset((p), 0, 64)
+#define dcbf(p) RTE_SET_USED(p)
+#define dcbf_64(p) dcbf(p)
+#define dccivac(p) RTE_SET_USED(p)
+#define dcbit_ro(p) RTE_SET_USED(p)
+
+#else
+#define dcbz(p) RTE_SET_USED(p)
+#define dcbz_64(p) dcbz(p)
+#define dcbf(p) RTE_SET_USED(p)
+#define dcbf_64(p) dcbf(p)
+#define dccivac(p) RTE_SET_USED(p)
+#define dcbit_ro(p) RTE_SET_USED(p)
+#endif
+
#define barrier() { asm volatile ("" : : : "memory"); }
#define cpu_relax barrier
+#if defined(RTE_ARCH_ARM64)
static inline uint64_t mfatb(void)
{
uint64_t ret, ret_new, timeout = 200;
@@ -160,6 +187,11 @@ static inline uint64_t mfatb(void)
DPAA_BUG_ON(!timeout && (ret != ret_new));
return ret * 64;
}
+#else
+
+#define mfatb rte_rdtsc
+
+#endif
/* Spin for a few cycles without bothering the bus */
static inline void cpu_spin(int cycles)
diff --git a/drivers/bus/dpaa/include/fsl_fman.h b/drivers/bus/dpaa/include/fsl_fman.h
index c0ef1bff..1d1ce867 100644
--- a/drivers/bus/dpaa/include/fsl_fman.h
+++ b/drivers/bus/dpaa/include/fsl_fman.h
@@ -108,6 +108,12 @@ int fman_if_get_fdoff(struct fman_if *fm_if);
/* Set interface fd->offset value */
void fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset);
+/* Get interface SG enable status value */
+int fman_if_get_sg_enable(struct fman_if *fm_if);
+
+/* Set interface SG support mode */
+void fman_if_set_sg(struct fman_if *fm_if, int enable);
+
/* Get interface Max Frame length (MTU) */
uint16_t fman_if_get_maxfrm(struct fman_if *fm_if);
diff --git a/drivers/bus/dpaa/include/fsl_qman.h b/drivers/bus/dpaa/include/fsl_qman.h
index e9793f30..b18cf037 100644
--- a/drivers/bus/dpaa/include/fsl_qman.h
+++ b/drivers/bus/dpaa/include/fsl_qman.h
@@ -284,20 +284,20 @@ static inline dma_addr_t qm_sg_addr(const struct qm_sg_entry *sg)
} while (0)
/* See 1.5.8.1: "Enqueue Command" */
-struct qm_eqcr_entry {
+struct __rte_aligned(8) qm_eqcr_entry {
u8 __dont_write_directly__verb;
u8 dca;
u16 seqnum;
u32 orp; /* 24-bit */
u32 fqid; /* 24-bit */
u32 tag;
- struct qm_fd fd;
+ struct qm_fd fd; /* this has alignment 8 */
u8 __reserved3[32];
} __packed;
/* "Frame Dequeue Response" */
-struct qm_dqrr_entry {
+struct __rte_aligned(8) qm_dqrr_entry {
u8 verb;
u8 stat;
u16 seqnum; /* 15-bit */
@@ -305,7 +305,7 @@ struct qm_dqrr_entry {
u8 __reserved2[3];
u32 fqid; /* 24-bit */
u32 contextB;
- struct qm_fd fd;
+ struct qm_fd fd; /* this has alignment 8 */
u8 __reserved4[32];
};
@@ -323,18 +323,19 @@ struct qm_dqrr_entry {
/* "ERN Message Response" */
/* "FQ State Change Notification" */
struct qm_mr_entry {
- u8 verb;
union {
struct {
+ u8 verb;
u8 dca;
u16 seqnum;
u8 rc; /* Rejection Code */
u32 orp:24;
u32 fqid; /* 24-bit */
u32 tag;
- struct qm_fd fd;
- } __packed ern;
+ struct qm_fd fd; /* this has alignment 8 */
+ } __packed __rte_aligned(8) ern;
struct {
+ u8 verb;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */
u8 __reserved1:4;
@@ -349,18 +350,19 @@ struct qm_mr_entry {
u32 __reserved3:24;
u32 fqid; /* 24-bit */
u32 tag;
- struct qm_fd fd;
- } __packed dcern;
+ struct qm_fd fd; /* this has alignment 8 */
+ } __packed __rte_aligned(8) dcern;
struct {
+ u8 verb;
u8 fqs; /* Frame Queue Status */
u8 __reserved1[6];
u32 fqid; /* 24-bit */
u32 contextB;
u8 __reserved2[16];
- } __packed fq; /* FQRN/FQRNI/FQRL/FQPN */
+ } __packed __rte_aligned(8) fq; /* FQRN/FQRNI/FQRL/FQPN */
};
u8 __reserved2[32];
-} __packed;
+} __packed __rte_aligned(8);
#define QM_MR_VERB_VBIT 0x80
/*
* ERNs originating from direct-connect portals ("dcern") use 0x20 as a verb
@@ -1330,10 +1332,11 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
* qman_set_vdq - Issue a volatile dequeue command
* @fq: Frame Queue on which the volatile dequeue command is issued
* @num: Number of Frames requested for volatile dequeue
+ * @vdqcr_flags: QM_VDQCR_EXACT flag to for VDQCR command
*
* This function will issue a volatile dequeue command to the QMAN.
*/
-int qman_set_vdq(struct qman_fq *fq, u16 num);
+int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags);
/**
* qman_dequeue - Get the DQRR entry after volatile dequeue command
diff --git a/drivers/bus/dpaa/include/of.h b/drivers/bus/dpaa/include/of.h
index 151be5a3..7ea7608f 100644
--- a/drivers/bus/dpaa/include/of.h
+++ b/drivers/bus/dpaa/include/of.h
@@ -109,6 +109,8 @@ const struct device_node *of_get_parent(const struct device_node *dev_node);
const struct device_node *of_get_next_child(const struct device_node *dev_node,
const struct device_node *prev);
+const void *of_get_mac_address(const struct device_node *np);
+
#define for_each_child_node(parent, child) \
for (child = of_get_next_child(parent, NULL); child != NULL; \
child = of_get_next_child(parent, child))
diff --git a/drivers/bus/dpaa/meson.build b/drivers/bus/dpaa/meson.build
new file mode 100644
index 00000000..d10b62c0
--- /dev/null
+++ b/drivers/bus/dpaa/meson.build
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+
+if host_machine.system() != 'linux'
+ build = false
+endif
+
+deps += ['eventdev']
+sources = files('base/fman/fman.c',
+ 'base/fman/fman_hw.c',
+ 'base/fman/netcfg_layer.c',
+ 'base/fman/of.c',
+ 'base/qbman/bman.c',
+ 'base/qbman/bman_driver.c',
+ 'base/qbman/dpaa_alloc.c',
+ 'base/qbman/dpaa_sys.c',
+ 'base/qbman/process.c',
+ 'base/qbman/qman.c',
+ 'base/qbman/qman_driver.c',
+ 'dpaa_bus.c')
+
+allow_experimental_apis = true
+
+if cc.has_argument('-Wno-cast-qual')
+ cflags += '-Wno-cast-qual'
+endif
+
+includes += include_directories('include', 'base/qbman')
+cflags += ['-D_GNU_SOURCE']
diff --git a/drivers/bus/dpaa/rte_bus_dpaa_version.map b/drivers/bus/dpaa/rte_bus_dpaa_version.map
index 8d902854..7d6d6243 100644
--- a/drivers/bus/dpaa/rte_bus_dpaa_version.map
+++ b/drivers/bus/dpaa/rte_bus_dpaa_version.map
@@ -92,3 +92,13 @@ DPDK_18.02 {
local: *;
} DPDK_17.11;
+
+DPDK_18.08 {
+ global:
+
+ fman_if_get_sg_enable;
+ fman_if_set_sg;
+ of_get_mac_address;
+
+ local: *;
+} DPDK_18.02;
diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h
index 718701b1..15dc6a4a 100644
--- a/drivers/bus/dpaa/rte_dpaa_bus.h
+++ b/drivers/bus/dpaa/rte_dpaa_bus.h
@@ -15,8 +15,6 @@
#include <of.h>
#include <netcfg.h>
-#define FSL_DPAA_BUS_NAME "FSL_DPAA_BUS"
-
#define DPAA_MEMPOOL_OPS_NAME "dpaa"
#define DEV_TO_DPAA_DEVICE(ptr) \
@@ -95,20 +93,35 @@ struct dpaa_portal {
uint64_t tid;/**< Parent Thread id for this portal */
};
-/* TODO - this is costly, need to write a fast coversion routine */
+/* Various structures representing contiguous memory maps */
+struct dpaa_memseg {
+ TAILQ_ENTRY(dpaa_memseg) next;
+ char *vaddr;
+ rte_iova_t iova;
+ size_t len;
+};
+
+TAILQ_HEAD(dpaa_memseg_list, dpaa_memseg);
+extern struct dpaa_memseg_list rte_dpaa_memsegs;
+
+/* Either iterate over the list of internal memseg references or fallback to
+ * EAL memseg based iova2virt.
+ */
static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr)
{
- const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
- int i;
-
- for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr != NULL; i++) {
- if (paddr >= memseg[i].iova && paddr <
- memseg[i].iova + memseg[i].len)
- return (uint8_t *)(memseg[i].addr) +
- (paddr - memseg[i].iova);
+ struct dpaa_memseg *ms;
+
+ /* Check if the address is already part of the memseg list internally
+ * maintained by the dpaa driver.
+ */
+ TAILQ_FOREACH(ms, &rte_dpaa_memsegs, next) {
+ if (paddr >= ms->iova && paddr <
+ ms->iova + ms->len)
+ return RTE_PTR_ADD(ms->vaddr, (uintptr_t)(paddr - ms->iova));
}
- return NULL;
+ /* If not, Fallback to full memseg list searching */
+ return rte_mem_iova2virt(paddr);
}
/**
@@ -151,8 +164,7 @@ void dpaa_portal_finish(void *arg);
/** Helper for DPAA device registration from driver (eth, crypto) instance */
#define RTE_PMD_REGISTER_DPAA(nm, dpaa_drv) \
-RTE_INIT(dpaainitfn_ ##nm); \
-static void dpaainitfn_ ##nm(void) \
+RTE_INIT(dpaainitfn_ ##nm) \
{\
(dpaa_drv).driver.name = RTE_STR(nm);\
rte_dpaa_driver_register(&dpaa_drv); \
diff --git a/drivers/bus/dpaa/rte_dpaa_logs.h b/drivers/bus/dpaa/rte_dpaa_logs.h
index 0fd70bbf..e4143543 100644
--- a/drivers/bus/dpaa/rte_dpaa_logs.h
+++ b/drivers/bus/dpaa/rte_dpaa_logs.h
@@ -15,10 +15,7 @@ extern int dpaa_logtype_pmd;
extern int dpaa_logtype_eventdev;
#define DPAA_BUS_LOG(level, fmt, args...) \
- rte_log(RTE_LOG_ ## level, dpaa_logtype_bus, "%s(): " fmt "\n", \
- __func__, ##args)
-
-#define BUS_INIT_FUNC_TRACE() DPAA_BUS_LOG(DEBUG, " >>")
+ rte_log(RTE_LOG_ ## level, dpaa_logtype_bus, "dpaa: " fmt "\n", ##args)
#ifdef RTE_LIBRTE_DPAA_DEBUG_BUS
#define DPAA_BUS_HWWARN(cond, fmt, args...) \
@@ -31,7 +28,11 @@ extern int dpaa_logtype_eventdev;
#endif
#define DPAA_BUS_DEBUG(fmt, args...) \
- DPAA_BUS_LOG(DEBUG, fmt, ## args)
+ rte_log(RTE_LOG_DEBUG, dpaa_logtype_bus, "dpaa: %s(): " fmt "\n", \
+ __func__, ##args)
+
+#define BUS_INIT_FUNC_TRACE() DPAA_BUS_DEBUG(" >>")
+
#define DPAA_BUS_INFO(fmt, args...) \
DPAA_BUS_LOG(INFO, fmt, ## args)
#define DPAA_BUS_ERR(fmt, args...) \
diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile
index de237f0f..515d0f53 100644
--- a/drivers/bus/fslmc/Makefile
+++ b/drivers/bus/fslmc/Makefile
@@ -9,23 +9,13 @@ include $(RTE_SDK)/mk/rte.vars.mk
#
LIB = librte_bus_fslmc.a
-ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_PMD),y)
-CONFIG_RTE_LIBRTE_FSLMC_BUS = $(CONFIG_RTE_LIBRTE_DPAA2_PMD)
-endif
-
CFLAGS += -DALLOW_EXPERIMENTAL_API
-ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y)
-CFLAGS += -O0 -g
-CFLAGS += "-Wno-error"
-else
CFLAGS += -O3
CFLAGS += $(WERROR_FLAGS)
-endif
CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc
CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc
CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include
-CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
LDLIBS += -lrte_ethdev
@@ -42,11 +32,12 @@ SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \
SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \
mc/dpmng.c \
- mc/dpbp.c \
- mc/dpio.c \
- mc/mc_sys.c \
+ mc/dpbp.c \
+ mc/dpio.c \
+ mc/mc_sys.c \
mc/dpcon.c \
- mc/dpci.c
+ mc/dpci.c \
+ mc/dpdmai.c
SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpio.c
SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpbp.c
diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index 5ee0beb8..d2900edc 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -18,11 +18,12 @@
#include <rte_fslmc.h>
#include <fslmc_vfio.h>
+#include "fslmc_logs.h"
-#define FSLMC_BUS_LOG(level, fmt, args...) \
- RTE_LOG(level, EAL, fmt "\n", ##args)
+int dpaa2_logtype_bus;
#define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups"
+#define FSLMC_BUS_NAME fslmc
struct rte_fslmc_bus rte_fslmc_bus;
uint8_t dpaa2_virt_mode;
@@ -93,6 +94,41 @@ insert_in_device_list(struct rte_dpaa2_device *newdev)
TAILQ_INSERT_TAIL(&rte_fslmc_bus.device_list, newdev, next);
}
+static struct rte_devargs *
+fslmc_devargs_lookup(struct rte_dpaa2_device *dev)
+{
+ struct rte_devargs *devargs;
+ char dev_name[32];
+
+ RTE_EAL_DEVARGS_FOREACH("fslmc", devargs) {
+ devargs->bus->parse(devargs->name, &dev_name);
+ if (strcmp(dev_name, dev->device.name) == 0) {
+ DPAA2_BUS_INFO("**Devargs matched %s", dev_name);
+ return devargs;
+ }
+ }
+ return NULL;
+}
+
+static void
+dump_device_list(void)
+{
+ struct rte_dpaa2_device *dev;
+ uint32_t global_log_level;
+ int local_log_level;
+
+ /* Only if the log level has been set to Debugging, print list */
+ global_log_level = rte_log_get_global_level();
+ local_log_level = rte_log_get_level(dpaa2_logtype_bus);
+ if (global_log_level == RTE_LOG_DEBUG ||
+ local_log_level == RTE_LOG_DEBUG) {
+ DPAA2_BUS_LOG(DEBUG, "List of devices scanned on bus:");
+ TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
+ DPAA2_BUS_LOG(DEBUG, "\t\t%s", dev->device.name);
+ }
+ }
+}
+
static int
scan_one_fslmc_device(char *dev_name)
{
@@ -109,7 +145,7 @@ scan_one_fslmc_device(char *dev_name)
/* Creating a temporary copy to perform cut-parse over string */
dup_dev_name = strdup(dev_name);
if (!dup_dev_name) {
- FSLMC_BUS_LOG(ERR, "Out of memory.");
+ DPAA2_BUS_ERR("Unable to allocate device name memory");
return -ENOMEM;
}
@@ -120,7 +156,7 @@ scan_one_fslmc_device(char *dev_name)
*/
dev = calloc(1, sizeof(struct rte_dpaa2_device));
if (!dev) {
- FSLMC_BUS_LOG(ERR, "Out of memory.");
+ DPAA2_BUS_ERR("Unable to allocate device object");
free(dup_dev_name);
return -ENOMEM;
}
@@ -128,7 +164,7 @@ scan_one_fslmc_device(char *dev_name)
/* Parse the device name and ID */
t_ptr = strtok(dup_dev_name, ".");
if (!t_ptr) {
- FSLMC_BUS_LOG(ERR, "Incorrect device string observed.");
+ DPAA2_BUS_ERR("Incorrect device name observed");
goto cleanup;
}
if (!strncmp("dpni", t_ptr, 4))
@@ -145,6 +181,8 @@ scan_one_fslmc_device(char *dev_name)
dev->dev_type = DPAA2_CI;
else if (!strncmp("dpmcp", t_ptr, 5))
dev->dev_type = DPAA2_MPORTAL;
+ else if (!strncmp("dpdmai", t_ptr, 6))
+ dev->dev_type = DPAA2_QDMA;
else
dev->dev_type = DPAA2_UNKNOWN;
@@ -153,17 +191,17 @@ scan_one_fslmc_device(char *dev_name)
t_ptr = strtok(NULL, ".");
if (!t_ptr) {
- FSLMC_BUS_LOG(ERR, "Incorrect device string observed (%s).",
- t_ptr);
+ DPAA2_BUS_ERR("Incorrect device string observed (%s)", t_ptr);
goto cleanup;
}
sscanf(t_ptr, "%hu", &dev->object_id);
dev->device.name = strdup(dev_name);
if (!dev->device.name) {
- FSLMC_BUS_LOG(ERR, "Out of memory.");
+ DPAA2_BUS_ERR("Unable to clone device name. Out of memory");
goto cleanup;
}
+ dev->device.devargs = fslmc_devargs_lookup(dev);
/* Add device in the fslmc device list */
insert_in_device_list(dev);
@@ -182,6 +220,54 @@ cleanup:
}
static int
+rte_fslmc_parse(const char *name, void *addr)
+{
+ uint16_t dev_id;
+ char *t_ptr;
+ char *sep = strchr(name, ':');
+
+ if (strncmp(name, RTE_STR(FSLMC_BUS_NAME),
+ strlen(RTE_STR(FSLMC_BUS_NAME)))) {
+ return -EINVAL;
+ }
+
+ if (!sep) {
+ DPAA2_BUS_ERR("Incorrect device name observed");
+ return -EINVAL;
+ }
+
+ t_ptr = (char *)(sep + 1);
+
+ if (strncmp("dpni", t_ptr, 4) &&
+ strncmp("dpseci", t_ptr, 6) &&
+ strncmp("dpcon", t_ptr, 5) &&
+ strncmp("dpbp", t_ptr, 4) &&
+ strncmp("dpio", t_ptr, 4) &&
+ strncmp("dpci", t_ptr, 4) &&
+ strncmp("dpmcp", t_ptr, 5) &&
+ strncmp("dpdmai", t_ptr, 6)) {
+ DPAA2_BUS_ERR("Unknown or unsupported device");
+ return -EINVAL;
+ }
+
+ t_ptr = strchr(name, '.');
+ if (!t_ptr) {
+ DPAA2_BUS_ERR("Incorrect device string observed (%s)", t_ptr);
+ return -EINVAL;
+ }
+
+ t_ptr = (char *)(t_ptr + 1);
+ if (sscanf(t_ptr, "%hu", &dev_id) <= 0) {
+ DPAA2_BUS_ERR("Incorrect device string observed (%s)", t_ptr);
+ return -EINVAL;
+ }
+
+ if (addr)
+ strcpy(addr, (char *)(sep + 1));
+ return 0;
+}
+
+static int
rte_fslmc_scan(void)
{
int ret;
@@ -193,8 +279,7 @@ rte_fslmc_scan(void)
int groupid;
if (process_once) {
- FSLMC_BUS_LOG(DEBUG,
- "Fslmc bus already scanned. Not rescanning");
+ DPAA2_BUS_DEBUG("Fslmc bus already scanned. Not rescanning");
return 0;
}
process_once = 1;
@@ -208,7 +293,7 @@ rte_fslmc_scan(void)
groupid);
dir = opendir(fslmc_dirpath);
if (!dir) {
- FSLMC_BUS_LOG(ERR, "Unable to open VFIO group dir.");
+ DPAA2_BUS_ERR("Unable to open VFIO group directory");
goto scan_fail;
}
@@ -224,9 +309,12 @@ rte_fslmc_scan(void)
device_count += 1;
}
- FSLMC_BUS_LOG(INFO, "fslmc: Bus scan completed");
-
closedir(dir);
+
+ DPAA2_BUS_INFO("FSLMC Bus scan completed");
+ /* If debugging is enabled, device list is dumped to log output */
+ dump_device_list();
+
return 0;
scan_fail_cleanup:
@@ -235,7 +323,7 @@ scan_fail_cleanup:
/* Remove all devices in the list */
cleanup_fslmc_device_list();
scan_fail:
- FSLMC_BUS_LOG(DEBUG, "FSLMC Bus Not Available. Skipping.");
+ DPAA2_BUS_DEBUG("FSLMC Bus Not Available. Skipping");
/* Irrespective of failure, scan only return success */
return 0;
}
@@ -254,6 +342,8 @@ static int
rte_fslmc_probe(void)
{
int ret = 0;
+ int probe_all;
+
struct rte_dpaa2_device *dev;
struct rte_dpaa2_driver *drv;
@@ -262,16 +352,29 @@ rte_fslmc_probe(void)
ret = fslmc_vfio_setup_group();
if (ret) {
- FSLMC_BUS_LOG(ERR, "Unable to setup VFIO %d", ret);
+ DPAA2_BUS_ERR("Unable to setup VFIO %d", ret);
+ return 0;
+ }
+
+ /* Map existing segments as well as, in case of hotpluggable memory,
+ * install callback handler.
+ */
+ ret = rte_fslmc_vfio_dmamap();
+ if (ret) {
+ DPAA2_BUS_ERR("Unable to DMA map existing VAs: (%d)", ret);
+ /* Not continuing ahead */
+ DPAA2_BUS_ERR("FSLMC VFIO Mapping failed");
return 0;
}
ret = fslmc_vfio_process_group();
if (ret) {
- FSLMC_BUS_LOG(ERR, "Unable to setup devices %d", ret);
+ DPAA2_BUS_ERR("Unable to setup devices %d", ret);
return 0;
}
+ probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST;
+
TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
ret = rte_fslmc_match(drv, dev);
@@ -281,9 +384,21 @@ rte_fslmc_probe(void)
if (!drv->probe)
continue;
- ret = drv->probe(drv, dev);
- if (ret)
- FSLMC_BUS_LOG(ERR, "Unable to probe.\n");
+ if (dev->device.devargs &&
+ dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
+ DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
+ dev->device.name);
+ continue;
+ }
+
+ if (probe_all ||
+ (dev->device.devargs &&
+ dev->device.devargs->policy ==
+ RTE_DEV_WHITELISTED)) {
+ ret = drv->probe(drv, dev);
+ if (ret)
+ DPAA2_BUS_ERR("Unable to probe");
+ }
break;
}
}
@@ -298,16 +413,19 @@ static struct rte_device *
rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
const void *data)
{
+ const struct rte_dpaa2_device *dstart;
struct rte_dpaa2_device *dev;
- TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
- if (start && &dev->device == start) {
- start = NULL; /* starting point found */
- continue;
- }
-
+ if (start != NULL) {
+ dstart = RTE_DEV_TO_FSLMC_CONST(start);
+ dev = TAILQ_NEXT(dstart, next);
+ } else {
+ dev = TAILQ_FIRST(&rte_fslmc_bus.device_list);
+ }
+ while (dev != NULL) {
if (cmp(&dev->device, data) == 0)
return &dev->device;
+ dev = TAILQ_NEXT(dev, next);
}
return NULL;
@@ -390,6 +508,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
.bus = {
.scan = rte_fslmc_scan,
.probe = rte_fslmc_probe,
+ .parse = rte_fslmc_parse,
.find_device = rte_fslmc_find_device,
.get_iommu_class = rte_dpaa2_get_iommu_class,
},
@@ -398,4 +517,12 @@ struct rte_fslmc_bus rte_fslmc_bus = {
.device_count = {0},
};
-RTE_REGISTER_BUS(fslmc, rte_fslmc_bus.bus);
+RTE_REGISTER_BUS(FSLMC_BUS_NAME, rte_fslmc_bus.bus);
+
+RTE_INIT(fslmc_init_log)
+{
+ /* Bus level logs */
+ dpaa2_logtype_bus = rte_log_register("bus.fslmc");
+ if (dpaa2_logtype_bus >= 0)
+ rte_log_set_level(dpaa2_logtype_bus, RTE_LOG_NOTICE);
+}
diff --git a/drivers/bus/fslmc/fslmc_logs.h b/drivers/bus/fslmc/fslmc_logs.h
index d87b6386..dd74cb7d 100644
--- a/drivers/bus/fslmc/fslmc_logs.h
+++ b/drivers/bus/fslmc/fslmc_logs.h
@@ -7,44 +7,35 @@
#ifndef _FSLMC_LOGS_H_
#define _FSLMC_LOGS_H_
-#define PMD_INIT_LOG(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ##args)
-
-#ifdef RTE_LIBRTE_DPAA2_DEBUG_INIT
-#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
-#else
-#define PMD_INIT_FUNC_TRACE() do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_DPAA2_DEBUG_RX
-#define PMD_RX_LOG(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
-#else
-#define PMD_RX_LOG(level, fmt, args...) do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_DPAA2_DEBUG_TX
-#define PMD_TX_LOG(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
-#else
-#define PMD_TX_LOG(level, fmt, args...) do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_DPAA2_DEBUG_TX_FREE
-#define PMD_TX_FREE_LOG(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
-#else
-#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
-#define PMD_DRV_LOG_RAW(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args)
-#else
-#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0)
-#endif
-
-#define PMD_DRV_LOG(level, fmt, args...) \
- PMD_DRV_LOG_RAW(level, fmt "\n", ## args)
+extern int dpaa2_logtype_bus;
+
+#define DPAA2_BUS_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, dpaa2_logtype_bus, "fslmc: " fmt "\n", \
+ ##args)
+
+/* Debug logs are with Function names */
+#define DPAA2_BUS_DEBUG(fmt, args...) \
+ rte_log(RTE_LOG_DEBUG, dpaa2_logtype_bus, "fslmc: %s(): " fmt "\n", \
+ __func__, ##args)
+
+#define BUS_INIT_FUNC_TRACE() DPAA2_BUS_DEBUG(" >>")
+
+#define DPAA2_BUS_INFO(fmt, args...) \
+ DPAA2_BUS_LOG(INFO, fmt, ## args)
+#define DPAA2_BUS_ERR(fmt, args...) \
+ DPAA2_BUS_LOG(ERR, fmt, ## args)
+#define DPAA2_BUS_WARN(fmt, args...) \
+ DPAA2_BUS_LOG(WARNING, fmt, ## args)
+
+/* DP Logs, toggled out at compile time if level lower than current level */
+#define DPAA2_BUS_DP_LOG(level, fmt, args...) \
+ RTE_LOG_DP(level, PMD, fmt, ## args)
+
+#define DPAA2_BUS_DP_DEBUG(fmt, args...) \
+ DPAA2_BUS_DP_LOG(DEBUG, fmt, ## args)
+#define DPAA2_BUS_DP_INFO(fmt, args...) \
+ DPAA2_BUS_DP_LOG(INFO, fmt, ## args)
+#define DPAA2_BUS_DP_WARN(fmt, args...) \
+ DPAA2_BUS_DP_LOG(WARNING, fmt, ## args)
#endif /* _FSLMC_LOGS_H_ */
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 1241295b..4c2cd2a8 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -30,17 +30,16 @@
#include <rte_kvargs.h>
#include <rte_dev.h>
#include <rte_bus.h>
+#include <rte_eal_memconfig.h>
#include "rte_fslmc.h"
#include "fslmc_vfio.h"
+#include "fslmc_logs.h"
#include <mc/fsl_dpmng.h>
#include "portal/dpaa2_hw_pvt.h"
#include "portal/dpaa2_hw_dpio.h"
-#define FSLMC_VFIO_LOG(level, fmt, args...) \
- RTE_LOG(level, EAL, fmt "\n", ##args)
-
/** Pathname of FSL-MC devices directory. */
#define SYSFS_FSL_MC_DEVICES "/sys/bus/fsl-mc/devices"
@@ -53,7 +52,6 @@ static int container_device_fd;
static char *g_container;
static uint32_t *msi_intr_vaddr;
void *(*rte_mcp_ptr_list);
-static int is_dma_done;
static struct rte_dpaa2_object_list dpaa2_obj_list =
TAILQ_HEAD_INITIALIZER(dpaa2_obj_list);
@@ -76,33 +74,32 @@ fslmc_get_container_group(int *groupid)
if (!g_container) {
container = getenv("DPRC");
if (container == NULL) {
- RTE_LOG(WARNING, EAL, "DPAA2: DPRC not available\n");
+ DPAA2_BUS_DEBUG("DPAA2: DPRC not available");
return -EINVAL;
}
if (strlen(container) >= FSLMC_CONTAINER_MAX_LEN) {
- FSLMC_VFIO_LOG(ERR, "Invalid container name: %s\n",
- container);
+ DPAA2_BUS_ERR("Invalid container name: %s", container);
return -1;
}
g_container = strdup(container);
if (!g_container) {
- FSLMC_VFIO_LOG(ERR, "Out of memory.");
+ DPAA2_BUS_ERR("Mem alloc failure; Container name");
return -ENOMEM;
}
}
/* get group number */
- ret = vfio_get_group_no(SYSFS_FSL_MC_DEVICES, g_container, groupid);
+ ret = rte_vfio_get_group_num(SYSFS_FSL_MC_DEVICES,
+ g_container, groupid);
if (ret <= 0) {
- FSLMC_VFIO_LOG(ERR, "Unable to find %s IOMMU group",
- g_container);
+ DPAA2_BUS_ERR("Unable to find %s IOMMU group", g_container);
return -1;
}
- FSLMC_VFIO_LOG(DEBUG, "Container: %s has VFIO iommu group id = %d",
- g_container, *groupid);
+ DPAA2_BUS_DEBUG("Container: %s has VFIO iommu group id = %d",
+ g_container, *groupid);
return 0;
}
@@ -113,14 +110,14 @@ vfio_connect_container(void)
int fd, ret;
if (vfio_container.used) {
- FSLMC_VFIO_LOG(DEBUG, "No container available.");
+ DPAA2_BUS_DEBUG("No container available");
return -1;
}
/* Try connecting to vfio container if already created */
if (!ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER,
&vfio_container.fd)) {
- FSLMC_VFIO_LOG(INFO,
+ DPAA2_BUS_DEBUG(
"Container pre-exists with FD[0x%x] for this group",
vfio_container.fd);
vfio_group.container = &vfio_container;
@@ -128,9 +125,9 @@ vfio_connect_container(void)
}
/* Opens main vfio file descriptor which represents the "container" */
- fd = vfio_get_container_fd();
+ fd = rte_vfio_get_container_fd();
if (fd < 0) {
- FSLMC_VFIO_LOG(ERR, "Failed to open VFIO container");
+ DPAA2_BUS_ERR("Failed to open VFIO container");
return -errno;
}
@@ -139,19 +136,19 @@ vfio_connect_container(void)
/* Connect group to container */
ret = ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER, &fd);
if (ret) {
- FSLMC_VFIO_LOG(ERR, "Failed to setup group container");
+ DPAA2_BUS_ERR("Failed to setup group container");
close(fd);
return -errno;
}
ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
if (ret) {
- FSLMC_VFIO_LOG(ERR, "Failed to setup VFIO iommu");
+ DPAA2_BUS_ERR("Failed to setup VFIO iommu");
close(fd);
return -errno;
}
} else {
- FSLMC_VFIO_LOG(ERR, "No supported IOMMU available");
+ DPAA2_BUS_ERR("No supported IOMMU available");
close(fd);
return -EINVAL;
}
@@ -179,7 +176,7 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group)
vaddr = (unsigned long *)mmap(NULL, 0x1000, PROT_WRITE |
PROT_READ, MAP_SHARED, container_device_fd, 0x6030000);
if (vaddr == MAP_FAILED) {
- FSLMC_VFIO_LOG(ERR, "Unable to map region (errno = %d)", errno);
+ DPAA2_BUS_ERR("Unable to map region (errno = %d)", errno);
return -errno;
}
@@ -189,88 +186,195 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group)
if (ret == 0)
return 0;
- FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA fails (errno = %d)", errno);
+ DPAA2_BUS_ERR("Unable to map DMA address (errno = %d)", errno);
return -errno;
}
-int rte_fslmc_vfio_dmamap(void)
+static int fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
+static int fslmc_unmap_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
+
+static void
+fslmc_memevent_cb(enum rte_mem_event type, const void *addr, size_t len,
+ void *arg __rte_unused)
{
+ struct rte_memseg_list *msl;
+ struct rte_memseg *ms;
+ size_t cur_len = 0, map_len = 0;
+ uint64_t virt_addr;
+ rte_iova_t iova_addr;
int ret;
+
+ msl = rte_mem_virt2memseg_list(addr);
+
+ while (cur_len < len) {
+ const void *va = RTE_PTR_ADD(addr, cur_len);
+
+ ms = rte_mem_virt2memseg(va, msl);
+ iova_addr = ms->iova;
+ virt_addr = ms->addr_64;
+ map_len = ms->len;
+
+ DPAA2_BUS_DEBUG("Request for %s, va=%p, "
+ "virt_addr=0x%" PRIx64 ", "
+ "iova=0x%" PRIx64 ", map_len=%zu",
+ type == RTE_MEM_EVENT_ALLOC ?
+ "alloc" : "dealloc",
+ va, virt_addr, iova_addr, map_len);
+
+ if (type == RTE_MEM_EVENT_ALLOC)
+ ret = fslmc_map_dma(virt_addr, iova_addr, map_len);
+ else
+ ret = fslmc_unmap_dma(virt_addr, iova_addr, map_len);
+
+ if (ret != 0) {
+ DPAA2_BUS_ERR("DMA Mapping/Unmapping failed. "
+ "Map=%d, addr=%p, len=%zu, err:(%d)",
+ type, va, map_len, ret);
+ return;
+ }
+
+ cur_len += map_len;
+ }
+
+ if (type == RTE_MEM_EVENT_ALLOC)
+ DPAA2_BUS_DEBUG("Total Mapped: addr=%p, len=%zu",
+ addr, len);
+ else
+ DPAA2_BUS_DEBUG("Total Unmapped: addr=%p, len=%zu",
+ addr, len);
+}
+
+static int
+fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr __rte_unused, size_t len)
+{
struct fslmc_vfio_group *group;
struct vfio_iommu_type1_dma_map dma_map = {
.argsz = sizeof(struct vfio_iommu_type1_dma_map),
.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
};
+ int ret;
- int i;
- const struct rte_memseg *memseg;
+ dma_map.size = len;
+ dma_map.vaddr = vaddr;
- if (is_dma_done)
- return 0;
+#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
+ dma_map.iova = iovaddr;
+#else
+ dma_map.iova = dma_map.vaddr;
+#endif
- memseg = rte_eal_get_physmem_layout();
- if (memseg == NULL) {
- FSLMC_VFIO_LOG(ERR, "Cannot get physical layout.");
- return -ENODEV;
+ /* SET DMA MAP for IOMMU */
+ group = &vfio_group;
+
+ if (!group->container) {
+ DPAA2_BUS_ERR("Container is not connected ");
+ return -1;
}
- for (i = 0; i < RTE_MAX_MEMSEG; i++) {
- if (memseg[i].addr == NULL && memseg[i].len == 0) {
- FSLMC_VFIO_LOG(DEBUG, "Total %d segments found.", i);
- break;
- }
+ DPAA2_BUS_DEBUG("--> Map address: 0x%"PRIx64", size: %"PRIu64"",
+ (uint64_t)dma_map.vaddr, (uint64_t)dma_map.size);
+ ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &dma_map);
+ if (ret) {
+ DPAA2_BUS_ERR("VFIO_IOMMU_MAP_DMA API(errno = %d)",
+ errno);
+ return -1;
+ }
- dma_map.size = memseg[i].len;
- dma_map.vaddr = memseg[i].addr_64;
-#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
- if (rte_eal_iova_mode() == RTE_IOVA_VA)
- dma_map.iova = dma_map.vaddr;
- else
- dma_map.iova = memseg[i].iova;
-#else
- dma_map.iova = dma_map.vaddr;
-#endif
+ return 0;
+}
- /* SET DMA MAP for IOMMU */
- group = &vfio_group;
+static int
+fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len)
+{
+ struct fslmc_vfio_group *group;
+ struct vfio_iommu_type1_dma_unmap dma_unmap = {
+ .argsz = sizeof(struct vfio_iommu_type1_dma_unmap),
+ .flags = 0,
+ };
+ int ret;
- if (!group->container) {
- FSLMC_VFIO_LOG(ERR, "Container is not connected ");
- return -1;
- }
+ dma_unmap.size = len;
+ dma_unmap.iova = vaddr;
- FSLMC_VFIO_LOG(DEBUG, "-->Initial SHM Virtual ADDR %llX",
- dma_map.vaddr);
- FSLMC_VFIO_LOG(DEBUG, "-----> DMA size 0x%llX", dma_map.size);
- ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA,
- &dma_map);
- if (ret) {
- FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA API(errno = %d)",
- errno);
- return ret;
- }
+ /* SET DMA MAP for IOMMU */
+ group = &vfio_group;
+
+ if (!group->container) {
+ DPAA2_BUS_ERR("Container is not connected ");
+ return -1;
}
- /* Verifying that at least single segment is available */
- if (i <= 0) {
- FSLMC_VFIO_LOG(ERR, "No Segments found for VFIO Mapping");
+ DPAA2_BUS_DEBUG("--> Unmap address: 0x%"PRIx64", size: %"PRIu64"",
+ (uint64_t)dma_unmap.iova, (uint64_t)dma_unmap.size);
+ ret = ioctl(group->container->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap);
+ if (ret) {
+ DPAA2_BUS_ERR("VFIO_IOMMU_UNMAP_DMA API(errno = %d)",
+ errno);
return -1;
}
+ return 0;
+}
+
+static int
+fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused,
+ const struct rte_memseg *ms, void *arg)
+{
+ int *n_segs = arg;
+ int ret;
+
+ ret = fslmc_map_dma(ms->addr_64, ms->iova, ms->len);
+ if (ret)
+ DPAA2_BUS_ERR("Unable to VFIO map (addr=%p, len=%zu)",
+ ms->addr, ms->len);
+ else
+ (*n_segs)++;
+
+ return ret;
+}
+
+int rte_fslmc_vfio_dmamap(void)
+{
+ int i = 0, ret;
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ rte_rwlock_t *mem_lock = &mcfg->memory_hotplug_lock;
+
+ /* Lock before parsing and registering callback to memory subsystem */
+ rte_rwlock_read_lock(mem_lock);
+
+ if (rte_memseg_walk(fslmc_dmamap_seg, &i) < 0) {
+ rte_rwlock_read_unlock(mem_lock);
+ return -1;
+ }
+
+ ret = rte_mem_event_callback_register("fslmc_memevent_clb",
+ fslmc_memevent_cb, NULL);
+ if (ret && rte_errno == ENOTSUP)
+ DPAA2_BUS_DEBUG("Memory event callbacks not supported");
+ else if (ret)
+ DPAA2_BUS_DEBUG("Unable to install memory handler");
+ else
+ DPAA2_BUS_DEBUG("Installed memory callback handler");
+
+ DPAA2_BUS_DEBUG("Total %d segments found.", i);
+
/* TODO - This is a W.A. as VFIO currently does not add the mapping of
* the interrupt region to SMMU. This should be removed once the
* support is added in the Kernel.
*/
- vfio_map_irq_region(group);
+ vfio_map_irq_region(&vfio_group);
- is_dma_done = 1;
+ /* Existing segments have been mapped and memory callback for hotplug
+ * has been installed.
+ */
+ rte_rwlock_read_unlock(mem_lock);
return 0;
}
static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj)
{
- int64_t v_addr = (int64_t)MAP_FAILED;
+ intptr_t v_addr = (intptr_t)MAP_FAILED;
int32_t ret, mc_fd;
struct vfio_device_info d_info = { .argsz = sizeof(d_info) };
@@ -279,29 +383,26 @@ static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj)
/* getting the mcp object's fd*/
mc_fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, mcp_obj);
if (mc_fd < 0) {
- FSLMC_VFIO_LOG(ERR, "error in VFIO get dev %s fd from group %d",
- mcp_obj, group->fd);
+ DPAA2_BUS_ERR("Error in VFIO get dev %s fd from group %d",
+ mcp_obj, group->fd);
return v_addr;
}
/* getting device info*/
ret = ioctl(mc_fd, VFIO_DEVICE_GET_INFO, &d_info);
if (ret < 0) {
- FSLMC_VFIO_LOG(ERR, "error in VFIO getting DEVICE_INFO");
+ DPAA2_BUS_ERR("Error in VFIO getting DEVICE_INFO");
goto MC_FAILURE;
}
/* getting device region info*/
ret = ioctl(mc_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
if (ret < 0) {
- FSLMC_VFIO_LOG(ERR, "error in VFIO getting REGION_INFO");
+ DPAA2_BUS_ERR("Error in VFIO getting REGION_INFO");
goto MC_FAILURE;
}
- FSLMC_VFIO_LOG(DEBUG, "region offset = %llx , region size = %llx",
- reg_info.offset, reg_info.size);
-
- v_addr = (uint64_t)mmap(NULL, reg_info.size,
+ v_addr = (size_t)mmap(NULL, reg_info.size,
PROT_WRITE | PROT_READ, MAP_SHARED,
mc_fd, reg_info.offset);
@@ -334,8 +435,8 @@ int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index)
ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
if (ret) {
- RTE_LOG(ERR, EAL, "Error:dpaa2 SET IRQs fd=%d, err = %d(%s)\n",
- intr_handle->fd, errno, strerror(errno));
+ DPAA2_BUS_ERR("Error:dpaa2 SET IRQs fd=%d, err = %d(%s)",
+ intr_handle->fd, errno, strerror(errno));
return ret;
}
@@ -359,8 +460,8 @@ int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index)
ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
if (ret)
- RTE_LOG(ERR, EAL,
- "Error disabling dpaa2 interrupts for fd %d\n",
+ DPAA2_BUS_ERR(
+ "Error disabling dpaa2 interrupts for fd %d",
intr_handle->fd);
return ret;
@@ -383,9 +484,8 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
if (ret < 0) {
- FSLMC_VFIO_LOG(ERR,
- "cannot get IRQ(%d) info, error %i (%s)",
- i, errno, strerror(errno));
+ DPAA2_BUS_ERR("Cannot get IRQ(%d) info, error %i (%s)",
+ i, errno, strerror(errno));
return -1;
}
@@ -399,9 +499,8 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
/* set up an eventfd for interrupts */
fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
if (fd < 0) {
- FSLMC_VFIO_LOG(ERR,
- "cannot set up eventfd, error %i (%s)\n",
- errno, strerror(errno));
+ DPAA2_BUS_ERR("Cannot set up eventfd, error %i (%s)",
+ errno, strerror(errno));
return -1;
}
@@ -430,13 +529,14 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev)
dev_fd = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD,
dev->device.name);
if (dev_fd <= 0) {
- FSLMC_VFIO_LOG(ERR, "Unable to obtain device FD for device:%s",
- dev->device.name);
+ DPAA2_BUS_ERR("Unable to obtain device FD for device:%s",
+ dev->device.name);
return -1;
}
if (ioctl(dev_fd, VFIO_DEVICE_GET_INFO, &device_info)) {
- FSLMC_VFIO_LOG(ERR, "DPAA2 VFIO_DEVICE_GET_INFO fail");
+ DPAA2_BUS_ERR("Unable to obtain information for device:%s",
+ dev->device.name);
return -1;
}
@@ -461,61 +561,74 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev)
break;
}
- FSLMC_VFIO_LOG(DEBUG, "Device (%s) abstracted from VFIO",
- dev->device.name);
+ DPAA2_BUS_LOG(DEBUG, "Device (%s) abstracted from VFIO",
+ dev->device.name);
return 0;
}
static int
fslmc_process_mcp(struct rte_dpaa2_device *dev)
{
- int64_t v_addr;
- char *dev_name;
+ int ret;
+ intptr_t v_addr;
+ char *dev_name = NULL;
struct fsl_mc_io dpmng = {0};
struct mc_version mc_ver_info = {0};
rte_mcp_ptr_list = malloc(sizeof(void *) * 1);
if (!rte_mcp_ptr_list) {
- FSLMC_VFIO_LOG(ERR, "Out of memory");
- return -ENOMEM;
+ DPAA2_BUS_ERR("Unable to allocate MC portal memory");
+ ret = -ENOMEM;
+ goto cleanup;
}
dev_name = strdup(dev->device.name);
if (!dev_name) {
- FSLMC_VFIO_LOG(ERR, "Out of memory.");
- free(rte_mcp_ptr_list);
- rte_mcp_ptr_list = NULL;
- return -ENOMEM;
+ DPAA2_BUS_ERR("Unable to allocate MC device name memory");
+ ret = -ENOMEM;
+ goto cleanup;
}
v_addr = vfio_map_mcp_obj(&vfio_group, dev_name);
- if (v_addr == (int64_t)MAP_FAILED) {
- FSLMC_VFIO_LOG(ERR, "Error mapping region (errno = %d)",
- errno);
- free(rte_mcp_ptr_list);
- rte_mcp_ptr_list = NULL;
- return -1;
+ if (v_addr == (intptr_t)MAP_FAILED) {
+ DPAA2_BUS_ERR("Error mapping region (errno = %d)", errno);
+ ret = -1;
+ goto cleanup;
}
/* check the MC version compatibility */
dpmng.regs = (void *)v_addr;
- if (mc_get_version(&dpmng, CMD_PRI_LOW, &mc_ver_info))
- RTE_LOG(WARNING, PMD, "\tmc_get_version failed\n");
+ if (mc_get_version(&dpmng, CMD_PRI_LOW, &mc_ver_info)) {
+ DPAA2_BUS_ERR("Unable to obtain MC version");
+ ret = -1;
+ goto cleanup;
+ }
if ((mc_ver_info.major != MC_VER_MAJOR) ||
(mc_ver_info.minor < MC_VER_MINOR)) {
- RTE_LOG(ERR, PMD, "DPAA2 MC version not compatible!"
- " Expected %d.%d.x, Detected %d.%d.%d\n",
- MC_VER_MAJOR, MC_VER_MINOR,
- mc_ver_info.major, mc_ver_info.minor,
- mc_ver_info.revision);
- free(rte_mcp_ptr_list);
- rte_mcp_ptr_list = NULL;
- return -1;
+ DPAA2_BUS_ERR("DPAA2 MC version not compatible!"
+ " Expected %d.%d.x, Detected %d.%d.%d",
+ MC_VER_MAJOR, MC_VER_MINOR,
+ mc_ver_info.major, mc_ver_info.minor,
+ mc_ver_info.revision);
+ ret = -1;
+ goto cleanup;
}
rte_mcp_ptr_list[0] = (void *)v_addr;
+ free(dev_name);
return 0;
+
+cleanup:
+ if (dev_name)
+ free(dev_name);
+
+ if (rte_mcp_ptr_list) {
+ free(rte_mcp_ptr_list);
+ rte_mcp_ptr_list = NULL;
+ }
+
+ return ret;
}
int
@@ -530,7 +643,7 @@ fslmc_vfio_process_group(void)
if (dev->dev_type == DPAA2_MPORTAL) {
ret = fslmc_process_mcp(dev);
if (ret) {
- FSLMC_VFIO_LOG(DEBUG, "Unable to map Portal.");
+ DPAA2_BUS_ERR("Unable to map MC Portal");
return -1;
}
if (!found_mportal)
@@ -547,23 +660,19 @@ fslmc_vfio_process_group(void)
/* Cannot continue if there is not even a single mportal */
if (!found_mportal) {
- FSLMC_VFIO_LOG(DEBUG,
- "No MC Portal device found. Not continuing.");
+ DPAA2_BUS_ERR("No MC Portal device found. Not continuing");
return -1;
}
TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
- if (!dev)
- break;
-
switch (dev->dev_type) {
case DPAA2_ETH:
case DPAA2_CRYPTO:
+ case DPAA2_QDMA:
ret = fslmc_process_iodevices(dev);
if (ret) {
- FSLMC_VFIO_LOG(DEBUG,
- "Dev (%s) init failed.",
- dev->device.name);
+ DPAA2_BUS_DEBUG("Dev (%s) init failed",
+ dev->device.name);
return ret;
}
break;
@@ -576,9 +685,8 @@ fslmc_vfio_process_group(void)
*/
ret = fslmc_process_iodevices(dev);
if (ret) {
- FSLMC_VFIO_LOG(DEBUG,
- "Dev (%s) init failed.",
- dev->device.name);
+ DPAA2_BUS_DEBUG("Dev (%s) init failed",
+ dev->device.name);
return -1;
}
@@ -592,8 +700,8 @@ fslmc_vfio_process_group(void)
case DPAA2_UNKNOWN:
default:
/* Unknown - ignore */
- FSLMC_VFIO_LOG(DEBUG, "Found unknown device (%s).",
- dev->device.name);
+ DPAA2_BUS_DEBUG("Found unknown device (%s)",
+ dev->device.name);
TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
free(dev);
dev = NULL;
@@ -622,12 +730,12 @@ fslmc_vfio_setup_group(void)
* processing.
*/
if (vfio_group.groupid == groupid) {
- FSLMC_VFIO_LOG(ERR, "groupid already exists %d", groupid);
+ DPAA2_BUS_ERR("groupid already exists %d", groupid);
return 0;
}
/* Get the actual group fd */
- ret = vfio_get_group_fd(groupid);
+ ret = rte_vfio_get_group_fd(groupid);
if (ret < 0)
return ret;
vfio_group.fd = ret;
@@ -635,14 +743,14 @@ fslmc_vfio_setup_group(void)
/* Check group viability */
ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_STATUS, &status);
if (ret) {
- FSLMC_VFIO_LOG(ERR, "VFIO error getting group status");
+ DPAA2_BUS_ERR("VFIO error getting group status");
close(vfio_group.fd);
rte_vfio_clear_group(vfio_group.fd);
return ret;
}
if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
- FSLMC_VFIO_LOG(ERR, "VFIO group not viable");
+ DPAA2_BUS_ERR("VFIO group not viable");
close(vfio_group.fd);
rte_vfio_clear_group(vfio_group.fd);
return -EPERM;
@@ -655,7 +763,7 @@ fslmc_vfio_setup_group(void)
/* Now connect this IOMMU group to given container */
ret = vfio_connect_container();
if (ret) {
- FSLMC_VFIO_LOG(ERR,
+ DPAA2_BUS_ERR(
"Error connecting container with groupid %d",
groupid);
close(vfio_group.fd);
@@ -667,15 +775,15 @@ fslmc_vfio_setup_group(void)
/* Get Device information */
ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD, g_container);
if (ret < 0) {
- FSLMC_VFIO_LOG(ERR, "Error getting device %s fd from group %d",
- g_container, vfio_group.groupid);
+ DPAA2_BUS_ERR("Error getting device %s fd from group %d",
+ g_container, vfio_group.groupid);
close(vfio_group.fd);
rte_vfio_clear_group(vfio_group.fd);
return ret;
}
container_device_fd = ret;
- FSLMC_VFIO_LOG(DEBUG, "VFIO Container FD is [0x%X]",
- container_device_fd);
+ DPAA2_BUS_DEBUG("VFIO Container FD is [0x%X]",
+ container_device_fd);
return 0;
}
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index e8fb3445..9e2c4fee 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -10,8 +10,6 @@
#include <rte_vfio.h>
-#include "eal_vfio.h"
-
#define DPAA2_MC_DPNI_DEVID 7
#define DPAA2_MC_DPSECI_DEVID 3
#define DPAA2_MC_DPCON_DEVID 5
diff --git a/drivers/bus/fslmc/mc/dpdmai.c b/drivers/bus/fslmc/mc/dpdmai.c
new file mode 100644
index 00000000..528889df
--- /dev/null
+++ b/drivers/bus/fslmc/mc/dpdmai.c
@@ -0,0 +1,429 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#include <fsl_mc_sys.h>
+#include <fsl_mc_cmd.h>
+#include <fsl_dpdmai.h>
+#include <fsl_dpdmai_cmd.h>
+
+/**
+ * dpdmai_open() - Open a control session for the specified object
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @dpdmai_id: DPDMAI unique ID
+ * @token: Returned token; use in subsequent API calls
+ *
+ * This function can be used to open a control session for an
+ * already created object; an object may have been declared in
+ * the DPL or by calling the dpdmai_create() function.
+ * This function returns a unique authentication token,
+ * associated with the specific object ID and the specific MC
+ * portal; this token must be used in all subsequent commands for
+ * this specific object.
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_open(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ int dpdmai_id,
+ uint16_t *token)
+{
+ struct dpdmai_cmd_open *cmd_params;
+ struct mc_command cmd = { 0 };
+ int err;
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN,
+ cmd_flags,
+ 0);
+ cmd_params = (struct dpdmai_cmd_open *)cmd.params;
+ cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id);
+
+ /* send command to mc*/
+ err = mc_send_command(mc_io, &cmd);
+ if (err)
+ return err;
+
+ /* retrieve response parameters */
+ *token = mc_cmd_hdr_read_token(&cmd);
+
+ return 0;
+}
+
+/**
+ * dpdmai_close() - Close the control session of the object
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ *
+ * After this function is called, no further operations are
+ * allowed on the object without opening a new control session.
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_close(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token)
+{
+ struct mc_command cmd = { 0 };
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE,
+ cmd_flags, token);
+
+ /* send command to mc*/
+ return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpdmai_create() - Create the DPDMAI object
+ * @mc_io: Pointer to MC portal's I/O object
+ * @dprc_token: Parent container token; '0' for default container
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @cfg: Configuration structure
+ * @obj_id: Returned object id
+ *
+ * Create the DPDMAI object, allocate required resources and
+ * perform required initialization.
+ *
+ * The object can be created either by declaring it in the
+ * DPL file, or by calling this function.
+ *
+ * The function accepts an authentication token of a parent
+ * container that this object should be assigned to. The token
+ * can be '0' so the object will be assigned to the default container.
+ * The newly created object can be opened with the returned
+ * object id and using the container's associated tokens and MC portals.
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_create(struct fsl_mc_io *mc_io,
+ uint16_t dprc_token,
+ uint32_t cmd_flags,
+ const struct dpdmai_cfg *cfg,
+ uint32_t *obj_id)
+{
+ struct dpdmai_cmd_create *cmd_params;
+ struct mc_command cmd = { 0 };
+ int err;
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE,
+ cmd_flags,
+ dprc_token);
+ cmd_params = (struct dpdmai_cmd_create *)cmd.params;
+ cmd_params->priorities[0] = cfg->priorities[0];
+ cmd_params->priorities[1] = cfg->priorities[1];
+
+ /* send command to mc*/
+ err = mc_send_command(mc_io, &cmd);
+ if (err)
+ return err;
+
+ /* retrieve response parameters */
+ *obj_id = mc_cmd_read_object_id(&cmd);
+
+ return 0;
+}
+
+/**
+ * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @dprc_token: Parent container token; '0' for default container
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @object_id: The object id; it must be a valid id within the container that
+ * created this object;
+ *
+ * The function accepts the authentication token of the parent container that
+ * created the object (not the one that currently owns the object). The object
+ * is searched within parent using the provided 'object_id'.
+ * All tokens to the object must be closed before calling destroy.
+ *
+ * Return: '0' on Success; error code otherwise.
+ */
+int dpdmai_destroy(struct fsl_mc_io *mc_io,
+ uint16_t dprc_token,
+ uint32_t cmd_flags,
+ uint32_t object_id)
+{
+ struct dpdmai_cmd_destroy *cmd_params;
+ struct mc_command cmd = { 0 };
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY,
+ cmd_flags,
+ dprc_token);
+ cmd_params = (struct dpdmai_cmd_destroy *)cmd.params;
+ cmd_params->dpdmai_id = cpu_to_le32(object_id);
+
+ /* send command to mc*/
+ return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_enable(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token)
+{
+ struct mc_command cmd = { 0 };
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE,
+ cmd_flags,
+ token);
+
+ /* send command to mc*/
+ return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_disable(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token)
+{
+ struct mc_command cmd = { 0 };
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE,
+ cmd_flags,
+ token);
+
+ /* send command to mc*/
+ return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpdmai_is_enabled() - Check if the DPDMAI is enabled.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ * @en: Returns '1' if object is enabled; '0' otherwise
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ int *en)
+{
+ struct dpdmai_rsp_is_enabled *rsp_params;
+ struct mc_command cmd = { 0 };
+ int err;
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_IS_ENABLED,
+ cmd_flags,
+ token);
+
+ /* send command to mc*/
+ err = mc_send_command(mc_io, &cmd);
+ if (err)
+ return err;
+
+ /* retrieve response parameters */
+ rsp_params = (struct dpdmai_rsp_is_enabled *)cmd.params;
+ *en = dpdmai_get_field(rsp_params->en, ENABLE);
+
+ return 0;
+}
+
+/**
+ * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_reset(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token)
+{
+ struct mc_command cmd = { 0 };
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET,
+ cmd_flags,
+ token);
+
+ /* send command to mc*/
+ return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpdmai_get_attributes() - Retrieve DPDMAI attributes.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ * @attr: Returned object's attributes
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ struct dpdmai_attr *attr)
+{
+ struct dpdmai_rsp_get_attr *rsp_params;
+ struct mc_command cmd = { 0 };
+ int err;
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR,
+ cmd_flags,
+ token);
+
+ /* send command to mc*/
+ err = mc_send_command(mc_io, &cmd);
+ if (err)
+ return err;
+
+ /* retrieve response parameters */
+ rsp_params = (struct dpdmai_rsp_get_attr *)cmd.params;
+ attr->id = le32_to_cpu(rsp_params->id);
+ attr->num_of_priorities = rsp_params->num_of_priorities;
+
+ return 0;
+}
+
+/**
+ * dpdmai_set_rx_queue() - Set Rx queue configuration
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ * @priority: Select the queue relative to number of
+ * priorities configured at DPDMAI creation; use
+ * DPDMAI_ALL_QUEUES to configure all Rx queues
+ * identically.
+ * @cfg: Rx queue configuration
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ uint8_t priority,
+ const struct dpdmai_rx_queue_cfg *cfg)
+{
+ struct dpdmai_cmd_set_rx_queue *cmd_params;
+ struct mc_command cmd = { 0 };
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE,
+ cmd_flags,
+ token);
+ cmd_params = (struct dpdmai_cmd_set_rx_queue *)cmd.params;
+ cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
+ cmd_params->dest_priority = cfg->dest_cfg.priority;
+ cmd_params->priority = priority;
+ cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx);
+ cmd_params->options = cpu_to_le32(cfg->options);
+ dpdmai_set_field(cmd_params->dest_type,
+ DEST_TYPE,
+ cfg->dest_cfg.dest_type);
+
+ /* send command to mc*/
+ return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpdmai_get_rx_queue() - Retrieve Rx queue attributes.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ * @priority: Select the queue relative to number of
+ * priorities configured at DPDMAI creation
+ * @attr: Returned Rx queue attributes
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ uint8_t priority,
+ struct dpdmai_rx_queue_attr *attr)
+{
+ struct dpdmai_cmd_get_queue *cmd_params;
+ struct dpdmai_rsp_get_rx_queue *rsp_params;
+ struct mc_command cmd = { 0 };
+ int err;
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE,
+ cmd_flags,
+ token);
+ cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params;
+ cmd_params->priority = priority;
+
+ /* send command to mc*/
+ err = mc_send_command(mc_io, &cmd);
+ if (err)
+ return err;
+
+ /* retrieve response parameters */
+ rsp_params = (struct dpdmai_rsp_get_rx_queue *)cmd.params;
+ attr->user_ctx = le64_to_cpu(rsp_params->user_ctx);
+ attr->fqid = le32_to_cpu(rsp_params->fqid);
+ attr->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id);
+ attr->dest_cfg.priority = le32_to_cpu(rsp_params->dest_priority);
+ attr->dest_cfg.dest_type = dpdmai_get_field(rsp_params->dest_type,
+ DEST_TYPE);
+
+ return 0;
+}
+
+/**
+ * dpdmai_get_tx_queue() - Retrieve Tx queue attributes.
+ * @mc_io: Pointer to MC portal's I/O object
+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token: Token of DPDMAI object
+ * @priority: Select the queue relative to number of
+ * priorities configured at DPDMAI creation
+ * @attr: Returned Tx queue attributes
+ *
+ * Return: '0' on Success; Error code otherwise.
+ */
+int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ uint8_t priority,
+ struct dpdmai_tx_queue_attr *attr)
+{
+ struct dpdmai_cmd_get_queue *cmd_params;
+ struct dpdmai_rsp_get_tx_queue *rsp_params;
+ struct mc_command cmd = { 0 };
+ int err;
+
+ /* prepare command */
+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE,
+ cmd_flags,
+ token);
+ cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params;
+ cmd_params->priority = priority;
+
+ /* send command to mc*/
+ err = mc_send_command(mc_io, &cmd);
+ if (err)
+ return err;
+
+ /* retrieve response parameters */
+ rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params;
+ attr->fqid = le32_to_cpu(rsp_params->fqid);
+
+ return 0;
+}
diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai.h b/drivers/bus/fslmc/mc/fsl_dpdmai.h
new file mode 100644
index 00000000..03e46ec1
--- /dev/null
+++ b/drivers/bus/fslmc/mc/fsl_dpdmai.h
@@ -0,0 +1,189 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef __FSL_DPDMAI_H
+#define __FSL_DPDMAI_H
+
+struct fsl_mc_io;
+
+/* Data Path DMA Interface API
+ * Contains initialization APIs and runtime control APIs for DPDMAI
+ */
+
+/* General DPDMAI macros */
+
+/**
+ * Maximum number of Tx/Rx priorities per DPDMAI object
+ */
+#define DPDMAI_PRIO_NUM 2
+
+/**
+ * All queues considered; see dpdmai_set_rx_queue()
+ */
+#define DPDMAI_ALL_QUEUES (uint8_t)(-1)
+
+int dpdmai_open(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ int dpdmai_id,
+ uint16_t *token);
+
+int dpdmai_close(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token);
+
+/**
+ * struct dpdmai_cfg - Structure representing DPDMAI configuration
+ * @priorities: Priorities for the DMA hardware processing; valid priorities are
+ * configured with values 1-8; the entry following last valid entry
+ * should be configured with 0
+ */
+struct dpdmai_cfg {
+ uint8_t priorities[DPDMAI_PRIO_NUM];
+};
+
+int dpdmai_create(struct fsl_mc_io *mc_io,
+ uint16_t dprc_token,
+ uint32_t cmd_flags,
+ const struct dpdmai_cfg *cfg,
+ uint32_t *obj_id);
+
+int dpdmai_destroy(struct fsl_mc_io *mc_io,
+ uint16_t dprc_token,
+ uint32_t cmd_flags,
+ uint32_t object_id);
+
+int dpdmai_enable(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token);
+
+int dpdmai_disable(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token);
+
+int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ int *en);
+
+int dpdmai_reset(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token);
+
+/**
+ * struct dpdmai_attr - Structure representing DPDMAI attributes
+ * @id: DPDMAI object ID
+ * @num_of_priorities: number of priorities
+ */
+struct dpdmai_attr {
+ int id;
+ uint8_t num_of_priorities;
+};
+
+int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ struct dpdmai_attr *attr);
+
+/**
+ * enum dpdmai_dest - DPDMAI destination types
+ * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode
+ * and does not generate FQDAN notifications; user is expected to dequeue
+ * from the queue based on polling or other user-defined method
+ * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN
+ * notifications to the specified DPIO; user is expected to dequeue
+ * from the queue only after notification is received
+ * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate
+ * FQDAN notifications, but is connected to the specified DPCON object;
+ * user is expected to dequeue from the DPCON channel
+ */
+enum dpdmai_dest {
+ DPDMAI_DEST_NONE = 0,
+ DPDMAI_DEST_DPIO = 1,
+ DPDMAI_DEST_DPCON = 2
+};
+
+/**
+ * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters
+ * @dest_type: Destination type
+ * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type
+ * @priority: Priority selection within the DPIO or DPCON channel; valid values
+ * are 0-1 or 0-7, depending on the number of priorities in that
+ * channel; not relevant for 'DPDMAI_DEST_NONE' option
+ */
+struct dpdmai_dest_cfg {
+ enum dpdmai_dest dest_type;
+ int dest_id;
+ uint8_t priority;
+};
+
+/* DPDMAI queue modification options */
+
+/**
+ * Select to modify the user's context associated with the queue
+ */
+#define DPDMAI_QUEUE_OPT_USER_CTX 0x00000001
+
+/**
+ * Select to modify the queue's destination
+ */
+#define DPDMAI_QUEUE_OPT_DEST 0x00000002
+
+/**
+ * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration
+ * @options: Flags representing the suggested modifications to the queue;
+ * Use any combination of 'DPDMAI_QUEUE_OPT_<X>' flags
+ * @user_ctx: User context value provided in the frame descriptor of each
+ * dequeued frame;
+ * valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options'
+ * @dest_cfg: Queue destination parameters;
+ * valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options'
+ */
+struct dpdmai_rx_queue_cfg {
+ uint32_t options;
+ uint64_t user_ctx;
+ struct dpdmai_dest_cfg dest_cfg;
+
+};
+
+int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ uint8_t priority,
+ const struct dpdmai_rx_queue_cfg *cfg);
+
+/**
+ * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues
+ * @user_ctx: User context value provided in the frame descriptor of each
+ * dequeued frame
+ * @dest_cfg: Queue destination configuration
+ * @fqid: Virtual FQID value to be used for dequeue operations
+ */
+struct dpdmai_rx_queue_attr {
+ uint64_t user_ctx;
+ struct dpdmai_dest_cfg dest_cfg;
+ uint32_t fqid;
+};
+
+int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ uint8_t priority,
+ struct dpdmai_rx_queue_attr *attr);
+
+/**
+ * struct dpdmai_tx_queue_attr - Structure representing attributes of Tx queues
+ * @fqid: Virtual FQID to be used for sending frames to DMA hardware
+ */
+
+struct dpdmai_tx_queue_attr {
+ uint32_t fqid;
+};
+
+int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
+ uint32_t cmd_flags,
+ uint16_t token,
+ uint8_t priority,
+ struct dpdmai_tx_queue_attr *attr);
+
+#endif /* __FSL_DPDMAI_H */
diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h
new file mode 100644
index 00000000..618e19ea
--- /dev/null
+++ b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h
@@ -0,0 +1,107 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _FSL_DPDMAI_CMD_H
+#define _FSL_DPDMAI_CMD_H
+
+/* DPDMAI Version */
+#define DPDMAI_VER_MAJOR 3
+#define DPDMAI_VER_MINOR 2
+
+/* Command versioning */
+#define DPDMAI_CMD_BASE_VERSION 1
+#define DPDMAI_CMD_ID_OFFSET 4
+
+#define DPDMAI_CMD(id) ((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
+
+/* Command IDs */
+#define DPDMAI_CMDID_CLOSE DPDMAI_CMD(0x800)
+#define DPDMAI_CMDID_OPEN DPDMAI_CMD(0x80E)
+#define DPDMAI_CMDID_CREATE DPDMAI_CMD(0x90E)
+#define DPDMAI_CMDID_DESTROY DPDMAI_CMD(0x98E)
+#define DPDMAI_CMDID_GET_API_VERSION DPDMAI_CMD(0xa0E)
+
+#define DPDMAI_CMDID_ENABLE DPDMAI_CMD(0x002)
+#define DPDMAI_CMDID_DISABLE DPDMAI_CMD(0x003)
+#define DPDMAI_CMDID_GET_ATTR DPDMAI_CMD(0x004)
+#define DPDMAI_CMDID_RESET DPDMAI_CMD(0x005)
+#define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMD(0x006)
+
+#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMD(0x1A0)
+#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMD(0x1A1)
+#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMD(0x1A2)
+
+/* Macros for accessing command fields smaller than 1byte */
+#define DPDMAI_MASK(field) \
+ GENMASK(DPDMAI_##field##_SHIFT + DPDMAI_##field##_SIZE - 1, \
+ DPDMAI_##field##_SHIFT)
+#define dpdmai_set_field(var, field, val) \
+ ((var) |= (((val) << DPDMAI_##field##_SHIFT) & DPDMAI_MASK(field)))
+#define dpdmai_get_field(var, field) \
+ (((var) & DPDMAI_MASK(field)) >> DPDMAI_##field##_SHIFT)
+
+#pragma pack(push, 1)
+struct dpdmai_cmd_open {
+ uint32_t dpdmai_id;
+};
+
+struct dpdmai_cmd_create {
+ uint8_t pad;
+ uint8_t priorities[2];
+};
+
+struct dpdmai_cmd_destroy {
+ uint32_t dpdmai_id;
+};
+
+#define DPDMAI_ENABLE_SHIFT 0
+#define DPDMAI_ENABLE_SIZE 1
+
+struct dpdmai_rsp_is_enabled {
+ /* only the LSB bit */
+ uint8_t en;
+};
+
+struct dpdmai_rsp_get_attr {
+ uint32_t id;
+ uint8_t num_of_priorities;
+};
+
+#define DPDMAI_DEST_TYPE_SHIFT 0
+#define DPDMAI_DEST_TYPE_SIZE 4
+
+struct dpdmai_cmd_set_rx_queue {
+ uint32_t dest_id;
+ uint8_t dest_priority;
+ uint8_t priority;
+ /* from LSB: dest_type:4 */
+ uint8_t dest_type;
+ uint8_t pad;
+ uint64_t user_ctx;
+ uint32_t options;
+};
+
+struct dpdmai_cmd_get_queue {
+ uint8_t pad[5];
+ uint8_t priority;
+};
+
+struct dpdmai_rsp_get_rx_queue {
+ uint32_t dest_id;
+ uint8_t dest_priority;
+ uint8_t pad1;
+ /* from LSB: dest_type:4 */
+ uint8_t dest_type;
+ uint8_t pad2;
+ uint64_t user_ctx;
+ uint32_t fqid;
+};
+
+struct dpdmai_rsp_get_tx_queue {
+ uint64_t pad;
+ uint32_t fqid;
+};
+
+#pragma pack(pop)
+#endif /* _FSL_DPDMAI_CMD_H */
diff --git a/drivers/bus/fslmc/mc/fsl_mc_cmd.h b/drivers/bus/fslmc/mc/fsl_mc_cmd.h
index a3c3e798..ac919610 100644
--- a/drivers/bus/fslmc/mc/fsl_mc_cmd.h
+++ b/drivers/bus/fslmc/mc/fsl_mc_cmd.h
@@ -27,7 +27,7 @@
#define le32_to_cpu rte_le_to_cpu_32
#define le16_to_cpu rte_le_to_cpu_16
-#define BITS_PER_LONG 64
+#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
#define GENMASK(h, l) \
(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
diff --git a/drivers/bus/fslmc/meson.build b/drivers/bus/fslmc/meson.build
new file mode 100644
index 00000000..22a56a6f
--- /dev/null
+++ b/drivers/bus/fslmc/meson.build
@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+
+if host_machine.system() != 'linux'
+ build = false
+endif
+
+deps += ['eventdev', 'kvargs']
+sources = files('fslmc_bus.c',
+ 'fslmc_vfio.c',
+ 'mc/dpbp.c',
+ 'mc/dpci.c',
+ 'mc/dpcon.c',
+ 'mc/dpdmai.c',
+ 'mc/dpio.c',
+ 'mc/dpmng.c',
+ 'mc/mc_sys.c',
+ 'portal/dpaa2_hw_dpbp.c',
+ 'portal/dpaa2_hw_dpci.c',
+ 'portal/dpaa2_hw_dpio.c',
+ 'qbman/qbman_portal.c',
+ 'qbman/qbman_debug.c')
+
+allow_experimental_apis = true
+
+includes += include_directories('mc', 'qbman/include', 'portal')
+cflags += ['-D_GNU_SOURCE']
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index f1f14e29..39c5adf9 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -44,7 +44,7 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
/* Allocate DPAA2 dpbp handle */
dpbp_node = rte_malloc(NULL, sizeof(struct dpaa2_dpbp_dev), 0);
if (!dpbp_node) {
- PMD_INIT_LOG(ERR, "Memory allocation failed for DPBP Device");
+ DPAA2_BUS_ERR("Memory allocation failed for DPBP Device");
return -1;
}
@@ -53,8 +53,8 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
ret = dpbp_open(&dpbp_node->dpbp,
CMD_PRI_LOW, dpbp_id, &dpbp_node->token);
if (ret) {
- PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d",
- ret);
+ DPAA2_BUS_ERR("Unable to open buffer pool object: err(%d)",
+ ret);
rte_free(dpbp_node);
return -1;
}
@@ -62,8 +62,8 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
/* Clean the device first */
ret = dpbp_reset(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token);
if (ret) {
- PMD_INIT_LOG(ERR, "Failure cleaning dpbp device with"
- " error code %d\n", ret);
+ DPAA2_BUS_ERR("Unable to reset buffer pool device. err(%d)",
+ ret);
dpbp_close(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token);
rte_free(dpbp_node);
return -1;
@@ -74,8 +74,6 @@ dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
TAILQ_INSERT_TAIL(&dpbp_dev_list, dpbp_node, next);
- RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpbp.%d]\n", dpbp_id);
-
if (!register_once) {
rte_mbuf_set_platform_mempool_ops(DPAA2_MEMPOOL_OPS_NAME);
register_once = 1;
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
index fb28e497..5ad0374d 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
@@ -39,13 +39,14 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
struct dpci_attr attr;
struct dpci_rx_queue_cfg rx_queue_cfg;
struct dpci_rx_queue_attr rx_attr;
+ struct dpci_tx_queue_attr tx_attr;
int ret, i;
/* Allocate DPAA2 dpci handle */
dpci_node = rte_malloc(NULL, sizeof(struct dpaa2_dpci_dev), 0);
if (!dpci_node) {
- PMD_INIT_LOG(ERR, "Memory allocation failed for DPCI Device");
- return -1;
+ DPAA2_BUS_ERR("Memory allocation failed for DPCI Device");
+ return -ENOMEM;
}
/* Open the dpci object */
@@ -53,43 +54,57 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
ret = dpci_open(&dpci_node->dpci,
CMD_PRI_LOW, dpci_id, &dpci_node->token);
if (ret) {
- PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d",
- ret);
- rte_free(dpci_node);
- return -1;
+ DPAA2_BUS_ERR("Resource alloc failure with err code: %d", ret);
+ goto err;
}
/* Get the device attributes */
ret = dpci_get_attributes(&dpci_node->dpci,
CMD_PRI_LOW, dpci_node->token, &attr);
if (ret != 0) {
- PMD_INIT_LOG(ERR, "Reading device failed with err code: %d",
- ret);
- rte_free(dpci_node);
- return -1;
+ DPAA2_BUS_ERR("Reading device failed with err code: %d", ret);
+ goto err;
}
- /* Set up the Rx Queue */
- memset(&rx_queue_cfg, 0, sizeof(struct dpci_rx_queue_cfg));
- ret = dpci_set_rx_queue(&dpci_node->dpci,
- CMD_PRI_LOW,
- dpci_node->token,
- 0, &rx_queue_cfg);
- if (ret) {
- PMD_INIT_LOG(ERR, "Setting Rx queue failed with err code: %d",
- ret);
- rte_free(dpci_node);
- return -1;
+ for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) {
+ struct dpaa2_queue *rxq;
+
+ memset(&rx_queue_cfg, 0, sizeof(struct dpci_rx_queue_cfg));
+ ret = dpci_set_rx_queue(&dpci_node->dpci,
+ CMD_PRI_LOW,
+ dpci_node->token,
+ i, &rx_queue_cfg);
+ if (ret) {
+ DPAA2_BUS_ERR("Setting Rx queue failed with err code: %d",
+ ret);
+ goto err;
+ }
+
+ /* Allocate DQ storage for the DPCI Rx queues */
+ rxq = &(dpci_node->rx_queue[i]);
+ rxq->q_storage = rte_malloc("dq_storage",
+ sizeof(struct queue_storage_info_t),
+ RTE_CACHE_LINE_SIZE);
+ if (!rxq->q_storage) {
+ DPAA2_BUS_ERR("q_storage allocation failed\n");
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ memset(rxq->q_storage, 0, sizeof(struct queue_storage_info_t));
+ ret = dpaa2_alloc_dq_storage(rxq->q_storage);
+ if (ret) {
+ DPAA2_BUS_ERR("dpaa2_alloc_dq_storage failed\n");
+ goto err;
+ }
}
/* Enable the device */
ret = dpci_enable(&dpci_node->dpci,
CMD_PRI_LOW, dpci_node->token);
if (ret != 0) {
- PMD_INIT_LOG(ERR, "Enabling device failed with err code: %d",
- ret);
- rte_free(dpci_node);
- return -1;
+ DPAA2_BUS_ERR("Enabling device failed with err code: %d", ret);
+ goto err;
}
for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) {
@@ -99,14 +114,22 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
dpci_node->token, i,
&rx_attr);
if (ret != 0) {
- PMD_INIT_LOG(ERR,
- "Reading device failed with err code: %d",
- ret);
- rte_free(dpci_node);
- return -1;
+ DPAA2_BUS_ERR("Rx queue fetch failed with err code: %d",
+ ret);
+ goto err;
}
+ dpci_node->rx_queue[i].fqid = rx_attr.fqid;
- dpci_node->queue[i].fqid = rx_attr.fqid;
+ ret = dpci_get_tx_queue(&dpci_node->dpci,
+ CMD_PRI_LOW,
+ dpci_node->token, i,
+ &tx_attr);
+ if (ret != 0) {
+ DPAA2_BUS_ERR("Reading device failed with err code: %d",
+ ret);
+ goto err;
+ }
+ dpci_node->tx_queue[i].fqid = tx_attr.fqid;
}
dpci_node->dpci_id = dpci_id;
@@ -114,9 +137,20 @@ rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
TAILQ_INSERT_TAIL(&dpci_dev_list, dpci_node, next);
- RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpci.%d]\n", dpci_id);
-
return 0;
+
+err:
+ for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) {
+ struct dpaa2_queue *rxq = &(dpci_node->rx_queue[i]);
+
+ if (rxq->q_storage) {
+ dpaa2_free_dq_storage(rxq->q_storage);
+ rte_free(rxq->q_storage);
+ }
+ }
+ rte_free(dpci_node);
+
+ return ret;
}
struct dpaa2_dpci_dev *rte_dpaa2_alloc_dpci_dev(void)
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index eefde155..99f70be1 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -101,7 +101,7 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id)
snprintf(string, STRING_LEN, "dpio.%d", dpio_id);
file = fopen("/proc/interrupts", "r");
if (!file) {
- PMD_DRV_LOG(WARNING, "Failed to open /proc/interrupts file\n");
+ DPAA2_BUS_WARN("Failed to open /proc/interrupts file");
return;
}
while (getline(&temp, &len, file) != -1) {
@@ -112,8 +112,8 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id)
}
if (!token) {
- PMD_DRV_LOG(WARNING, "Failed to get interrupt id for dpio.%d\n",
- dpio_id);
+ DPAA2_BUS_WARN("Failed to get interrupt id for dpio.%d",
+ dpio_id);
if (temp)
free(temp);
fclose(file);
@@ -125,10 +125,10 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id)
cpu_mask, token);
ret = system(command);
if (ret < 0)
- PMD_DRV_LOG(WARNING,
- "Failed to affine interrupts on respective core\n");
+ DPAA2_BUS_WARN(
+ "Failed to affine interrupts on respective core");
else
- PMD_DRV_LOG(WARNING, " %s command is executed\n", command);
+ DPAA2_BUS_DEBUG(" %s command is executed", command);
free(temp);
fclose(file);
@@ -143,7 +143,7 @@ static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev)
dpio_epoll_fd = epoll_create(1);
ret = rte_dpaa2_intr_enable(&dpio_dev->intr_handle, 0);
if (ret) {
- PMD_DRV_LOG(ERR, "Interrupt registeration failed\n");
+ DPAA2_BUS_ERR("Interrupt registeration failed");
return -1;
}
@@ -166,7 +166,7 @@ static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev)
ret = epoll_ctl(dpio_epoll_fd, EPOLL_CTL_ADD, eventfd, &epoll_ev);
if (ret < 0) {
- PMD_DRV_LOG(ERR, "epoll_ctl failed\n");
+ DPAA2_BUS_ERR("epoll_ctl failed");
return -1;
}
dpio_dev->epoll_fd = dpio_epoll_fd;
@@ -185,28 +185,27 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev)
dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io));
if (!dpio_dev->dpio) {
- PMD_INIT_LOG(ERR, "Memory allocation failure\n");
+ DPAA2_BUS_ERR("Memory allocation failure");
return -1;
}
- PMD_DRV_LOG(DEBUG, "Allocated DPIO Portal[%p]", dpio_dev->dpio);
dpio_dev->dpio->regs = dpio_dev->mc_portal;
if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id,
&dpio_dev->token)) {
- PMD_INIT_LOG(ERR, "Failed to allocate IO space\n");
+ DPAA2_BUS_ERR("Failed to allocate IO space");
free(dpio_dev->dpio);
return -1;
}
if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) {
- PMD_INIT_LOG(ERR, "Failed to reset dpio\n");
+ DPAA2_BUS_ERR("Failed to reset dpio");
dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
free(dpio_dev->dpio);
return -1;
}
if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) {
- PMD_INIT_LOG(ERR, "Failed to Enable dpio\n");
+ DPAA2_BUS_ERR("Failed to Enable dpio");
dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
free(dpio_dev->dpio);
return -1;
@@ -214,7 +213,7 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev)
if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW,
dpio_dev->token, &attr)) {
- PMD_INIT_LOG(ERR, "DPIO Get attribute failed\n");
+ DPAA2_BUS_ERR("DPIO Get attribute failed");
dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
free(dpio_dev->dpio);
@@ -231,7 +230,7 @@ configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev)
dpio_dev->sw_portal = qbman_swp_init(&p_des);
if (dpio_dev->sw_portal == NULL) {
- PMD_DRV_LOG(ERR, " QBMan SW Portal Init failed\n");
+ DPAA2_BUS_ERR("QBMan SW Portal Init failed");
dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
free(dpio_dev->dpio);
return -1;
@@ -249,7 +248,7 @@ dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id)
if (cpu_id < 0) {
cpu_id = rte_get_master_lcore();
if (cpu_id < 0) {
- RTE_LOG(ERR, PMD, "\tGetting CPU Index failed\n");
+ DPAA2_BUS_ERR("Getting CPU Index failed");
return -1;
}
}
@@ -258,19 +257,19 @@ dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id)
*/
sdest = dpaa2_core_cluster_sdest(cpu_id);
- PMD_DRV_LOG(DEBUG, "Portal= %d CPU= %u SDEST= %d",
- dpio_dev->index, cpu_id, sdest);
+ DPAA2_BUS_DEBUG("Portal= %d CPU= %u SDEST= %d",
+ dpio_dev->index, cpu_id, sdest);
ret = dpio_set_stashing_destination(dpio_dev->dpio, CMD_PRI_LOW,
dpio_dev->token, sdest);
if (ret) {
- PMD_DRV_LOG(ERR, "%d ERROR in SDEST\n", ret);
+ DPAA2_BUS_ERR("%d ERROR in SDEST", ret);
return -1;
}
#ifdef RTE_LIBRTE_PMD_DPAA2_EVENTDEV
if (dpaa2_dpio_intr_init(dpio_dev)) {
- PMD_DRV_LOG(ERR, "Interrupt registration failed for dpio\n");
+ DPAA2_BUS_ERR("Interrupt registration failed for dpio");
return -1;
}
#endif
@@ -291,12 +290,12 @@ struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id)
if (!dpio_dev)
return NULL;
- PMD_DRV_LOG(DEBUG, "New Portal=0x%x (%d) affined thread - %lu",
- dpio_dev, dpio_dev->index, syscall(SYS_gettid));
+ DPAA2_BUS_DEBUG("New Portal %p (%d) affined thread - %lu",
+ dpio_dev, dpio_dev->index, syscall(SYS_gettid));
ret = dpaa2_configure_stashing(dpio_dev, cpu_id);
if (ret)
- PMD_DRV_LOG(ERR, "dpaa2_configure_stashing failed");
+ DPAA2_BUS_ERR("dpaa2_configure_stashing failed");
return dpio_dev;
}
@@ -314,8 +313,9 @@ dpaa2_affine_qbman_swp(void)
return -1;
if (dpaa2_io_portal[lcore_id].dpio_dev) {
- PMD_DRV_LOG(INFO, "DPAA Portal=0x%x (%d) is being shared"
- " between thread %lu and current %lu",
+ DPAA2_BUS_DP_INFO("DPAA Portal=%p (%d) is being shared"
+ " between thread %" PRIu64 " and current "
+ "%" PRIu64 "\n",
dpaa2_io_portal[lcore_id].dpio_dev,
dpaa2_io_portal[lcore_id].dpio_dev->index,
dpaa2_io_portal[lcore_id].net_tid,
@@ -326,7 +326,8 @@ dpaa2_affine_qbman_swp(void)
[lcore_id].dpio_dev->ref_count);
dpaa2_io_portal[lcore_id].net_tid = tid;
- PMD_DRV_LOG(DEBUG, "Old Portal=0x%x (%d) affined thread - %lu",
+ DPAA2_BUS_DP_DEBUG("Old Portal=%p (%d) affined thread - "
+ "%" PRIu64 "\n",
dpaa2_io_portal[lcore_id].dpio_dev,
dpaa2_io_portal[lcore_id].dpio_dev->index,
tid);
@@ -348,7 +349,7 @@ dpaa2_affine_qbman_swp(void)
}
int
-dpaa2_affine_qbman_swp_sec(void)
+dpaa2_affine_qbman_ethrx_swp(void)
{
unsigned int lcore_id = rte_lcore_id();
uint64_t tid = syscall(SYS_gettid);
@@ -359,32 +360,36 @@ dpaa2_affine_qbman_swp_sec(void)
else if (lcore_id >= RTE_MAX_LCORE)
return -1;
- if (dpaa2_io_portal[lcore_id].sec_dpio_dev) {
- PMD_DRV_LOG(INFO, "DPAA Portal=0x%x (%d) is being shared"
- " between thread %lu and current %lu",
- dpaa2_io_portal[lcore_id].sec_dpio_dev,
- dpaa2_io_portal[lcore_id].sec_dpio_dev->index,
- dpaa2_io_portal[lcore_id].sec_tid,
- tid);
- RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev
- = dpaa2_io_portal[lcore_id].sec_dpio_dev;
+ if (dpaa2_io_portal[lcore_id].ethrx_dpio_dev) {
+ DPAA2_BUS_DP_INFO(
+ "DPAA Portal=%p (%d) is being shared between thread"
+ " %" PRIu64 " and current %" PRIu64 "\n",
+ dpaa2_io_portal[lcore_id].ethrx_dpio_dev,
+ dpaa2_io_portal[lcore_id].ethrx_dpio_dev->index,
+ dpaa2_io_portal[lcore_id].sec_tid,
+ tid);
+ RTE_PER_LCORE(_dpaa2_io).ethrx_dpio_dev
+ = dpaa2_io_portal[lcore_id].ethrx_dpio_dev;
rte_atomic16_inc(&dpaa2_io_portal
- [lcore_id].sec_dpio_dev->ref_count);
+ [lcore_id].ethrx_dpio_dev->ref_count);
dpaa2_io_portal[lcore_id].sec_tid = tid;
- PMD_DRV_LOG(DEBUG, "Old Portal=0x%x (%d) affined thread - %lu",
- dpaa2_io_portal[lcore_id].sec_dpio_dev,
- dpaa2_io_portal[lcore_id].sec_dpio_dev->index,
- tid);
+ DPAA2_BUS_DP_DEBUG(
+ "Old Portal=%p (%d) affined thread"
+ " - %" PRIu64 "\n",
+ dpaa2_io_portal[lcore_id].ethrx_dpio_dev,
+ dpaa2_io_portal[lcore_id].ethrx_dpio_dev->index,
+ tid);
return 0;
}
/* Populate the dpaa2_io_portal structure */
- dpaa2_io_portal[lcore_id].sec_dpio_dev = dpaa2_get_qbman_swp(lcore_id);
+ dpaa2_io_portal[lcore_id].ethrx_dpio_dev =
+ dpaa2_get_qbman_swp(lcore_id);
- if (dpaa2_io_portal[lcore_id].sec_dpio_dev) {
- RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev
- = dpaa2_io_portal[lcore_id].sec_dpio_dev;
+ if (dpaa2_io_portal[lcore_id].ethrx_dpio_dev) {
+ RTE_PER_LCORE(_dpaa2_io).ethrx_dpio_dev
+ = dpaa2_io_portal[lcore_id].ethrx_dpio_dev;
dpaa2_io_portal[lcore_id].sec_tid = tid;
return 0;
} else {
@@ -401,15 +406,14 @@ dpaa2_create_dpio_device(int vdev_fd,
struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)};
if (obj_info->num_regions < NUM_DPIO_REGIONS) {
- PMD_INIT_LOG(ERR, "ERROR, Not sufficient number "
- "of DPIO regions.\n");
+ DPAA2_BUS_ERR("Not sufficient number of DPIO regions");
return -1;
}
dpio_dev = rte_malloc(NULL, sizeof(struct dpaa2_dpio_dev),
RTE_CACHE_LINE_SIZE);
if (!dpio_dev) {
- PMD_INIT_LOG(ERR, "Memory allocation failed for DPIO Device\n");
+ DPAA2_BUS_ERR("Memory allocation failed for DPIO Device");
return -1;
}
@@ -421,31 +425,31 @@ dpaa2_create_dpio_device(int vdev_fd,
reg_info.index = 0;
if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
- PMD_INIT_LOG(ERR, "vfio: error getting region info\n");
+ DPAA2_BUS_ERR("vfio: error getting region info");
rte_free(dpio_dev);
return -1;
}
dpio_dev->ce_size = reg_info.size;
- dpio_dev->qbman_portal_ce_paddr = (uint64_t)mmap(NULL, reg_info.size,
+ dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size,
PROT_WRITE | PROT_READ, MAP_SHARED,
vdev_fd, reg_info.offset);
reg_info.index = 1;
if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
- PMD_INIT_LOG(ERR, "vfio: error getting region info\n");
+ DPAA2_BUS_ERR("vfio: error getting region info");
rte_free(dpio_dev);
return -1;
}
dpio_dev->ci_size = reg_info.size;
- dpio_dev->qbman_portal_ci_paddr = (uint64_t)mmap(NULL, reg_info.size,
+ dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size,
PROT_WRITE | PROT_READ, MAP_SHARED,
vdev_fd, reg_info.offset);
if (configure_dpio_qbman_swp(dpio_dev)) {
- PMD_INIT_LOG(ERR,
- "Fail to configure the dpio qbman portal for %d\n",
+ DPAA2_BUS_ERR(
+ "Fail to configure the dpio qbman portal for %d",
dpio_dev->hw_id);
rte_free(dpio_dev);
return -1;
@@ -455,8 +459,8 @@ dpaa2_create_dpio_device(int vdev_fd,
dpio_dev->index = io_space_count;
if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) {
- PMD_INIT_LOG(ERR, "Fail to setup interrupt for %d\n",
- dpio_dev->hw_id);
+ DPAA2_BUS_ERR("Fail to setup interrupt for %d",
+ dpio_dev->hw_id);
rte_free(dpio_dev);
}
@@ -466,21 +470,20 @@ dpaa2_create_dpio_device(int vdev_fd,
if (mc_get_soc_version(dpio_dev->dpio,
CMD_PRI_LOW, &mc_plat_info)) {
- PMD_INIT_LOG(ERR, "\tmc_get_soc_version failed\n");
+ DPAA2_BUS_ERR("Unable to get SoC version information");
} else if ((mc_plat_info.svr & 0xffff0000) == SVR_LS1080A) {
dpaa2_core_cluster_base = 0x02;
dpaa2_cluster_sz = 4;
- PMD_INIT_LOG(DEBUG, "\tLS108x (A53) Platform Detected");
+ DPAA2_BUS_DEBUG("LS108x (A53) Platform Detected");
} else if ((mc_plat_info.svr & 0xffff0000) == SVR_LX2160A) {
dpaa2_core_cluster_base = 0x00;
dpaa2_cluster_sz = 2;
- PMD_INIT_LOG(DEBUG, "\tLX2160 Platform Detected");
+ DPAA2_BUS_DEBUG("LX2160 Platform Detected");
}
dpaa2_svr_family = (mc_plat_info.svr & 0xffff0000);
}
TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next);
- RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpio.%d]\n", object_id);
return 0;
}
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
index c0bd8782..d593eea7 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
@@ -13,7 +13,7 @@
struct dpaa2_io_portal_t {
struct dpaa2_dpio_dev *dpio_dev;
- struct dpaa2_dpio_dev *sec_dpio_dev;
+ struct dpaa2_dpio_dev *ethrx_dpio_dev;
uint64_t net_tid;
uint64_t sec_tid;
void *eventdev;
@@ -25,8 +25,8 @@ RTE_DECLARE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io);
#define DPAA2_PER_LCORE_DPIO RTE_PER_LCORE(_dpaa2_io).dpio_dev
#define DPAA2_PER_LCORE_PORTAL DPAA2_PER_LCORE_DPIO->sw_portal
-#define DPAA2_PER_LCORE_SEC_DPIO RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev
-#define DPAA2_PER_LCORE_SEC_PORTAL DPAA2_PER_LCORE_SEC_DPIO->sw_portal
+#define DPAA2_PER_LCORE_ETHRX_DPIO RTE_PER_LCORE(_dpaa2_io).ethrx_dpio_dev
+#define DPAA2_PER_LCORE_ETHRX_PORTAL DPAA2_PER_LCORE_ETHRX_DPIO->sw_portal
/* Variable to store DPAA2 platform type */
extern uint32_t dpaa2_svr_family;
@@ -39,7 +39,7 @@ struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id);
int dpaa2_affine_qbman_swp(void);
/* Affine additional DPIO portal to current crypto processing thread */
-int dpaa2_affine_qbman_swp_sec(void);
+int dpaa2_affine_qbman_ethrx_swp(void);
/* allocate memory for FQ - dq storage */
int
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
index d421dbf0..82075936 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
@@ -142,7 +142,8 @@ struct dpaa2_dpci_dev {
uint16_t token;
rte_atomic16_t in_use;
uint32_t dpci_id; /*HW ID for DPCI object */
- struct dpaa2_queue queue[DPAA2_DPCI_MAX_QUEUES];
+ struct dpaa2_queue rx_queue[DPAA2_DPCI_MAX_QUEUES];
+ struct dpaa2_queue tx_queue[DPAA2_DPCI_MAX_QUEUES];
};
/*! Global MCP list */
@@ -174,7 +175,7 @@ enum qbman_fd_format {
};
/*Macros to define operations on FD*/
#define DPAA2_SET_FD_ADDR(fd, addr) do { \
- (fd)->simple.addr_lo = lower_32_bits((uint64_t)(addr)); \
+ (fd)->simple.addr_lo = lower_32_bits((size_t)(addr)); \
(fd)->simple.addr_hi = upper_32_bits((uint64_t)(addr)); \
} while (0)
#define DPAA2_SET_FD_LEN(fd, length) ((fd)->simple.len = length)
@@ -188,50 +189,55 @@ enum qbman_fd_format {
((fd)->simple.frc = (0x80000000 | (len)))
#define DPAA2_GET_FD_FRC_PARSE_SUM(fd) \
((uint16_t)(((fd)->simple.frc & 0xffff0000) >> 16))
-#define DPAA2_SET_FD_FRC(fd, frc) ((fd)->simple.frc = frc)
+#define DPAA2_SET_FD_FRC(fd, _frc) ((fd)->simple.frc = _frc)
#define DPAA2_RESET_FD_CTRL(fd) ((fd)->simple.ctrl = 0)
#define DPAA2_SET_FD_ASAL(fd, asal) ((fd)->simple.ctrl |= (asal << 16))
#define DPAA2_SET_FD_FLC(fd, addr) do { \
- (fd)->simple.flc_lo = lower_32_bits((uint64_t)(addr)); \
+ (fd)->simple.flc_lo = lower_32_bits((size_t)(addr)); \
(fd)->simple.flc_hi = upper_32_bits((uint64_t)(addr)); \
} while (0)
#define DPAA2_SET_FLE_INTERNAL_JD(fle, len) ((fle)->frc = (0x80000000 | (len)))
#define DPAA2_GET_FLE_ADDR(fle) \
- (uint64_t)((((uint64_t)((fle)->addr_hi)) << 32) + (fle)->addr_lo)
+ (size_t)((((uint64_t)((fle)->addr_hi)) << 32) + (fle)->addr_lo)
#define DPAA2_SET_FLE_ADDR(fle, addr) do { \
- (fle)->addr_lo = lower_32_bits((uint64_t)addr); \
- (fle)->addr_hi = upper_32_bits((uint64_t)addr); \
+ (fle)->addr_lo = lower_32_bits((size_t)addr); \
+ (fle)->addr_hi = upper_32_bits((uint64_t)addr); \
} while (0)
#define DPAA2_GET_FLE_CTXT(fle) \
- (uint64_t)((((uint64_t)((fle)->reserved[1])) << 32) + \
- (fle)->reserved[0])
+ ((((uint64_t)((fle)->reserved[1])) << 32) + (fle)->reserved[0])
#define DPAA2_FLE_SAVE_CTXT(fle, addr) do { \
- (fle)->reserved[0] = lower_32_bits((uint64_t)addr); \
- (fle)->reserved[1] = upper_32_bits((uint64_t)addr); \
+ (fle)->reserved[0] = lower_32_bits((size_t)addr); \
+ (fle)->reserved[1] = upper_32_bits((uint64_t)addr); \
} while (0)
#define DPAA2_SET_FLE_OFFSET(fle, offset) \
((fle)->fin_bpid_offset |= (uint32_t)(offset) << 16)
-#define DPAA2_SET_FLE_BPID(fle, bpid) ((fle)->fin_bpid_offset |= (uint64_t)bpid)
+#define DPAA2_SET_FLE_LEN(fle, len) ((fle)->length = len)
+#define DPAA2_SET_FLE_BPID(fle, bpid) ((fle)->fin_bpid_offset |= (size_t)bpid)
#define DPAA2_GET_FLE_BPID(fle) ((fle)->fin_bpid_offset & 0x000000ff)
-#define DPAA2_SET_FLE_FIN(fle) ((fle)->fin_bpid_offset |= (uint64_t)1 << 31)
+#define DPAA2_SET_FLE_FIN(fle) ((fle)->fin_bpid_offset |= 1 << 31)
#define DPAA2_SET_FLE_IVP(fle) (((fle)->fin_bpid_offset |= 0x00004000))
+#define DPAA2_SET_FLE_BMT(fle) (((fle)->fin_bpid_offset |= 0x00008000))
#define DPAA2_SET_FD_COMPOUND_FMT(fd) \
((fd)->simple.bpid_offset |= (uint32_t)1 << 28)
#define DPAA2_GET_FD_ADDR(fd) \
-((uint64_t)((((uint64_t)((fd)->simple.addr_hi)) << 32) + (fd)->simple.addr_lo))
+(((((uint64_t)((fd)->simple.addr_hi)) << 32) + (fd)->simple.addr_lo))
#define DPAA2_GET_FD_LEN(fd) ((fd)->simple.len)
#define DPAA2_GET_FD_BPID(fd) (((fd)->simple.bpid_offset & 0x00003FFF))
#define DPAA2_GET_FD_IVP(fd) (((fd)->simple.bpid_offset & 0x00004000) >> 14)
#define DPAA2_GET_FD_OFFSET(fd) (((fd)->simple.bpid_offset & 0x0FFF0000) >> 16)
+#define DPAA2_GET_FD_FRC(fd) ((fd)->simple.frc)
+#define DPAA2_GET_FD_FLC(fd) \
+ (((uint64_t)((fd)->simple.flc_hi) << 32) + (fd)->simple.flc_lo)
+#define DPAA2_GET_FD_ERR(fd) ((fd)->simple.bpid_offset & 0x000000FF)
#define DPAA2_GET_FLE_OFFSET(fle) (((fle)->fin_bpid_offset & 0x0FFF0000) >> 16)
#define DPAA2_SET_FLE_SG_EXT(fle) ((fle)->fin_bpid_offset |= (uint64_t)1 << 29)
#define DPAA2_IS_SET_FLE_SG_EXT(fle) \
(((fle)->fin_bpid_offset & ((uint64_t)1 << 29)) ? 1 : 0)
#define DPAA2_INLINE_MBUF_FROM_BUF(buf, meta_data_size) \
- ((struct rte_mbuf *)((uint64_t)(buf) - (meta_data_size)))
+ ((struct rte_mbuf *)((size_t)(buf) - (meta_data_size)))
#define DPAA2_ASAL_VAL (DPAA2_MBUF_HW_ANNOTATION / 64)
@@ -255,47 +261,53 @@ enum qbman_fd_format {
*/
#define DPAA2_EQ_RESP_ALWAYS 1
+/* Various structures representing contiguous memory maps */
+struct dpaa2_memseg {
+ TAILQ_ENTRY(dpaa2_memseg) next;
+ char *vaddr;
+ rte_iova_t iova;
+ size_t len;
+};
+
+TAILQ_HEAD(dpaa2_memseg_list, dpaa2_memseg);
+extern struct dpaa2_memseg_list rte_dpaa2_memsegs;
+
#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
extern uint8_t dpaa2_virt_mode;
static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused));
/* todo - this is costly, need to write a fast coversion routine */
static void *dpaa2_mem_ptov(phys_addr_t paddr)
{
- const struct rte_memseg *memseg;
- int i;
+ struct dpaa2_memseg *ms;
if (dpaa2_virt_mode)
- return (void *)paddr;
-
- memseg = rte_eal_get_physmem_layout();
-
- for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
- if (paddr >= memseg[i].iova &&
- (char *)paddr < (char *)memseg[i].iova + memseg[i].len)
- return (void *)(memseg[i].addr_64
- + (paddr - memseg[i].iova));
+ return (void *)(size_t)paddr;
+
+ /* Check if the address is already part of the memseg list internally
+ * maintained by the dpaa2 driver.
+ */
+ TAILQ_FOREACH(ms, &rte_dpaa2_memsegs, next) {
+ if (paddr >= ms->iova && paddr <
+ ms->iova + ms->len)
+ return RTE_PTR_ADD(ms->vaddr, (uintptr_t)(paddr - ms->iova));
}
- return NULL;
+
+ /* If not, Fallback to full memseg list searching */
+ return rte_mem_iova2virt(paddr);
}
static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused));
static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr)
{
const struct rte_memseg *memseg;
- int i;
if (dpaa2_virt_mode)
return vaddr;
- memseg = rte_eal_get_physmem_layout();
-
- for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
- if (vaddr >= memseg[i].addr_64 &&
- vaddr < memseg[i].addr_64 + memseg[i].len)
- return memseg[i].iova
- + (vaddr - memseg[i].addr_64);
- }
- return (phys_addr_t)(NULL);
+ memseg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr, NULL);
+ if (memseg)
+ return memseg->phys_addr + RTE_PTR_DIFF(vaddr, memseg->addr);
+ return (size_t)NULL;
}
/**
@@ -306,28 +318,26 @@ static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr)
*/
#define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_iova)
-#define DPAA2_OP_VADDR_TO_IOVA(op) (op->phys_addr)
/**
* macro to convert Virtual address to IOVA
*/
-#define DPAA2_VADDR_TO_IOVA(_vaddr) dpaa2_mem_vtop((uint64_t)(_vaddr))
+#define DPAA2_VADDR_TO_IOVA(_vaddr) dpaa2_mem_vtop((size_t)(_vaddr))
/**
* macro to convert IOVA to Virtual address
*/
-#define DPAA2_IOVA_TO_VADDR(_iova) dpaa2_mem_ptov((phys_addr_t)(_iova))
+#define DPAA2_IOVA_TO_VADDR(_iova) dpaa2_mem_ptov((size_t)(_iova))
/**
* macro to convert modify the memory containing IOVA to Virtual address
*/
#define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type) \
- {_mem = (_type)(dpaa2_mem_ptov((phys_addr_t)(_mem))); }
+ {_mem = (_type)(dpaa2_mem_ptov((size_t)(_mem))); }
#else /* RTE_LIBRTE_DPAA2_USE_PHYS_IOVA */
#define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_addr)
-#define DPAA2_OP_VADDR_TO_IOVA(op) (op)
#define DPAA2_VADDR_TO_IOVA(_vaddr) (_vaddr)
#define DPAA2_IOVA_TO_VADDR(_iova) (_iova)
#define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type)
diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h
index 96269ed4..bb60a98f 100644
--- a/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h
+++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h
@@ -6,8 +6,6 @@
#ifndef _FSL_QBMAN_BASE_H
#define _FSL_QBMAN_BASE_H
-typedef uint64_t dma_addr_t;
-
/**
* DOC: QBMan basic structures
*
diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c
index e2217335..07145005 100644
--- a/drivers/bus/fslmc/qbman/qbman_portal.c
+++ b/drivers/bus/fslmc/qbman/qbman_portal.c
@@ -122,8 +122,7 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
p->vdq.valid_bit = QB_VALID_BIT;
p->dqrr.next_idx = 0;
p->dqrr.valid_bit = QB_VALID_BIT;
- qman_version = p->desc.qman_version;
- if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) {
+ if ((p->desc.qman_version & 0xFFFF0000) < QMAN_REV_4100) {
p->dqrr.dqrr_size = 4;
p->dqrr.reset_bug = 1;
} else {
@@ -553,10 +552,9 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s,
/* Flush all the cacheline without load/store in between */
eqcr_pi = s->eqcr.pi;
- addr_cena = (uint64_t)s->sys.addr_cena;
+ addr_cena = (size_t)s->sys.addr_cena;
for (i = 0; i < num_enqueued; i++) {
- dcbf((uint64_t *)(addr_cena +
- QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
+ dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
eqcr_pi++;
eqcr_pi &= 0xF;
}
@@ -620,10 +618,9 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
/* Flush all the cacheline without load/store in between */
eqcr_pi = s->eqcr.pi;
- addr_cena = (uint64_t)s->sys.addr_cena;
+ addr_cena = (size_t)s->sys.addr_cena;
for (i = 0; i < num_enqueued; i++) {
- dcbf((uint64_t *)(addr_cena +
- QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
+ dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
eqcr_pi++;
eqcr_pi &= 0xF;
}
@@ -690,7 +687,7 @@ void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
dma_addr_t storage_phys,
int stash)
{
- d->pull.rsp_addr_virt = (uint64_t)storage;
+ d->pull.rsp_addr_virt = (size_t)storage;
if (!storage) {
d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
@@ -749,7 +746,7 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
}
d->pull.tok = s->sys.idx + 1;
- s->vdq.storage = (void *)d->pull.rsp_addr_virt;
+ s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
memcpy(&p[1], &cl[1], 12);
diff --git a/drivers/bus/fslmc/qbman/qbman_portal.h b/drivers/bus/fslmc/qbman/qbman_portal.h
index 8bff0b4f..dbea22a1 100644
--- a/drivers/bus/fslmc/qbman/qbman_portal.h
+++ b/drivers/bus/fslmc/qbman/qbman_portal.h
@@ -7,7 +7,6 @@
#include "qbman_sys.h"
#include <fsl_qbman_portal.h>
-uint32_t qman_version;
#define QMAN_REV_4000 0x04000000
#define QMAN_REV_4100 0x04010000
#define QMAN_REV_4101 0x04010001
diff --git a/drivers/bus/fslmc/qbman/qbman_sys.h b/drivers/bus/fslmc/qbman/qbman_sys.h
index 846788ea..2bd33ea5 100644
--- a/drivers/bus/fslmc/qbman/qbman_sys.h
+++ b/drivers/bus/fslmc/qbman/qbman_sys.h
@@ -20,6 +20,9 @@
#include "qbman_sys_decl.h"
+#define CENA_WRITE_ENABLE 0
+#define CINH_WRITE_ENABLE 1
+
/* Debugging assists */
static inline void __hexdump(unsigned long start, unsigned long end,
unsigned long p, size_t sz, const unsigned char *c)
@@ -178,7 +181,11 @@ static inline void *qbman_cena_write_start_wo_shadow(struct qbman_swp_sys *s,
s->addr_cena, s->idx, offset);
#endif
QBMAN_BUG_ON(offset & 63);
+#ifdef RTE_ARCH_64
return (s->addr_cena + offset);
+#else
+ return (s->addr_cinh + offset);
+#endif
}
static inline void qbman_cena_write_complete(struct qbman_swp_sys *s,
@@ -191,11 +198,19 @@ static inline void qbman_cena_write_complete(struct qbman_swp_sys *s,
s->addr_cena, s->idx, offset, shadow);
hexdump(cmd, 64);
#endif
+#ifdef RTE_ARCH_64
for (loop = 15; loop >= 1; loop--)
__raw_writel(shadow[loop], s->addr_cena +
offset + loop * 4);
lwsync();
__raw_writel(shadow[0], s->addr_cena + offset);
+#else
+ for (loop = 15; loop >= 1; loop--)
+ __raw_writel(shadow[loop], s->addr_cinh +
+ offset + loop * 4);
+ lwsync();
+ __raw_writel(shadow[0], s->addr_cinh + offset);
+#endif
dcbf(s->addr_cena + offset);
}
@@ -224,9 +239,15 @@ static inline void *qbman_cena_read(struct qbman_swp_sys *s, uint32_t offset)
s->addr_cena, s->idx, offset, shadow);
#endif
+#ifdef RTE_ARCH_64
for (loop = 0; loop < 16; loop++)
shadow[loop] = __raw_readl(s->addr_cena + offset
+ loop * 4);
+#else
+ for (loop = 0; loop < 16; loop++)
+ shadow[loop] = __raw_readl(s->addr_cinh + offset
+ + loop * 4);
+#endif
#ifdef QBMAN_CENA_TRACE
hexdump(shadow, 64);
#endif
@@ -313,6 +334,11 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
uint8_t dqrr_size)
{
uint32_t reg;
+#ifdef RTE_ARCH_64
+ uint8_t wn = CENA_WRITE_ENABLE;
+#else
+ uint8_t wn = CINH_WRITE_ENABLE;
+#endif
s->addr_cena = d->cena_bar;
s->addr_cinh = d->cinh_bar;
@@ -333,10 +359,10 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
QBMAN_BUG_ON(reg);
#endif
if (s->eqcr_mode == qman_eqcr_vb_array)
- reg = qbman_set_swp_cfg(dqrr_size, 0, 0, 3, 2, 3, 1, 1, 1, 1,
+ reg = qbman_set_swp_cfg(dqrr_size, wn, 0, 3, 2, 3, 1, 1, 1, 1,
1, 1);
else
- reg = qbman_set_swp_cfg(dqrr_size, 0, 1, 3, 2, 2, 1, 1, 1, 1,
+ reg = qbman_set_swp_cfg(dqrr_size, wn, 1, 3, 2, 2, 1, 1, 1, 1,
1, 1);
qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg);
reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG);
diff --git a/drivers/bus/fslmc/qbman/qbman_sys_decl.h b/drivers/bus/fslmc/qbman/qbman_sys_decl.h
index f82bb18c..fa6977fe 100644
--- a/drivers/bus/fslmc/qbman/qbman_sys_decl.h
+++ b/drivers/bus/fslmc/qbman/qbman_sys_decl.h
@@ -15,6 +15,7 @@
/****************/
/* arch assists */
/****************/
+#if defined(RTE_ARCH_ARM64)
#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); }
#define lwsync() { asm volatile("dmb st" : : : "memory"); }
#define dcbf(p) { asm volatile("dc cvac, %0" : : "r"(p) : "memory"); }
@@ -28,3 +29,25 @@ static inline void prefetch_for_store(void *p)
{
asm volatile("prfm pstl1keep, [%0, #0]" : : "r" (p));
}
+#elif defined(RTE_ARCH_ARM)
+#define dcbz(p) memset(p, 0, 64)
+#define lwsync() { asm volatile("dmb st" : : : "memory"); }
+#define dcbf(p) RTE_SET_USED(p)
+#define dccivac(p) RTE_SET_USED(p)
+#define prefetch_for_load(p) { asm volatile ("pld [%0]" : : "r" (p)); }
+#define prefetch_for_store(p) { asm volatile ("pld [%0]" : : "r" (p)); }
+
+#else
+#define dcbz(p) RTE_SET_USED(p)
+#define lwsync()
+#define dcbf(p) RTE_SET_USED(p)
+#define dccivac(p) RTE_SET_USED(p)
+static inline void prefetch_for_load(void *p)
+{
+ RTE_SET_USED(p);
+}
+static inline void prefetch_for_store(void *p)
+{
+ RTE_SET_USED(p);
+}
+#endif
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index b7db0741..fe45a113 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -2,7 +2,6 @@ DPDK_17.05 {
global:
dpaa2_affine_qbman_swp;
- dpaa2_affine_qbman_swp_sec;
dpaa2_alloc_dpbp_dev;
dpaa2_alloc_dq_storage;
dpaa2_free_dpbp_dev;
@@ -101,3 +100,19 @@ DPDK_18.02 {
rte_fslmc_get_device_count;
} DPDK_17.11;
+
+DPDK_18.05 {
+ global:
+
+ dpaa2_affine_qbman_ethrx_swp;
+ dpdmai_close;
+ dpdmai_disable;
+ dpdmai_enable;
+ dpdmai_get_attributes;
+ dpdmai_get_rx_queue;
+ dpdmai_get_tx_queue;
+ dpdmai_open;
+ dpdmai_set_rx_queue;
+ rte_dpaa2_free_dpci_dev;
+
+} DPDK_18.02;
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 69d0fec7..cea5b78f 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -31,6 +31,7 @@ extern "C" {
#include <rte_dev.h>
#include <rte_bus.h>
#include <rte_tailq.h>
+#include <rte_devargs.h>
#include <fslmc_vfio.h>
@@ -49,6 +50,9 @@ struct rte_dpaa2_driver;
TAILQ_HEAD(rte_fslmc_device_list, rte_dpaa2_device);
TAILQ_HEAD(rte_fslmc_driver_list, rte_dpaa2_driver);
+#define RTE_DEV_TO_FSLMC_CONST(ptr) \
+ container_of(ptr, const struct rte_dpaa2_device, device)
+
extern struct rte_fslmc_bus rte_fslmc_bus;
enum rte_dpaa2_dev_type {
@@ -61,6 +65,7 @@ enum rte_dpaa2_dev_type {
DPAA2_IO, /**< DPIO type device */
DPAA2_CI, /**< DPCI type device */
DPAA2_MPORTAL, /**< DPMCP type device */
+ DPAA2_QDMA, /**< DPDMAI type device */
/* Unknown device placeholder */
DPAA2_UNKNOWN,
DPAA2_DEVTYPE_MAX,
@@ -91,6 +96,7 @@ struct rte_dpaa2_device {
union {
struct rte_eth_dev *eth_dev; /**< ethernet device */
struct rte_cryptodev *cryptodev; /**< Crypto Device */
+ struct rte_rawdev *rawdev; /**< Raw Device */
};
enum rte_dpaa2_dev_type dev_type; /**< Device Type */
uint16_t object_id; /**< DPAA2 Object ID */
@@ -167,8 +173,7 @@ void rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver);
/** Helper for DPAA2 device registration from driver (eth, crypto) instance */
#define RTE_PMD_REGISTER_DPAA2(nm, dpaa2_drv) \
-RTE_INIT(dpaa2initfn_ ##nm); \
-static void dpaa2initfn_ ##nm(void) \
+RTE_INIT(dpaa2initfn_ ##nm) \
{\
(dpaa2_drv).driver.name = RTE_STR(nm);\
rte_fslmc_driver_register(&dpaa2_drv); \
@@ -197,8 +202,7 @@ uint32_t rte_fslmc_get_device_count(enum rte_dpaa2_dev_type device_type);
/** Helper for DPAA2 object registration */
#define RTE_PMD_REGISTER_DPAA2_OBJECT(nm, dpaa2_obj) \
-RTE_INIT(dpaa2objinitfn_ ##nm); \
-static void dpaa2objinitfn_ ##nm(void) \
+RTE_INIT(dpaa2objinitfn_ ##nm) \
{\
(dpaa2_obj).name = RTE_STR(nm);\
rte_fslmc_object_register(&dpaa2_obj); \
diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile
new file mode 100644
index 00000000..3ff3bdb8
--- /dev/null
+++ b/drivers/bus/ifpga/Makefile
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_bus_ifpga.a
+
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+LDLIBS += -lrte_eal
+LDLIBS += -lrte_rawdev
+LDLIBS += -lrte_kvargs
+
+# versioning export map
+EXPORT_MAP := rte_bus_ifpga_version.map
+
+# library version
+LIBABIVER := 1
+
+SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_bus.c
+SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_common.c
+
+#
+# Export include files
+#
+SYMLINK-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)-include += rte_bus_ifpga.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c
new file mode 100644
index 00000000..b324872e
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_bus.c
@@ -0,0 +1,467 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <rte_errno.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+#include <rte_string_fns.h>
+
+#include "rte_rawdev.h"
+#include "rte_rawdev_pmd.h"
+#include "rte_bus_ifpga.h"
+#include "ifpga_logs.h"
+#include "ifpga_common.h"
+
+int ifpga_bus_logtype;
+
+/* Forward declaration to access Intel FPGA bus
+ * on which iFPGA devices are connected
+ */
+static struct rte_bus rte_ifpga_bus;
+
+static struct ifpga_afu_dev_list ifpga_afu_dev_list =
+ TAILQ_HEAD_INITIALIZER(ifpga_afu_dev_list);
+static struct ifpga_afu_drv_list ifpga_afu_drv_list =
+ TAILQ_HEAD_INITIALIZER(ifpga_afu_drv_list);
+
+
+/* register a ifpga bus based driver */
+void rte_ifpga_driver_register(struct rte_afu_driver *driver)
+{
+ RTE_VERIFY(driver);
+
+ TAILQ_INSERT_TAIL(&ifpga_afu_drv_list, driver, next);
+}
+
+/* un-register a fpga bus based driver */
+void rte_ifpga_driver_unregister(struct rte_afu_driver *driver)
+{
+ TAILQ_REMOVE(&ifpga_afu_drv_list, driver, next);
+}
+
+static struct rte_afu_device *
+ifpga_find_afu_dev(const struct rte_rawdev *rdev,
+ const struct rte_afu_id *afu_id)
+{
+ struct rte_afu_device *afu_dev = NULL;
+
+ TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) {
+ if (afu_dev &&
+ afu_dev->rawdev == rdev &&
+ !ifpga_afu_id_cmp(&afu_dev->id, afu_id))
+ return afu_dev;
+ }
+ return NULL;
+}
+
+static const char * const valid_args[] = {
+#define IFPGA_ARG_NAME "ifpga"
+ IFPGA_ARG_NAME,
+#define IFPGA_ARG_PORT "port"
+ IFPGA_ARG_PORT,
+#define IFPGA_AFU_BTS "afu_bts"
+ IFPGA_AFU_BTS,
+ NULL
+};
+
+/*
+ * Scan the content of the FPGA bus, and the devices in the devices
+ * list
+ */
+static struct rte_afu_device *
+ifpga_scan_one(struct rte_rawdev *rawdev,
+ struct rte_devargs *devargs)
+{
+ struct rte_kvargs *kvlist = NULL;
+ struct rte_afu_device *afu_dev = NULL;
+ struct rte_afu_pr_conf afu_pr_conf;
+ int ret = 0;
+ char *path = NULL;
+
+ memset(&afu_pr_conf, 0, sizeof(struct rte_afu_pr_conf));
+
+ kvlist = rte_kvargs_parse(devargs->args, valid_args);
+ if (!kvlist) {
+ IFPGA_BUS_ERR("error when parsing param");
+ goto end;
+ }
+
+ if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
+ if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT,
+ &rte_ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) {
+ IFPGA_BUS_ERR("error to parse %s",
+ IFPGA_ARG_PORT);
+ goto end;
+ }
+ } else {
+ IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+ IFPGA_ARG_PORT);
+ goto end;
+ }
+
+ if (rte_kvargs_count(kvlist, IFPGA_AFU_BTS) == 1) {
+ if (rte_kvargs_process(kvlist, IFPGA_AFU_BTS,
+ &rte_ifpga_get_string_arg, &path) < 0) {
+ IFPGA_BUS_ERR("Failed to parse %s",
+ IFPGA_AFU_BTS);
+ goto end;
+ }
+ } else {
+ IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+ IFPGA_AFU_BTS);
+ goto end;
+ }
+
+ afu_pr_conf.afu_id.uuid.uuid_low = 0;
+ afu_pr_conf.afu_id.uuid.uuid_high = 0;
+ afu_pr_conf.pr_enable = path?1:0;
+
+ if (ifpga_find_afu_dev(rawdev, &afu_pr_conf.afu_id))
+ goto end;
+
+ afu_dev = calloc(1, sizeof(*afu_dev));
+ if (!afu_dev)
+ goto end;
+
+ afu_dev->device.devargs = devargs;
+ afu_dev->device.numa_node = SOCKET_ID_ANY;
+ afu_dev->device.name = devargs->name;
+ afu_dev->rawdev = rawdev;
+ afu_dev->id.uuid.uuid_low = 0;
+ afu_dev->id.uuid.uuid_high = 0;
+ afu_dev->id.port = afu_pr_conf.afu_id.port;
+
+ if (rawdev->dev_ops && rawdev->dev_ops->dev_info_get)
+ rawdev->dev_ops->dev_info_get(rawdev, afu_dev);
+
+ if (rawdev->dev_ops &&
+ rawdev->dev_ops->dev_start &&
+ rawdev->dev_ops->dev_start(rawdev))
+ goto end;
+
+ strlcpy(afu_pr_conf.bs_path, path, sizeof(afu_pr_conf.bs_path));
+ if (rawdev->dev_ops &&
+ rawdev->dev_ops->firmware_load &&
+ rawdev->dev_ops->firmware_load(rawdev,
+ &afu_pr_conf)){
+ IFPGA_BUS_ERR("firmware load error %d\n", ret);
+ goto end;
+ }
+ afu_dev->id.uuid.uuid_low = afu_pr_conf.afu_id.uuid.uuid_low;
+ afu_dev->id.uuid.uuid_high = afu_pr_conf.afu_id.uuid.uuid_high;
+
+ rte_kvargs_free(kvlist);
+ free(path);
+ return afu_dev;
+
+end:
+ if (kvlist)
+ rte_kvargs_free(kvlist);
+ if (path)
+ free(path);
+ if (afu_dev)
+ free(afu_dev);
+
+ return NULL;
+}
+
+/*
+ * Scan the content of the FPGA bus, and the devices in the devices
+ * list
+ */
+static int
+ifpga_scan(void)
+{
+ struct rte_devargs *devargs;
+ struct rte_kvargs *kvlist = NULL;
+ struct rte_rawdev *rawdev = NULL;
+ char *name = NULL;
+ char name1[RTE_RAWDEV_NAME_MAX_LEN];
+ struct rte_afu_device *afu_dev = NULL;
+
+ /* for FPGA devices we scan the devargs_list populated via cmdline */
+ RTE_EAL_DEVARGS_FOREACH(IFPGA_ARG_NAME, devargs) {
+ if (devargs->bus != &rte_ifpga_bus)
+ continue;
+
+ kvlist = rte_kvargs_parse(devargs->args, valid_args);
+ if (!kvlist) {
+ IFPGA_BUS_ERR("error when parsing param");
+ goto end;
+ }
+
+ if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
+ if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
+ &rte_ifpga_get_string_arg, &name) < 0) {
+ IFPGA_BUS_ERR("error to parse %s",
+ IFPGA_ARG_NAME);
+ goto end;
+ }
+ } else {
+ IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+ IFPGA_ARG_NAME);
+ goto end;
+ }
+
+ memset(name1, 0, sizeof(name1));
+ snprintf(name1, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+
+ rawdev = rte_rawdev_pmd_get_named_dev(name1);
+ if (!rawdev)
+ goto end;
+
+ afu_dev = ifpga_scan_one(rawdev, devargs);
+ if (afu_dev != NULL)
+ TAILQ_INSERT_TAIL(&ifpga_afu_dev_list, afu_dev, next);
+ }
+
+end:
+ if (kvlist)
+ rte_kvargs_free(kvlist);
+ if (name)
+ free(name);
+
+ return 0;
+}
+
+/*
+ * Match the AFU Driver and AFU Device using the ID Table
+ */
+static int
+rte_afu_match(const struct rte_afu_driver *afu_drv,
+ const struct rte_afu_device *afu_dev)
+{
+ const struct rte_afu_uuid *id_table;
+
+ for (id_table = afu_drv->id_table;
+ ((id_table->uuid_low != 0) && (id_table->uuid_high != 0));
+ id_table++) {
+ /* check if device's identifiers match the driver's ones */
+ if ((id_table->uuid_low != afu_dev->id.uuid.uuid_low) ||
+ id_table->uuid_high !=
+ afu_dev->id.uuid.uuid_high)
+ continue;
+
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+ifpga_probe_one_driver(struct rte_afu_driver *drv,
+ struct rte_afu_device *afu_dev)
+{
+ int ret;
+
+ if (!rte_afu_match(drv, afu_dev))
+ /* Match of device and driver failed */
+ return 1;
+
+ /* reference driver structure */
+ afu_dev->driver = drv;
+ afu_dev->device.driver = &drv->driver;
+
+ /* call the driver probe() function */
+ ret = drv->probe(afu_dev);
+ if (ret) {
+ afu_dev->driver = NULL;
+ afu_dev->device.driver = NULL;
+ }
+
+ return ret;
+}
+
+static int
+ifpga_probe_all_drivers(struct rte_afu_device *afu_dev)
+{
+ struct rte_afu_driver *drv = NULL;
+ int ret = 0;
+
+ if (afu_dev == NULL)
+ return -1;
+
+ /* Check if a driver is already loaded */
+ if (afu_dev->driver != NULL)
+ return 0;
+
+ TAILQ_FOREACH(drv, &ifpga_afu_drv_list, next) {
+ if (ifpga_probe_one_driver(drv, afu_dev)) {
+ ret = -1;
+ break;
+ }
+ }
+ return ret;
+}
+
+/*
+ * Scan the content of the Intel FPGA bus, and call the probe() function for
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+ifpga_probe(void)
+{
+ struct rte_afu_device *afu_dev = NULL;
+ int ret = 0;
+
+ TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) {
+ if (afu_dev->device.driver)
+ continue;
+
+ ret = ifpga_probe_all_drivers(afu_dev);
+ if (ret < 0)
+ IFPGA_BUS_ERR("failed to initialize %s device\n",
+ rte_ifpga_device_name(afu_dev));
+ }
+
+ return ret;
+}
+
+static int
+ifpga_plug(struct rte_device *dev)
+{
+ return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev));
+}
+
+static int
+ifpga_remove_driver(struct rte_afu_device *afu_dev)
+{
+ const char *name;
+ const struct rte_afu_driver *driver;
+
+ name = rte_ifpga_device_name(afu_dev);
+ if (!afu_dev->device.driver) {
+ IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
+ return 1;
+ }
+
+ driver = RTE_DRV_TO_AFU_CONST(afu_dev->device.driver);
+ return driver->remove(afu_dev);
+}
+
+static int
+ifpga_unplug(struct rte_device *dev)
+{
+ struct rte_afu_device *afu_dev = NULL;
+ struct rte_devargs *devargs = NULL;
+ int ret;
+
+ if (dev == NULL)
+ return -EINVAL;
+
+ afu_dev = RTE_DEV_TO_AFU(dev);
+ if (!afu_dev)
+ return -ENOENT;
+
+ devargs = dev->devargs;
+
+ ret = ifpga_remove_driver(afu_dev);
+ if (ret)
+ return ret;
+
+ TAILQ_REMOVE(&ifpga_afu_dev_list, afu_dev, next);
+
+ rte_devargs_remove(devargs->bus->name, devargs->name);
+ free(afu_dev);
+ return 0;
+
+}
+
+static struct rte_device *
+ifpga_find_device(const struct rte_device *start,
+ rte_dev_cmp_t cmp, const void *data)
+{
+ struct rte_afu_device *afu_dev;
+
+ TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) {
+ if (start && &afu_dev->device == start) {
+ start = NULL;
+ continue;
+ }
+ if (cmp(&afu_dev->device, data) == 0)
+ return &afu_dev->device;
+ }
+
+ return NULL;
+}
+static int
+ifpga_parse(const char *name, void *addr)
+{
+ int *out = addr;
+ struct rte_rawdev *rawdev = NULL;
+ char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN];
+ char *c1 = NULL;
+ char *c2 = NULL;
+ int port = IFPGA_BUS_DEV_PORT_MAX;
+ char str_port[8];
+ int str_port_len = 0;
+ int ret;
+
+ memset(str_port, 0, 8);
+ c1 = strchr(name, '|');
+ if (c1 != NULL) {
+ str_port_len = c1 - name;
+ c2 = c1 + 1;
+ }
+
+ if (str_port_len < 8 &&
+ str_port_len > 0) {
+ memcpy(str_port, name, str_port_len);
+ ret = sscanf(str_port, "%d", &port);
+ if (ret == -1)
+ return 0;
+ }
+
+ memset(rawdev_name, 0, sizeof(rawdev_name));
+ snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", c2);
+ rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name);
+
+ if ((port < IFPGA_BUS_DEV_PORT_MAX) &&
+ rawdev &&
+ (addr != NULL))
+ *out = port;
+
+ if ((port < IFPGA_BUS_DEV_PORT_MAX) &&
+ rawdev)
+ return 0;
+ else
+ return 1;
+}
+
+static struct rte_bus rte_ifpga_bus = {
+ .scan = ifpga_scan,
+ .probe = ifpga_probe,
+ .find_device = ifpga_find_device,
+ .plug = ifpga_plug,
+ .unplug = ifpga_unplug,
+ .parse = ifpga_parse,
+};
+
+RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus);
+
+RTE_INIT(ifpga_init_log)
+{
+ ifpga_bus_logtype = rte_log_register("bus.ifpga");
+ if (ifpga_bus_logtype >= 0)
+ rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE);
+}
diff --git a/drivers/bus/ifpga/ifpga_common.c b/drivers/bus/ifpga/ifpga_common.c
new file mode 100644
index 00000000..78e2eaee
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_common.c
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <rte_errno.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+
+#include <rte_devargs.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+
+#include "rte_bus_ifpga.h"
+#include "ifpga_logs.h"
+#include "ifpga_common.h"
+
+int rte_ifpga_get_string_arg(const char *key __rte_unused,
+ const char *value, void *extra_args)
+{
+ if (!value || !extra_args)
+ return -EINVAL;
+
+ *(char **)extra_args = strdup(value);
+
+ if (!*(char **)extra_args)
+ return -ENOMEM;
+
+ return 0;
+}
+int rte_ifpga_get_integer32_arg(const char *key __rte_unused,
+ const char *value, void *extra_args)
+{
+ if (!value || !extra_args)
+ return -EINVAL;
+
+ *(int *)extra_args = strtoull(value, NULL, 0);
+
+ return 0;
+}
+int ifpga_get_integer64_arg(const char *key __rte_unused,
+ const char *value, void *extra_args)
+{
+ if (!value || !extra_args)
+ return -EINVAL;
+
+ *(uint64_t *)extra_args = strtoull(value, NULL, 0);
+
+ return 0;
+}
+int ifpga_get_unsigned_long(const char *str, int base)
+{
+ unsigned long num;
+ char *end = NULL;
+
+ errno = 0;
+
+ num = strtoul(str, &end, base);
+ if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
+ return -1;
+
+ return num;
+}
+
+int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0,
+ const struct rte_afu_id *afu_id1)
+{
+ if ((afu_id0->uuid.uuid_low == afu_id1->uuid.uuid_low) &&
+ (afu_id0->uuid.uuid_high == afu_id1->uuid.uuid_high) &&
+ (afu_id0->port == afu_id1->port)) {
+ return 0;
+ } else
+ return 1;
+}
diff --git a/drivers/bus/ifpga/ifpga_common.h b/drivers/bus/ifpga/ifpga_common.h
new file mode 100644
index 00000000..f9254b9d
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_common.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef _IFPGA_COMMON_H_
+#define _IFPGA_COMMON_H_
+
+int rte_ifpga_get_string_arg(const char *key __rte_unused,
+ const char *value, void *extra_args);
+int rte_ifpga_get_integer32_arg(const char *key __rte_unused,
+ const char *value, void *extra_args);
+int ifpga_get_integer64_arg(const char *key __rte_unused,
+ const char *value, void *extra_args);
+int ifpga_get_unsigned_long(const char *str, int base);
+int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0,
+ const struct rte_afu_id *afu_id1);
+
+#endif /* _IFPGA_COMMON_H_ */
diff --git a/drivers/bus/ifpga/ifpga_logs.h b/drivers/bus/ifpga/ifpga_logs.h
new file mode 100644
index 00000000..873e0a4f
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_logs.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef _IFPGA_LOGS_H_
+#define _IFPGA_LOGS_H_
+
+#include <rte_log.h>
+
+extern int ifpga_bus_logtype;
+
+#define IFPGA_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
+ __func__, ##args)
+
+#define IFPGA_BUS_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
+ __func__, ##args)
+
+#define IFPGA_BUS_FUNC_TRACE() IFPGA_BUS_LOG(DEBUG, ">>")
+
+#define IFPGA_BUS_DEBUG(fmt, args...) \
+ IFPGA_BUS_LOG(DEBUG, fmt, ## args)
+#define IFPGA_BUS_INFO(fmt, args...) \
+ IFPGA_BUS_LOG(INFO, fmt, ## args)
+#define IFPGA_BUS_ERR(fmt, args...) \
+ IFPGA_BUS_LOG(ERR, fmt, ## args)
+#define IFPGA_BUS_WARN(fmt, args...) \
+ IFPGA_BUS_LOG(WARNING, fmt, ## args)
+
+#endif /* _IFPGA_BUS_LOGS_H_ */
diff --git a/drivers/bus/ifpga/meson.build b/drivers/bus/ifpga/meson.build
new file mode 100644
index 00000000..c9b08c86
--- /dev/null
+++ b/drivers/bus/ifpga/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2018 Intel Corporation
+
+deps += ['pci', 'kvargs', 'rawdev']
+install_headers('rte_bus_ifpga.h')
+sources = files('ifpga_common.c', 'ifpga_bus.c')
+
+allow_experimental_apis = true
diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h
new file mode 100644
index 00000000..51d5ae0d
--- /dev/null
+++ b/drivers/bus/ifpga/rte_bus_ifpga.h
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef _RTE_BUS_IFPGA_H_
+#define _RTE_BUS_IFPGA_H_
+
+/**
+ * @file
+ *
+ * RTE Intel FPGA Bus Interface
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_bus.h>
+#include <rte_pci.h>
+
+/** Name of Intel FPGA Bus */
+#define IFPGA_BUS_NAME ifpga
+
+/* Forward declarations */
+struct rte_afu_device;
+struct rte_afu_driver;
+
+/** Double linked list of Intel FPGA AFU device. */
+TAILQ_HEAD(ifpga_afu_dev_list, rte_afu_device);
+/** Double linked list of Intel FPGA AFU device drivers. */
+TAILQ_HEAD(ifpga_afu_drv_list, rte_afu_driver);
+
+#define IFPGA_BUS_BITSTREAM_PATH_MAX_LEN 256
+
+struct rte_afu_uuid {
+ uint64_t uuid_low;
+ uint64_t uuid_high;
+} __attribute__ ((packed));
+
+#define IFPGA_BUS_DEV_PORT_MAX 4
+
+/**
+ * A structure describing an ID for a AFU driver. Each driver provides a
+ * table of these IDs for each device that it supports.
+ */
+struct rte_afu_id {
+ struct rte_afu_uuid uuid;
+ int port; /**< port number */
+} __attribute__ ((packed));
+
+/**
+ * A structure PR (Partial Reconfiguration) configuration AFU driver.
+ */
+
+struct rte_afu_pr_conf {
+ struct rte_afu_id afu_id;
+ int pr_enable;
+ char bs_path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
+};
+
+#define AFU_PRI_STR_SIZE (PCI_PRI_STR_SIZE + 8)
+
+/**
+ * A structure describing a AFU device.
+ */
+struct rte_afu_device {
+ TAILQ_ENTRY(rte_afu_device) next; /**< Next in device list. */
+ struct rte_device device; /**< Inherit core device */
+ struct rte_rawdev *rawdev; /**< Point Rawdev */
+ struct rte_afu_id id; /**< AFU id within FPGA. */
+ uint32_t num_region; /**< number of regions found */
+ struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE];
+ /**< AFU Memory Resource */
+ struct rte_intr_handle intr_handle; /**< Interrupt handle */
+ struct rte_afu_driver *driver; /**< Associated driver */
+ char path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
+} __attribute__ ((packed));
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_afu_device.
+ */
+#define RTE_DEV_TO_AFU(ptr) \
+ container_of(ptr, struct rte_afu_device, device)
+
+#define RTE_DRV_TO_AFU_CONST(ptr) \
+ container_of(ptr, const struct rte_afu_driver, driver)
+
+/**
+ * Initialization function for the driver called during FPGA BUS probing.
+ */
+typedef int (afu_probe_t)(struct rte_afu_device *);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ */
+typedef int (afu_remove_t)(struct rte_afu_device *);
+
+/**
+ * A structure describing a AFU device.
+ */
+struct rte_afu_driver {
+ TAILQ_ENTRY(rte_afu_driver) next; /**< Next afu driver. */
+ struct rte_driver driver; /**< Inherit core driver. */
+ afu_probe_t *probe; /**< Device Probe function. */
+ afu_remove_t *remove; /**< Device Remove function. */
+ const struct rte_afu_uuid *id_table; /**< AFU uuid within FPGA. */
+};
+
+static inline const char *
+rte_ifpga_device_name(const struct rte_afu_device *afu)
+{
+ if (afu && afu->device.name)
+ return afu->device.name;
+ return NULL;
+}
+
+/**
+ * Register a ifpga afu device driver.
+ *
+ * @param driver
+ * A pointer to a rte_afu_driver structure describing the driver
+ * to be registered.
+ */
+void rte_ifpga_driver_register(struct rte_afu_driver *driver);
+
+/**
+ * Unregister a ifpga afu device driver.
+ *
+ * @param driver
+ * A pointer to a rte_afu_driver structure describing the driver
+ * to be unregistered.
+ */
+void rte_ifpga_driver_unregister(struct rte_afu_driver *driver);
+
+#define RTE_PMD_REGISTER_AFU(nm, afudrv)\
+static const char *afudrvinit_ ## nm ## _alias;\
+RTE_INIT(afudrvinitfn_ ##afudrv)\
+{\
+ (afudrv).driver.name = RTE_STR(nm);\
+ (afudrv).driver.alias = afudrvinit_ ## nm ## _alias;\
+ rte_ifpga_driver_register(&afudrv);\
+} \
+RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+#define RTE_PMD_REGISTER_AFU_ALIAS(nm, alias)\
+static const char *afudrvinit_ ## nm ## _alias = RTE_STR(alias)
+
+#endif /* _RTE_BUS_IFPGA_H_ */
diff --git a/drivers/bus/ifpga/rte_bus_ifpga_version.map b/drivers/bus/ifpga/rte_bus_ifpga_version.map
new file mode 100644
index 00000000..a0279797
--- /dev/null
+++ b/drivers/bus/ifpga/rte_bus_ifpga_version.map
@@ -0,0 +1,10 @@
+DPDK_18.05 {
+ global:
+
+ rte_ifpga_get_integer32_arg;
+ rte_ifpga_get_string_arg;
+ rte_ifpga_driver_register;
+ rte_ifpga_driver_unregister;
+
+ local: *;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index c6af500e..80de2d91 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2017 Intel Corporation
-drivers = ['pci', 'vdev']
+drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
std_deps = ['eal']
config_flag_fmt = 'RTE_LIBRTE_@0@_BUS'
driver_name_fmt = 'rte_bus_@0@'
diff --git a/drivers/bus/pci/Makefile b/drivers/bus/pci/Makefile
index f3df1c4c..cf373068 100644
--- a/drivers/bus/pci/Makefile
+++ b/drivers/bus/pci/Makefile
@@ -1,33 +1,5 @@
-# BSD LICENSE
-#
-# Copyright(c) 2017 6WIND S.A.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in
-# the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of 6WIND nor the names of its
-# contributors may be used to endorse or promote products derived
-# from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 6WIND S.A.
include $(RTE_SDK)/mk/rte.vars.mk
@@ -37,6 +9,7 @@ EXPORT_MAP := rte_bus_pci_version.map
CFLAGS := -I$(SRCDIR) $(CFLAGS)
CFLAGS += -O3 $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),)
SYSTEM := linux
@@ -49,6 +22,9 @@ CFLAGS += -I$(RTE_SDK)/drivers/bus/pci/$(SYSTEM)
CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal
+# memseg walk is not part of stable API yet
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
LDLIBS += -lrte_ethdev -lrte_pci
diff --git a/drivers/bus/pci/bsd/Makefile b/drivers/bus/pci/bsd/Makefile
index 4450913e..c1b54c05 100644
--- a/drivers/bus/pci/bsd/Makefile
+++ b/drivers/bus/pci/bsd/Makefile
@@ -1,32 +1,4 @@
-# BSD LICENSE
-#
-# Copyright(c) 2017 6WIND S.A.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in
-# the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of 6WIND nor the names of its
-# contributors may be used to endorse or promote products derived
-# from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 6WIND S.A.
SRCS += pci.c
diff --git a/drivers/bus/pci/linux/Makefile b/drivers/bus/pci/linux/Makefile
index 77c5f970..96ea1d54 100644
--- a/drivers/bus/pci/linux/Makefile
+++ b/drivers/bus/pci/linux/Makefile
@@ -1,33 +1,5 @@
-# BSD LICENSE
-#
-# Copyright(c) 2017 6WIND S.A.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in
-# the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of 6WIND nor the names of its
-# contributors may be used to endorse or promote products derived
-# from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 6WIND S.A.
SRCS += pci.c
SRCS += pci_uio.c
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index abde6411..04648ac9 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -15,7 +15,6 @@
#include <rte_memcpy.h>
#include <rte_vfio.h>
-#include "eal_private.h"
#include "eal_filesystem.h"
#include "private.h"
@@ -33,7 +32,8 @@
extern struct rte_pci_bus rte_pci_bus;
static int
-pci_get_kernel_driver_by_path(const char *filename, char *dri_name)
+pci_get_kernel_driver_by_path(const char *filename, char *dri_name,
+ size_t len)
{
int count;
char path[PATH_MAX];
@@ -54,7 +54,7 @@ pci_get_kernel_driver_by_path(const char *filename, char *dri_name)
name = strrchr(path, '/');
if (name) {
- strncpy(dri_name, name + 1, strlen(name + 1) + 1);
+ strlcpy(dri_name, name + 1, len);
return 0;
}
@@ -116,24 +116,28 @@ rte_pci_unmap_device(struct rte_pci_device *dev)
}
}
-void *
-pci_find_max_end_va(void)
+static int
+find_max_end_va(const struct rte_memseg_list *msl, void *arg)
{
- const struct rte_memseg *seg = rte_eal_get_physmem_layout();
- const struct rte_memseg *last = seg;
- unsigned i = 0;
+ size_t sz = msl->memseg_arr.len * msl->page_sz;
+ void *end_va = RTE_PTR_ADD(msl->base_va, sz);
+ void **max_va = arg;
- for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) {
- if (seg->addr == NULL)
- break;
+ if (*max_va < end_va)
+ *max_va = end_va;
+ return 0;
+}
- if (seg->addr > last->addr)
- last = seg;
+void *
+pci_find_max_end_va(void)
+{
+ void *va = NULL;
- }
- return RTE_PTR_ADD(last->addr, last->len);
+ rte_memseg_list_walk(find_max_end_va, &va);
+ return va;
}
+
/* parse one line of the "resource" sysfs file (note that the 'line'
* string is modified)
*/
@@ -310,7 +314,7 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
/* parse driver */
snprintf(filename, sizeof(filename), "%s/driver", dirname);
- ret = pci_get_kernel_driver_by_path(filename, driver);
+ ret = pci_get_kernel_driver_by_path(filename, driver, sizeof(driver));
if (ret < 0) {
RTE_LOG(ERR, EAL, "Fail to get kernel driver\n");
free(dev);
diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c
index d423e4bb..a7c14421 100644
--- a/drivers/bus/pci/linux/pci_uio.c
+++ b/drivers/bus/pci/linux/pci_uio.c
@@ -282,22 +282,19 @@ int
pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx,
struct mapped_pci_resource *uio_res, int map_idx)
{
- int fd;
+ int fd = -1;
char devname[PATH_MAX];
void *mapaddr;
struct rte_pci_addr *loc;
struct pci_map *maps;
+ int wc_activate = 0;
+
+ if (dev->driver != NULL)
+ wc_activate = dev->driver->drv_flags & RTE_PCI_DRV_WC_ACTIVATE;
loc = &dev->addr;
maps = uio_res->maps;
- /* update devname for mmap */
- snprintf(devname, sizeof(devname),
- "%s/" PCI_PRI_FMT "/resource%d",
- rte_pci_get_sysfs_path(),
- loc->domain, loc->bus, loc->devid,
- loc->function, res_idx);
-
/* allocate memory to keep path */
maps[map_idx].path = rte_malloc(NULL, strlen(devname) + 1, 0);
if (maps[map_idx].path == NULL) {
@@ -309,11 +306,37 @@ pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx,
/*
* open resource file, to mmap it
*/
- fd = open(devname, O_RDWR);
- if (fd < 0) {
- RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
+ if (wc_activate) {
+ /* update devname for mmap */
+ snprintf(devname, sizeof(devname),
+ "%s/" PCI_PRI_FMT "/resource%d_wc",
+ rte_pci_get_sysfs_path(),
+ loc->domain, loc->bus, loc->devid,
+ loc->function, res_idx);
+
+ if (access(devname, R_OK|W_OK) != -1) {
+ fd = open(devname, O_RDWR);
+ if (fd < 0)
+ RTE_LOG(INFO, EAL, "%s cannot be mapped. "
+ "Fall-back to non prefetchable mode.\n",
+ devname);
+ }
+ }
+
+ if (!wc_activate || fd < 0) {
+ snprintf(devname, sizeof(devname),
+ "%s/" PCI_PRI_FMT "/resource%d",
+ rte_pci_get_sysfs_path(),
+ loc->domain, loc->bus, loc->devid,
+ loc->function, res_idx);
+
+ /* then try to map resource file */
+ fd = open(devname, O_RDWR);
+ if (fd < 0) {
+ RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
devname, strerror(errno));
- goto error;
+ goto error;
+ }
}
/* try mapping somewhere close to the end of hugepages */
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index aeeaa9ed..686386d6 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -584,6 +584,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
dev->mem_resource[i].addr = maps[i].addr;
}
+ /* we need save vfio_dev_fd, so it can be used during release */
+ dev->intr_handle.vfio_dev_fd = vfio_dev_fd;
+
return 0;
err_vfio_dev_fd:
close(vfio_dev_fd);
@@ -603,22 +606,58 @@ pci_vfio_map_resource(struct rte_pci_device *dev)
return pci_vfio_map_resource_secondary(dev);
}
-int
-pci_vfio_unmap_resource(struct rte_pci_device *dev)
+static struct mapped_pci_resource *
+find_and_unmap_vfio_resource(struct mapped_pci_res_list *vfio_res_list,
+ struct rte_pci_device *dev,
+ const char *pci_addr)
+{
+ struct mapped_pci_resource *vfio_res = NULL;
+ struct pci_map *maps;
+ int i;
+
+ /* Get vfio_res */
+ TAILQ_FOREACH(vfio_res, vfio_res_list, next) {
+ if (rte_pci_addr_cmp(&vfio_res->pci_addr, &dev->addr))
+ continue;
+ break;
+ }
+
+ if (vfio_res == NULL)
+ return vfio_res;
+
+ RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n",
+ pci_addr);
+
+ maps = vfio_res->maps;
+ for (i = 0; i < (int) vfio_res->nb_maps; i++) {
+
+ /*
+ * We do not need to be aware of MSI-X table BAR mappings as
+ * when mapping. Just using current maps array is enough
+ */
+ if (maps[i].addr) {
+ RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n",
+ pci_addr, maps[i].addr);
+ pci_unmap_resource(maps[i].addr, maps[i].size);
+ }
+ }
+
+ return vfio_res;
+}
+
+static int
+pci_vfio_unmap_resource_primary(struct rte_pci_device *dev)
{
char pci_addr[PATH_MAX] = {0};
struct rte_pci_addr *loc = &dev->addr;
- int i, ret;
struct mapped_pci_resource *vfio_res = NULL;
struct mapped_pci_res_list *vfio_res_list;
-
- struct pci_map *maps;
+ int ret;
/* store PCI address string */
snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
loc->domain, loc->bus, loc->devid, loc->function);
-
if (close(dev->intr_handle.fd) < 0) {
RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n",
pci_addr);
@@ -639,13 +678,10 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev)
return ret;
}
- vfio_res_list = RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
- /* Get vfio_res */
- TAILQ_FOREACH(vfio_res, vfio_res_list, next) {
- if (memcmp(&vfio_res->pci_addr, &dev->addr, sizeof(dev->addr)))
- continue;
- break;
- }
+ vfio_res_list =
+ RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
+ vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr);
+
/* if we haven't found our tailq entry, something's wrong */
if (vfio_res == NULL) {
RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n",
@@ -653,30 +689,56 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev)
return -1;
}
- /* unmap BARs */
- maps = vfio_res->maps;
+ TAILQ_REMOVE(vfio_res_list, vfio_res, next);
- RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n",
- pci_addr);
- for (i = 0; i < (int) vfio_res->nb_maps; i++) {
+ return 0;
+}
- /*
- * We do not need to be aware of MSI-X table BAR mappings as
- * when mapping. Just using current maps array is enough
- */
- if (maps[i].addr) {
- RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n",
- pci_addr, maps[i].addr);
- pci_unmap_resource(maps[i].addr, maps[i].size);
- }
+static int
+pci_vfio_unmap_resource_secondary(struct rte_pci_device *dev)
+{
+ char pci_addr[PATH_MAX] = {0};
+ struct rte_pci_addr *loc = &dev->addr;
+ struct mapped_pci_resource *vfio_res = NULL;
+ struct mapped_pci_res_list *vfio_res_list;
+ int ret;
+
+ /* store PCI address string */
+ snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
+ loc->domain, loc->bus, loc->devid, loc->function);
+
+ ret = rte_vfio_release_device(rte_pci_get_sysfs_path(), pci_addr,
+ dev->intr_handle.vfio_dev_fd);
+ if (ret < 0) {
+ RTE_LOG(ERR, EAL,
+ "%s(): cannot release device\n", __func__);
+ return ret;
}
- TAILQ_REMOVE(vfio_res_list, vfio_res, next);
+ vfio_res_list =
+ RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
+ vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr);
+
+ /* if we haven't found our tailq entry, something's wrong */
+ if (vfio_res == NULL) {
+ RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n",
+ pci_addr);
+ return -1;
+ }
return 0;
}
int
+pci_vfio_unmap_resource(struct rte_pci_device *dev)
+{
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ return pci_vfio_unmap_resource_primary(dev);
+ else
+ return pci_vfio_unmap_resource_secondary(dev);
+}
+
+int
pci_vfio_ioport_map(struct rte_pci_device *dev, int bar,
struct rte_pci_ioport *p)
{
diff --git a/drivers/bus/pci/meson.build b/drivers/bus/pci/meson.build
index 12756a4d..72939e59 100644
--- a/drivers/bus/pci/meson.build
+++ b/drivers/bus/pci/meson.build
@@ -14,3 +14,6 @@ else
sources += files('bsd/pci.c')
includes += include_directories('bsd')
endif
+
+# memseg walk is not part of stable API yet
+allow_experimental_apis = true
diff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c
index 2a00f365..7736b3f9 100644
--- a/drivers/bus/pci/pci_common.c
+++ b/drivers/bus/pci/pci_common.c
@@ -26,6 +26,7 @@
#include "private.h"
+
extern struct rte_pci_bus rte_pci_bus;
#define SYSFS_PCI_DEVICES "/sys/bus/pci/devices"
@@ -45,12 +46,8 @@ static struct rte_devargs *pci_devargs_lookup(struct rte_pci_device *dev)
{
struct rte_devargs *devargs;
struct rte_pci_addr addr;
- struct rte_bus *pbus;
- pbus = rte_bus_find_by_name("pci");
- TAILQ_FOREACH(devargs, &devargs_list, next) {
- if (devargs->bus != pbus)
- continue;
+ RTE_EAL_DEVARGS_FOREACH("pci", devargs) {
devargs->bus->parse(devargs->name, &addr);
if (!rte_pci_addr_cmp(&dev->addr, &addr))
return devargs;
@@ -73,7 +70,7 @@ pci_name_set(struct rte_pci_device *dev)
*/
if (devargs != NULL)
/* If an rte_devargs exists, the generic rte_device uses the
- * given name as its namea
+ * given name as its name.
*/
dev->device.name = dev->device.devargs->name;
else
@@ -159,17 +156,24 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
RTE_LOG(INFO, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id,
dev->id.device_id, dr->driver.name);
+ /*
+ * reference driver structure
+ * This needs to be before rte_pci_map_device(), as it enables to use
+ * driver flags for adjusting configuration.
+ */
+ dev->driver = dr;
+ dev->device.driver = &dr->driver;
+
if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
/* map resources for devices that use igb_uio */
ret = rte_pci_map_device(dev);
- if (ret != 0)
+ if (ret != 0) {
+ dev->driver = NULL;
+ dev->device.driver = NULL;
return ret;
+ }
}
- /* reference driver structure */
- dev->driver = dr;
- dev->device.driver = &dr->driver;
-
/* call the driver probe() function */
ret = dr->probe(dr, dev);
if (ret) {
@@ -259,81 +263,6 @@ pci_probe_all_drivers(struct rte_pci_device *dev)
}
/*
- * Find the pci device specified by pci address, then invoke probe function of
- * the driver of the device.
- */
-int
-rte_pci_probe_one(const struct rte_pci_addr *addr)
-{
- struct rte_pci_device *dev = NULL;
-
- int ret = 0;
-
- if (addr == NULL)
- return -1;
-
- /* update current pci device in global list, kernel bindings might have
- * changed since last time we looked at it.
- */
- if (pci_update_device(addr) < 0)
- goto err_return;
-
- FOREACH_DEVICE_ON_PCIBUS(dev) {
- if (rte_pci_addr_cmp(&dev->addr, addr))
- continue;
-
- ret = pci_probe_all_drivers(dev);
- if (ret)
- goto err_return;
- return 0;
- }
- return -1;
-
-err_return:
- RTE_LOG(WARNING, EAL,
- "Requested device " PCI_PRI_FMT " cannot be used\n",
- addr->domain, addr->bus, addr->devid, addr->function);
- return -1;
-}
-
-/*
- * Detach device specified by its pci address.
- */
-int
-rte_pci_detach(const struct rte_pci_addr *addr)
-{
- struct rte_pci_device *dev = NULL;
- int ret = 0;
-
- if (addr == NULL)
- return -1;
-
- FOREACH_DEVICE_ON_PCIBUS(dev) {
- if (rte_pci_addr_cmp(&dev->addr, addr))
- continue;
-
- ret = rte_pci_detach_dev(dev);
- if (ret < 0)
- /* negative value is an error */
- goto err_return;
- if (ret > 0)
- /* positive value means driver doesn't support it */
- continue;
-
- rte_pci_remove_device(dev);
- free(dev);
- return 0;
- }
- return -1;
-
-err_return:
- RTE_LOG(WARNING, EAL, "Requested device " PCI_PRI_FMT
- " cannot be used\n", dev->addr.domain, dev->addr.bus,
- dev->addr.devid, dev->addr.function);
- return -1;
-}
-
-/*
* Scan the content of the PCI bus, and call the probe() function for
* all registered drivers that have a matching entry in its id_table
* for discovered devices.
@@ -449,7 +378,7 @@ rte_pci_insert_device(struct rte_pci_device *exist_pci_dev,
}
/* Remove a device from PCI bus */
-void
+static void
rte_pci_remove_device(struct rte_pci_device *pci_dev)
{
TAILQ_REMOVE(&rte_pci_bus.device_list, pci_dev, next);
@@ -459,17 +388,20 @@ static struct rte_device *
pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
const void *data)
{
- struct rte_pci_device *dev;
+ const struct rte_pci_device *pstart;
+ struct rte_pci_device *pdev;
- FOREACH_DEVICE_ON_PCIBUS(dev) {
- if (start && &dev->device == start) {
- start = NULL; /* starting point found */
- continue;
- }
- if (cmp(&dev->device, data) == 0)
- return &dev->device;
+ if (start != NULL) {
+ pstart = RTE_DEV_TO_PCI_CONST(start);
+ pdev = TAILQ_NEXT(pstart, next);
+ } else {
+ pdev = TAILQ_FIRST(&rte_pci_bus.device_list);
+ }
+ while (pdev != NULL) {
+ if (cmp(&pdev->device, data) == 0)
+ return &pdev->device;
+ pdev = TAILQ_NEXT(pdev, next);
}
-
return NULL;
}
diff --git a/drivers/bus/pci/private.h b/drivers/bus/pci/private.h
index 88fa587e..8ddd03e1 100644
--- a/drivers/bus/pci/private.h
+++ b/drivers/bus/pci/private.h
@@ -33,36 +33,6 @@ rte_pci_probe(void);
int rte_pci_scan(void);
/**
- * Probe the single PCI device.
- *
- * Scan the content of the PCI bus, and find the pci device specified by pci
- * address, then call the probe() function for registered driver that has a
- * matching entry in its id_table for discovered device.
- *
- * @param addr
- * The PCI Bus-Device-Function address to probe.
- * @return
- * - 0 on success.
- * - Negative on error.
- */
-int rte_pci_probe_one(const struct rte_pci_addr *addr);
-
-/**
- * Close the single PCI device.
- *
- * Scan the content of the PCI bus, and find the pci device specified by pci
- * address, then call the remove() function for registered driver that has a
- * matching entry in its id_table for discovered device.
- *
- * @param addr
- * The PCI Bus-Device-Function address to close.
- * @return
- * - 0 on success.
- * - Negative on error.
- */
-int rte_pci_detach(const struct rte_pci_addr *addr);
-
-/**
* Find the name of a PCI device.
*/
void
@@ -94,16 +64,6 @@ void rte_pci_insert_device(struct rte_pci_device *exist_pci_dev,
struct rte_pci_device *new_pci_dev);
/**
- * Remove a PCI device from the PCI Bus. This sets to NULL the bus references
- * in the PCI device object as well as the generic device object.
- *
- * @param pci_device
- * PCI device to be removed from PCI Bus
- * @return void
- */
-void rte_pci_remove_device(struct rte_pci_device *pci_device);
-
-/**
* Update a pci device object by asking the kernel for the latest information.
*
* This function is private to EAL.
@@ -117,16 +77,6 @@ void rte_pci_remove_device(struct rte_pci_device *pci_device);
int pci_update_device(const struct rte_pci_addr *addr);
/**
- * Unbind kernel driver for this device
- *
- * This function is private to EAL.
- *
- * @return
- * 0 on success, negative on error
- */
-int pci_unbind_kernel_driver(struct rte_pci_device *dev);
-
-/**
* Map the PCI resource of a PCI device in virtual memory
*
* This function is private to EAL.
diff --git a/drivers/bus/pci/rte_bus_pci.h b/drivers/bus/pci/rte_bus_pci.h
index 357afb91..0d1955ff 100644
--- a/drivers/bus/pci/rte_bus_pci.h
+++ b/drivers/bus/pci/rte_bus_pci.h
@@ -74,6 +74,9 @@ struct rte_pci_device {
*/
#define RTE_DEV_TO_PCI(ptr) container_of(ptr, struct rte_pci_device, device)
+#define RTE_DEV_TO_PCI_CONST(ptr) \
+ container_of(ptr, const struct rte_pci_device, device)
+
#define RTE_ETH_DEV_TO_PCI(eth_dev) RTE_DEV_TO_PCI((eth_dev)->device)
/** Any PCI device identifier (vendor, device, ...) */
@@ -132,6 +135,8 @@ struct rte_pci_bus {
/** Device needs PCI BAR mapping (done with either IGB_UIO or VFIO) */
#define RTE_PCI_DRV_NEED_MAPPING 0x0001
+/** Device needs PCI BAR mapping with enabled write combining (wc) */
+#define RTE_PCI_DRV_WC_ACTIVATE 0x0002
/** Device driver supports link state interrupt */
#define RTE_PCI_DRV_INTR_LSC 0x0008
/** Device driver supports device removal interrupt */
@@ -186,8 +191,7 @@ void rte_pci_register(struct rte_pci_driver *driver);
/** Helper for PCI device registration from driver (eth, crypto) instance */
#define RTE_PMD_REGISTER_PCI(nm, pci_drv) \
-RTE_INIT(pciinitfn_ ##nm); \
-static void pciinitfn_ ##nm(void) \
+RTE_INIT(pciinitfn_ ##nm) \
{\
(pci_drv).driver.name = RTE_STR(nm);\
rte_pci_register(&pci_drv); \
diff --git a/drivers/bus/vdev/Makefile b/drivers/bus/vdev/Makefile
index 24d424a3..bd0bb895 100644
--- a/drivers/bus/vdev/Makefile
+++ b/drivers/bus/vdev/Makefile
@@ -10,6 +10,7 @@ LIB = librte_bus_vdev.a
CFLAGS += -O3
CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
# versioning export map
EXPORT_MAP := rte_bus_vdev_version.map
diff --git a/drivers/bus/vdev/meson.build b/drivers/bus/vdev/meson.build
index 03cb87eb..2ee648b4 100644
--- a/drivers/bus/vdev/meson.build
+++ b/drivers/bus/vdev/meson.build
@@ -3,3 +3,5 @@
sources = files('vdev.c')
install_headers('rte_bus_vdev.h')
+
+allow_experimental_apis = true
diff --git a/drivers/bus/vdev/rte_bus_vdev.h b/drivers/bus/vdev/rte_bus_vdev.h
index f9d8a238..9ae3eaae 100644
--- a/drivers/bus/vdev/rte_bus_vdev.h
+++ b/drivers/bus/vdev/rte_bus_vdev.h
@@ -1,33 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2016 RehiveTech. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of RehiveTech nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016 RehiveTech. All rights reserved.
*/
#ifndef RTE_VDEV_H
@@ -53,6 +25,9 @@ struct rte_vdev_device {
#define RTE_DEV_TO_VDEV(ptr) \
container_of(ptr, struct rte_vdev_device, device)
+#define RTE_DEV_TO_VDEV_CONST(ptr) \
+ container_of(ptr, const struct rte_vdev_device, device)
+
static inline const char *
rte_vdev_device_name(const struct rte_vdev_device *dev)
{
@@ -111,9 +86,8 @@ void rte_vdev_register(struct rte_vdev_driver *driver);
void rte_vdev_unregister(struct rte_vdev_driver *driver);
#define RTE_PMD_REGISTER_VDEV(nm, vdrv)\
-RTE_INIT(vdrvinitfn_ ##vdrv);\
static const char *vdrvinit_ ## nm ## _alias;\
-static void vdrvinitfn_ ##vdrv(void)\
+RTE_INIT(vdrvinitfn_ ##vdrv)\
{\
(vdrv).driver.name = RTE_STR(nm);\
(vdrv).driver.alias = vdrvinit_ ## nm ## _alias;\
diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c
index e4bc7246..6139dd55 100644
--- a/drivers/bus/vdev/vdev.c
+++ b/drivers/bus/vdev/vdev.c
@@ -1,33 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2016 RehiveTech. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of RehiveTech nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016 RehiveTech. All rights reserved.
*/
#include <string.h>
@@ -46,11 +18,14 @@
#include <rte_memory.h>
#include <rte_tailq.h>
#include <rte_spinlock.h>
+#include <rte_string_fns.h>
#include <rte_errno.h>
#include "rte_bus_vdev.h"
#include "vdev_logs.h"
+#define VDEV_MP_KEY "bus_vdev_mp"
+
int vdev_logtype_bus;
/* Forward declare to access virtual bus name */
@@ -61,6 +36,10 @@ TAILQ_HEAD(vdev_device_list, rte_vdev_device);
static struct vdev_device_list vdev_device_list =
TAILQ_HEAD_INITIALIZER(vdev_device_list);
+/* The lock needs to be recursive because a vdev can manage another vdev. */
+static rte_spinlock_recursive_t vdev_device_list_lock =
+ RTE_SPINLOCK_RECURSIVE_INITIALIZER;
+
struct vdev_driver_list vdev_driver_list =
TAILQ_HEAD_INITIALIZER(vdev_driver_list);
@@ -165,7 +144,7 @@ vdev_probe_all_drivers(struct rte_vdev_device *dev)
name = rte_vdev_device_name(dev);
- VDEV_LOG(DEBUG, "Search driver %s to probe device %s\n", name,
+ VDEV_LOG(DEBUG, "Search driver %s to probe device %s", name,
rte_vdev_device_name(dev));
if (vdev_parse(name, &driver))
@@ -177,6 +156,7 @@ vdev_probe_all_drivers(struct rte_vdev_device *dev)
return ret;
}
+/* The caller shall be responsible for thread-safe */
static struct rte_vdev_device *
find_vdev(const char *name)
{
@@ -188,7 +168,7 @@ find_vdev(const char *name)
TAILQ_FOREACH(dev, &vdev_device_list, next) {
const char *devname = rte_vdev_device_name(dev);
- if (!strncmp(devname, name, strlen(name)))
+ if (!strcmp(devname, name))
return dev;
}
@@ -221,8 +201,8 @@ alloc_devargs(const char *name, const char *args)
return devargs;
}
-int
-rte_vdev_init(const char *name, const char *args)
+static int
+insert_vdev(const char *name, const char *args, struct rte_vdev_device **p_dev)
{
struct rte_vdev_device *dev;
struct rte_devargs *devargs;
@@ -231,10 +211,6 @@ rte_vdev_init(const char *name, const char *args)
if (name == NULL)
return -EINVAL;
- dev = find_vdev(name);
- if (dev)
- return -EEXIST;
-
devargs = alloc_devargs(name, args);
if (!devargs)
return -ENOMEM;
@@ -249,18 +225,18 @@ rte_vdev_init(const char *name, const char *args)
dev->device.numa_node = SOCKET_ID_ANY;
dev->device.name = devargs->name;
- ret = vdev_probe_all_drivers(dev);
- if (ret) {
- if (ret > 0)
- VDEV_LOG(ERR, "no driver found for %s\n", name);
+ if (find_vdev(name)) {
+ ret = -EEXIST;
goto fail;
}
- TAILQ_INSERT_TAIL(&devargs_list, devargs, next);
-
TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
- return 0;
+ rte_devargs_insert(devargs);
+
+ if (p_dev)
+ *p_dev = dev;
+ return 0;
fail:
free(devargs->args);
free(devargs);
@@ -268,6 +244,31 @@ fail:
return ret;
}
+int
+rte_vdev_init(const char *name, const char *args)
+{
+ struct rte_vdev_device *dev;
+ struct rte_devargs *devargs;
+ int ret;
+
+ rte_spinlock_recursive_lock(&vdev_device_list_lock);
+ ret = insert_vdev(name, args, &dev);
+ if (ret == 0) {
+ ret = vdev_probe_all_drivers(dev);
+ if (ret) {
+ if (ret > 0)
+ VDEV_LOG(ERR, "no driver found for %s", name);
+ /* If fails, remove it from vdev list */
+ devargs = dev->device.devargs;
+ TAILQ_REMOVE(&vdev_device_list, dev, next);
+ rte_devargs_remove(devargs->bus->name, devargs->name);
+ free(dev);
+ }
+ }
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
+ return ret;
+}
+
static int
vdev_remove_driver(struct rte_vdev_device *dev)
{
@@ -275,7 +276,7 @@ vdev_remove_driver(struct rte_vdev_device *dev)
const struct rte_vdev_driver *driver;
if (!dev->device.driver) {
- VDEV_LOG(DEBUG, "no driver attach to device %s\n", name);
+ VDEV_LOG(DEBUG, "no driver attach to device %s", name);
return 1;
}
@@ -294,23 +295,98 @@ rte_vdev_uninit(const char *name)
if (name == NULL)
return -EINVAL;
- dev = find_vdev(name);
- if (!dev)
- return -ENOENT;
+ rte_spinlock_recursive_lock(&vdev_device_list_lock);
- devargs = dev->device.devargs;
+ dev = find_vdev(name);
+ if (!dev) {
+ ret = -ENOENT;
+ goto unlock;
+ }
ret = vdev_remove_driver(dev);
if (ret)
- return ret;
+ goto unlock;
TAILQ_REMOVE(&vdev_device_list, dev, next);
+ devargs = dev->device.devargs;
+ rte_devargs_remove(devargs->bus->name, devargs->name);
+ free(dev);
- TAILQ_REMOVE(&devargs_list, devargs, next);
+unlock:
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
+ return ret;
+}
+
+struct vdev_param {
+#define VDEV_SCAN_REQ 1
+#define VDEV_SCAN_ONE 2
+#define VDEV_SCAN_REP 3
+ int type;
+ int num;
+ char name[RTE_DEV_NAME_MAX_LEN];
+};
+
+static int vdev_plug(struct rte_device *dev);
+
+/**
+ * This function works as the action for both primary and secondary process
+ * for static vdev discovery when a secondary process is booting.
+ *
+ * step 1, secondary process sends a sync request to ask for vdev in primary;
+ * step 2, primary process receives the request, and send vdevs one by one;
+ * step 3, primary process sends back reply, which indicates how many vdevs
+ * are sent.
+ */
+static int
+vdev_action(const struct rte_mp_msg *mp_msg, const void *peer)
+{
+ struct rte_vdev_device *dev;
+ struct rte_mp_msg mp_resp;
+ struct vdev_param *ou = (struct vdev_param *)&mp_resp.param;
+ const struct vdev_param *in = (const struct vdev_param *)mp_msg->param;
+ const char *devname;
+ int num;
+
+ strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name));
+ mp_resp.len_param = sizeof(*ou);
+ mp_resp.num_fds = 0;
+
+ switch (in->type) {
+ case VDEV_SCAN_REQ:
+ ou->type = VDEV_SCAN_ONE;
+ ou->num = 1;
+ num = 0;
+
+ rte_spinlock_recursive_lock(&vdev_device_list_lock);
+ TAILQ_FOREACH(dev, &vdev_device_list, next) {
+ devname = rte_vdev_device_name(dev);
+ if (strlen(devname) == 0) {
+ VDEV_LOG(INFO, "vdev with no name is not sent");
+ continue;
+ }
+ VDEV_LOG(INFO, "send vdev, %s", devname);
+ strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN);
+ if (rte_mp_sendmsg(&mp_resp) < 0)
+ VDEV_LOG(ERR, "send vdev, %s, failed, %s",
+ devname, strerror(rte_errno));
+ num++;
+ }
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
+
+ ou->type = VDEV_SCAN_REP;
+ ou->num = num;
+ if (rte_mp_reply(&mp_resp, peer) < 0)
+ VDEV_LOG(ERR, "Failed to reply a scan request");
+ break;
+ case VDEV_SCAN_ONE:
+ VDEV_LOG(INFO, "receive vdev, %s", in->name);
+ if (insert_vdev(in->name, NULL, NULL) < 0)
+ VDEV_LOG(ERR, "failed to add vdev, %s", in->name);
+ break;
+ default:
+ VDEV_LOG(ERR, "vdev cannot recognize this message");
+ }
- free(devargs->args);
- free(devargs);
- free(dev);
return 0;
}
@@ -321,13 +397,41 @@ vdev_scan(void)
struct rte_devargs *devargs;
struct vdev_custom_scan *custom_scan;
+ if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 &&
+ rte_errno != EEXIST) {
+ VDEV_LOG(ERR, "Failed to add vdev mp action");
+ return -1;
+ }
+
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+ struct rte_mp_msg mp_req, *mp_rep;
+ struct rte_mp_reply mp_reply;
+ struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
+ struct vdev_param *req = (struct vdev_param *)mp_req.param;
+ struct vdev_param *resp;
+
+ strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name));
+ mp_req.len_param = sizeof(*req);
+ mp_req.num_fds = 0;
+ req->type = VDEV_SCAN_REQ;
+ if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
+ mp_reply.nb_received == 1) {
+ mp_rep = &mp_reply.msgs[0];
+ resp = (struct vdev_param *)mp_rep->param;
+ VDEV_LOG(INFO, "Received %d vdevs", resp->num);
+ } else
+ VDEV_LOG(ERR, "Failed to request vdev from primary");
+
+ /* Fall through to allow private vdevs in secondary process */
+ }
+
/* call custom scan callbacks if any */
rte_spinlock_lock(&vdev_custom_scan_lock);
TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
if (custom_scan->callback != NULL)
/*
* the callback should update devargs list
- * by calling rte_eal_devargs_insert() with
+ * by calling rte_devargs_insert() with
* devargs.bus = rte_bus_find_by_name("vdev");
* devargs.type = RTE_DEVTYPE_VIRTUAL;
* devargs.policy = RTE_DEV_WHITELISTED;
@@ -337,24 +441,27 @@ vdev_scan(void)
rte_spinlock_unlock(&vdev_custom_scan_lock);
/* for virtual devices we scan the devargs_list populated via cmdline */
- TAILQ_FOREACH(devargs, &devargs_list, next) {
-
- if (devargs->bus != &rte_vdev_bus)
- continue;
-
- dev = find_vdev(devargs->name);
- if (dev)
- continue;
+ RTE_EAL_DEVARGS_FOREACH("vdev", devargs) {
dev = calloc(1, sizeof(*dev));
if (!dev)
return -1;
+ rte_spinlock_recursive_lock(&vdev_device_list_lock);
+
+ if (find_vdev(devargs->name)) {
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
+ free(dev);
+ continue;
+ }
+
dev->device.devargs = devargs;
dev->device.numa_node = SOCKET_ID_ANY;
dev->device.name = devargs->name;
TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
+
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
}
return 0;
@@ -368,12 +475,16 @@ vdev_probe(void)
/* call the init function for each virtual device */
TAILQ_FOREACH(dev, &vdev_device_list, next) {
+ /* we don't use the vdev lock here, as it's only used in DPDK
+ * initialization; and we don't want to hold such a lock when
+ * we call each driver probe.
+ */
if (dev->device.driver)
continue;
if (vdev_probe_all_drivers(dev)) {
- VDEV_LOG(ERR, "failed to initialize %s device\n",
+ VDEV_LOG(ERR, "failed to initialize %s device",
rte_vdev_device_name(dev));
ret = -1;
}
@@ -386,17 +497,24 @@ static struct rte_device *
vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
const void *data)
{
+ const struct rte_vdev_device *vstart;
struct rte_vdev_device *dev;
- TAILQ_FOREACH(dev, &vdev_device_list, next) {
- if (start && &dev->device == start) {
- start = NULL;
- continue;
- }
+ rte_spinlock_recursive_lock(&vdev_device_list_lock);
+ if (start != NULL) {
+ vstart = RTE_DEV_TO_VDEV_CONST(start);
+ dev = TAILQ_NEXT(vstart, next);
+ } else {
+ dev = TAILQ_FIRST(&vdev_device_list);
+ }
+ while (dev != NULL) {
if (cmp(&dev->device, data) == 0)
- return &dev->device;
+ break;
+ dev = TAILQ_NEXT(dev, next);
}
- return NULL;
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
+
+ return dev ? &dev->device : NULL;
}
static int
diff --git a/drivers/bus/vmbus/Makefile b/drivers/bus/vmbus/Makefile
new file mode 100644
index 00000000..deee9dd1
--- /dev/null
+++ b/drivers/bus/vmbus/Makefile
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+LIB = librte_bus_vmbus.a
+LIBABIVER := 1
+EXPORT_MAP := rte_bus_vmbus_version.map
+
+CFLAGS += -I$(SRCDIR)
+CFLAGS += -O3 $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),)
+SYSTEM := linux
+endif
+ifneq ($(CONFIG_RTE_EXEC_ENV_BSDAPP),)
+$(error "VMBUS not implemented for BSD yet")
+endif
+
+CFLAGS += -I$(RTE_SDK)/drivers/bus/vmbus/$(SYSTEM)
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_ethdev
+
+include $(RTE_SDK)/drivers/bus/vmbus/$(SYSTEM)/Makefile
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) := $(addprefix $(SYSTEM)/,$(SRCS))
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_common.c
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_channel.c vmbus_bufring.c
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_common_uio.c
+
+SYMLINK-$(CONFIG_RTE_LIBRTE_VMBUS)-include += rte_bus_vmbus.h
+SYMLINK-$(CONFIG_RTE_LIBRTE_VMBUS)-include += rte_vmbus_reg.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/bus/vmbus/linux/Makefile b/drivers/bus/vmbus/linux/Makefile
new file mode 100644
index 00000000..ef0d30b2
--- /dev/null
+++ b/drivers/bus/vmbus/linux/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+SRCS += vmbus_bus.c vmbus_uio.c
diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c
new file mode 100644
index 00000000..52d6a3c0
--- /dev/null
+++ b/drivers/bus/vmbus/linux/vmbus_bus.c
@@ -0,0 +1,355 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <string.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include <rte_eal.h>
+#include <rte_uuid.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_devargs.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+#include <rte_bus_vmbus.h>
+
+#include "eal_filesystem.h"
+#include "private.h"
+
+/** Pathname of VMBUS devices directory. */
+#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices"
+
+extern struct rte_vmbus_bus rte_vmbus_bus;
+
+/* Read sysfs file to get UUID */
+static int
+parse_sysfs_uuid(const char *filename, rte_uuid_t uu)
+{
+ char buf[BUFSIZ];
+ char *cp, *in = buf;
+ FILE *f;
+
+ f = fopen(filename, "r");
+ if (f == NULL) {
+ VMBUS_LOG(ERR, "cannot open sysfs value %s: %s",
+ filename, strerror(errno));
+ return -1;
+ }
+
+ if (fgets(buf, sizeof(buf), f) == NULL) {
+ VMBUS_LOG(ERR, "cannot read sysfs value %s",
+ filename);
+ fclose(f);
+ return -1;
+ }
+ fclose(f);
+
+ cp = strchr(buf, '\n');
+ if (cp)
+ *cp = '\0';
+
+ /* strip { } notation */
+ if (buf[0] == '{') {
+ in = buf + 1;
+ cp = strchr(in, '}');
+ if (cp)
+ *cp = '\0';
+ }
+
+ if (rte_uuid_parse(in, uu) < 0) {
+ VMBUS_LOG(ERR, "%s %s not a valid UUID",
+ filename, buf);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+get_sysfs_string(const char *filename, char *buf, size_t buflen)
+{
+ char *cp;
+ FILE *f;
+
+ f = fopen(filename, "r");
+ if (f == NULL) {
+ VMBUS_LOG(ERR, "cannot open sysfs value %s:%s",
+ filename, strerror(errno));
+ return -1;
+ }
+
+ if (fgets(buf, buflen, f) == NULL) {
+ VMBUS_LOG(ERR, "cannot read sysfs value %s",
+ filename);
+ fclose(f);
+ return -1;
+ }
+ fclose(f);
+
+ /* remove trailing newline */
+ cp = memchr(buf, '\n', buflen);
+ if (cp)
+ *cp = '\0';
+
+ return 0;
+}
+
+static int
+vmbus_get_uio_dev(const struct rte_vmbus_device *dev,
+ char *dstbuf, size_t buflen)
+{
+ char dirname[PATH_MAX];
+ unsigned int uio_num;
+ struct dirent *e;
+ DIR *dir;
+
+ /* Assume recent kernel where uio is in uio/uioX */
+ snprintf(dirname, sizeof(dirname),
+ SYSFS_VMBUS_DEVICES "/%s/uio", dev->device.name);
+
+ dir = opendir(dirname);
+ if (dir == NULL)
+ return -1; /* Not a UIO device */
+
+ /* take the first file starting with "uio" */
+ while ((e = readdir(dir)) != NULL) {
+ const int prefix_len = 3;
+ char *endptr;
+
+ if (strncmp(e->d_name, "uio", prefix_len) != 0)
+ continue;
+
+ /* try uio%d */
+ errno = 0;
+ uio_num = strtoull(e->d_name + prefix_len, &endptr, 10);
+ if (errno == 0 && endptr != (e->d_name + prefix_len)) {
+ snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num);
+ break;
+ }
+ }
+ closedir(dir);
+
+ if (e == NULL)
+ return -1;
+
+ return uio_num;
+}
+
+/* Check map names with kernel names */
+static const char *map_names[VMBUS_MAX_RESOURCE] = {
+ [HV_TXRX_RING_MAP] = "txrx_rings",
+ [HV_INT_PAGE_MAP] = "int_page",
+ [HV_MON_PAGE_MAP] = "monitor_page",
+ [HV_RECV_BUF_MAP] = "recv:",
+ [HV_SEND_BUF_MAP] = "send:",
+};
+
+
+/* map the resources of a vmbus device in virtual memory */
+int
+rte_vmbus_map_device(struct rte_vmbus_device *dev)
+{
+ char uioname[PATH_MAX], filename[PATH_MAX];
+ char dirname[PATH_MAX], mapname[64];
+ int i;
+
+ dev->uio_num = vmbus_get_uio_dev(dev, uioname, sizeof(uioname));
+ if (dev->uio_num < 0) {
+ VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped");
+ return 1;
+ }
+
+ /* Extract resource value */
+ for (i = 0; i < VMBUS_MAX_RESOURCE; i++) {
+ struct rte_mem_resource *res = &dev->resource[i];
+ unsigned long len, gpad = 0;
+ char *cp;
+
+ snprintf(dirname, sizeof(dirname),
+ "%s/maps/map%d", uioname, i);
+
+ snprintf(filename, sizeof(filename),
+ "%s/name", dirname);
+
+ if (get_sysfs_string(filename, mapname, sizeof(mapname)) < 0) {
+ VMBUS_LOG(ERR, "could not read %s", filename);
+ return -1;
+ }
+
+ if (strncmp(map_names[i], mapname, strlen(map_names[i])) != 0) {
+ VMBUS_LOG(ERR,
+ "unexpected resource %s (expected %s)",
+ mapname, map_names[i]);
+ return -1;
+ }
+
+ snprintf(filename, sizeof(filename),
+ "%s/size", dirname);
+ if (eal_parse_sysfs_value(filename, &len) < 0) {
+ VMBUS_LOG(ERR,
+ "could not read %s", filename);
+ return -1;
+ }
+ res->len = len;
+
+ /* both send and receive buffers have gpad in name */
+ cp = memchr(mapname, ':', sizeof(mapname));
+ if (cp)
+ gpad = strtoul(cp+1, NULL, 0);
+
+ /* put the GPAD value in physical address */
+ res->phys_addr = gpad;
+ }
+
+ return vmbus_uio_map_resource(dev);
+}
+
+void
+rte_vmbus_unmap_device(struct rte_vmbus_device *dev)
+{
+ vmbus_uio_unmap_resource(dev);
+}
+
+/* Scan one vmbus sysfs entry, and fill the devices list from it. */
+static int
+vmbus_scan_one(const char *name)
+{
+ struct rte_vmbus_device *dev, *dev2;
+ char filename[PATH_MAX];
+ char dirname[PATH_MAX];
+ unsigned long tmp;
+
+ dev = calloc(1, sizeof(*dev));
+ if (dev == NULL)
+ return -1;
+
+ dev->device.name = strdup(name);
+ if (!dev->device.name)
+ goto error;
+
+ /* sysfs base directory
+ * /sys/bus/vmbus/devices/7a08391f-f5a0-4ac0-9802-d13fd964f8df
+ * or on older kernel
+ * /sys/bus/vmbus/devices/vmbus_1
+ */
+ snprintf(dirname, sizeof(dirname), "%s/%s",
+ SYSFS_VMBUS_DEVICES, name);
+
+ /* get device id */
+ snprintf(filename, sizeof(filename), "%s/device_id", dirname);
+ if (parse_sysfs_uuid(filename, dev->device_id) < 0)
+ goto error;
+
+ /* get device class */
+ snprintf(filename, sizeof(filename), "%s/class_id", dirname);
+ if (parse_sysfs_uuid(filename, dev->class_id) < 0)
+ goto error;
+
+ /* get relid */
+ snprintf(filename, sizeof(filename), "%s/id", dirname);
+ if (eal_parse_sysfs_value(filename, &tmp) < 0)
+ goto error;
+ dev->relid = tmp;
+
+ /* get monitor id */
+ snprintf(filename, sizeof(filename), "%s/monitor_id", dirname);
+ if (eal_parse_sysfs_value(filename, &tmp) < 0)
+ goto error;
+ dev->monitor_id = tmp;
+
+ /* get numa node (if present) */
+ snprintf(filename, sizeof(filename), "%s/numa_node",
+ dirname);
+
+ if (access(filename, R_OK) == 0) {
+ if (eal_parse_sysfs_value(filename, &tmp) < 0)
+ goto error;
+ dev->device.numa_node = tmp;
+ } else {
+ /* if no NUMA support, set default to 0 */
+ dev->device.numa_node = SOCKET_ID_ANY;
+ }
+
+ /* device is valid, add in list (sorted) */
+ VMBUS_LOG(DEBUG, "Adding vmbus device %s", name);
+
+ TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) {
+ int ret;
+
+ ret = rte_uuid_compare(dev->device_id, dev2->device_id);
+ if (ret > 0)
+ continue;
+
+ if (ret < 0) {
+ vmbus_insert_device(dev2, dev);
+ } else { /* already registered */
+ VMBUS_LOG(NOTICE,
+ "%s already registered", name);
+ free(dev);
+ }
+ return 0;
+ }
+
+ vmbus_add_device(dev);
+ return 0;
+error:
+ VMBUS_LOG(DEBUG, "failed");
+
+ free(dev);
+ return -1;
+}
+
+/*
+ * Scan the content of the vmbus, and the devices in the devices list
+ */
+int
+rte_vmbus_scan(void)
+{
+ struct dirent *e;
+ DIR *dir;
+
+ dir = opendir(SYSFS_VMBUS_DEVICES);
+ if (dir == NULL) {
+ if (errno == ENOENT)
+ return 0;
+
+ VMBUS_LOG(ERR, "opendir %s failed: %s",
+ SYSFS_VMBUS_DEVICES, strerror(errno));
+ return -1;
+ }
+
+ while ((e = readdir(dir)) != NULL) {
+ if (e->d_name[0] == '.')
+ continue;
+
+ if (vmbus_scan_one(e->d_name) < 0)
+ goto error;
+ }
+ closedir(dir);
+ return 0;
+
+error:
+ closedir(dir);
+ return -1;
+}
+
+void rte_vmbus_irq_mask(struct rte_vmbus_device *device)
+{
+ vmbus_uio_irq_control(device, 1);
+}
+
+void rte_vmbus_irq_unmask(struct rte_vmbus_device *device)
+{
+ vmbus_uio_irq_control(device, 0);
+}
+
+int rte_vmbus_irq_read(struct rte_vmbus_device *device)
+{
+ return vmbus_uio_irq_read(device);
+}
diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c
new file mode 100644
index 00000000..856c6d66
--- /dev/null
+++ b/drivers/bus/vmbus/linux/vmbus_uio.c
@@ -0,0 +1,398 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_memory.h>
+#include <rte_eal_memconfig.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_bus_vmbus.h>
+#include <rte_string_fns.h>
+
+#include "private.h"
+
+/** Pathname of VMBUS devices directory. */
+#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices"
+
+static void *vmbus_map_addr;
+
+/* Control interrupts */
+void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff)
+{
+ if (write(dev->intr_handle.fd, &onoff, sizeof(onoff)) < 0) {
+ VMBUS_LOG(ERR, "cannot write to %d:%s",
+ dev->intr_handle.fd, strerror(errno));
+ }
+}
+
+int vmbus_uio_irq_read(struct rte_vmbus_device *dev)
+{
+ int32_t count;
+ int cc;
+
+ cc = read(dev->intr_handle.fd, &count, sizeof(count));
+ if (cc < (int)sizeof(count)) {
+ if (cc < 0) {
+ VMBUS_LOG(ERR, "IRQ read failed %s",
+ strerror(errno));
+ return -errno;
+ }
+ VMBUS_LOG(ERR, "can't read IRQ count");
+ return -EINVAL;
+ }
+
+ return count;
+}
+
+void
+vmbus_uio_free_resource(struct rte_vmbus_device *dev,
+ struct mapped_vmbus_resource *uio_res)
+{
+ rte_free(uio_res);
+
+ if (dev->intr_handle.uio_cfg_fd >= 0) {
+ close(dev->intr_handle.uio_cfg_fd);
+ dev->intr_handle.uio_cfg_fd = -1;
+ }
+
+ if (dev->intr_handle.fd >= 0) {
+ close(dev->intr_handle.fd);
+ dev->intr_handle.fd = -1;
+ dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+ }
+}
+
+int
+vmbus_uio_alloc_resource(struct rte_vmbus_device *dev,
+ struct mapped_vmbus_resource **uio_res)
+{
+ char devname[PATH_MAX]; /* contains the /dev/uioX */
+
+ /* save fd if in primary process */
+ snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num);
+ dev->intr_handle.fd = open(devname, O_RDWR);
+ if (dev->intr_handle.fd < 0) {
+ VMBUS_LOG(ERR, "Cannot open %s: %s",
+ devname, strerror(errno));
+ goto error;
+ }
+ dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX;
+
+ /* allocate the mapping details for secondary processes*/
+ *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0);
+ if (*uio_res == NULL) {
+ VMBUS_LOG(ERR, "cannot store uio mmap details");
+ goto error;
+ }
+
+ strlcpy((*uio_res)->path, devname, PATH_MAX);
+ rte_uuid_copy((*uio_res)->id, dev->device_id);
+
+ return 0;
+
+error:
+ vmbus_uio_free_resource(dev, *uio_res);
+ return -1;
+}
+
+static int
+find_max_end_va(const struct rte_memseg_list *msl, void *arg)
+{
+ size_t sz = msl->memseg_arr.len * msl->page_sz;
+ void *end_va = RTE_PTR_ADD(msl->base_va, sz);
+ void **max_va = arg;
+
+ if (*max_va < end_va)
+ *max_va = end_va;
+ return 0;
+}
+
+/*
+ * TODO: this should be part of memseg api.
+ * code is duplicated from PCI.
+ */
+static void *
+vmbus_find_max_end_va(void)
+{
+ void *va = NULL;
+
+ rte_memseg_list_walk(find_max_end_va, &va);
+ return va;
+}
+
+int
+vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx,
+ struct mapped_vmbus_resource *uio_res,
+ int flags)
+{
+ size_t size = dev->resource[idx].len;
+ struct vmbus_map *maps = uio_res->maps;
+ void *mapaddr;
+ off_t offset;
+ int fd;
+
+ /* devname for mmap */
+ fd = open(uio_res->path, O_RDWR);
+ if (fd < 0) {
+ VMBUS_LOG(ERR, "Cannot open %s: %s",
+ uio_res->path, strerror(errno));
+ return -1;
+ }
+
+ /* try mapping somewhere close to the end of hugepages */
+ if (vmbus_map_addr == NULL)
+ vmbus_map_addr = vmbus_find_max_end_va();
+
+ /* offset is special in uio it indicates which resource */
+ offset = idx * PAGE_SIZE;
+
+ mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags);
+ close(fd);
+
+ if (mapaddr == MAP_FAILED)
+ return -1;
+
+ dev->resource[idx].addr = mapaddr;
+ vmbus_map_addr = RTE_PTR_ADD(mapaddr, size);
+
+ /* Record result of sucessful mapping for use by secondary */
+ maps[idx].addr = mapaddr;
+ maps[idx].size = size;
+
+ return 0;
+}
+
+static int vmbus_uio_map_primary(struct vmbus_channel *chan,
+ void **ring_buf, uint32_t *ring_size)
+{
+ struct mapped_vmbus_resource *uio_res;
+
+ uio_res = vmbus_uio_find_resource(chan->device);
+ if (!uio_res) {
+ VMBUS_LOG(ERR, "can not find resources!");
+ return -ENOMEM;
+ }
+
+ if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) {
+ VMBUS_LOG(ERR, "VMBUS: only %u resources found!",
+ uio_res->nb_maps);
+ return -EINVAL;
+ }
+
+ *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2;
+ *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr;
+ return 0;
+}
+
+static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev,
+ const struct vmbus_channel *chan,
+ void **ring_buf, uint32_t *ring_size)
+{
+ char ring_path[PATH_MAX];
+ size_t file_size;
+ struct stat sb;
+ int fd;
+
+ snprintf(ring_path, sizeof(ring_path),
+ "%s/%s/channels/%u/ring",
+ SYSFS_VMBUS_DEVICES, dev->device.name,
+ chan->relid);
+
+ fd = open(ring_path, O_RDWR);
+ if (fd < 0) {
+ VMBUS_LOG(ERR, "Cannot open %s: %s",
+ ring_path, strerror(errno));
+ return -errno;
+ }
+
+ if (fstat(fd, &sb) < 0) {
+ VMBUS_LOG(ERR, "Cannot state %s: %s",
+ ring_path, strerror(errno));
+ close(fd);
+ return -errno;
+ }
+ file_size = sb.st_size;
+
+ if (file_size == 0 || (file_size & (PAGE_SIZE - 1))) {
+ VMBUS_LOG(ERR, "incorrect size %s: %zu",
+ ring_path, file_size);
+
+ close(fd);
+ return -EINVAL;
+ }
+
+ *ring_size = file_size / 2;
+ *ring_buf = vmbus_map_resource(vmbus_map_addr, fd,
+ 0, sb.st_size, 0);
+ close(fd);
+
+ if (ring_buf == MAP_FAILED)
+ return -EIO;
+
+ vmbus_map_addr = RTE_PTR_ADD(ring_buf, file_size);
+ return 0;
+}
+
+int vmbus_uio_map_rings(struct vmbus_channel *chan)
+{
+ const struct rte_vmbus_device *dev = chan->device;
+ uint32_t ring_size;
+ void *ring_buf;
+ int ret;
+
+ /* Primary channel */
+ if (chan->subchannel_id == 0)
+ ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size);
+ else
+ ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size);
+
+ if (ret)
+ return ret;
+
+ vmbus_br_setup(&chan->txbr, ring_buf, ring_size);
+ vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size);
+ return 0;
+}
+
+static int vmbus_uio_sysfs_read(const char *dir, const char *name,
+ unsigned long *val, unsigned long max_range)
+{
+ char path[PATH_MAX];
+ FILE *f;
+ int ret;
+
+ snprintf(path, sizeof(path), "%s/%s", dir, name);
+ f = fopen(path, "r");
+ if (!f) {
+ VMBUS_LOG(ERR, "can't open %s:%s",
+ path, strerror(errno));
+ return -errno;
+ }
+
+ if (fscanf(f, "%lu", val) != 1)
+ ret = -EIO;
+ else if (*val > max_range)
+ ret = -ERANGE;
+ else
+ ret = 0;
+ fclose(f);
+
+ return ret;
+}
+
+static bool vmbus_uio_ring_present(const struct rte_vmbus_device *dev,
+ uint32_t relid)
+{
+ char ring_path[PATH_MAX];
+
+ /* Check if kernel has subchannel sysfs files */
+ snprintf(ring_path, sizeof(ring_path),
+ "%s/%s/channels/%u/ring",
+ SYSFS_VMBUS_DEVICES, dev->device.name, relid);
+
+ return access(ring_path, R_OK|W_OK) == 0;
+}
+
+bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev,
+ const struct vmbus_channel *chan)
+{
+ return vmbus_uio_ring_present(dev, chan->relid);
+}
+
+static bool vmbus_isnew_subchannel(struct vmbus_channel *primary,
+ unsigned long id)
+{
+ const struct vmbus_channel *c;
+
+ STAILQ_FOREACH(c, &primary->subchannel_list, next) {
+ if (c->relid == id)
+ return false;
+ }
+ return true;
+}
+
+int vmbus_uio_get_subchan(struct vmbus_channel *primary,
+ struct vmbus_channel **subchan)
+{
+ const struct rte_vmbus_device *dev = primary->device;
+ char chan_path[PATH_MAX], subchan_path[PATH_MAX];
+ struct dirent *ent;
+ DIR *chan_dir;
+
+ snprintf(chan_path, sizeof(chan_path),
+ "%s/%s/channels",
+ SYSFS_VMBUS_DEVICES, dev->device.name);
+
+ chan_dir = opendir(chan_path);
+ if (!chan_dir) {
+ VMBUS_LOG(ERR, "cannot open %s: %s",
+ chan_path, strerror(errno));
+ return -errno;
+ }
+
+ while ((ent = readdir(chan_dir))) {
+ unsigned long relid, subid, monid;
+ char *endp;
+ int err;
+
+ if (ent->d_name[0] == '.')
+ continue;
+
+ errno = 0;
+ relid = strtoul(ent->d_name, &endp, 0);
+ if (*endp || errno != 0 || relid > UINT16_MAX) {
+ VMBUS_LOG(NOTICE, "not a valid channel relid: %s",
+ ent->d_name);
+ continue;
+ }
+
+ snprintf(subchan_path, sizeof(subchan_path), "%s/%lu",
+ chan_path, relid);
+ err = vmbus_uio_sysfs_read(subchan_path, "subchannel_id",
+ &subid, UINT16_MAX);
+ if (err) {
+ VMBUS_LOG(NOTICE, "invalid subchannel id %lu",
+ subid);
+ closedir(chan_dir);
+ return err;
+ }
+
+ if (subid == 0)
+ continue; /* skip primary channel */
+
+ if (!vmbus_isnew_subchannel(primary, relid))
+ continue;
+
+ if (!vmbus_uio_ring_present(dev, relid))
+ continue; /* Ring may not be ready yet */
+
+ err = vmbus_uio_sysfs_read(subchan_path, "monitor_id",
+ &monid, UINT8_MAX);
+ if (err) {
+ VMBUS_LOG(NOTICE, "invalid monitor id %lu",
+ monid);
+ return err;
+ }
+
+ err = vmbus_chan_create(dev, relid, subid, monid, subchan);
+ if (err) {
+ VMBUS_LOG(NOTICE, "subchannel setup failed");
+ return err;
+ }
+ break;
+ }
+ closedir(chan_dir);
+
+ return (ent == NULL) ? -ENOENT : 0;
+}
diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build
new file mode 100644
index 00000000..18daabec
--- /dev/null
+++ b/drivers/bus/vmbus/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+allow_experimental_apis = true
+
+install_headers('rte_bus_vmbus.h','rte_vmbus_reg.h')
+
+sources = files('vmbus_common.c',
+ 'vmbus_channel.c',
+ 'vmbus_bufring.c',
+ 'vmbus_common_uio.c')
+
+if host_machine.system() == 'linux'
+ sources += files('linux/vmbus_bus.c',
+ 'linux/vmbus_uio.c')
+ includes += include_directories('linux')
+else
+ build = false
+endif
diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h
new file mode 100644
index 00000000..9964fc42
--- /dev/null
+++ b/drivers/bus/vmbus/private.h
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#ifndef _VMBUS_PRIVATE_H_
+#define _VMBUS_PRIVATE_H_
+
+#include <stdbool.h>
+#include <sys/uio.h>
+#include <rte_log.h>
+#include <rte_vmbus_reg.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE 4096
+#endif
+
+extern int vmbus_logtype_bus;
+#define VMBUS_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, vmbus_logtype_bus, "%s(): " fmt "\n", \
+ __func__, ##args)
+
+struct vmbus_br {
+ struct vmbus_bufring *vbr;
+ uint32_t dsize;
+ uint32_t windex; /* next available location */
+};
+
+#define UIO_NAME_MAX 64
+
+struct vmbus_map {
+ void *addr; /* user mmap of resource */
+ uint64_t size; /* length */
+};
+
+/*
+ * For multi-process we need to reproduce all vmbus mappings in secondary
+ * processes, so save them in a tailq.
+ */
+struct mapped_vmbus_resource {
+ TAILQ_ENTRY(mapped_vmbus_resource) next;
+
+ rte_uuid_t id;
+ int nb_maps;
+ struct vmbus_map maps[VMBUS_MAX_RESOURCE];
+ char path[PATH_MAX];
+};
+
+TAILQ_HEAD(mapped_vmbus_res_list, mapped_vmbus_resource);
+
+#define HV_MON_TRIG_LEN 32
+#define HV_MON_TRIG_MAX 4
+
+struct vmbus_channel {
+ STAILQ_HEAD(, vmbus_channel) subchannel_list;
+ STAILQ_ENTRY(vmbus_channel) next;
+ const struct rte_vmbus_device *device;
+
+ struct vmbus_br rxbr;
+ struct vmbus_br txbr;
+
+ uint16_t relid;
+ uint16_t subchannel_id;
+ uint8_t monitor_id;
+};
+
+#define VMBUS_MAX_CHANNELS 64
+
+int vmbus_chan_create(const struct rte_vmbus_device *device,
+ uint16_t relid, uint16_t subid, uint8_t monitor_id,
+ struct vmbus_channel **new_chan);
+
+void vmbus_add_device(struct rte_vmbus_device *vmbus_dev);
+void vmbus_insert_device(struct rte_vmbus_device *exist_vmbus_dev,
+ struct rte_vmbus_device *new_vmbus_dev);
+void vmbus_remove_device(struct rte_vmbus_device *vmbus_device);
+
+void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff);
+int vmbus_uio_irq_read(struct rte_vmbus_device *dev);
+
+int vmbus_uio_map_resource(struct rte_vmbus_device *dev);
+void vmbus_uio_unmap_resource(struct rte_vmbus_device *dev);
+
+int vmbus_uio_alloc_resource(struct rte_vmbus_device *dev,
+ struct mapped_vmbus_resource **uio_res);
+void vmbus_uio_free_resource(struct rte_vmbus_device *dev,
+ struct mapped_vmbus_resource *uio_res);
+
+struct mapped_vmbus_resource *
+vmbus_uio_find_resource(const struct rte_vmbus_device *dev);
+int vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int res_idx,
+ struct mapped_vmbus_resource *uio_res,
+ int flags);
+
+void *vmbus_map_resource(void *requested_addr, int fd, off_t offset,
+ size_t size, int additional_flags);
+void vmbus_unmap_resource(void *requested_addr, size_t size);
+
+bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev,
+ const struct vmbus_channel *chan);
+int vmbus_uio_get_subchan(struct vmbus_channel *primary,
+ struct vmbus_channel **subchan);
+int vmbus_uio_map_rings(struct vmbus_channel *chan);
+
+void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen);
+
+/* Amount of space available for write */
+static inline uint32_t
+vmbus_br_availwrite(const struct vmbus_br *br, uint32_t windex)
+{
+ uint32_t rindex = br->vbr->rindex;
+
+ if (windex >= rindex)
+ return br->dsize - (windex - rindex);
+ else
+ return rindex - windex;
+}
+
+static inline uint32_t
+vmbus_br_availread(const struct vmbus_br *br)
+{
+ return br->dsize - vmbus_br_availwrite(br, br->vbr->windex);
+}
+
+int vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
+ bool *need_sig);
+
+int vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen);
+
+int vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t hlen);
+
+#endif /* _VMBUS_PRIVATE_H_ */
diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h
new file mode 100644
index 00000000..4a2c1f6f
--- /dev/null
+++ b/drivers/bus/vmbus/rte_bus_vmbus.h
@@ -0,0 +1,407 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#ifndef _VMBUS_H_
+#define _VMBUS_H_
+
+/**
+ * @file
+ *
+ * VMBUS Interface
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_compat.h>
+#include <rte_uuid.h>
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_vmbus_reg.h>
+
+/* Forward declarations */
+struct rte_vmbus_device;
+struct rte_vmbus_driver;
+struct rte_vmbus_bus;
+struct vmbus_channel;
+struct vmbus_mon_page;
+
+TAILQ_HEAD(rte_vmbus_device_list, rte_vmbus_device);
+TAILQ_HEAD(rte_vmbus_driver_list, rte_vmbus_driver);
+
+/* VMBus iterators */
+#define FOREACH_DEVICE_ON_VMBUS(p) \
+ TAILQ_FOREACH(p, &(rte_vmbus_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_VMBUS(p) \
+ TAILQ_FOREACH(p, &(rte_vmbus_bus.driver_list), next)
+
+/** Maximum number of VMBUS resources. */
+enum hv_uio_map {
+ HV_TXRX_RING_MAP = 0,
+ HV_INT_PAGE_MAP,
+ HV_MON_PAGE_MAP,
+ HV_RECV_BUF_MAP,
+ HV_SEND_BUF_MAP
+};
+#define VMBUS_MAX_RESOURCE 5
+
+/**
+ * A structure describing a VMBUS device.
+ */
+struct rte_vmbus_device {
+ TAILQ_ENTRY(rte_vmbus_device) next; /**< Next probed VMBUS device */
+ const struct rte_vmbus_driver *driver; /**< Associated driver */
+ struct rte_device device; /**< Inherit core device */
+ rte_uuid_t device_id; /**< VMBUS device id */
+ rte_uuid_t class_id; /**< VMBUS device type */
+ uint32_t relid; /**< id for primary */
+ uint8_t monitor_id; /**< monitor page */
+ int uio_num; /**< UIO device number */
+ uint32_t *int_page; /**< VMBUS interrupt page */
+ struct vmbus_channel *primary; /**< VMBUS primary channel */
+ struct vmbus_mon_page *monitor_page; /**< VMBUS monitor page */
+
+ struct rte_intr_handle intr_handle; /**< Interrupt handle */
+ struct rte_mem_resource resource[VMBUS_MAX_RESOURCE];
+};
+
+/**
+ * Initialization function for the driver called during VMBUS probing.
+ */
+typedef int (vmbus_probe_t)(struct rte_vmbus_driver *,
+ struct rte_vmbus_device *);
+
+/**
+ * Initialization function for the driver called during hot plugging.
+ */
+typedef int (vmbus_remove_t)(struct rte_vmbus_device *);
+
+/**
+ * A structure describing a VMBUS driver.
+ */
+struct rte_vmbus_driver {
+ TAILQ_ENTRY(rte_vmbus_driver) next; /**< Next in list. */
+ struct rte_driver driver;
+ struct rte_vmbus_bus *bus; /**< VM bus reference. */
+ vmbus_probe_t *probe; /**< Device Probe function. */
+ vmbus_remove_t *remove; /**< Device Remove function. */
+
+ const rte_uuid_t *id_table; /**< ID table. */
+};
+
+
+/**
+ * Structure describing the VM bus
+ */
+struct rte_vmbus_bus {
+ struct rte_bus bus; /**< Inherit the generic class */
+ struct rte_vmbus_device_list device_list; /**< List of devices */
+ struct rte_vmbus_driver_list driver_list; /**< List of drivers */
+};
+
+/**
+ * Scan the content of the VMBUS bus, and the devices in the devices
+ * list
+ *
+ * @return
+ * 0 on success, negative on error
+ */
+int rte_vmbus_scan(void);
+
+/**
+ * Probe the VMBUS bus
+ *
+ * @return
+ * - 0 on success.
+ * - !0 on error.
+ */
+int rte_vmbus_probe(void);
+
+/**
+ * Map the VMBUS device resources in user space virtual memory address
+ *
+ * @param dev
+ * A pointer to a rte_vmbus_device structure describing the device
+ * to use
+ *
+ * @return
+ * 0 on success, negative on error and positive if no driver
+ * is found for the device.
+ */
+int rte_vmbus_map_device(struct rte_vmbus_device *dev);
+
+/**
+ * Unmap this device
+ *
+ * @param dev
+ * A pointer to a rte_vmbus_device structure describing the device
+ * to use
+ */
+void rte_vmbus_unmap_device(struct rte_vmbus_device *dev);
+
+/**
+ * Get connection to primary VMBUS channel
+ *
+ * @param device
+ * A pointer to a rte_vmbus_device structure describing the device
+ * @param chan
+ * A pointer to a VMBUS channel pointer that will be filled.
+ * @return
+ * - 0 Success; channel opened.
+ * - -ENOMEM: Not enough memory available.
+ * - -EINVAL: Regions could not be mapped.
+ */
+int rte_vmbus_chan_open(struct rte_vmbus_device *device,
+ struct vmbus_channel **chan);
+
+/**
+ * Free connection to VMBUS channel
+ *
+ * @param chan
+ * VMBUS channel
+ */
+void rte_vmbus_chan_close(struct vmbus_channel *chan);
+
+/**
+ * Gets the maximum number of channels supported on device
+ *
+ * @param device
+ * A pointer to a rte_vmbus_device structure describing the device
+ * @return
+ * Number of channels available.
+ */
+int rte_vmbus_max_channels(const struct rte_vmbus_device *device);
+
+/**
+ * Get a connection to new secondary vmbus channel
+ *
+ * @param primary
+ * A pointer to primary VMBUS channel
+ * @param chan
+ * A pointer to a secondary VMBUS channel pointer that will be filled.
+ * @return
+ * - 0 Success; channel opened.
+ * - -ENOMEM: Not enough memory available.
+ * - -EINVAL: Regions could not be mapped.
+ */
+int rte_vmbus_subchan_open(struct vmbus_channel *primary,
+ struct vmbus_channel **new_chan);
+
+/**
+ * Disable IRQ for device
+ *
+ * @param device
+ * VMBUS device
+ */
+void rte_vmbus_irq_mask(struct rte_vmbus_device *device);
+
+/**
+ * Enable IRQ for device
+ *
+ * @param device
+ * VMBUS device
+ */
+void rte_vmbus_irq_unmask(struct rte_vmbus_device *device);
+
+/**
+ * Read (and wait) for IRQ
+ *
+ * @param device
+ * VMBUS device
+ */
+int rte_vmbus_irq_read(struct rte_vmbus_device *device);
+
+/**
+ * Test if channel is empty
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ * @return
+ * Return true if no data present in incoming ring.
+ */
+bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel);
+
+/**
+ * Send the specified buffer on the given channel
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ * @param type
+ * Type of packet that is being send e.g. negotiate, time
+ * packet etc.
+ * @param data
+ * Pointer to the buffer to send
+ * @param dlen
+ * Number of bytes of data to send
+ * @param xact
+ * Identifier of the request
+ * @param flags
+ * Message type inband, rxbuf, gpa
+ * @param need_sig
+ * Is host signal tx is required (optional)
+ *
+ * Sends data in buffer directly to hyper-v via the vmbus
+ */
+int rte_vmbus_chan_send(struct vmbus_channel *channel, uint16_t type,
+ void *data, uint32_t dlen,
+ uint64_t xact, uint32_t flags, bool *need_sig);
+
+/**
+ * Explicitly signal host that data is available
+ *
+ * @param
+ * Pointer to vmbus_channel structure.
+ *
+ * Used when batching multiple sends and only signaling host
+ * after the last send.
+ */
+void rte_vmbus_chan_signal_tx(const struct vmbus_channel *channel);
+
+/* Structure for scatter/gather I/O */
+struct iova_list {
+ rte_iova_t addr;
+ uint32_t len;
+};
+#define MAX_PAGE_BUFFER_COUNT 32
+
+/**
+ * Send a scattered buffer on the given channel
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ * @param type
+ * Type of packet that is being send e.g. negotiate, time
+ * packet etc.
+ * @param gpa
+ * Array of buffers to send
+ * @param gpacnt
+ * Number of elements in iov
+ * @param data
+ * Pointer to the buffer additional data to send
+ * @param dlen
+ * Maximum size of what the the buffer will hold
+ * @param xact
+ * Identifier of the request
+ * @param flags
+ * Message type inband, rxbuf, gpa
+ * @param need_sig
+ * Is host signal tx is required (optional)
+ *
+ * Sends data in buffer directly to hyper-v via the vmbus
+ */
+int rte_vmbus_chan_send_sglist(struct vmbus_channel *channel,
+ struct vmbus_gpa gpa[], uint32_t gpacnt,
+ void *data, uint32_t dlen,
+ uint64_t xact, bool *need_sig);
+/**
+ * Receive response to request on the given channel
+ * skips the channel header.
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ * @param data
+ * Pointer to the buffer you want to receive the data into.
+ * @param len
+ * Pointer to size of receive buffer (in/out)
+ * @param
+ * Pointer to received transaction_id
+ * @return
+ * On success, returns 0
+ * On failure, returns negative errno.
+ */
+int rte_vmbus_chan_recv(struct vmbus_channel *chan,
+ void *data, uint32_t *len,
+ uint64_t *request_id);
+
+/**
+ * Receive response to request on the given channel
+ * includes the channel header.
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ * @param data
+ * Pointer to the buffer you want to receive the data into.
+ * @param len
+ * Pointer to size of receive buffer (in/out)
+ * @return
+ * On success, returns number of bytes read.
+ * On failure, returns negative errno.
+ */
+int rte_vmbus_chan_recv_raw(struct vmbus_channel *chan,
+ void *data, uint32_t *len);
+
+/**
+ * Notify host of bytes read (after recv_raw)
+ * Signals host if required.
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ * @param bytes_read
+ * Number of bytes read since last signal
+ */
+void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read);
+
+/**
+ * Determine sub channel index of the given channel
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ * @return
+ * Sub channel index (0 for primary)
+ */
+uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan);
+
+/**
+ * Register a VMBUS driver.
+ *
+ * @param driver
+ * A pointer to a rte_vmbus_driver structure describing the driver
+ * to be registered.
+ */
+void rte_vmbus_register(struct rte_vmbus_driver *driver);
+
+/**
+ * For debug dump contents of ring buffer.
+ *
+ * @param channel
+ * Pointer to vmbus_channel structure.
+ */
+void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan);
+
+/**
+ * Unregister a VMBUS driver.
+ *
+ * @param driver
+ * A pointer to a rte_vmbus_driver structure describing the driver
+ * to be unregistered.
+ */
+void rte_vmbus_unregister(struct rte_vmbus_driver *driver);
+
+/** Helper for VMBUS device registration from driver instance */
+#define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \
+ RTE_INIT(vmbusinitfn_ ##nm); \
+ static void vmbusinitfn_ ##nm(void) \
+ { \
+ (vmbus_drv).driver.name = RTE_STR(nm); \
+ rte_vmbus_register(&vmbus_drv); \
+ } \
+ RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _VMBUS_H_ */
diff --git a/drivers/bus/vmbus/rte_bus_vmbus_version.map b/drivers/bus/vmbus/rte_bus_vmbus_version.map
new file mode 100644
index 00000000..dabb9203
--- /dev/null
+++ b/drivers/bus/vmbus/rte_bus_vmbus_version.map
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+
+DPDK_18.08 {
+ global:
+
+ rte_vmbus_chan_close;
+ rte_vmbus_chan_open;
+ rte_vmbus_chan_recv;
+ rte_vmbus_chan_recv_raw;
+ rte_vmbus_chan_rx_empty;
+ rte_vmbus_chan_send;
+ rte_vmbus_chan_send_sglist;
+ rte_vmbus_chan_signal_read;
+ rte_vmbus_chan_signal_tx;
+ rte_vmbus_irq_mask;
+ rte_vmbus_irq_read;
+ rte_vmbus_irq_unmask;
+ rte_vmbus_map_device;
+ rte_vmbus_max_channels;
+ rte_vmbus_probe;
+ rte_vmbus_register;
+ rte_vmbus_scan;
+ rte_vmbus_sub_channel_index;
+ rte_vmbus_subchan_open;
+ rte_vmbus_unmap_device;
+ rte_vmbus_unregister;
+
+ local: *;
+};
diff --git a/drivers/bus/vmbus/rte_vmbus_reg.h b/drivers/bus/vmbus/rte_vmbus_reg.h
new file mode 100644
index 00000000..f5a0693d
--- /dev/null
+++ b/drivers/bus/vmbus/rte_vmbus_reg.h
@@ -0,0 +1,344 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#ifndef _VMBUS_REG_H_
+#define _VMBUS_REG_H_
+
+/*
+ * Hyper-V SynIC message format.
+ */
+#define VMBUS_MSG_DSIZE_MAX 240
+#define VMBUS_MSG_SIZE 256
+
+struct vmbus_message {
+ uint32_t type; /* HYPERV_MSGTYPE_ */
+ uint8_t dsize; /* data size */
+ uint8_t flags; /* VMBUS_MSGFLAG_ */
+ uint16_t rsvd;
+ uint64_t id;
+ uint8_t data[VMBUS_MSG_DSIZE_MAX];
+} __rte_packed;
+
+#define VMBUS_MSGFLAG_PENDING 0x01
+
+/*
+ * Hyper-V Monitor Notification Facility
+ */
+
+struct vmbus_mon_trig {
+ uint32_t pending;
+ uint32_t armed;
+} __rte_packed;
+
+#define VMBUS_MONTRIGS_MAX 4
+#define VMBUS_MONTRIG_LEN 32
+
+/*
+ * Hyper-V Monitor Notification Facility
+ */
+struct hyperv_mon_param {
+ uint32_t connid;
+ uint16_t evtflag_ofs;
+ uint16_t rsvd;
+} __rte_packed;
+
+struct vmbus_mon_page {
+ uint32_t state;
+ uint32_t rsvd1;
+
+ struct vmbus_mon_trig trigs[VMBUS_MONTRIGS_MAX];
+ uint8_t rsvd2[536];
+
+ uint16_t lat[VMBUS_MONTRIGS_MAX][VMBUS_MONTRIG_LEN];
+ uint8_t rsvd3[256];
+
+ struct hyperv_mon_param
+ param[VMBUS_MONTRIGS_MAX][VMBUS_MONTRIG_LEN];
+ uint8_t rsvd4[1984];
+} __rte_packed;
+
+/*
+ * Buffer ring
+ */
+
+struct vmbus_bufring {
+ volatile uint32_t windex;
+ volatile uint32_t rindex;
+
+ /*
+ * Interrupt mask {0,1}
+ *
+ * For TX bufring, host set this to 1, when it is processing
+ * the TX bufring, so that we can safely skip the TX event
+ * notification to host.
+ *
+ * For RX bufring, once this is set to 1 by us, host will not
+ * further dispatch interrupts to us, even if there are data
+ * pending on the RX bufring. This effectively disables the
+ * interrupt of the channel to which this RX bufring is attached.
+ */
+ volatile uint32_t imask;
+
+ /*
+ * Win8 uses some of the reserved bits to implement
+ * interrupt driven flow management. On the send side
+ * we can request that the receiver interrupt the sender
+ * when the ring transitions from being full to being able
+ * to handle a message of size "pending_send_sz".
+ *
+ * Add necessary state for this enhancement.
+ */
+ volatile uint32_t pending_send;
+ uint32_t reserved1[12];
+
+ union {
+ struct {
+ uint32_t feat_pending_send_sz:1;
+ };
+ uint32_t value;
+ } feature_bits;
+
+ /* Pad it to PAGE_SIZE so that data starts on page boundary */
+ uint8_t reserved2[4028];
+
+ /*
+ * Ring data starts here + RingDataStartOffset
+ * !!! DO NOT place any fields below this !!!
+ */
+ uint8_t data[0];
+} __rte_packed;
+
+/*
+ * Channel packets
+ */
+
+/* Channel packet flags */
+#define VMBUS_CHANPKT_TYPE_INBAND 0x0006
+#define VMBUS_CHANPKT_TYPE_RXBUF 0x0007
+#define VMBUS_CHANPKT_TYPE_GPA 0x0009
+#define VMBUS_CHANPKT_TYPE_COMP 0x000b
+
+#define VMBUS_CHANPKT_FLAG_NONE 0
+#define VMBUS_CHANPKT_FLAG_RC 0x0001 /* report completion */
+
+#define VMBUS_CHANPKT_SIZE_SHIFT 3
+#define VMBUS_CHANPKT_SIZE_ALIGN (1 << VMBUS_CHANPKT_SIZE_SHIFT)
+#define VMBUS_CHANPKT_HLEN_MIN \
+ (sizeof(struct vmbus_chanpkt_hdr) >> VMBUS_CHANPKT_SIZE_SHIFT)
+
+static inline uint32_t
+vmbus_chanpkt_getlen(uint16_t pktlen)
+{
+ return (uint32_t)pktlen << VMBUS_CHANPKT_SIZE_SHIFT;
+}
+
+/*
+ * GPA stuffs.
+ */
+struct vmbus_gpa_range {
+ uint32_t len;
+ uint32_t ofs;
+ uint64_t page[0];
+} __rte_packed;
+
+/* This is actually vmbus_gpa_range.gpa_page[1] */
+struct vmbus_gpa {
+ uint32_t len;
+ uint32_t ofs;
+ uint64_t page;
+} __rte_packed;
+
+struct vmbus_chanpkt_hdr {
+ uint16_t type; /* VMBUS_CHANPKT_TYPE_ */
+ uint16_t hlen; /* header len, in 8 bytes */
+ uint16_t tlen; /* total len, in 8 bytes */
+ uint16_t flags; /* VMBUS_CHANPKT_FLAG_ */
+ uint64_t xactid;
+} __rte_packed;
+
+static inline uint32_t
+vmbus_chanpkt_datalen(const struct vmbus_chanpkt_hdr *pkt)
+{
+ return vmbus_chanpkt_getlen(pkt->tlen)
+ - vmbus_chanpkt_getlen(pkt->hlen);
+}
+
+struct vmbus_chanpkt {
+ struct vmbus_chanpkt_hdr hdr;
+} __rte_packed;
+
+struct vmbus_rxbuf_desc {
+ uint32_t len;
+ uint32_t ofs;
+} __rte_packed;
+
+struct vmbus_chanpkt_rxbuf {
+ struct vmbus_chanpkt_hdr hdr;
+ uint16_t rxbuf_id;
+ uint16_t rsvd;
+ uint32_t rxbuf_cnt;
+ struct vmbus_rxbuf_desc rxbuf[];
+} __rte_packed;
+
+struct vmbus_chanpkt_sglist {
+ struct vmbus_chanpkt_hdr hdr;
+ uint32_t rsvd;
+ uint32_t gpa_cnt;
+ struct vmbus_gpa gpa[];
+} __rte_packed;
+
+/*
+ * Channel messages
+ * - Embedded in vmbus_message.msg_data, e.g. response and notification.
+ * - Embedded in hypercall_postmsg_in.hc_data, e.g. request.
+ */
+
+#define VMBUS_CHANMSG_TYPE_CHOFFER 1 /* NOTE */
+#define VMBUS_CHANMSG_TYPE_CHRESCIND 2 /* NOTE */
+#define VMBUS_CHANMSG_TYPE_CHREQUEST 3 /* REQ */
+#define VMBUS_CHANMSG_TYPE_CHOFFER_DONE 4 /* NOTE */
+#define VMBUS_CHANMSG_TYPE_CHOPEN 5 /* REQ */
+#define VMBUS_CHANMSG_TYPE_CHOPEN_RESP 6 /* RESP */
+#define VMBUS_CHANMSG_TYPE_CHCLOSE 7 /* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_CONN 8 /* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_SUBCONN 9 /* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_CONNRESP 10 /* RESP */
+#define VMBUS_CHANMSG_TYPE_GPADL_DISCONN 11 /* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_DISCONNRESP 12 /* RESP */
+#define VMBUS_CHANMSG_TYPE_CHFREE 13 /* REQ */
+#define VMBUS_CHANMSG_TYPE_CONNECT 14 /* REQ */
+#define VMBUS_CHANMSG_TYPE_CONNECT_RESP 15 /* RESP */
+#define VMBUS_CHANMSG_TYPE_DISCONNECT 16 /* REQ */
+#define VMBUS_CHANMSG_TYPE_MAX 22
+
+struct vmbus_chanmsg_hdr {
+ uint32_t type; /* VMBUS_CHANMSG_TYPE_ */
+ uint32_t rsvd;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CONNECT */
+struct vmbus_chanmsg_connect {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t ver;
+ uint32_t rsvd;
+ uint64_t evtflags;
+ uint64_t mnf1;
+ uint64_t mnf2;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CONNECT_RESP */
+struct vmbus_chanmsg_connect_resp {
+ struct vmbus_chanmsg_hdr hdr;
+ uint8_t done;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHREQUEST */
+struct vmbus_chanmsg_chrequest {
+ struct vmbus_chanmsg_hdr hdr;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_DISCONNECT */
+struct vmbus_chanmsg_disconnect {
+ struct vmbus_chanmsg_hdr hdr;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHOPEN */
+struct vmbus_chanmsg_chopen {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+ uint32_t openid;
+ uint32_t gpadl;
+ uint32_t vcpuid;
+ uint32_t txbr_pgcnt;
+#define VMBUS_CHANMSG_CHOPEN_UDATA_SIZE 120
+ uint8_t udata[VMBUS_CHANMSG_CHOPEN_UDATA_SIZE];
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHOPEN_RESP */
+struct vmbus_chanmsg_chopen_resp {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+ uint32_t openid;
+ uint32_t status;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_GPADL_CONN */
+struct vmbus_chanmsg_gpadl_conn {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+ uint32_t gpadl;
+ uint16_t range_len;
+ uint16_t range_cnt;
+ struct vmbus_gpa_range range;
+} __rte_packed;
+
+#define VMBUS_CHANMSG_GPADL_CONN_PGMAX 26
+
+/* VMBUS_CHANMSG_TYPE_GPADL_SUBCONN */
+struct vmbus_chanmsg_gpadl_subconn {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t msgno;
+ uint32_t gpadl;
+ uint64_t gpa_page[];
+} __rte_packed;
+
+#define VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX 28
+
+/* VMBUS_CHANMSG_TYPE_GPADL_CONNRESP */
+struct vmbus_chanmsg_gpadl_connresp {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+ uint32_t gpadl;
+ uint32_t status;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHCLOSE */
+struct vmbus_chanmsg_chclose {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_GPADL_DISCONN */
+struct vmbus_chanmsg_gpadl_disconn {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+ uint32_t gpadl;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHFREE */
+struct vmbus_chanmsg_chfree {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHRESCIND */
+struct vmbus_chanmsg_chrescind {
+ struct vmbus_chanmsg_hdr hdr;
+ uint32_t chanid;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHOFFER */
+struct vmbus_chanmsg_choffer {
+ struct vmbus_chanmsg_hdr hdr;
+ rte_uuid_t chtype;
+ rte_uuid_t chinst;
+ uint64_t chlat; /* unit: 100ns */
+ uint32_t chrev;
+ uint32_t svrctx_sz;
+ uint16_t chflags;
+ uint16_t mmio_sz; /* unit: MB */
+ uint8_t udata[120];
+ uint16_t subidx;
+ uint16_t rsvd;
+ uint32_t chanid;
+ uint8_t montrig;
+ uint8_t flags1; /* VMBUS_CHOFFER_FLAG1_ */
+ uint16_t flags2;
+ uint32_t connid;
+} __rte_packed;
+
+#define VMBUS_CHOFFER_FLAG1_HASMNF 0x01
+
+#endif /* !_VMBUS_REG_H_ */
diff --git a/drivers/bus/vmbus/vmbus_bufring.c b/drivers/bus/vmbus/vmbus_bufring.c
new file mode 100644
index 00000000..c8800160
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_bufring.c
@@ -0,0 +1,244 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2009-2012,2016 Microsoft Corp.
+ * Copyright (c) 2012 NetApp Inc.
+ * Copyright (c) 2012 Citrix Inc.
+ * All rights reserved.
+ */
+
+#include <unistd.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <sys/uio.h>
+
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_pause.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+/* Increase bufring index by inc with wraparound */
+static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz)
+{
+ idx += inc;
+ if (idx >= sz)
+ idx -= sz;
+
+ return idx;
+}
+
+void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen)
+{
+ br->vbr = buf;
+ br->windex = br->vbr->windex;
+ br->dsize = blen - sizeof(struct vmbus_bufring);
+}
+
+/*
+ * When we write to the ring buffer, check if the host needs to be
+ * signaled.
+ *
+ * The contract:
+ * - The host guarantees that while it is draining the TX bufring,
+ * it will set the br_imask to indicate it does not need to be
+ * interrupted when new data are added.
+ * - The host guarantees that it will completely drain the TX bufring
+ * before exiting the read loop. Further, once the TX bufring is
+ * empty, it will clear the br_imask and re-check to see if new
+ * data have arrived.
+ */
+static inline bool
+vmbus_txbr_need_signal(const struct vmbus_br *tbr, uint32_t old_windex)
+{
+ rte_smp_mb();
+ if (tbr->vbr->imask)
+ return false;
+
+ rte_smp_rmb();
+
+ /*
+ * This is the only case we need to signal when the
+ * ring transitions from being empty to non-empty.
+ */
+ return old_windex == tbr->vbr->rindex;
+}
+
+static inline uint32_t
+vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex,
+ const void *src0, uint32_t cplen)
+{
+ uint8_t *br_data = tbr->vbr->data;
+ uint32_t br_dsize = tbr->dsize;
+ const uint8_t *src = src0;
+
+ /* XXX use double mapping like Linux kernel? */
+ if (cplen > br_dsize - windex) {
+ uint32_t fraglen = br_dsize - windex;
+
+ /* Wrap-around detected */
+ memcpy(br_data + windex, src, fraglen);
+ memcpy(br_data, src + fraglen, cplen - fraglen);
+ } else {
+ memcpy(br_data + windex, src, cplen);
+ }
+
+ return vmbus_br_idxinc(windex, cplen, br_dsize);
+}
+
+/*
+ * Write scattered channel packet to TX bufring.
+ *
+ * The offset of this channel packet is written as a 64bits value
+ * immediately after this channel packet.
+ *
+ * The write goes through three stages:
+ * 1. Reserve space in ring buffer for the new data.
+ * Writer atomically moves priv_write_index.
+ * 2. Copy the new data into the ring.
+ * 3. Update the tail of the ring (visible to host) that indicates
+ * next read location. Writer updates write_index
+ */
+int
+vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
+ bool *need_sig)
+{
+ struct vmbus_bufring *vbr = tbr->vbr;
+ uint32_t ring_size = tbr->dsize;
+ uint32_t old_windex, next_windex, windex, total;
+ uint64_t save_windex;
+ int i;
+
+ total = 0;
+ for (i = 0; i < iovlen; i++)
+ total += iov[i].iov_len;
+ total += sizeof(save_windex);
+
+ /* Reserve space in ring */
+ do {
+ uint32_t avail;
+
+ /* Get current free location */
+ old_windex = tbr->windex;
+
+ /* Prevent compiler reordering this with calculation */
+ rte_compiler_barrier();
+
+ avail = vmbus_br_availwrite(tbr, old_windex);
+
+ /* If not enough space in ring, then tell caller. */
+ if (avail <= total)
+ return -EAGAIN;
+
+ next_windex = vmbus_br_idxinc(old_windex, total, ring_size);
+
+ /* Atomic update of next write_index for other threads */
+ } while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex));
+
+ /* Space from old..new is now reserved */
+ windex = old_windex;
+ for (i = 0; i < iovlen; i++) {
+ windex = vmbus_txbr_copyto(tbr, windex,
+ iov[i].iov_base, iov[i].iov_len);
+ }
+
+ /* Set the offset of the current channel packet. */
+ save_windex = ((uint64_t)old_windex) << 32;
+ windex = vmbus_txbr_copyto(tbr, windex, &save_windex,
+ sizeof(save_windex));
+
+ /* The region reserved should match region used */
+ RTE_ASSERT(windex == next_windex);
+
+ /* Ensure that data is available before updating host index */
+ rte_smp_wmb();
+
+ /* Checkin for our reservation. wait for our turn to update host */
+ while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
+ rte_pause();
+
+ /* If host had read all data before this, then need to signal */
+ *need_sig |= vmbus_txbr_need_signal(tbr, old_windex);
+ return 0;
+}
+
+static inline uint32_t
+vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex,
+ void *dst0, size_t cplen)
+{
+ const uint8_t *br_data = rbr->vbr->data;
+ uint32_t br_dsize = rbr->dsize;
+ uint8_t *dst = dst0;
+
+ if (cplen > br_dsize - rindex) {
+ uint32_t fraglen = br_dsize - rindex;
+
+ /* Wrap-around detected. */
+ memcpy(dst, br_data + rindex, fraglen);
+ memcpy(dst + fraglen, br_data, cplen - fraglen);
+ } else {
+ memcpy(dst, br_data + rindex, cplen);
+ }
+
+ return vmbus_br_idxinc(rindex, cplen, br_dsize);
+}
+
+/* Copy data from receive ring but don't change index */
+int
+vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen)
+{
+ uint32_t avail;
+
+ /*
+ * The requested data and the 64bits channel packet
+ * offset should be there at least.
+ */
+ avail = vmbus_br_availread(rbr);
+ if (avail < dlen + sizeof(uint64_t))
+ return -EAGAIN;
+
+ vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen);
+ return 0;
+}
+
+/*
+ * Copy data from receive ring and change index
+ * NOTE:
+ * We assume (dlen + skip) == sizeof(channel packet).
+ */
+int
+vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip)
+{
+ struct vmbus_bufring *vbr = rbr->vbr;
+ uint32_t br_dsize = rbr->dsize;
+ uint32_t rindex;
+
+ if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t))
+ return -EAGAIN;
+
+ /* Record where host was when we started read (for debug) */
+ rbr->windex = rbr->vbr->windex;
+
+ /*
+ * Copy channel packet from RX bufring.
+ */
+ rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize);
+ rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen);
+
+ /*
+ * Discard this channel packet's 64bits offset, which is useless to us.
+ */
+ rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize);
+
+ /* Update the read index _after_ the channel packet is fetched. */
+ rte_compiler_barrier();
+
+ vbr->rindex = rindex;
+
+ return 0;
+}
diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c
new file mode 100644
index 00000000..cc5f3e83
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_channel.c
@@ -0,0 +1,405 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <unistd.h>
+#include <stdint.h>
+#include <string.h>
+#include <sys/uio.h>
+
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+static inline void
+vmbus_sync_set_bit(volatile uint32_t *addr, uint32_t mask)
+{
+ /* Use GCC builtin which atomic does atomic OR operation */
+ __sync_or_and_fetch(addr, mask);
+}
+
+static inline void
+vmbus_send_interrupt(const struct rte_vmbus_device *dev, uint32_t relid)
+{
+ uint32_t *int_addr;
+ uint32_t int_mask;
+
+ int_addr = dev->int_page + relid / 32;
+ int_mask = 1u << (relid % 32);
+
+ vmbus_sync_set_bit(int_addr, int_mask);
+}
+
+static inline void
+vmbus_set_monitor(const struct rte_vmbus_device *dev, uint32_t monitor_id)
+{
+ uint32_t *monitor_addr, monitor_mask;
+ unsigned int trigger_index;
+
+ trigger_index = monitor_id / HV_MON_TRIG_LEN;
+ monitor_mask = 1u << (monitor_id % HV_MON_TRIG_LEN);
+
+ monitor_addr = &dev->monitor_page->trigs[trigger_index].pending;
+ vmbus_sync_set_bit(monitor_addr, monitor_mask);
+}
+
+static void
+vmbus_set_event(const struct rte_vmbus_device *dev,
+ const struct vmbus_channel *chan)
+{
+ vmbus_send_interrupt(dev, chan->relid);
+ vmbus_set_monitor(dev, chan->monitor_id);
+}
+
+/*
+ * Notify host that there are data pending on our TX bufring.
+ *
+ * Since this in userspace, rely on the monitor page.
+ * Can't do a hypercall from userspace.
+ */
+void
+rte_vmbus_chan_signal_tx(const struct vmbus_channel *chan)
+{
+ const struct rte_vmbus_device *dev = chan->device;
+ const struct vmbus_br *tbr = &chan->txbr;
+
+ /* Make sure all updates are done before signaling host */
+ rte_smp_wmb();
+
+ /* If host is ignoring interrupts? */
+ if (tbr->vbr->imask)
+ return;
+
+ vmbus_set_event(dev, chan);
+}
+
+
+/* Do a simple send directly using transmit ring. */
+int rte_vmbus_chan_send(struct vmbus_channel *chan, uint16_t type,
+ void *data, uint32_t dlen,
+ uint64_t xactid, uint32_t flags, bool *need_sig)
+{
+ struct vmbus_chanpkt pkt;
+ unsigned int pktlen, pad_pktlen;
+ const uint32_t hlen = sizeof(pkt);
+ bool send_evt = false;
+ uint64_t pad = 0;
+ struct iovec iov[3];
+ int error;
+
+ pktlen = hlen + dlen;
+ pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t));
+
+ pkt.hdr.type = type;
+ pkt.hdr.flags = flags;
+ pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+ pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+ pkt.hdr.xactid = xactid;
+
+ iov[0].iov_base = &pkt;
+ iov[0].iov_len = hlen;
+ iov[1].iov_base = data;
+ iov[1].iov_len = dlen;
+ iov[2].iov_base = &pad;
+ iov[2].iov_len = pad_pktlen - pktlen;
+
+ error = vmbus_txbr_write(&chan->txbr, iov, 3, &send_evt);
+
+ /*
+ * caller sets need_sig to non-NULL if it will handle
+ * signaling if required later.
+ * if need_sig is NULL, signal now if needed.
+ */
+ if (need_sig)
+ *need_sig |= send_evt;
+ else if (error == 0 && send_evt)
+ rte_vmbus_chan_signal_tx(chan);
+ return error;
+}
+
+/* Do a scatter/gather send where the descriptor points to data. */
+int rte_vmbus_chan_send_sglist(struct vmbus_channel *chan,
+ struct vmbus_gpa sg[], uint32_t sglen,
+ void *data, uint32_t dlen,
+ uint64_t xactid, bool *need_sig)
+{
+ struct vmbus_chanpkt_sglist pkt;
+ unsigned int pktlen, pad_pktlen, hlen;
+ bool send_evt = false;
+ struct iovec iov[4];
+ uint64_t pad = 0;
+ int error;
+
+ hlen = offsetof(struct vmbus_chanpkt_sglist, gpa[sglen]);
+ pktlen = hlen + dlen;
+ pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t));
+
+ pkt.hdr.type = VMBUS_CHANPKT_TYPE_GPA;
+ pkt.hdr.flags = VMBUS_CHANPKT_FLAG_RC;
+ pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+ pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+ pkt.hdr.xactid = xactid;
+ pkt.rsvd = 0;
+ pkt.gpa_cnt = sglen;
+
+ iov[0].iov_base = &pkt;
+ iov[0].iov_len = sizeof(pkt);
+ iov[1].iov_base = sg;
+ iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen;
+ iov[2].iov_base = data;
+ iov[2].iov_len = dlen;
+ iov[3].iov_base = &pad;
+ iov[3].iov_len = pad_pktlen - pktlen;
+
+ error = vmbus_txbr_write(&chan->txbr, iov, 4, &send_evt);
+
+ /* if caller is batching, just propagate the status */
+ if (need_sig)
+ *need_sig |= send_evt;
+ else if (error == 0 && send_evt)
+ rte_vmbus_chan_signal_tx(chan);
+ return error;
+}
+
+bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel)
+{
+ const struct vmbus_br *br = &channel->rxbr;
+
+ return br->vbr->rindex == br->vbr->windex;
+}
+
+/* Signal host after reading N bytes */
+void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read)
+{
+ struct vmbus_br *rbr = &chan->rxbr;
+ uint32_t write_sz, pending_sz;
+
+ /* No need for signaling on older versions */
+ if (!rbr->vbr->feature_bits.feat_pending_send_sz)
+ return;
+
+ /* Make sure reading of pending happens after new read index */
+ rte_mb();
+
+ pending_sz = rbr->vbr->pending_send;
+ if (!pending_sz)
+ return;
+
+ rte_smp_rmb();
+ write_sz = vmbus_br_availwrite(rbr, rbr->vbr->windex);
+
+ /* If there was space before then host was not blocked */
+ if (write_sz - bytes_read > pending_sz)
+ return;
+
+ /* If pending write will not fit */
+ if (write_sz <= pending_sz)
+ return;
+
+ vmbus_set_event(chan->device, chan);
+}
+
+int rte_vmbus_chan_recv(struct vmbus_channel *chan, void *data, uint32_t *len,
+ uint64_t *request_id)
+{
+ struct vmbus_chanpkt_hdr pkt;
+ uint32_t dlen, hlen, bufferlen = *len;
+ int error;
+
+ *len = 0;
+
+ error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt));
+ if (error)
+ return error;
+
+ if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) {
+ VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen);
+ /* XXX this channel is dead actually. */
+ return -EIO;
+ }
+
+ if (unlikely(pkt.hlen > pkt.tlen)) {
+ VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u",
+ pkt.hlen, pkt.tlen);
+ return -EIO;
+ }
+
+ /* Length are in quad words */
+ hlen = pkt.hlen << VMBUS_CHANPKT_SIZE_SHIFT;
+ dlen = (pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT) - hlen;
+ *len = dlen;
+
+ /* If caller buffer is not large enough */
+ if (unlikely(dlen > bufferlen))
+ return -ENOBUFS;
+
+ if (request_id)
+ *request_id = pkt.xactid;
+
+ /* Read data and skip packet header */
+ error = vmbus_rxbr_read(&chan->rxbr, data, dlen, hlen);
+ if (error)
+ return error;
+
+ rte_vmbus_chan_signal_read(chan, dlen + hlen + sizeof(uint64_t));
+ return 0;
+}
+
+/* TODO: replace this with inplace ring buffer (no copy) */
+int rte_vmbus_chan_recv_raw(struct vmbus_channel *chan,
+ void *data, uint32_t *len)
+{
+ struct vmbus_chanpkt_hdr pkt;
+ uint32_t dlen, bufferlen = *len;
+ int error;
+
+ error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt));
+ if (error)
+ return error;
+
+ if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) {
+ VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen);
+ /* XXX this channel is dead actually. */
+ return -EIO;
+ }
+
+ if (unlikely(pkt.hlen > pkt.tlen)) {
+ VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u",
+ pkt.hlen, pkt.tlen);
+ return -EIO;
+ }
+
+ /* Length are in quad words */
+ dlen = pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT;
+ *len = dlen;
+
+ /* If caller buffer is not large enough */
+ if (unlikely(dlen > bufferlen))
+ return -ENOBUFS;
+
+ /* Read data and skip packet header */
+ error = vmbus_rxbr_read(&chan->rxbr, data, dlen, 0);
+ if (error)
+ return error;
+
+ /* Return the number of bytes read */
+ return dlen + sizeof(uint64_t);
+}
+
+int vmbus_chan_create(const struct rte_vmbus_device *device,
+ uint16_t relid, uint16_t subid, uint8_t monitor_id,
+ struct vmbus_channel **new_chan)
+{
+ struct vmbus_channel *chan;
+ int err;
+
+ chan = rte_zmalloc_socket("VMBUS", sizeof(*chan), RTE_CACHE_LINE_SIZE,
+ device->device.numa_node);
+ if (!chan)
+ return -ENOMEM;
+
+ STAILQ_INIT(&chan->subchannel_list);
+ chan->device = device;
+ chan->subchannel_id = subid;
+ chan->relid = relid;
+ chan->monitor_id = monitor_id;
+ *new_chan = chan;
+
+ err = vmbus_uio_map_rings(chan);
+ if (err) {
+ rte_free(chan);
+ return err;
+ }
+
+ return 0;
+}
+
+/* Setup the primary channel */
+int rte_vmbus_chan_open(struct rte_vmbus_device *device,
+ struct vmbus_channel **new_chan)
+{
+ int err;
+
+ err = vmbus_chan_create(device, device->relid, 0,
+ device->monitor_id, new_chan);
+ if (!err)
+ device->primary = *new_chan;
+
+ return err;
+}
+
+int rte_vmbus_max_channels(const struct rte_vmbus_device *device)
+{
+ if (vmbus_uio_subchannels_supported(device, device->primary))
+ return VMBUS_MAX_CHANNELS;
+ else
+ return 1;
+}
+
+/* Setup secondary channel */
+int rte_vmbus_subchan_open(struct vmbus_channel *primary,
+ struct vmbus_channel **new_chan)
+{
+ struct vmbus_channel *chan;
+ int err;
+
+ err = vmbus_uio_get_subchan(primary, &chan);
+ if (err)
+ return err;
+
+ STAILQ_INSERT_TAIL(&primary->subchannel_list, chan, next);
+ *new_chan = chan;
+ return 0;
+}
+
+uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan)
+{
+ return chan->subchannel_id;
+}
+
+void rte_vmbus_chan_close(struct vmbus_channel *chan)
+{
+ const struct rte_vmbus_device *device = chan->device;
+ struct vmbus_channel *primary = device->primary;
+
+ if (chan != primary)
+ STAILQ_REMOVE(&primary->subchannel_list, chan,
+ vmbus_channel, next);
+
+ rte_free(chan);
+}
+
+static void vmbus_dump_ring(FILE *f, const char *id, const struct vmbus_br *br)
+{
+ const struct vmbus_bufring *vbr = br->vbr;
+ struct vmbus_chanpkt_hdr pkt;
+
+ fprintf(f, "%s windex=%u rindex=%u mask=%u pending=%u feature=%#x\n",
+ id, vbr->windex, vbr->rindex, vbr->imask,
+ vbr->pending_send, vbr->feature_bits.value);
+ fprintf(f, " size=%u avail write=%u read=%u\n",
+ br->dsize, vmbus_br_availwrite(br, vbr->windex),
+ vmbus_br_availread(br));
+
+ if (vmbus_rxbr_peek(br, &pkt, sizeof(pkt)) == 0)
+ fprintf(f, " pkt type %#x len %u flags %#x xactid %#"PRIx64"\n",
+ pkt.type,
+ pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT,
+ pkt.flags, pkt.xactid);
+}
+
+void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan)
+{
+ fprintf(f, "channel[%u] relid=%u monitor=%u\n",
+ chan->subchannel_id, chan->relid, chan->monitor_id);
+ vmbus_dump_ring(f, "rxbr", &chan->rxbr);
+ vmbus_dump_ring(f, "txbr", &chan->txbr);
+}
diff --git a/drivers/bus/vmbus/vmbus_common.c b/drivers/bus/vmbus/vmbus_common.c
new file mode 100644
index 00000000..c7165ad5
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_common.c
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <string.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_devargs.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include <rte_memory.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+int vmbus_logtype_bus;
+extern struct rte_vmbus_bus rte_vmbus_bus;
+
+/* map a particular resource from a file */
+void *
+vmbus_map_resource(void *requested_addr, int fd, off_t offset, size_t size,
+ int flags)
+{
+ void *mapaddr;
+
+ /* Map the memory resource of device */
+ mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | flags, fd, offset);
+ if (mapaddr == MAP_FAILED) {
+ VMBUS_LOG(ERR,
+ "mmap(%d, %p, %zu, %ld) failed: %s",
+ fd, requested_addr, size, (long)offset,
+ strerror(errno));
+ }
+ return mapaddr;
+}
+
+/* unmap a particular resource */
+void
+vmbus_unmap_resource(void *requested_addr, size_t size)
+{
+ if (requested_addr == NULL)
+ return;
+
+ /* Unmap the VMBUS memory resource of device */
+ if (munmap(requested_addr, size)) {
+ VMBUS_LOG(ERR, "munmap(%p, 0x%lx) failed: %s",
+ requested_addr, (unsigned long)size,
+ strerror(errno));
+ } else
+ VMBUS_LOG(DEBUG, " VMBUS memory unmapped at %p",
+ requested_addr);
+}
+
+/**
+ * Match the VMBUS driver and device using UUID table
+ *
+ * @param drv
+ * VMBUS driver from which ID table would be extracted
+ * @param pci_dev
+ * VMBUS device to match against the driver
+ * @return
+ * true for successful match
+ * false for unsuccessful match
+ */
+static bool
+vmbus_match(const struct rte_vmbus_driver *dr,
+ const struct rte_vmbus_device *dev)
+{
+ const rte_uuid_t *id_table;
+
+ for (id_table = dr->id_table; !rte_uuid_is_null(*id_table); ++id_table) {
+ if (rte_uuid_compare(*id_table, dev->class_id) == 0)
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * If device ID match, call the devinit() function of the driver.
+ */
+static int
+vmbus_probe_one_driver(struct rte_vmbus_driver *dr,
+ struct rte_vmbus_device *dev)
+{
+ char guid[RTE_UUID_STRLEN];
+ int ret;
+
+ if (!vmbus_match(dr, dev))
+ return 1; /* not supported */
+
+ rte_uuid_unparse(dev->device_id, guid, sizeof(guid));
+ VMBUS_LOG(INFO, "VMBUS device %s on NUMA socket %i",
+ guid, dev->device.numa_node);
+
+ /* TODO add blacklisted */
+
+ /* map resources for device */
+ ret = rte_vmbus_map_device(dev);
+ if (ret != 0)
+ return ret;
+
+ /* reference driver structure */
+ dev->driver = dr;
+ dev->device.driver = &dr->driver;
+
+ if (dev->device.numa_node < 0) {
+ VMBUS_LOG(WARNING, " Invalid NUMA socket, default to 0");
+ dev->device.numa_node = 0;
+ }
+
+ /* call the driver probe() function */
+ VMBUS_LOG(INFO, " probe driver: %s", dr->driver.name);
+ ret = dr->probe(dr, dev);
+ if (ret) {
+ dev->driver = NULL;
+ rte_vmbus_unmap_device(dev);
+ }
+
+ return ret;
+}
+
+/*
+ * IF device class GUID mathces, call the probe function of
+ * registere drivers for the vmbus device.
+ * Return -1 if initialization failed,
+ * and 1 if no driver found for this device.
+ */
+static int
+vmbus_probe_all_drivers(struct rte_vmbus_device *dev)
+{
+ struct rte_vmbus_driver *dr;
+ int rc;
+
+ /* Check if a driver is already loaded */
+ if (dev->driver != NULL) {
+ VMBUS_LOG(DEBUG, "VMBUS driver already loaded");
+ return 0;
+ }
+
+ FOREACH_DRIVER_ON_VMBUS(dr) {
+ rc = vmbus_probe_one_driver(dr, dev);
+ if (rc < 0) /* negative is an error */
+ return -1;
+
+ if (rc > 0) /* positive driver doesn't support it */
+ continue;
+
+ return 0;
+ }
+ return 1;
+}
+
+/*
+ * Scan the vmbus, and call the devinit() function for
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+int
+rte_vmbus_probe(void)
+{
+ struct rte_vmbus_device *dev;
+ size_t probed = 0, failed = 0;
+ char ubuf[RTE_UUID_STRLEN];
+
+ FOREACH_DEVICE_ON_VMBUS(dev) {
+ probed++;
+
+ rte_uuid_unparse(dev->device_id, ubuf, sizeof(ubuf));
+
+ /* TODO: add whitelist/blacklist */
+
+ if (vmbus_probe_all_drivers(dev) < 0) {
+ VMBUS_LOG(NOTICE,
+ "Requested device %s cannot be used", ubuf);
+ rte_errno = errno;
+ failed++;
+ }
+ }
+
+ return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+vmbus_parse(const char *name, void *addr)
+{
+ rte_uuid_t guid;
+ int ret;
+
+ ret = rte_uuid_parse(name, guid);
+ if (ret == 0 && addr)
+ memcpy(addr, &guid, sizeof(guid));
+
+ return ret;
+}
+
+/* register vmbus driver */
+void
+rte_vmbus_register(struct rte_vmbus_driver *driver)
+{
+ VMBUS_LOG(DEBUG,
+ "Registered driver %s", driver->driver.name);
+
+ TAILQ_INSERT_TAIL(&rte_vmbus_bus.driver_list, driver, next);
+ driver->bus = &rte_vmbus_bus;
+}
+
+/* unregister vmbus driver */
+void
+rte_vmbus_unregister(struct rte_vmbus_driver *driver)
+{
+ TAILQ_REMOVE(&rte_vmbus_bus.driver_list, driver, next);
+ driver->bus = NULL;
+}
+
+/* Add a device to VMBUS bus */
+void
+vmbus_add_device(struct rte_vmbus_device *vmbus_dev)
+{
+ TAILQ_INSERT_TAIL(&rte_vmbus_bus.device_list, vmbus_dev, next);
+}
+
+/* Insert a device into a predefined position in VMBUS bus */
+void
+vmbus_insert_device(struct rte_vmbus_device *exist_vmbus_dev,
+ struct rte_vmbus_device *new_vmbus_dev)
+{
+ TAILQ_INSERT_BEFORE(exist_vmbus_dev, new_vmbus_dev, next);
+}
+
+/* Remove a device from VMBUS bus */
+void
+vmbus_remove_device(struct rte_vmbus_device *vmbus_dev)
+{
+ TAILQ_REMOVE(&rte_vmbus_bus.device_list, vmbus_dev, next);
+}
+
+/* VMBUS doesn't support hotplug */
+static struct rte_device *
+vmbus_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+ const void *data)
+{
+ struct rte_vmbus_device *dev;
+
+ FOREACH_DEVICE_ON_VMBUS(dev) {
+ if (start && &dev->device == start) {
+ start = NULL;
+ continue;
+ }
+ if (cmp(&dev->device, data) == 0)
+ return &dev->device;
+ }
+
+ return NULL;
+}
+
+
+struct rte_vmbus_bus rte_vmbus_bus = {
+ .bus = {
+ .scan = rte_vmbus_scan,
+ .probe = rte_vmbus_probe,
+ .find_device = vmbus_find_device,
+ .parse = vmbus_parse,
+ },
+ .device_list = TAILQ_HEAD_INITIALIZER(rte_vmbus_bus.device_list),
+ .driver_list = TAILQ_HEAD_INITIALIZER(rte_vmbus_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(vmbus, rte_vmbus_bus.bus);
+
+RTE_INIT(vmbus_init_log)
+{
+ vmbus_logtype_bus = rte_log_register("bus.vmbus");
+ if (vmbus_logtype_bus >= 0)
+ rte_log_set_level(vmbus_logtype_bus, RTE_LOG_NOTICE);
+}
diff --git a/drivers/bus/vmbus/vmbus_common_uio.c b/drivers/bus/vmbus/vmbus_common_uio.c
new file mode 100644
index 00000000..5ddd36ab
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_common_uio.c
@@ -0,0 +1,232 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+static struct rte_tailq_elem vmbus_tailq = {
+ .name = "VMBUS_RESOURCE_LIST",
+};
+EAL_REGISTER_TAILQ(vmbus_tailq)
+
+static int
+vmbus_uio_map_secondary(struct rte_vmbus_device *dev)
+{
+ int fd, i;
+ struct mapped_vmbus_resource *uio_res;
+ struct mapped_vmbus_res_list *uio_res_list
+ = RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+ TAILQ_FOREACH(uio_res, uio_res_list, next) {
+
+ /* skip this element if it doesn't match our UUID */
+ if (rte_uuid_compare(uio_res->id, dev->device_id) != 0)
+ continue;
+
+ /* open /dev/uioX */
+ fd = open(uio_res->path, O_RDWR);
+ if (fd < 0) {
+ VMBUS_LOG(ERR, "Cannot open %s: %s",
+ uio_res->path, strerror(errno));
+ return -1;
+ }
+
+ for (i = 0; i != uio_res->nb_maps; i++) {
+ void *mapaddr;
+
+ mapaddr = vmbus_map_resource(uio_res->maps[i].addr,
+ fd, 0,
+ uio_res->maps[i].size, 0);
+
+ if (mapaddr == uio_res->maps[i].addr)
+ continue;
+
+ VMBUS_LOG(ERR,
+ "Cannot mmap device resource file %s to address: %p",
+ uio_res->path, uio_res->maps[i].addr);
+
+ if (mapaddr != MAP_FAILED)
+ /* unmap addr wrongly mapped */
+ vmbus_unmap_resource(mapaddr,
+ (size_t)uio_res->maps[i].size);
+
+ /* unmap addrs correctly mapped */
+ while (--i >= 0)
+ vmbus_unmap_resource(uio_res->maps[i].addr,
+ (size_t)uio_res->maps[i].size);
+
+ close(fd);
+ return -1;
+ }
+
+ /* fd is not needed in slave process, close it */
+ close(fd);
+ return 0;
+ }
+
+ VMBUS_LOG(ERR, "Cannot find resource for device");
+ return 1;
+}
+
+static int
+vmbus_uio_map_primary(struct rte_vmbus_device *dev)
+{
+ int i, ret;
+ struct mapped_vmbus_resource *uio_res = NULL;
+ struct mapped_vmbus_res_list *uio_res_list =
+ RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+ /* allocate uio resource */
+ ret = vmbus_uio_alloc_resource(dev, &uio_res);
+ if (ret)
+ return ret;
+
+ /* Map the resources */
+ for (i = 0; i < VMBUS_MAX_RESOURCE; i++) {
+ /* skip empty BAR */
+ if (dev->resource[i].len == 0)
+ continue;
+
+ ret = vmbus_uio_map_resource_by_index(dev, i, uio_res, 0);
+ if (ret)
+ goto error;
+ }
+
+ uio_res->nb_maps = i;
+
+ TAILQ_INSERT_TAIL(uio_res_list, uio_res, next);
+
+ return 0;
+error:
+ while (--i >= 0) {
+ vmbus_unmap_resource(uio_res->maps[i].addr,
+ (size_t)uio_res->maps[i].size);
+ }
+ vmbus_uio_free_resource(dev, uio_res);
+ return -1;
+}
+
+
+struct mapped_vmbus_resource *
+vmbus_uio_find_resource(const struct rte_vmbus_device *dev)
+{
+ struct mapped_vmbus_resource *uio_res;
+ struct mapped_vmbus_res_list *uio_res_list =
+ RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+ if (dev == NULL)
+ return NULL;
+
+ TAILQ_FOREACH(uio_res, uio_res_list, next) {
+ /* skip this element if it doesn't match our VMBUS address */
+ if (rte_uuid_compare(uio_res->id, dev->device_id) == 0)
+ return uio_res;
+ }
+ return NULL;
+}
+
+/* map the VMBUS resource of a VMBUS device in virtual memory */
+int
+vmbus_uio_map_resource(struct rte_vmbus_device *dev)
+{
+ struct mapped_vmbus_resource *uio_res;
+ int ret;
+
+ /* TODO: handle rescind */
+ dev->intr_handle.fd = -1;
+ dev->intr_handle.uio_cfg_fd = -1;
+ dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+
+ /* secondary processes - use already recorded details */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ ret = vmbus_uio_map_secondary(dev);
+ else
+ ret = vmbus_uio_map_primary(dev);
+
+ if (ret != 0)
+ return ret;
+
+ uio_res = vmbus_uio_find_resource(dev);
+ if (!uio_res) {
+ VMBUS_LOG(ERR, "can not find resources!");
+ return -EIO;
+ }
+
+ if (uio_res->nb_maps <= HV_MON_PAGE_MAP) {
+ VMBUS_LOG(ERR, "VMBUS: only %u resources found!",
+ uio_res->nb_maps);
+ return -EINVAL;
+ }
+
+ dev->int_page = (uint32_t *)((char *)uio_res->maps[HV_INT_PAGE_MAP].addr
+ + (PAGE_SIZE >> 1));
+ dev->monitor_page = uio_res->maps[HV_MON_PAGE_MAP].addr;
+ return 0;
+}
+
+static void
+vmbus_uio_unmap(struct mapped_vmbus_resource *uio_res)
+{
+ int i;
+
+ if (uio_res == NULL)
+ return;
+
+ for (i = 0; i != uio_res->nb_maps; i++) {
+ vmbus_unmap_resource(uio_res->maps[i].addr,
+ (size_t)uio_res->maps[i].size);
+ }
+}
+
+/* unmap the VMBUS resource of a VMBUS device in virtual memory */
+void
+vmbus_uio_unmap_resource(struct rte_vmbus_device *dev)
+{
+ struct mapped_vmbus_resource *uio_res;
+ struct mapped_vmbus_res_list *uio_res_list =
+ RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+ if (dev == NULL)
+ return;
+
+ /* find an entry for the device */
+ uio_res = vmbus_uio_find_resource(dev);
+ if (uio_res == NULL)
+ return;
+
+ /* secondary processes - just free maps */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return vmbus_uio_unmap(uio_res);
+
+ TAILQ_REMOVE(uio_res_list, uio_res, next);
+
+ /* unmap all resources */
+ vmbus_uio_unmap(uio_res);
+
+ /* free uio resource */
+ rte_free(uio_res);
+
+ /* close fd if in primary process */
+ close(dev->intr_handle.fd);
+ if (dev->intr_handle.uio_cfg_fd >= 0) {
+ close(dev->intr_handle.uio_cfg_fd);
+ dev->intr_handle.uio_cfg_fd = -1;
+ }
+
+ dev->intr_handle.fd = -1;
+ dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+}