From 8d01b9cd70a67cdafd5b965a70420c3bd7fb3f82 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 1 Nov 2018 11:59:50 +0000 Subject: New upstream version 18.11-rc1 Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb Signed-off-by: Luca Boccassi --- drivers/bus/fslmc/Makefile | 3 +- drivers/bus/fslmc/fslmc_bus.c | 35 +- drivers/bus/fslmc/fslmc_vfio.c | 13 +- drivers/bus/fslmc/mc/dpbp.c | 10 + drivers/bus/fslmc/mc/dpci.c | 197 ++++++ drivers/bus/fslmc/mc/dpcon.c | 30 + drivers/bus/fslmc/mc/dpdmai.c | 14 + drivers/bus/fslmc/mc/dpio.c | 9 + drivers/bus/fslmc/mc/fsl_dpbp.h | 1 + drivers/bus/fslmc/mc/fsl_dpbp_cmd.h | 16 +- drivers/bus/fslmc/mc/fsl_dpci.h | 47 +- drivers/bus/fslmc/mc/fsl_dpci_cmd.h | 62 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 19 + drivers/bus/fslmc/mc/fsl_dpdmai.h | 5 + drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h | 20 +- drivers/bus/fslmc/mc/fsl_dpmng.h | 2 +- drivers/bus/fslmc/mc/fsl_dpopr.h | 85 +++ drivers/bus/fslmc/meson.build | 5 +- drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 7 + drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 197 +++--- drivers/bus/fslmc/portal/dpaa2_hw_dpio.h | 4 + drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 53 +- drivers/bus/fslmc/qbman/include/compat.h | 3 +- drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h | 33 +- drivers/bus/fslmc/qbman/qbman_portal.c | 764 +++++++++++++++++---- drivers/bus/fslmc/qbman/qbman_portal.h | 30 +- drivers/bus/fslmc/qbman/qbman_sys.h | 100 ++- drivers/bus/fslmc/qbman/qbman_sys_decl.h | 4 + drivers/bus/fslmc/rte_bus_fslmc_version.map | 13 + 29 files changed, 1490 insertions(+), 291 deletions(-) create mode 100644 drivers/bus/fslmc/mc/fsl_dpopr.h (limited to 'drivers/bus/fslmc') diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile index 515d0f53..218d9bd2 100644 --- a/drivers/bus/fslmc/Makefile +++ b/drivers/bus/fslmc/Makefile @@ -19,12 +19,13 @@ CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev +LDLIBS += -lrte_common_dpaax # versioning export map EXPORT_MAP := rte_bus_fslmc_version.map # library version -LIBABIVER := 1 +LIBABIVER := 2 SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ qbman/qbman_portal.c \ diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c index d2900edc..89af9385 100644 --- a/drivers/bus/fslmc/fslmc_bus.c +++ b/drivers/bus/fslmc/fslmc_bus.c @@ -20,6 +20,8 @@ #include #include "fslmc_logs.h" +#include + int dpaa2_logtype_bus; #define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups" @@ -161,6 +163,8 @@ scan_one_fslmc_device(char *dev_name) return -ENOMEM; } + dev->device.bus = &rte_fslmc_bus.bus; + /* Parse the device name and ID */ t_ptr = strtok(dup_dev_name, "."); if (!t_ptr) { @@ -375,6 +379,19 @@ rte_fslmc_probe(void) probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST; + /* In case of PA, the FD addresses returned by qbman APIs are physical + * addresses, which need conversion into equivalent VA address for + * rte_mbuf. For that, a table (a serial array, in memory) is used to + * increase translation efficiency. + * This has to be done before probe as some device initialization + * (during) probe allocate memory (dpaa2_sec) which needs to be pinned + * to this table. + * + * Error is ignored as relevant logs are handled within dpaax and + * handling for unavailable dpaax table too is transparent to caller. + */ + dpaax_iova_table_populate(); + TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { ret = rte_fslmc_match(drv, dev); @@ -384,6 +401,9 @@ rte_fslmc_probe(void) if (!drv->probe) continue; + if (rte_dev_is_probed(&dev->device)) + continue; + if (dev->device.devargs && dev->device.devargs->policy == RTE_DEV_BLACKLISTED) { DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping", @@ -396,8 +416,12 @@ rte_fslmc_probe(void) dev->device.devargs->policy == RTE_DEV_WHITELISTED)) { ret = drv->probe(drv, dev); - if (ret) + if (ret) { DPAA2_BUS_ERR("Unable to probe"); + } else { + dev->driver = drv; + dev->device.driver = &drv->driver; + } } break; } @@ -450,6 +474,11 @@ rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver) fslmc_bus = driver->fslmc_bus; + /* Cleanup the PA->VA Translation table; From whereever this function + * is called from. + */ + dpaax_iova_table_depopulate(); + TAILQ_REMOVE(&fslmc_bus->driver_list, driver, next); /* Update Bus references */ driver->fslmc_bus = NULL; @@ -490,6 +519,10 @@ rte_dpaa2_get_iommu_class(void) if (TAILQ_EMPTY(&rte_fslmc_bus.device_list)) return RTE_IOVA_DC; +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + return RTE_IOVA_PA; +#endif + /* check if all devices on the bus support Virtual addressing or not */ has_iova_va = fslmc_all_device_support_iova(); diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c index 4c2cd2a8..493b6e5b 100644 --- a/drivers/bus/fslmc/fslmc_vfio.c +++ b/drivers/bus/fslmc/fslmc_vfio.c @@ -221,6 +221,13 @@ fslmc_memevent_cb(enum rte_mem_event type, const void *addr, size_t len, "alloc" : "dealloc", va, virt_addr, iova_addr, map_len); + /* iova_addr may be set to RTE_BAD_IOVA */ + if (iova_addr == RTE_BAD_IOVA) { + DPAA2_BUS_DEBUG("Segment has invalid iova, skipping\n"); + cur_len += map_len; + continue; + } + if (type == RTE_MEM_EVENT_ALLOC) ret = fslmc_map_dma(virt_addr, iova_addr, map_len); else @@ -318,11 +325,15 @@ fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len) static int fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused, - const struct rte_memseg *ms, void *arg) + const struct rte_memseg *ms, void *arg) { int *n_segs = arg; int ret; + /* if IOVA address is invalid, skip */ + if (ms->iova == RTE_BAD_IOVA) + return 0; + ret = fslmc_map_dma(ms->addr_64, ms->iova, ms->len); if (ret) DPAA2_BUS_ERR("Unable to VFIO map (addr=%p, len=%zu)", diff --git a/drivers/bus/fslmc/mc/dpbp.c b/drivers/bus/fslmc/mc/dpbp.c index 0215d22d..d9103409 100644 --- a/drivers/bus/fslmc/mc/dpbp.c +++ b/drivers/bus/fslmc/mc/dpbp.c @@ -248,6 +248,16 @@ int dpbp_reset(struct fsl_mc_io *mc_io, /* send command to mc*/ return mc_send_command(mc_io, &cmd); } +/** + * dpbp_get_attributes - Retrieve DPBP attributes. + * + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ int dpbp_get_attributes(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, diff --git a/drivers/bus/fslmc/mc/dpci.c b/drivers/bus/fslmc/mc/dpci.c index ff366bfa..95edae9d 100644 --- a/drivers/bus/fslmc/mc/dpci.c +++ b/drivers/bus/fslmc/mc/dpci.c @@ -265,6 +265,15 @@ int dpci_reset(struct fsl_mc_io *mc_io, return mc_send_command(mc_io, &cmd); } +/** + * dpci_get_attributes() - Retrieve DPCI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ int dpci_get_attributes(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, @@ -292,6 +301,94 @@ int dpci_get_attributes(struct fsl_mc_io *mc_io, return 0; } +/** + * dpci_get_peer_attributes() - Retrieve peer DPCI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @attr: Returned peer attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_peer_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_peer_attr *attr) +{ + struct dpci_rsp_get_peer_attr *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_PEER_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_peer_attr *)cmd.params; + attr->peer_id = le32_to_cpu(rsp_params->id); + attr->num_of_priorities = rsp_params->num_of_priorities; + + return 0; +} + +/** + * dpci_get_link_state() - Retrieve the DPCI link state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @up: Returned link state; returns '1' if link is up, '0' otherwise + * + * DPCI can be connected to another DPCI, together they + * create a 'link'. In order to use the DPCI Tx and Rx queues, + * both objects must be enabled. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_link_state(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *up) +{ + struct dpci_rsp_get_link_state *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_LINK_STATE, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_link_state *)cmd.params; + *up = dpci_get_field(rsp_params->up, UP); + + return 0; +} + +/** + * dpci_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @priority: Select the queue relative to number of + * priorities configured at DPCI creation; use + * DPCI_ALL_QUEUES to configure all Rx queues + * identically. + * @cfg: Rx queue configuration + * + * Return: '0' on Success; Error code otherwise. + */ int dpci_set_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, @@ -314,6 +411,9 @@ int dpci_set_rx_queue(struct fsl_mc_io *mc_io, dpci_set_field(cmd_params->dest_type, DEST_TYPE, cfg->dest_cfg.dest_type); + dpci_set_field(cmd_params->dest_type, + ORDER_PRESERVATION, + cfg->order_preservation_en); /* send command to mc*/ return mc_send_command(mc_io, &cmd); @@ -438,3 +538,100 @@ int dpci_get_api_version(struct fsl_mc_io *mc_io, return 0; } + +/** + * dpci_set_opr() - Set Order Restoration configuration. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @index: The queue index + * @options: Configuration mode options + * can be OPR_OPT_CREATE or OPR_OPT_RETIRE + * @cfg: Configuration options for the OPR + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_set_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + uint8_t options, + struct opr_cfg *cfg) +{ + struct dpci_cmd_set_opr *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_SET_OPR, + cmd_flags, + token); + cmd_params = (struct dpci_cmd_set_opr *)cmd.params; + cmd_params->index = index; + cmd_params->options = options; + cmd_params->oloe = cfg->oloe; + cmd_params->oeane = cfg->oeane; + cmd_params->olws = cfg->olws; + cmd_params->oa = cfg->oa; + cmd_params->oprrws = cfg->oprrws; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpci_get_opr() - Retrieve Order Restoration config and query. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @index: The queue index + * @cfg: Returned OPR configuration + * @qry: Returned OPR query + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + struct opr_cfg *cfg, + struct opr_qry *qry) +{ + struct dpci_rsp_get_opr *rsp_params; + struct dpci_cmd_get_opr *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_OPR, + cmd_flags, + token); + cmd_params = (struct dpci_cmd_get_opr *)cmd.params; + cmd_params->index = index; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_opr *)cmd.params; + cfg->oloe = rsp_params->oloe; + cfg->oeane = rsp_params->oeane; + cfg->olws = rsp_params->olws; + cfg->oa = rsp_params->oa; + cfg->oprrws = rsp_params->oprrws; + qry->rip = dpci_get_field(rsp_params->flags, RIP); + qry->enable = dpci_get_field(rsp_params->flags, OPR_ENABLE); + qry->nesn = le16_to_cpu(rsp_params->nesn); + qry->ndsn = le16_to_cpu(rsp_params->ndsn); + qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq); + qry->tseq_nlis = dpci_get_field(rsp_params->tseq_nlis, TSEQ_NLIS); + qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq); + qry->hseq_nlis = dpci_get_field(rsp_params->hseq_nlis, HSEQ_NLIS); + qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr); + qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr); + qry->opr_vid = le16_to_cpu(rsp_params->opr_vid); + qry->opr_id = le16_to_cpu(rsp_params->opr_id); + + return 0; +} diff --git a/drivers/bus/fslmc/mc/dpcon.c b/drivers/bus/fslmc/mc/dpcon.c index 3f6e04b9..92bd2651 100644 --- a/drivers/bus/fslmc/mc/dpcon.c +++ b/drivers/bus/fslmc/mc/dpcon.c @@ -295,6 +295,36 @@ int dpcon_get_attributes(struct fsl_mc_io *mc_io, return 0; } +/** + * dpcon_set_notification() - Set DPCON notification destination + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * @cfg: Notification parameters + * + * Return: '0' on Success; Error code otherwise + */ +int dpcon_set_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_notification_cfg *cfg) +{ + struct dpcon_cmd_set_notification *dpcon_cmd; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_SET_NOTIFICATION, + cmd_flags, + token); + dpcon_cmd = (struct dpcon_cmd_set_notification *)cmd.params; + dpcon_cmd->dpio_id = cpu_to_le32(cfg->dpio_id); + dpcon_cmd->priority = cfg->priority; + dpcon_cmd->user_ctx = cpu_to_le64(cfg->user_ctx); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + /** * dpcon_get_api_version - Get Data Path Concentrator API version * @mc_io: Pointer to MC portal's DPCON object diff --git a/drivers/bus/fslmc/mc/dpdmai.c b/drivers/bus/fslmc/mc/dpdmai.c index 528889df..dcb9d516 100644 --- a/drivers/bus/fslmc/mc/dpdmai.c +++ b/drivers/bus/fslmc/mc/dpdmai.c @@ -113,6 +113,7 @@ int dpdmai_create(struct fsl_mc_io *mc_io, cmd_flags, dprc_token); cmd_params = (struct dpdmai_cmd_create *)cmd.params; + cmd_params->num_queues = cfg->num_queues; cmd_params->priorities[0] = cfg->priorities[0]; cmd_params->priorities[1] = cfg->priorities[1]; @@ -297,6 +298,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, rsp_params = (struct dpdmai_rsp_get_attr *)cmd.params; attr->id = le32_to_cpu(rsp_params->id); attr->num_of_priorities = rsp_params->num_of_priorities; + attr->num_of_queues = rsp_params->num_of_queues; return 0; } @@ -306,6 +308,8 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: Rx queue index. Accepted values are form 0 to num_queues + * parameter provided in dpdmai_create * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation; use * DPDMAI_ALL_QUEUES to configure all Rx queues @@ -317,6 +321,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, const struct dpdmai_rx_queue_cfg *cfg) { @@ -331,6 +336,7 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); cmd_params->dest_priority = cfg->dest_cfg.priority; cmd_params->priority = priority; + cmd_params->queue_idx = queue_idx; cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); cmd_params->options = cpu_to_le32(cfg->options); dpdmai_set_field(cmd_params->dest_type, @@ -346,6 +352,8 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: Rx queue index. Accepted values are form 0 to num_queues + * parameter provided in dpdmai_create * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @attr: Returned Rx queue attributes @@ -355,6 +363,7 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_rx_queue_attr *attr) { @@ -369,6 +378,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, token); cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params; cmd_params->priority = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -392,6 +402,8 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: Tx queue index. Accepted values are form 0 to num_queues + * parameter provided in dpdmai_create * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @attr: Returned Tx queue attributes @@ -401,6 +413,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_tx_queue_attr *attr) { @@ -415,6 +428,7 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, token); cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params; cmd_params->priority = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); diff --git a/drivers/bus/fslmc/mc/dpio.c b/drivers/bus/fslmc/mc/dpio.c index 966277cc..a3382ed1 100644 --- a/drivers/bus/fslmc/mc/dpio.c +++ b/drivers/bus/fslmc/mc/dpio.c @@ -268,6 +268,15 @@ int dpio_reset(struct fsl_mc_io *mc_io, return mc_send_command(mc_io, &cmd); } +/** + * dpio_get_attributes() - Retrieve DPIO attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise + */ int dpio_get_attributes(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, diff --git a/drivers/bus/fslmc/mc/fsl_dpbp.h b/drivers/bus/fslmc/mc/fsl_dpbp.h index 11183626..9d405b42 100644 --- a/drivers/bus/fslmc/mc/fsl_dpbp.h +++ b/drivers/bus/fslmc/mc/fsl_dpbp.h @@ -82,6 +82,7 @@ int dpbp_get_attributes(struct fsl_mc_io *mc_io, /** * BPSCN write will attempt to allocate into a cache (coherent write) */ +#define DPBP_NOTIF_OPT_COHERENT_WRITE 0x00000001 int dpbp_get_api_version(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t *major_ver, diff --git a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h index 18402ced..55c9fc9b 100644 --- a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h @@ -9,13 +9,15 @@ /* DPBP Version */ #define DPBP_VER_MAJOR 3 -#define DPBP_VER_MINOR 3 +#define DPBP_VER_MINOR 4 /* Command versioning */ #define DPBP_CMD_BASE_VERSION 1 +#define DPBP_CMD_VERSION_2 2 #define DPBP_CMD_ID_OFFSET 4 #define DPBP_CMD(id) ((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_BASE_VERSION) +#define DPBP_CMD_V2(id) ((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_VERSION_2) /* Command IDs */ #define DPBP_CMDID_CLOSE DPBP_CMD(0x800) @@ -37,8 +39,8 @@ #define DPBP_CMDID_GET_IRQ_STATUS DPBP_CMD(0x016) #define DPBP_CMDID_CLEAR_IRQ_STATUS DPBP_CMD(0x017) -#define DPBP_CMDID_SET_NOTIFICATIONS DPBP_CMD(0x1b0) -#define DPBP_CMDID_GET_NOTIFICATIONS DPBP_CMD(0x1b1) +#define DPBP_CMDID_SET_NOTIFICATIONS DPBP_CMD_V2(0x1b0) +#define DPBP_CMDID_GET_NOTIFICATIONS DPBP_CMD_V2(0x1b1) #define DPBP_CMDID_GET_FREE_BUFFERS_NUM DPBP_CMD(0x1b2) @@ -68,8 +70,8 @@ struct dpbp_cmd_set_notifications { uint32_t depletion_exit; uint32_t surplus_entry; uint32_t surplus_exit; - uint16_t options; - uint16_t pad[3]; + uint32_t options; + uint16_t pad[2]; uint64_t message_ctx; uint64_t message_iova; }; @@ -79,8 +81,8 @@ struct dpbp_rsp_get_notifications { uint32_t depletion_exit; uint32_t surplus_entry; uint32_t surplus_exit; - uint16_t options; - uint16_t pad[3]; + uint32_t options; + uint16_t pad[2]; uint64_t message_ctx; uint64_t message_iova; }; diff --git a/drivers/bus/fslmc/mc/fsl_dpci.h b/drivers/bus/fslmc/mc/fsl_dpci.h index f69ed3f3..9af9097e 100644 --- a/drivers/bus/fslmc/mc/fsl_dpci.h +++ b/drivers/bus/fslmc/mc/fsl_dpci.h @@ -6,6 +6,8 @@ #ifndef __FSL_DPCI_H #define __FSL_DPCI_H +#include + /* Data Path Communication Interface API * Contains initialization APIs and runtime control APIs for DPCI */ @@ -17,7 +19,7 @@ struct fsl_mc_io; /** * Maximum number of Tx/Rx priorities per DPCI object */ -#define DPCI_PRIO_NUM 2 +#define DPCI_PRIO_NUM 4 /** * Indicates an invalid frame queue @@ -106,6 +108,27 @@ int dpci_get_attributes(struct fsl_mc_io *mc_io, uint16_t token, struct dpci_attr *attr); +/** + * struct dpci_peer_attr - Structure representing the peer DPCI attributes + * @peer_id: DPCI peer id; if no peer is connected returns (-1) + * @num_of_priorities: The pper's number of receive priorities; determines the + * number of transmit priorities for the local DPCI object + */ +struct dpci_peer_attr { + int peer_id; + uint8_t num_of_priorities; +}; + +int dpci_get_peer_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_peer_attr *attr); + +int dpci_get_link_state(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *up); + /** * enum dpci_dest - DPCI destination types * @DPCI_DEST_NONE: Unassigned destination; The queue is set in parked mode @@ -153,6 +176,11 @@ struct dpci_dest_cfg { */ #define DPCI_QUEUE_OPT_DEST 0x00000002 +/** + * Set the queue to hold active mode. + */ +#define DPCI_QUEUE_OPT_HOLD_ACTIVE 0x00000004 + /** * struct dpci_rx_queue_cfg - Structure representing RX queue configuration * @options: Flags representing the suggested modifications to the queue; @@ -163,11 +191,14 @@ struct dpci_dest_cfg { * 'options' * @dest_cfg: Queue destination parameters; * valid only if 'DPCI_QUEUE_OPT_DEST' is contained in 'options' + * @order_preservation_en: order preservation configuration for the rx queue + * valid only if 'DPCI_QUEUE_OPT_HOLD_ACTIVE' is contained in 'options' */ struct dpci_rx_queue_cfg { uint32_t options; uint64_t user_ctx; struct dpci_dest_cfg dest_cfg; + int order_preservation_en; }; int dpci_set_rx_queue(struct fsl_mc_io *mc_io, @@ -217,4 +248,18 @@ int dpci_get_api_version(struct fsl_mc_io *mc_io, uint16_t *major_ver, uint16_t *minor_ver); +int dpci_set_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + uint8_t options, + struct opr_cfg *cfg); + +int dpci_get_opr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t index, + struct opr_cfg *cfg, + struct opr_qry *qry); + #endif /* __FSL_DPCI_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpci_cmd.h b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h index 634248ac..92b85a82 100644 --- a/drivers/bus/fslmc/mc/fsl_dpci_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h @@ -8,7 +8,7 @@ /* DPCI Version */ #define DPCI_VER_MAJOR 3 -#define DPCI_VER_MINOR 3 +#define DPCI_VER_MINOR 4 #define DPCI_CMD_BASE_VERSION 1 #define DPCI_CMD_BASE_VERSION_V2 2 @@ -35,6 +35,8 @@ #define DPCI_CMDID_GET_PEER_ATTR DPCI_CMD_V1(0x0e2) #define DPCI_CMDID_GET_RX_QUEUE DPCI_CMD_V1(0x0e3) #define DPCI_CMDID_GET_TX_QUEUE DPCI_CMD_V1(0x0e4) +#define DPCI_CMDID_SET_OPR DPCI_CMD_V1(0x0e5) +#define DPCI_CMDID_GET_OPR DPCI_CMD_V1(0x0e6) /* Macros for accessing command fields smaller than 1byte */ #define DPCI_MASK(field) \ @@ -90,6 +92,8 @@ struct dpci_rsp_get_link_state { #define DPCI_DEST_TYPE_SHIFT 0 #define DPCI_DEST_TYPE_SIZE 4 +#define DPCI_ORDER_PRESERVATION_SHIFT 4 +#define DPCI_ORDER_PRESERVATION_SIZE 1 struct dpci_cmd_set_rx_queue { uint32_t dest_id; @@ -128,5 +132,61 @@ struct dpci_rsp_get_api_version { uint16_t minor; }; +struct dpci_cmd_set_opr { + uint16_t pad0; + uint8_t index; + uint8_t options; + uint8_t pad1[7]; + uint8_t oloe; + uint8_t oeane; + uint8_t olws; + uint8_t oa; + uint8_t oprrws; +}; + +struct dpci_cmd_get_opr { + uint16_t pad; + uint8_t index; +}; + +#define DPCI_RIP_SHIFT 0 +#define DPCI_RIP_SIZE 1 +#define DPCI_OPR_ENABLE_SHIFT 1 +#define DPCI_OPR_ENABLE_SIZE 1 +#define DPCI_TSEQ_NLIS_SHIFT 0 +#define DPCI_TSEQ_NLIS_SIZE 1 +#define DPCI_HSEQ_NLIS_SHIFT 0 +#define DPCI_HSEQ_NLIS_SIZE 1 + +struct dpci_rsp_get_opr { + uint64_t pad0; + /* from LSB: rip:1 enable:1 */ + uint8_t flags; + uint16_t pad1; + uint8_t oloe; + uint8_t oeane; + uint8_t olws; + uint8_t oa; + uint8_t oprrws; + uint16_t nesn; + uint16_t pad8; + uint16_t ndsn; + uint16_t pad2; + uint16_t ea_tseq; + /* only the LSB */ + uint8_t tseq_nlis; + uint8_t pad3; + uint16_t ea_hseq; + /* only the LSB */ + uint8_t hseq_nlis; + uint8_t pad4; + uint16_t ea_hptr; + uint16_t pad5; + uint16_t ea_tptr; + uint16_t pad6; + uint16_t opr_vid; + uint16_t pad7; + uint16_t opr_id; +}; #pragma pack(pop) #endif /* _FSL_DPCI_CMD_H */ diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h index 36dd5f3c..fc0430dc 100644 --- a/drivers/bus/fslmc/mc/fsl_dpcon.h +++ b/drivers/bus/fslmc/mc/fsl_dpcon.h @@ -81,6 +81,25 @@ int dpcon_get_attributes(struct fsl_mc_io *mc_io, uint16_t token, struct dpcon_attr *attr); +/** + * struct dpcon_notification_cfg - Structure representing notification params + * @dpio_id: DPIO object ID; must be configured with a notification channel; + * to disable notifications set it to 'DPCON_INVALID_DPIO_ID'; + * @priority: Priority selection within the DPIO channel; valid values + * are 0-7, depending on the number of priorities in that channel + * @user_ctx: User context value provided with each CDAN message + */ +struct dpcon_notification_cfg { + int dpio_id; + uint8_t priority; + uint64_t user_ctx; +}; + +int dpcon_set_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_notification_cfg *cfg); + int dpcon_get_api_version(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t *major_ver, diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai.h b/drivers/bus/fslmc/mc/fsl_dpdmai.h index 03e46ec1..40469cc1 100644 --- a/drivers/bus/fslmc/mc/fsl_dpdmai.h +++ b/drivers/bus/fslmc/mc/fsl_dpdmai.h @@ -39,6 +39,7 @@ int dpdmai_close(struct fsl_mc_io *mc_io, * should be configured with 0 */ struct dpdmai_cfg { + uint8_t num_queues; uint8_t priorities[DPDMAI_PRIO_NUM]; }; @@ -78,6 +79,7 @@ int dpdmai_reset(struct fsl_mc_io *mc_io, struct dpdmai_attr { int id; uint8_t num_of_priorities; + uint8_t num_of_queues; }; int dpdmai_get_attributes(struct fsl_mc_io *mc_io, @@ -149,6 +151,7 @@ struct dpdmai_rx_queue_cfg { int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, const struct dpdmai_rx_queue_cfg *cfg); @@ -168,6 +171,7 @@ struct dpdmai_rx_queue_attr { int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_rx_queue_attr *attr); @@ -183,6 +187,7 @@ struct dpdmai_tx_queue_attr { int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, + uint8_t queue_idx, uint8_t priority, struct dpdmai_tx_queue_attr *attr); diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h index 618e19ea..7e122de4 100644 --- a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h +++ b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h @@ -7,30 +7,32 @@ /* DPDMAI Version */ #define DPDMAI_VER_MAJOR 3 -#define DPDMAI_VER_MINOR 2 +#define DPDMAI_VER_MINOR 3 /* Command versioning */ #define DPDMAI_CMD_BASE_VERSION 1 +#define DPDMAI_CMD_VERSION_2 2 #define DPDMAI_CMD_ID_OFFSET 4 #define DPDMAI_CMD(id) ((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) +#define DPDMAI_CMD_V2(id) ((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_VERSION_2) /* Command IDs */ #define DPDMAI_CMDID_CLOSE DPDMAI_CMD(0x800) #define DPDMAI_CMDID_OPEN DPDMAI_CMD(0x80E) -#define DPDMAI_CMDID_CREATE DPDMAI_CMD(0x90E) +#define DPDMAI_CMDID_CREATE DPDMAI_CMD_V2(0x90E) #define DPDMAI_CMDID_DESTROY DPDMAI_CMD(0x98E) #define DPDMAI_CMDID_GET_API_VERSION DPDMAI_CMD(0xa0E) #define DPDMAI_CMDID_ENABLE DPDMAI_CMD(0x002) #define DPDMAI_CMDID_DISABLE DPDMAI_CMD(0x003) -#define DPDMAI_CMDID_GET_ATTR DPDMAI_CMD(0x004) +#define DPDMAI_CMDID_GET_ATTR DPDMAI_CMD_V2(0x004) #define DPDMAI_CMDID_RESET DPDMAI_CMD(0x005) #define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMD(0x006) -#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMD(0x1A0) -#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMD(0x1A1) -#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMD(0x1A2) +#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMD_V2(0x1A0) +#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMD_V2(0x1A1) +#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMD_V2(0x1A2) /* Macros for accessing command fields smaller than 1byte */ #define DPDMAI_MASK(field) \ @@ -47,7 +49,7 @@ struct dpdmai_cmd_open { }; struct dpdmai_cmd_create { - uint8_t pad; + uint8_t num_queues; uint8_t priorities[2]; }; @@ -66,6 +68,7 @@ struct dpdmai_rsp_is_enabled { struct dpdmai_rsp_get_attr { uint32_t id; uint8_t num_of_priorities; + uint8_t num_of_queues; }; #define DPDMAI_DEST_TYPE_SHIFT 0 @@ -77,7 +80,7 @@ struct dpdmai_cmd_set_rx_queue { uint8_t priority; /* from LSB: dest_type:4 */ uint8_t dest_type; - uint8_t pad; + uint8_t queue_idx; uint64_t user_ctx; uint32_t options; }; @@ -85,6 +88,7 @@ struct dpdmai_cmd_set_rx_queue { struct dpdmai_cmd_get_queue { uint8_t pad[5]; uint8_t priority; + uint8_t queue_idx; }; struct dpdmai_rsp_get_rx_queue { diff --git a/drivers/bus/fslmc/mc/fsl_dpmng.h b/drivers/bus/fslmc/mc/fsl_dpmng.h index afaf9b71..8559bef8 100644 --- a/drivers/bus/fslmc/mc/fsl_dpmng.h +++ b/drivers/bus/fslmc/mc/fsl_dpmng.h @@ -18,7 +18,7 @@ struct fsl_mc_io; * Management Complex firmware version information */ #define MC_VER_MAJOR 10 -#define MC_VER_MINOR 3 +#define MC_VER_MINOR 10 /** * struct mc_version diff --git a/drivers/bus/fslmc/mc/fsl_dpopr.h b/drivers/bus/fslmc/mc/fsl_dpopr.h new file mode 100644 index 00000000..fd727e01 --- /dev/null +++ b/drivers/bus/fslmc/mc/fsl_dpopr.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * Copyright 2018 NXP + * + */ +#ifndef __FSL_DPOPR_H_ +#define __FSL_DPOPR_H_ + +/** @addtogroup dpopr Data Path Order Restoration API + * Contains initialization APIs and runtime APIs for the Order Restoration + * @{ + */ + +/** Order Restoration properties */ + +/** + * Create a new Order Point Record option + */ +#define OPR_OPT_CREATE 0x1 +/** + * Retire an existing Order Point Record option + */ +#define OPR_OPT_RETIRE 0x2 + +/** + * struct opr_cfg - Structure representing OPR configuration + * @oprrws: Order point record (OPR) restoration window size (0 to 5) + * 0 - Window size is 32 frames. + * 1 - Window size is 64 frames. + * 2 - Window size is 128 frames. + * 3 - Window size is 256 frames. + * 4 - Window size is 512 frames. + * 5 - Window size is 1024 frames. + *@oa: OPR auto advance NESN window size (0 disabled, 1 enabled) + *@olws: OPR acceptable late arrival window size (0 to 3) + * 0 - Disabled. Late arrivals are always rejected. + * 1 - Window size is 32 frames. + * 2 - Window size is the same as the OPR restoration + * window size configured in the OPRRWS field. + * 3 - Window size is 8192 frames. + * Late arrivals are always accepted. + *@oeane: Order restoration list (ORL) resource exhaustion + * advance NESN enable (0 disabled, 1 enabled) + *@oloe: OPR loose ordering enable (0 disabled, 1 enabled) + */ +struct opr_cfg { + uint8_t oprrws; + uint8_t oa; + uint8_t olws; + uint8_t oeane; + uint8_t oloe; +}; + +/** + * struct opr_qry - Structure representing OPR configuration + * @enable: Enabled state + * @rip: Retirement In Progress + * @ndsn: Next dispensed sequence number + * @nesn: Next expected sequence number + * @ea_hseq: Early arrival head sequence number + * @hseq_nlis: HSEQ not last in sequence + * @ea_tseq: Early arrival tail sequence number + * @tseq_nlis: TSEQ not last in sequence + * @ea_tptr: Early arrival tail pointer + * @ea_hptr: Early arrival head pointer + * @opr_id: Order Point Record ID + * @opr_vid: Order Point Record Virtual ID + */ +struct opr_qry { + char enable; + char rip; + uint16_t ndsn; + uint16_t nesn; + uint16_t ea_hseq; + char hseq_nlis; + uint16_t ea_tseq; + char tseq_nlis; + uint16_t ea_tptr; + uint16_t ea_hptr; + uint16_t opr_id; + uint16_t opr_vid; +}; + +#endif /* __FSL_DPOPR_H_ */ diff --git a/drivers/bus/fslmc/meson.build b/drivers/bus/fslmc/meson.build index 22a56a6f..4b052157 100644 --- a/drivers/bus/fslmc/meson.build +++ b/drivers/bus/fslmc/meson.build @@ -1,11 +1,13 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright 2018 NXP +version = 2 + if host_machine.system() != 'linux' build = false endif -deps += ['eventdev', 'kvargs'] +deps += ['common_dpaax', 'eventdev', 'kvargs'] sources = files('fslmc_bus.c', 'fslmc_vfio.c', 'mc/dpbp.c', @@ -24,4 +26,3 @@ sources = files('fslmc_bus.c', allow_experimental_apis = true includes += include_directories('mc', 'qbman/include', 'portal') -cflags += ['-D_GNU_SOURCE'] diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c index 39c5adf9..db49d637 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c @@ -28,6 +28,13 @@ #include "portal/dpaa2_hw_pvt.h" #include "portal/dpaa2_hw_dpio.h" +/* List of all the memseg information locally maintained in dpaa2 driver. This + * is to optimize the PA_to_VA searches until a better mechanism (algo) is + * available. + */ +struct dpaa2_memseg_list rte_dpaa2_memsegs + = TAILQ_HEAD_INITIALIZER(rte_dpaa2_memsegs); + TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev); static struct dpbp_dev_list dpbp_dev_list = TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */ diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c index 99f70be1..ce069984 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright 2016 NXP + * Copyright 2016-2018 NXP * */ #include @@ -53,6 +53,11 @@ static uint32_t io_space_count; /* Variable to store DPAA2 platform type */ uint32_t dpaa2_svr_family; +/* Variable to store DPAA2 DQRR size */ +uint8_t dpaa2_dqrr_size; +/* Variable to store DPAA2 EQCR size */ +uint8_t dpaa2_eqcr_size; + /*Stashing Macros default for LS208x*/ static int dpaa2_core_cluster_base = 0x04; static int dpaa2_cluster_sz = 2; @@ -125,7 +130,7 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id) cpu_mask, token); ret = system(command); if (ret < 0) - DPAA2_BUS_WARN( + DPAA2_BUS_DEBUG( "Failed to affine interrupts on respective core"); else DPAA2_BUS_DEBUG(" %s command is executed", command); @@ -177,68 +182,6 @@ static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev) } #endif -static int -configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) -{ - struct qbman_swp_desc p_des; - struct dpio_attr attr; - - dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io)); - if (!dpio_dev->dpio) { - DPAA2_BUS_ERR("Memory allocation failure"); - return -1; - } - - dpio_dev->dpio->regs = dpio_dev->mc_portal; - if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id, - &dpio_dev->token)) { - DPAA2_BUS_ERR("Failed to allocate IO space"); - free(dpio_dev->dpio); - return -1; - } - - if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { - DPAA2_BUS_ERR("Failed to reset dpio"); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { - DPAA2_BUS_ERR("Failed to Enable dpio"); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW, - dpio_dev->token, &attr)) { - DPAA2_BUS_ERR("DPIO Get attribute failed"); - dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - /* Configure & setup SW portal */ - p_des.block = NULL; - p_des.idx = attr.qbman_portal_id; - p_des.cena_bar = (void *)(dpio_dev->qbman_portal_ce_paddr); - p_des.cinh_bar = (void *)(dpio_dev->qbman_portal_ci_paddr); - p_des.irq = -1; - p_des.qman_version = attr.qbman_version; - - dpio_dev->sw_portal = qbman_swp_init(&p_des); - if (dpio_dev->sw_portal == NULL) { - DPAA2_BUS_ERR("QBMan SW Portal Init failed"); - dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); - free(dpio_dev->dpio); - return -1; - } - - return 0; -} - static int dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id) { @@ -402,15 +345,17 @@ dpaa2_create_dpio_device(int vdev_fd, struct vfio_device_info *obj_info, int object_id) { - struct dpaa2_dpio_dev *dpio_dev; + struct dpaa2_dpio_dev *dpio_dev = NULL; struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)}; + struct qbman_swp_desc p_des; + struct dpio_attr attr; if (obj_info->num_regions < NUM_DPIO_REGIONS) { DPAA2_BUS_ERR("Not sufficient number of DPIO regions"); return -1; } - dpio_dev = rte_malloc(NULL, sizeof(struct dpaa2_dpio_dev), + dpio_dev = rte_zmalloc(NULL, sizeof(struct dpaa2_dpio_dev), RTE_CACHE_LINE_SIZE); if (!dpio_dev) { DPAA2_BUS_ERR("Memory allocation failed for DPIO Device"); @@ -423,45 +368,33 @@ dpaa2_create_dpio_device(int vdev_fd, /* Using single portal for all devices */ dpio_dev->mc_portal = rte_mcp_ptr_list[MC_PORTAL_INDEX]; - reg_info.index = 0; - if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { - DPAA2_BUS_ERR("vfio: error getting region info"); - rte_free(dpio_dev); - return -1; + dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io)); + if (!dpio_dev->dpio) { + DPAA2_BUS_ERR("Memory allocation failure"); + goto err; } - dpio_dev->ce_size = reg_info.size; - dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size, - PROT_WRITE | PROT_READ, MAP_SHARED, - vdev_fd, reg_info.offset); - - reg_info.index = 1; - if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { - DPAA2_BUS_ERR("vfio: error getting region info"); - rte_free(dpio_dev); - return -1; + dpio_dev->dpio->regs = dpio_dev->mc_portal; + if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id, + &dpio_dev->token)) { + DPAA2_BUS_ERR("Failed to allocate IO space"); + goto err; } - dpio_dev->ci_size = reg_info.size; - dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size, - PROT_WRITE | PROT_READ, MAP_SHARED, - vdev_fd, reg_info.offset); - - if (configure_dpio_qbman_swp(dpio_dev)) { - DPAA2_BUS_ERR( - "Fail to configure the dpio qbman portal for %d", - dpio_dev->hw_id); - rte_free(dpio_dev); - return -1; + if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { + DPAA2_BUS_ERR("Failed to reset dpio"); + goto err; } - io_space_count++; - dpio_dev->index = io_space_count; + if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { + DPAA2_BUS_ERR("Failed to Enable dpio"); + goto err; + } - if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) { - DPAA2_BUS_ERR("Fail to setup interrupt for %d", - dpio_dev->hw_id); - rte_free(dpio_dev); + if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW, + dpio_dev->token, &attr)) { + DPAA2_BUS_ERR("DPIO Get attribute failed"); + goto err; } /* find the SoC type for the first time */ @@ -481,11 +414,77 @@ dpaa2_create_dpio_device(int vdev_fd, DPAA2_BUS_DEBUG("LX2160 Platform Detected"); } dpaa2_svr_family = (mc_plat_info.svr & 0xffff0000); + + if (dpaa2_svr_family == SVR_LX2160A) { + dpaa2_dqrr_size = DPAA2_LX2_DQRR_RING_SIZE; + dpaa2_eqcr_size = DPAA2_LX2_EQCR_RING_SIZE; + } else { + dpaa2_dqrr_size = DPAA2_DQRR_RING_SIZE; + dpaa2_eqcr_size = DPAA2_EQCR_RING_SIZE; + } + } + + if (dpaa2_svr_family == SVR_LX2160A) + reg_info.index = DPAA2_SWP_CENA_MEM_REGION; + else + reg_info.index = DPAA2_SWP_CENA_REGION; + + if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + DPAA2_BUS_ERR("vfio: error getting region info"); + goto err; + } + + dpio_dev->ce_size = reg_info.size; + dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size, + PROT_WRITE | PROT_READ, MAP_SHARED, + vdev_fd, reg_info.offset); + + reg_info.index = DPAA2_SWP_CINH_REGION; + if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + DPAA2_BUS_ERR("vfio: error getting region info"); + goto err; + } + + dpio_dev->ci_size = reg_info.size; + dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size, + PROT_WRITE | PROT_READ, MAP_SHARED, + vdev_fd, reg_info.offset); + + /* Configure & setup SW portal */ + p_des.block = NULL; + p_des.idx = attr.qbman_portal_id; + p_des.cena_bar = (void *)(dpio_dev->qbman_portal_ce_paddr); + p_des.cinh_bar = (void *)(dpio_dev->qbman_portal_ci_paddr); + p_des.irq = -1; + p_des.qman_version = attr.qbman_version; + + dpio_dev->sw_portal = qbman_swp_init(&p_des); + if (dpio_dev->sw_portal == NULL) { + DPAA2_BUS_ERR("QBMan SW Portal Init failed"); + goto err; + } + + io_space_count++; + dpio_dev->index = io_space_count; + + if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) { + DPAA2_BUS_ERR("Fail to setup interrupt for %d", + dpio_dev->hw_id); + goto err; } TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next); return 0; + +err: + if (dpio_dev->dpio) { + dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + free(dpio_dev->dpio); + } + rte_free(dpio_dev); + return -1; } void @@ -506,7 +505,7 @@ dpaa2_alloc_dq_storage(struct queue_storage_info_t *q_storage) for (i = 0; i < NUM_DQS_PER_QUEUE; i++) { q_storage->dq_storage[i] = rte_malloc(NULL, - DPAA2_DQRR_RING_SIZE * sizeof(struct qbman_result), + dpaa2_dqrr_size * sizeof(struct qbman_result), RTE_CACHE_LINE_SIZE); if (!q_storage->dq_storage[i]) goto fail; diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h index d593eea7..462501a2 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h @@ -30,6 +30,10 @@ RTE_DECLARE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io); /* Variable to store DPAA2 platform type */ extern uint32_t dpaa2_svr_family; +/* Variable to store DPAA2 DQRR size */ +extern uint8_t dpaa2_dqrr_size; +/* Variable to store DPAA2 EQCR size */ +extern uint8_t dpaa2_eqcr_size; extern struct dpaa2_io_portal_t dpaa2_io_portal[RTE_MAX_LCORE]; diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h index 82075936..efbeebef 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright 2016 NXP + * Copyright 2016-2018 NXP * */ @@ -9,6 +9,7 @@ #define _DPAA2_HW_PVT_H_ #include +#include #include #include @@ -31,11 +32,27 @@ #define VLAN_TAG_SIZE 4 /** < Vlan Header Length */ #endif -#define MAX_TX_RING_SLOTS 8 - /** > 3) = 8 for LS2/LS2 */ +#define DPAA2_EQCR_SHIFT 3 +/* EQCR shift to get EQCR size for LX2 (2 >> 5) = 32 for LX2 */ +#define DPAA2_LX2_EQCR_SHIFT 5 + +#define DPAA2_SWP_CENA_REGION 0 +#define DPAA2_SWP_CINH_REGION 1 +#define DPAA2_SWP_CENA_MEM_REGION 2 #define MC_PORTAL_INDEX 0 #define NUM_DPIO_REGIONS 2 @@ -193,6 +210,12 @@ enum qbman_fd_format { #define DPAA2_RESET_FD_CTRL(fd) ((fd)->simple.ctrl = 0) #define DPAA2_SET_FD_ASAL(fd, asal) ((fd)->simple.ctrl |= (asal << 16)) + +#define DPAA2_RESET_FD_FLC(fd) do { \ + (fd)->simple.flc_lo = 0; \ + (fd)->simple.flc_hi = 0; \ +} while (0) + #define DPAA2_SET_FD_FLC(fd, addr) do { \ (fd)->simple.flc_lo = lower_32_bits((size_t)(addr)); \ (fd)->simple.flc_hi = upper_32_bits((uint64_t)(addr)); \ @@ -275,28 +298,26 @@ extern struct dpaa2_memseg_list rte_dpaa2_memsegs; #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA extern uint8_t dpaa2_virt_mode; static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused)); -/* todo - this is costly, need to write a fast coversion routine */ + static void *dpaa2_mem_ptov(phys_addr_t paddr) { - struct dpaa2_memseg *ms; + void *va; if (dpaa2_virt_mode) return (void *)(size_t)paddr; - /* Check if the address is already part of the memseg list internally - * maintained by the dpaa2 driver. - */ - TAILQ_FOREACH(ms, &rte_dpaa2_memsegs, next) { - if (paddr >= ms->iova && paddr < - ms->iova + ms->len) - return RTE_PTR_ADD(ms->vaddr, (uintptr_t)(paddr - ms->iova)); - } + va = (void *)dpaax_iova_table_get_va(paddr); + if (likely(va != NULL)) + return va; /* If not, Fallback to full memseg list searching */ - return rte_mem_iova2virt(paddr); + va = rte_mem_iova2virt(paddr); + + return va; } static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused)); + static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) { const struct rte_memseg *memseg; diff --git a/drivers/bus/fslmc/qbman/include/compat.h b/drivers/bus/fslmc/qbman/include/compat.h index 7be8f54c..655bff4b 100644 --- a/drivers/bus/fslmc/qbman/include/compat.h +++ b/drivers/bus/fslmc/qbman/include/compat.h @@ -78,13 +78,14 @@ do { \ #define lower_32_bits(x) ((uint32_t)(x)) #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) - #define __iomem #define __raw_readb(p) (*(const volatile unsigned char *)(p)) #define __raw_readl(p) (*(const volatile unsigned int *)(p)) #define __raw_writel(v, p) {*(volatile unsigned int *)(p) = (v); } +#define dma_wmb() rte_smp_mb() + #define atomic_t rte_atomic32_t #define atomic_read(v) rte_atomic32_read(v) #define atomic_set(v, i) rte_atomic32_set(v, i) diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h index 3e63db3a..10c72e04 100644 --- a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h +++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h @@ -42,6 +42,15 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d); */ void qbman_swp_finish(struct qbman_swp *p); +/** + * qbman_swp_invalidate() - Invalidate the cache enabled area of the QBMan + * portal. This is required to be called if a portal moved to another core + * because the QBMan portal area is non coherent + * @p: the qbman_swp object to be invalidated + * + */ +void qbman_swp_invalidate(struct qbman_swp *p); + /** * qbman_swp_get_desc() - Get the descriptor of the given portal object. * @p: the given portal object. @@ -172,7 +181,7 @@ void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit); /** * struct qbman_result - structure for qbman dequeue response and/or * notification. - * @donot_manipulate_directly: the 16 32bit data to represent the whole + * @dont_manipulate_directly: the 16 32bit data to represent the whole * possible qbman dequeue result. */ struct qbman_result { @@ -262,7 +271,7 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable); */ struct qbman_pull_desc { union { - uint32_t donot_manipulate_directly[16]; + uint32_t dont_manipulate_directly[16]; struct pull { uint8_t verb; uint8_t numf; @@ -355,6 +364,14 @@ void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid, void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, enum qbman_pull_type_e dct); +/** + * qbman_pull_desc_set_rad() - Decide whether reschedule the fq after dequeue + * + * @rad: 1 = Reschedule the FQ after dequeue. + * 0 = Allow the FQ to remain active after dequeue. + */ +void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad); + /** * qbman_swp_pull() - Issue the pull dequeue command * @s: the software portal object. @@ -775,7 +792,7 @@ uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn); /* struct qbman_eq_desc - structure of enqueue descriptor */ struct qbman_eq_desc { union { - uint32_t donot_manipulate_directly[8]; + uint32_t dont_manipulate_directly[8]; struct eq { uint8_t verb; uint8_t dca; @@ -796,11 +813,11 @@ struct qbman_eq_desc { /** * struct qbman_eq_response - structure of enqueue response - * @donot_manipulate_directly: the 16 32bit data to represent the whole + * @dont_manipulate_directly: the 16 32bit data to represent the whole * enqueue response. */ struct qbman_eq_response { - uint32_t donot_manipulate_directly[16]; + uint32_t dont_manipulate_directly[16]; }; /** @@ -958,6 +975,7 @@ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, * @s: the software portal used for enqueue. * @d: the enqueue descriptor. * @fd: the frame descriptor to be enqueued. + * @flags: bit-mask of QBMAN_ENQUEUE_FLAG_*** options * @num_frames: the number of the frames to be enqueued. * * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. @@ -973,7 +991,6 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, * @s: the software portal used for enqueue. * @d: the enqueue descriptor. * @fd: the frame descriptor to be enqueued. - * @flags: bit-mask of QBMAN_ENQUEUE_FLAG_*** options * @num_frames: the number of the frames to be enqueued. * * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. @@ -998,12 +1015,12 @@ int qbman_swp_enqueue_thresh(struct qbman_swp *s, unsigned int thresh); /*******************/ /** * struct qbman_release_desc - The structure for buffer release descriptor - * @donot_manipulate_directly: the 32bit data to represent the whole + * @dont_manipulate_directly: the 32bit data to represent the whole * possible settings of qbman release descriptor. */ struct qbman_release_desc { union { - uint32_t donot_manipulate_directly[16]; + uint32_t dont_manipulate_directly[16]; struct br { uint8_t verb; uint8_t reserved; diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c index 07145005..3380e54f 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.c +++ b/drivers/bus/fslmc/qbman/qbman_portal.c @@ -1,39 +1,17 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * Copyright 2018 NXP * */ +#include "qbman_sys.h" #include "qbman_portal.h" /* QBMan portal management command codes */ #define QBMAN_MC_ACQUIRE 0x30 #define QBMAN_WQCHAN_CONFIGURE 0x46 -/* CINH register offsets */ -#define QBMAN_CINH_SWP_EQCR_PI 0x800 -#define QBMAN_CINH_SWP_EQCR_CI 0x840 -#define QBMAN_CINH_SWP_EQAR 0x8c0 -#define QBMAN_CINH_SWP_DQPI 0xa00 -#define QBMAN_CINH_SWP_DCAP 0xac0 -#define QBMAN_CINH_SWP_SDQCR 0xb00 -#define QBMAN_CINH_SWP_RAR 0xcc0 -#define QBMAN_CINH_SWP_ISR 0xe00 -#define QBMAN_CINH_SWP_IER 0xe40 -#define QBMAN_CINH_SWP_ISDR 0xe80 -#define QBMAN_CINH_SWP_IIR 0xec0 -#define QBMAN_CINH_SWP_DQRR_ITR 0xa80 -#define QBMAN_CINH_SWP_ITPR 0xf40 - -/* CENA register offsets */ -#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6)) -#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6)) -#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((uint32_t)(n) << 6)) -#define QBMAN_CENA_SWP_CR 0x600 -#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((uint32_t)(vb) >> 1)) -#define QBMAN_CENA_SWP_VDQCR 0x780 -#define QBMAN_CENA_SWP_EQCR_CI 0x840 - /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */ #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6) @@ -83,6 +61,102 @@ enum qbman_sdqcr_fc { #define MAX_QBMAN_PORTALS 64 static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS]; +/* Internal Function declaration */ +static int +qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); +static int +qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); + +static int +qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); +static int +qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd); + +static int +qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames); +static int +qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames); + +static int +qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); +static int +qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); + +static int +qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d); +static int +qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d); + +const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s); +const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s); + +static int +qbman_swp_release_direct(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers); +static int +qbman_swp_release_mem_back(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers); + +/* Function pointers */ +static int (*qbman_swp_enqueue_array_mode_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) + = qbman_swp_enqueue_array_mode_direct; + +static int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) + = qbman_swp_enqueue_ring_mode_direct; + +static int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) + = qbman_swp_enqueue_multiple_direct; + +static int (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) + = qbman_swp_enqueue_multiple_desc_direct; + +static int (*qbman_swp_pull_ptr)(struct qbman_swp *s, + struct qbman_pull_desc *d) + = qbman_swp_pull_direct; + +const struct qbman_result *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s) + = qbman_swp_dqrr_next_direct; + +static int (*qbman_swp_release_ptr)(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers) + = qbman_swp_release_direct; + /*********************************/ /* Portal constructor/destructor */ /*********************************/ @@ -104,25 +178,30 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) { int ret; uint32_t eqcr_pi; + uint32_t mask_size; struct qbman_swp *p = malloc(sizeof(*p)); if (!p) return NULL; + + memset(p, 0, sizeof(struct qbman_swp)); + p->desc = *d; #ifdef QBMAN_CHECKING p->mc.check = swp_mc_can_start; #endif p->mc.valid_bit = QB_VALID_BIT; - p->sdq = 0; p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) + p->mr.valid_bit = QB_VALID_BIT; atomic_set(&p->vdq.busy, 1); p->vdq.valid_bit = QB_VALID_BIT; - p->dqrr.next_idx = 0; p->dqrr.valid_bit = QB_VALID_BIT; - if ((p->desc.qman_version & 0xFFFF0000) < QMAN_REV_4100) { + qman_version = p->desc.qman_version; + if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) { p->dqrr.dqrr_size = 4; p->dqrr.reset_bug = 1; } else { @@ -136,18 +215,54 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) pr_err("qbman_swp_sys_init() failed %d\n", ret); return NULL; } + + /* Verify that the DQRRPI is 0 - if it is not the portal isn't + * in default state which is an error + */ + if (qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQPI) & 0xF) { + pr_err("qbman DQRR PI is not zero, portal is not clean\n"); + free(p); + return NULL; + } + /* SDQCR needs to be initialized to 0 when no channels are * being dequeued from or else the QMan HW will indicate an * error. The values that were calculated above will be * applied when dequeues from a specific channel are enabled. */ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0); + + p->eqcr.pi_ring_size = 8; + if ((qman_version & 0xFFFF0000) >= QMAN_REV_5000) { + p->eqcr.pi_ring_size = 32; + qbman_swp_enqueue_array_mode_ptr = + qbman_swp_enqueue_array_mode_mem_back; + qbman_swp_enqueue_ring_mode_ptr = + qbman_swp_enqueue_ring_mode_mem_back; + qbman_swp_enqueue_multiple_ptr = + qbman_swp_enqueue_multiple_mem_back; + qbman_swp_enqueue_multiple_desc_ptr = + qbman_swp_enqueue_multiple_desc_mem_back; + qbman_swp_pull_ptr = qbman_swp_pull_mem_back; + qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back; + qbman_swp_release_ptr = qbman_swp_release_mem_back; + } + + for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) + p->eqcr.pi_mask = (p->eqcr.pi_mask<<1) + 1; eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI); - p->eqcr.pi = eqcr_pi & 0xF; + p->eqcr.pi = eqcr_pi & p->eqcr.pi_mask; p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT; - p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI) & 0xF; - p->eqcr.available = QBMAN_EQCR_SIZE - qm_cyc_diff(QBMAN_EQCR_SIZE, - p->eqcr.ci, p->eqcr.pi); + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) + p->eqcr.ci = qbman_cinh_read(&p->sys, + QBMAN_CINH_SWP_EQCR_CI) & p->eqcr.pi_mask; + else + p->eqcr.ci = qbman_cinh_read(&p->sys, + QBMAN_CINH_SWP_EQCR_PI) & p->eqcr.pi_mask; + p->eqcr.available = p->eqcr.pi_ring_size - + qm_cyc_diff(p->eqcr.pi_ring_size, + p->eqcr.ci & (p->eqcr.pi_mask<<1), + p->eqcr.pi & (p->eqcr.pi_mask<<1)); portal_idx_map[p->desc.idx] = p; return p; @@ -229,7 +344,8 @@ int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p) void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit) { - qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0); + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, + inhibit ? 0xffffffff : 0); } /***********************/ @@ -246,7 +362,10 @@ void *qbman_swp_mc_start(struct qbman_swp *p) #ifdef QBMAN_CHECKING QBMAN_BUG_ON(p->mc.check != swp_mc_can_start); #endif - ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR); + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) + ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR); + else + ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR_MEM); #ifdef QBMAN_CHECKING if (!ret) p->mc.check = swp_mc_can_submit; @@ -266,8 +385,17 @@ void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb) * caller wants to OR but has forgotten to do so. */ QBMAN_BUG_ON((*v & cmd_verb) != *v); - *v = cmd_verb | p->mc.valid_bit; - qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd); + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { + dma_wmb(); + *v = cmd_verb | p->mc.valid_bit; + qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd); + clean(cmd); + } else { + *v = cmd_verb | p->mr.valid_bit; + qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR_MEM, cmd); + dma_wmb(); + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE); + } #ifdef QBMAN_CHECKING p->mc.check = swp_mc_can_poll; #endif @@ -279,17 +407,34 @@ void *qbman_swp_mc_result(struct qbman_swp *p) #ifdef QBMAN_CHECKING QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll); #endif - qbman_cena_invalidate_prefetch(&p->sys, - QBMAN_CENA_SWP_RR(p->mc.valid_bit)); - ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); - /* Remove the valid-bit - command completed if the rest is non-zero */ - verb = ret[0] & ~QB_VALID_BIT; - if (!verb) - return NULL; + if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { + qbman_cena_invalidate_prefetch(&p->sys, + QBMAN_CENA_SWP_RR(p->mc.valid_bit)); + ret = qbman_cena_read(&p->sys, + QBMAN_CENA_SWP_RR(p->mc.valid_bit)); + /* Remove the valid-bit - + * command completed iff the rest is non-zero + */ + verb = ret[0] & ~QB_VALID_BIT; + if (!verb) + return NULL; + p->mc.valid_bit ^= QB_VALID_BIT; + } else { + ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR_MEM); + /* Command completed if the valid bit is toggled */ + if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT)) + return NULL; + /* Remove the valid-bit - + * command completed iff the rest is non-zero + */ + verb = ret[0] & ~QB_VALID_BIT; + if (!verb) + return NULL; + p->mr.valid_bit ^= QB_VALID_BIT; + } #ifdef QBMAN_CHECKING p->mc.check = swp_mc_can_start; #endif - p->mc.valid_bit ^= QB_VALID_BIT; return ret; } @@ -417,13 +562,26 @@ void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, } } -#define EQAR_IDX(eqar) ((eqar) & 0x7) +#define EQAR_IDX(eqar) ((eqar) & 0x1f) #define EQAR_VB(eqar) ((eqar) & 0x80) #define EQAR_SUCCESS(eqar) ((eqar) & 0x100) -static int qbman_swp_enqueue_array_mode(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd) +static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p, + uint8_t idx) +{ + if (idx < 16) + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4, + QMAN_RT_MODE); + else + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT2 + + (idx - 16) * 4, + QMAN_RT_MODE); +} + + +static int qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) { uint32_t *p; const uint32_t *cl = qb_cl(d); @@ -433,39 +591,69 @@ static int qbman_swp_enqueue_array_mode(struct qbman_swp *s, if (!EQAR_SUCCESS(eqar)) return -EBUSY; p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], fd, sizeof(*fd)); + /* Set the verb byte, have to substitute in the valid-bit */ - lwsync(); + dma_wmb(); p[0] = cl[0] | EQAR_VB(eqar); qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); return 0; } +static int qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR); -static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd) + pr_debug("EQAR=%08x\n", eqar); + if (!EQAR_SUCCESS(eqar)) + return -EBUSY; + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); + + /* Set the verb byte, have to substitute in the valid-bit */ + p[0] = cl[0] | EQAR_VB(eqar); + dma_wmb(); + qbman_write_eqcr_am_rt_register(s, EQAR_IDX(eqar)); + return 0; +} + +static inline int qbman_swp_enqueue_array_mode(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + return qbman_swp_enqueue_array_mode_ptr(s, d, fd); +} + +static int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) { uint32_t *p; const uint32_t *cl = qb_cl(d); - uint32_t eqcr_ci; - uint8_t diff; + uint32_t eqcr_ci, full_mask, half_mask; + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; if (!s->eqcr.available) { eqcr_ci = s->eqcr.ci; s->eqcr.ci = qbman_cena_read_reg(&s->sys, - QBMAN_CENA_SWP_EQCR_CI) & 0xF; - diff = qm_cyc_diff(QBMAN_EQCR_SIZE, - eqcr_ci, s->eqcr.ci); - s->eqcr.available += diff; - if (!diff) + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) return -EBUSY; } p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], fd, sizeof(*fd)); lwsync(); @@ -473,16 +661,61 @@ static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, /* Set the verb byte, have to substitute in the valid-bit */ p[0] = cl[0] | s->eqcr.pi_vb; qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); s->eqcr.pi++; - s->eqcr.pi &= 0xF; + s->eqcr.pi &= full_mask; s->eqcr.available--; - if (!(s->eqcr.pi & 7)) + if (!(s->eqcr.pi & half_mask)) s->eqcr.pi_vb ^= QB_VALID_BIT; return 0; } +static int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqcr_ci, full_mask, half_mask; + + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cinh_read(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) + return -EBUSY; + } + + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); + + /* Set the verb byte, have to substitute in the valid-bit */ + p[0] = cl[0] | s->eqcr.pi_vb; + s->eqcr.pi++; + s->eqcr.pi &= full_mask; + s->eqcr.available--; + if (!(s->eqcr.pi & half_mask)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); + return 0; +} + +static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + return qbman_swp_enqueue_ring_mode_ptr(s, d, fd); +} + int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, const struct qbman_fd *fd) { @@ -492,27 +725,27 @@ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, return qbman_swp_enqueue_ring_mode(s, d, fd); } -int qbman_swp_enqueue_multiple(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - uint32_t *flags, - int num_frames) +static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) { - uint32_t *p; + uint32_t *p = NULL; const uint32_t *cl = qb_cl(d); - uint32_t eqcr_ci, eqcr_pi; - uint8_t diff; + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; int i, num_enqueued = 0; uint64_t addr_cena; + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; if (!s->eqcr.available) { eqcr_ci = s->eqcr.ci; s->eqcr.ci = qbman_cena_read_reg(&s->sys, - QBMAN_CENA_SWP_EQCR_CI) & 0xF; - diff = qm_cyc_diff(QBMAN_EQCR_SIZE, - eqcr_ci, s->eqcr.ci); - s->eqcr.available += diff; - if (!diff) + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) return 0; } @@ -523,11 +756,10 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, /* Fill in the EQCR ring */ for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], &fd[i], sizeof(*fd)); eqcr_pi++; - eqcr_pi &= 0xF; } lwsync(); @@ -536,7 +768,7 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); p[0] = cl[0] | s->eqcr.pi_vb; if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; @@ -545,8 +777,7 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); } eqcr_pi++; - eqcr_pi &= 0xF; - if (!(eqcr_pi & 7)) + if (!(eqcr_pi & half_mask)) s->eqcr.pi_vb ^= QB_VALID_BIT; } @@ -554,35 +785,104 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; addr_cena = (size_t)s->sys.addr_cena; for (i = 0; i < num_enqueued; i++) { - dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + dcbf((uintptr_t)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask))); eqcr_pi++; - eqcr_pi &= 0xF; } - s->eqcr.pi = eqcr_pi; + s->eqcr.pi = eqcr_pi & full_mask; return num_enqueued; } -int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - int num_frames) +static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) +{ + uint32_t *p = NULL; + const uint32_t *cl = qb_cl(d); + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; + int i, num_enqueued = 0; + + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cinh_read(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) + return 0; + } + + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + } + + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + p[0] = cl[0] | s->eqcr.pi_vb; + if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { + struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; + + d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | + ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); + } + eqcr_pi++; + if (!(eqcr_pi & half_mask)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + } + s->eqcr.pi = eqcr_pi & full_mask; + + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); + return num_enqueued; +} + +inline int qbman_swp_enqueue_multiple(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + uint32_t *flags, + int num_frames) +{ + return qbman_swp_enqueue_multiple_ptr(s, d, fd, flags, num_frames); +} + +static int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) { uint32_t *p; const uint32_t *cl; - uint32_t eqcr_ci, eqcr_pi; - uint8_t diff; + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; int i, num_enqueued = 0; uint64_t addr_cena; + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; if (!s->eqcr.available) { eqcr_ci = s->eqcr.ci; s->eqcr.ci = qbman_cena_read_reg(&s->sys, - QBMAN_CENA_SWP_EQCR_CI) & 0xF; - diff = qm_cyc_diff(QBMAN_EQCR_SIZE, - eqcr_ci, s->eqcr.ci); - s->eqcr.available += diff; - if (!diff) + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) return 0; } @@ -593,12 +893,11 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, /* Fill in the EQCR ring */ for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); cl = qb_cl(&d[i]); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], &fd[i], sizeof(*fd)); eqcr_pi++; - eqcr_pi &= 0xF; } lwsync(); @@ -607,12 +906,11 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); cl = qb_cl(&d[i]); p[0] = cl[0] | s->eqcr.pi_vb; eqcr_pi++; - eqcr_pi &= 0xF; - if (!(eqcr_pi & 7)) + if (!(eqcr_pi & half_mask)) s->eqcr.pi_vb ^= QB_VALID_BIT; } @@ -620,14 +918,78 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, eqcr_pi = s->eqcr.pi; addr_cena = (size_t)s->sys.addr_cena; for (i = 0; i < num_enqueued; i++) { - dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + dcbf((uintptr_t)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask))); + eqcr_pi++; + } + s->eqcr.pi = eqcr_pi & full_mask; + + return num_enqueued; +} + +static int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) +{ + uint32_t *p; + const uint32_t *cl; + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; + int i, num_enqueued = 0; + + half_mask = (s->eqcr.pi_mask>>1); + full_mask = s->eqcr.pi_mask; + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cinh_read(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & full_mask; + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, + eqcr_ci, s->eqcr.ci); + if (!s->eqcr.available) + return 0; + } + + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + cl = qb_cl(&d[i]); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + } + + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); + cl = qb_cl(&d[i]); + p[0] = cl[0] | s->eqcr.pi_vb; eqcr_pi++; - eqcr_pi &= 0xF; + if (!(eqcr_pi & half_mask)) + s->eqcr.pi_vb ^= QB_VALID_BIT; } - s->eqcr.pi = eqcr_pi; + + s->eqcr.pi = eqcr_pi & full_mask; + + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); return num_enqueued; } +inline int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) +{ + return qbman_swp_enqueue_multiple_desc_ptr(s, d, fd, num_frames); +} /*************************/ /* Static (push) dequeue */ @@ -670,6 +1032,7 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable) #define QB_VDQCR_VERB_DT_SHIFT 2 #define QB_VDQCR_VERB_RLS_SHIFT 4 #define QB_VDQCR_VERB_WAE_SHIFT 5 +#define QB_VDQCR_VERB_RAD_SHIFT 6 enum qb_pull_dt_e { qb_pull_dt_channel, @@ -702,7 +1065,8 @@ void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, d->pull.rsp_addr = storage_phys; } -void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, uint8_t numframes) +void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, + uint8_t numframes) { d->pull.numf = numframes - 1; } @@ -735,7 +1099,20 @@ void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, d->pull.dq_src = chid; } -int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) +void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad) +{ + if (d->pull.verb & (1 << QB_VDQCR_VERB_RLS_SHIFT)) { + if (rad) + d->pull.verb |= 1 << QB_VDQCR_VERB_RAD_SHIFT; + else + d->pull.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT); + } else { + printf("The RAD feature is not valid when RLS = 0\n"); + } +} + +static int qbman_swp_pull_direct(struct qbman_swp *s, + struct qbman_pull_desc *d) { uint32_t *p; uint32_t *cl = qb_cl(d); @@ -759,6 +1136,36 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) return 0; } +static int qbman_swp_pull_mem_back(struct qbman_swp *s, + struct qbman_pull_desc *d) +{ + uint32_t *p; + uint32_t *cl = qb_cl(d); + + if (!atomic_dec_and_test(&s->vdq.busy)) { + atomic_inc(&s->vdq.busy); + return -EBUSY; + } + + d->pull.tok = s->sys.idx + 1; + s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt; + p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR_MEM); + memcpy(&p[1], &cl[1], 12); + + /* Set the verb byte, have to substitute in the valid-bit */ + p[0] = cl[0] | s->vdq.valid_bit; + s->vdq.valid_bit ^= QB_VALID_BIT; + dma_wmb(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE); + + return 0; +} + +inline int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) +{ + return qbman_swp_pull_ptr(s, d); +} + /****************/ /* Polling DQRR */ /****************/ @@ -791,7 +1198,12 @@ void qbman_swp_prefetch_dqrr_next(struct qbman_swp *s) * only once, so repeated calls can return a sequence of DQRR entries, without * requiring they be consumed immediately or in any particular order. */ -const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) +inline const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) +{ + return qbman_swp_dqrr_next_ptr(s); +} + +const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s) { uint32_t verb; uint32_t response_verb; @@ -801,7 +1213,7 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) /* Before using valid-bit to detect if something is there, we have to * handle the case of the DQRR reset bug... */ - if (unlikely(s->dqrr.reset_bug)) { + if (s->dqrr.reset_bug) { /* We pick up new entries by cache-inhibited producer index, * which means that a non-coherent mapping would require us to * invalidate and read *only* once that PI has indicated that @@ -833,7 +1245,8 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); } p = qbman_cena_read_wo_shadow(&s->sys, - QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + verb = p->dq.verb; /* If the valid-bit isn't of the expected polarity, nothing there. Note, @@ -867,11 +1280,54 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) return p; } +const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s) +{ + uint32_t verb; + uint32_t response_verb; + uint32_t flags; + const struct qbman_result *p; + + p = qbman_cena_read_wo_shadow(&s->sys, + QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx)); + + verb = p->dq.verb; + + /* If the valid-bit isn't of the expected polarity, nothing there. Note, + * in the DQRR reset bug workaround, we shouldn't need to skip these + * check, because we've already determined that a new entry is available + * and we've invalidated the cacheline before reading it, so the + * valid-bit behaviour is repaired and should tell us what we already + * knew from reading PI. + */ + if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) + return NULL; + + /* There's something there. Move "next_idx" attention to the next ring + * entry (and prefetch it) before returning what we found. + */ + s->dqrr.next_idx++; + if (s->dqrr.next_idx == s->dqrr.dqrr_size) { + s->dqrr.next_idx = 0; + s->dqrr.valid_bit ^= QB_VALID_BIT; + } + /* If this is the final response to a volatile dequeue command + * indicate that the vdq is no longer busy + */ + flags = p->dq.stat; + response_verb = verb & QBMAN_RESPONSE_VERB_MASK; + if ((response_verb == QBMAN_RESULT_DQ) && + (flags & QBMAN_DQ_STAT_VOLATILE) && + (flags & QBMAN_DQ_STAT_EXPIRED)) + atomic_inc(&s->vdq.busy); + return p; +} + /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct qbman_result *dq) { - qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); + qbman_cinh_write(&s->sys, + QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); } /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ @@ -884,6 +1340,7 @@ void qbman_swp_dqrr_idx_consume(struct qbman_swp *s, /*********************************/ /* Polling user-provided storage */ /*********************************/ + int qbman_result_has_new_result(struct qbman_swp *s, struct qbman_result *dq) { @@ -898,11 +1355,11 @@ int qbman_result_has_new_result(struct qbman_swp *s, ((struct qbman_result *)dq)->dq.tok = 0; /* - * VDQCR "no longer busy" hook - not quite the same as DQRR, because the - * fact "VDQCR" shows busy doesn't mean that we hold the result that - * makes it available. Eg. we may be looking at our 10th dequeue result, - * having released VDQCR after the 1st result and it is now busy due to - * some other command! + * VDQCR "no longer busy" hook - not quite the same as DQRR, because + * the fact "VDQCR" shows busy doesn't mean that we hold the result + * that makes it available. Eg. we may be looking at our 10th dequeue + * result, having released VDQCR after the 1st result and it is now + * busy due to some other command! */ if (s->vdq.storage == dq) { s->vdq.storage = NULL; @@ -936,11 +1393,11 @@ int qbman_check_command_complete(struct qbman_result *dq) s = portal_idx_map[dq->dq.tok - 1]; /* - * VDQCR "no longer busy" hook - not quite the same as DQRR, because the - * fact "VDQCR" shows busy doesn't mean that we hold the result that - * makes it available. Eg. we may be looking at our 10th dequeue result, - * having released VDQCR after the 1st result and it is now busy due to - * some other command! + * VDQCR "no longer busy" hook - not quite the same as DQRR, because + * the fact "VDQCR" shows busy doesn't mean that we hold the result + * that makes it available. Eg. we may be looking at our 10th dequeue + * result, having released VDQCR after the 1st result and it is now + * busy due to some other command! */ if (s->vdq.storage == dq) { s->vdq.storage = NULL; @@ -1142,8 +1599,10 @@ void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable) #define RAR_VB(rar) ((rar) & 0x80) #define RAR_SUCCESS(rar) ((rar) & 0x100) -int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, - const uint64_t *buffers, unsigned int num_buffers) +static int qbman_swp_release_direct(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, + unsigned int num_buffers) { uint32_t *p; const uint32_t *cl = qb_cl(d); @@ -1157,22 +1616,63 @@ int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, /* Start the release command */ p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); /* Copy the caller's buffer pointers to the command */ u64_to_le32_copy(&p[2], buffers, num_buffers); - /* Set the verb byte, have to substitute in the valid-bit and the number - * of buffers. + /* Set the verb byte, have to substitute in the valid-bit and the + * number of buffers. */ lwsync(); p[0] = cl[0] | RAR_VB(rar) | num_buffers; qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); return 0; } +static int qbman_swp_release_mem_back(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, + unsigned int num_buffers) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR); + + pr_debug("RAR=%08x\n", rar); + if (!RAR_SUCCESS(rar)) + return -EBUSY; + + QBMAN_BUG_ON(!num_buffers || (num_buffers > 7)); + + /* Start the release command */ + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar))); + + /* Copy the caller's buffer pointers to the command */ + u64_to_le32_copy(&p[2], buffers, num_buffers); + + /* Set the verb byte, have to substitute in the valid-bit and the + * number of buffers. + */ + p[0] = cl[0] | RAR_VB(rar) | num_buffers; + lwsync(); + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_RCR_AM_RT + + RAR_IDX(rar) * 4, QMAN_RT_MODE); + + return 0; +} + +inline int qbman_swp_release(struct qbman_swp *s, + const struct qbman_release_desc *d, + const uint64_t *buffers, + unsigned int num_buffers) +{ + return qbman_swp_release_ptr(s, d, buffers, num_buffers); +} + /*******************/ /* Buffer acquires */ /*******************/ @@ -1214,7 +1714,7 @@ int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, /* Complete the management command */ r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE); - if (unlikely(!r)) { + if (!r) { pr_err("qbman: acquire from BPID %d failed, no response\n", bpid); return -EIO; @@ -1224,7 +1724,7 @@ int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE); /* Determine success or failure */ - if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + if (r->rslt != QBMAN_MC_RSLT_OK) { pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n", bpid, r->rslt); return -EIO; @@ -1271,7 +1771,7 @@ static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid, /* Complete the management command */ r = qbman_swp_mc_complete(s, p, alt_fq_verb); - if (unlikely(!r)) { + if (!r) { pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n", alt_fq_verb); return -EIO; @@ -1281,7 +1781,7 @@ static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid, QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb); /* Determine success or failure */ - if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + if (r->rslt != QBMAN_MC_RSLT_OK) { pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n", fqid, alt_fq_verb, r->rslt); return -EIO; @@ -1362,7 +1862,7 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, /* Complete the management command */ r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE); - if (unlikely(!r)) { + if (!r) { pr_err("qbman: wqchan config failed, no response\n"); return -EIO; } @@ -1372,7 +1872,7 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, != QBMAN_WQCHAN_CONFIGURE); /* Determine success or failure */ - if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + if (r->rslt != QBMAN_MC_RSLT_OK) { pr_err("CDAN cQID %d failed: code = 0x%02x\n", channelid, r->rslt); return -EIO; diff --git a/drivers/bus/fslmc/qbman/qbman_portal.h b/drivers/bus/fslmc/qbman/qbman_portal.h index dbea22a1..3b0fc540 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.h +++ b/drivers/bus/fslmc/qbman/qbman_portal.h @@ -1,12 +1,17 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * Copyright 2018 NXP * */ +#ifndef _QBMAN_PORTAL_H_ +#define _QBMAN_PORTAL_H_ + #include "qbman_sys.h" #include +uint32_t qman_version; #define QMAN_REV_4000 0x04000000 #define QMAN_REV_4100 0x04010000 #define QMAN_REV_4101 0x04010001 @@ -14,13 +19,14 @@ /* All QBMan command and result structures use this "valid bit" encoding */ #define QB_VALID_BIT ((uint32_t)0x80) +/* All QBMan command use this "Read trigger bit" encoding */ +#define QB_RT_BIT ((uint32_t)0x100) + /* Management command result codes */ #define QBMAN_MC_RSLT_OK 0xf0 /* QBMan DQRR size is set at runtime in qbman_portal.c */ -#define QBMAN_EQCR_SIZE 8 - static inline uint8_t qm_cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last) { @@ -51,6 +57,10 @@ struct qbman_swp { #endif uint32_t valid_bit; /* 0x00 or 0x80 */ } mc; + /* Management response */ + struct { + uint32_t valid_bit; /* 0x00 or 0x80 */ + } mr; /* Push dequeues */ uint32_t sdq; /* Volatile dequeues */ @@ -87,6 +97,8 @@ struct qbman_swp { struct { uint32_t pi; uint32_t pi_vb; + uint32_t pi_ring_size; + uint32_t pi_mask; uint32_t ci; int available; } eqcr; @@ -141,4 +153,16 @@ static inline void *qbman_swp_mc_complete(struct qbman_swp *swp, void *cmd, * an inline) is necessary to work with different descriptor types and to work * correctly with const and non-const inputs (and similarly-qualified outputs). */ -#define qb_cl(d) (&(d)->donot_manipulate_directly[0]) +#define qb_cl(d) (&(d)->dont_manipulate_directly[0]) + +#ifdef RTE_ARCH_ARM64 + #define clean(p) \ + { asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); } + #define invalidate(p) \ + { asm volatile("dc ivac, %0" : : "r"(p) : "memory"); } +#else + #define clean(p) + #define invalidate(p) +#endif + +#endif diff --git a/drivers/bus/fslmc/qbman/qbman_sys.h b/drivers/bus/fslmc/qbman/qbman_sys.h index 2bd33ea5..d41af835 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys.h +++ b/drivers/bus/fslmc/qbman/qbman_sys.h @@ -18,11 +18,51 @@ * *not* to provide linux compatibility. */ +#ifndef _QBMAN_SYS_H_ +#define _QBMAN_SYS_H_ + #include "qbman_sys_decl.h" #define CENA_WRITE_ENABLE 0 #define CINH_WRITE_ENABLE 1 +/* CINH register offsets */ +#define QBMAN_CINH_SWP_EQCR_PI 0x800 +#define QBMAN_CINH_SWP_EQCR_CI 0x840 +#define QBMAN_CINH_SWP_EQAR 0x8c0 +#define QBMAN_CINH_SWP_CR_RT 0x900 +#define QBMAN_CINH_SWP_VDQCR_RT 0x940 +#define QBMAN_CINH_SWP_EQCR_AM_RT 0x980 +#define QBMAN_CINH_SWP_RCR_AM_RT 0x9c0 +#define QBMAN_CINH_SWP_DQPI 0xa00 +#define QBMAN_CINH_SWP_DQRR_ITR 0xa80 +#define QBMAN_CINH_SWP_DCAP 0xac0 +#define QBMAN_CINH_SWP_SDQCR 0xb00 +#define QBMAN_CINH_SWP_EQCR_AM_RT2 0xb40 +#define QBMAN_CINH_SWP_RCR_PI 0xc00 +#define QBMAN_CINH_SWP_RAR 0xcc0 +#define QBMAN_CINH_SWP_ISR 0xe00 +#define QBMAN_CINH_SWP_IER 0xe40 +#define QBMAN_CINH_SWP_ISDR 0xe80 +#define QBMAN_CINH_SWP_IIR 0xec0 +#define QBMAN_CINH_SWP_ITPR 0xf40 + +/* CENA register offsets */ +#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_CR 0x600 +#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((uint32_t)(vb) >> 1)) +#define QBMAN_CENA_SWP_VDQCR 0x780 +#define QBMAN_CENA_SWP_EQCR_CI 0x840 + +/* CENA register offsets in memory-backed mode */ +#define QBMAN_CENA_SWP_DQRR_MEM(n) (0x800 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_RCR_MEM(n) (0x1400 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_CR_MEM 0x1600 +#define QBMAN_CENA_SWP_RR_MEM 0x1680 +#define QBMAN_CENA_SWP_VDQCR_MEM 0x1780 + /* Debugging assists */ static inline void __hexdump(unsigned long start, unsigned long end, unsigned long p, size_t sz, const unsigned char *c) @@ -125,8 +165,8 @@ struct qbman_swp_sys { * place-holder. */ uint8_t *cena; - uint8_t __iomem *addr_cena; - uint8_t __iomem *addr_cinh; + uint8_t *addr_cena; + uint8_t *addr_cinh; uint32_t idx; enum qbman_eqcr_mode eqcr_mode; }; @@ -292,13 +332,16 @@ static inline void qbman_cena_prefetch(struct qbman_swp_sys *s, * qbman_portal.c. So use of it is declared locally here. */ #define QBMAN_CINH_SWP_CFG 0xd00 -#define QBMAN_CINH_SWP_CFG 0xd00 + #define SWP_CFG_DQRR_MF_SHIFT 20 #define SWP_CFG_EST_SHIFT 16 +#define SWP_CFG_CPBS_SHIFT 15 #define SWP_CFG_WN_SHIFT 14 #define SWP_CFG_RPM_SHIFT 12 #define SWP_CFG_DCM_SHIFT 10 #define SWP_CFG_EPM_SHIFT 8 +#define SWP_CFG_VPM_SHIFT 7 +#define SWP_CFG_CPM_SHIFT 6 #define SWP_CFG_SD_SHIFT 5 #define SWP_CFG_SP_SHIFT 4 #define SWP_CFG_SE_SHIFT 3 @@ -329,11 +372,20 @@ static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn, return reg; } +#define QMAN_RT_MODE 0x00000100 + +#define QMAN_REV_4000 0x04000000 +#define QMAN_REV_4100 0x04010000 +#define QMAN_REV_4101 0x04010001 +#define QMAN_REV_5000 0x05000000 +#define QMAN_REV_MASK 0xffff0000 + static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, const struct qbman_swp_desc *d, uint8_t dqrr_size) { uint32_t reg; + int i; #ifdef RTE_ARCH_64 uint8_t wn = CENA_WRITE_ENABLE; #else @@ -343,7 +395,7 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, s->addr_cena = d->cena_bar; s->addr_cinh = d->cinh_bar; s->idx = (uint32_t)d->idx; - s->cena = malloc(4096); + s->cena = malloc(64*1024); if (!s->cena) { pr_err("Could not allocate page for cena shadow\n"); return -1; @@ -358,12 +410,34 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); QBMAN_BUG_ON(reg); #endif + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) + memset(s->addr_cena, 0, 64*1024); + else { + /* Invalidate the portal memory. + * This ensures no stale cache lines + */ + for (i = 0; i < 0x1000; i += 64) + dccivac(s->addr_cena + i); + } + if (s->eqcr_mode == qman_eqcr_vb_array) - reg = qbman_set_swp_cfg(dqrr_size, wn, 0, 3, 2, 3, 1, 1, 1, 1, - 1, 1); - else - reg = qbman_set_swp_cfg(dqrr_size, wn, 1, 3, 2, 2, 1, 1, 1, 1, - 1, 1); + reg = qbman_set_swp_cfg(dqrr_size, wn, + 0, 3, 2, 3, 1, 1, 1, 1, 1, 1); + else { + if ((d->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) + reg = qbman_set_swp_cfg(dqrr_size, wn, + 1, 3, 2, 2, 1, 1, 1, 1, 1, 1); + else + reg = qbman_set_swp_cfg(dqrr_size, wn, + 1, 3, 2, 0, 1, 1, 1, 1, 1, 1); + } + + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { + reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */ + 1 << SWP_CFG_VPM_SHIFT | /* VDQCR read triggered mode */ + 1 << SWP_CFG_CPM_SHIFT; /* CR read triggered mode */ + } + qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg); reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); if (!reg) { @@ -371,6 +445,12 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, free(s->cena); return -1; } + + if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { + qbman_cinh_write(s, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE); + qbman_cinh_write(s, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE); + } + return 0; } @@ -378,3 +458,5 @@ static inline void qbman_swp_sys_finish(struct qbman_swp_sys *s) { free(s->cena); } + +#endif /* _QBMAN_SYS_H_ */ diff --git a/drivers/bus/fslmc/qbman/qbman_sys_decl.h b/drivers/bus/fslmc/qbman/qbman_sys_decl.h index fa6977fe..a29f5b46 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys_decl.h +++ b/drivers/bus/fslmc/qbman/qbman_sys_decl.h @@ -3,6 +3,9 @@ * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. * */ +#ifndef _QBMAN_SYS_DECL_H_ +#define _QBMAN_SYS_DECL_H_ + #include #include @@ -51,3 +54,4 @@ static inline void prefetch_for_store(void *p) RTE_SET_USED(p); } #endif +#endif /* _QBMAN_SYS_DECL_H_ */ diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map index fe45a113..dcc4e082 100644 --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map @@ -114,5 +114,18 @@ DPDK_18.05 { dpdmai_open; dpdmai_set_rx_queue; rte_dpaa2_free_dpci_dev; + rte_dpaa2_memsegs; } DPDK_18.02; + +DPDK_18.11 { + global: + + dpaa2_dqrr_size; + dpaa2_eqcr_size; + dpci_get_link_state; + dpci_get_opr; + dpci_get_peer_attributes; + dpci_set_opr; + +} DPDK_18.05; -- cgit 1.2.3-korg