/*- * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. * Copyright 2017 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the above-listed copyright holders nor the * names of any contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * GPL LICENSE SUMMARY * * ALTERNATIVELY, this software may be distributed under the terms of the * GNU General Public License ("GPL") as published by the Free Software * Foundation, either version 2 of that License or (at your option) any * later version. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "qman.h" #include /* Compilation constants */ #define DQRR_MAXFILL 15 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ #define IRQNAME "QMan portal %d" #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ /* maximum number of DQRR entries to process in qman_poll() */ #define FSL_QMAN_POLL_LIMIT 8 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about * inter-processor locking only. Note, FQLOCK() is always called either under a * local_irq_save() or from interrupt context - hence there's no need for irq * protection (and indeed, attempting to nest irq-protection doesn't work, as * the "irq en/disable" machinery isn't recursive...). */ #define FQLOCK(fq) \ do { \ struct qman_fq *__fq478 = (fq); \ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ spin_lock(&__fq478->fqlock); \ } while (0) #define FQUNLOCK(fq) \ do { \ struct qman_fq *__fq478 = (fq); \ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ spin_unlock(&__fq478->fqlock); \ } while (0) static inline void fq_set(struct qman_fq *fq, u32 mask) { dpaa_set_bits(mask, &fq->flags); } static inline void fq_clear(struct qman_fq *fq, u32 mask) { dpaa_clear_bits(mask, &fq->flags); } static inline int fq_isset(struct qman_fq *fq, u32 mask) { return fq->flags & mask; } static inline int fq_isclear(struct qman_fq *fq, u32 mask) { return !(fq->flags & mask); } struct qman_portal { struct qm_portal p; /* PORTAL_BITS_*** - dynamic, strictly internal */ unsigned long bits; /* interrupt sources processed by portal_isr(), configurable */ unsigned long irq_sources; u32 use_eqcr_ci_stashing; u32 slowpoll; /* only used when interrupts are off */ /* only 1 volatile dequeue at a time */ struct qman_fq *vdqcr_owned; u32 sdqcr; int dqrr_disable_ref; /* A portal-specific handler for DCP ERNs. If this is NULL, the global * handler is called instead. */ qman_cb_dc_ern cb_dc_ern; /* When the cpu-affine portal is activated, this is non-NULL */ const struct qm_portal_config *config; struct dpa_rbtree retire_table; char irqname[MAX_IRQNAME]; /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ struct qman_cgrs *cgrs; /* linked-list of CSCN handlers. */ struct list_head cgr_cbs; /* list lock */ spinlock_t cgr_lock; /* track if memory was allocated by the driver */ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ /* Keep a shadow copy of the DQRR on LE systems as the SW needs to * do byte swaps of DQRR read only memory. First entry must be aligned * to 2 ** 10 to ensure DQRR index calculations based shadow copy * address (6 bits for address shift + 4 bits for the DQRR size). */ struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] __attribute__((aligned(1024))); #endif }; /* Global handler for DCP ERNs. Used when the portal receiving the message does * not have a portal-specific handler. */ static qman_cb_dc_ern cb_dc_ern; static cpumask_t affine_mask; static DEFINE_SPINLOCK(affine_mask_lock); static u16 affine_channels[NR_CPUS]; static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal); static inline struct qman_portal *get_affine_portal(void) { return &RTE_PER_LCORE(qman_affine_portal); } /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux * retirement notifications (the fact they are sometimes h/w-consumed means that * contextB isn't always a s/w demux - and as we can't know which case it is * when looking at the notification, we have to use the slow lookup for all of * them). NB, it's possible to have multiple FQ objects refer to the same FQID * (though at most one of them should be the consumer), so this table isn't for * all FQs - FQs are added when retirement commands are issued, and removed when * they complete, which also massively reduces the size of this table. */ IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid); /* * This is what everything can wait on, even if it migrates to a different cpu * to the one whose affine portal it is waiting on. */ static DECLARE_WAIT_QUEUE_HEAD(affine_queue); static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) { int ret = fqtree_push(&p->retire_table, fq); if (ret) pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); return ret; } static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) { fqtree_del(&p->retire_table, fq); } static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) { return fqtree_find(&p->retire_table, fqid); } #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP static void **qman_fq_lookup_table; static size_t qman_fq_lookup_table_size; int qman_setup_fq_lookup_table(size_t num_entries) { num_entries++; /* Allocate 1 more entry since the first entry is not used */ qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *))); if (!qman_fq_lookup_table) { pr_err("QMan: Could not allocate fq lookup table\n"); return -ENOMEM; } memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *)); qman_fq_lookup_table_size = num_entries; pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n", qman_fq_lookup_table, (unsigned long)qman_fq_lookup_table_size); return 0; } /* global structure that maintains fq object mapping */ static DEFINE_SPINLOCK(fq_hash_table_lock); static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) { u32 i; spin_lock(&fq_hash_table_lock); /* Can't use index zero because this has special meaning * in context_b field. */ for (i = 1; i < qman_fq_lookup_table_size; i++) { if (qman_fq_lookup_table[i] == NULL) { *entry = i; qman_fq_lookup_table[i] = fq; spin_unlock(&fq_hash_table_lock); return 0; } } spin_unlock(&fq_hash_table_lock); return -ENOMEM; } static void clear_fq_table_entry(u32 entry) { spin_lock(&fq_hash_table_lock); DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); qman_fq_lookup_table[entry] = NULL; spin_unlock(&fq_hash_table_lock); } static inline struct qman_fq *get_fq_table_entry(u32 entry) { DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); return qman_fq_lookup_table[entry]; } #endif static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) { /* Byteswap the FQD to HW format */ fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); fqd->dest_wq = cpu_to_be16(fqd->dest_wq); fqd->ics_cred = cpu_to_be16(fqd->ics_cred); fqd->context_b = cpu_to_be32(fqd->context_b); fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); fqd->opaque_td = cpu_to_be16(fqd->opaque_td); } static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) { /* Byteswap the FQD to CPU format */ fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); fqd->dest_wq = be16_to_cpu(fqd->dest_wq); fqd->ics_cred = be16_to_cpu(fqd->ics_cred); fqd->context_b = be32_to_cpu(fqd->context_b); fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); } static inline void cpu_to_hw_fd(struct qm_fd *fd) { fd->addr = cpu_to_be40(fd->addr); fd->status = cpu_to_be32(fd->status); fd->opaque = cpu_to_be32(fd->opaque); } static inline void hw_fd_to_cpu(struct qm_fd *fd) { fd->addr = be40_to_cpu(fd->addr); fd->status = be32_to_cpu(fd->status); fd->opaque = be32_to_cpu(fd->opaque); } /* In the case that slow- and fast-path handling are both done by qman_poll() * (ie. because there is no interrupt handling), we ought to balance how often * we do the fast-path poll versus the slow-path poll. We'll use two decrementer * sources, so we call the fast poll 'n' times before calling the slow poll * once. The idle decrementer constant is used when the last slow-poll detected * no work to do, and the busy decrementer constant when the last slow-poll had * work to do. */ #define SLOW_POLL_IDLE 1000 #define SLOW_POLL_BUSY 10 static u32 __poll_portal_slow(struct qman_portal *p, u32 is); static inline unsigned int __poll_portal_fast(struct qman_portal *p, unsigned int poll_limit); /* Portal interrupt handler */ static irqreturn_t portal_isr(__always_unused int irq, void *ptr) { struct qman_portal *p = ptr; /* * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because * it could race against a Query Congestion State command also given * as part of the handling of this interrupt source. We mustn't * clear it a second time in this top-level function. */ u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); u32 is = qm_isr_status_read(&p->p) & p->irq_sources; /* DQRR-handling if it's interrupt-driven */ if (is & QM_PIRQ_DQRI) __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); /* Handling of anything else that's interrupt-driven */ clear |= __poll_portal_slow(p, is); qm_isr_status_clear(&p->p, clear); return IRQ_HANDLED; } /* This inner version is used privately by qman_create_affine_portal(), as well * as by the exported qman_stop_dequeues(). */ static inline void qman_stop_dequeues_ex(struct qman_portal *p) { if (!(p->dqrr_disable_ref++)) qm_dqrr_set_maxfill(&p->p, 0); } static int drain_mr_fqrni(struct qm_portal *p) { const struct qm_mr_entry *msg; loop: msg = qm_mr_current(p); if (!msg) { /* * if MR was full and h/w had other FQRNI entries to produce, we * need to allow it time to produce those entries once the * existing entries are consumed. A worst-case situation * (fully-loaded system) means h/w sequencers may have to do 3-4 * other things before servicing the portal's MR pump, each of * which (if slow) may take ~50 qman cycles (which is ~200 * processor cycles). So rounding up and then multiplying this * worst-case estimate by a factor of 10, just to be * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume * one entry at a time, so h/w has an opportunity to produce new * entries well before the ring has been fully consumed, so * we're being *really* paranoid here. */ u64 now, then = mfatb(); do { now = mfatb(); } while ((then + 10000) > now); msg = qm_mr_current(p); if (!msg) return 0; } if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { /* We aren't draining anything but FQRNIs */ pr_err("Found verb 0x%x in MR\n", msg->ern.verb); return -1; } qm_mr_next(p); qm_mr_cci_consume(p, 1); goto loop; } static inline int qm_eqcr_init(struct qm_portal *portal, enum qm_eqcr_pmode pmode, unsigned int eq_stash_thresh, int eq_stash_prio) { /* This use of 'register', as well as all other occurrences, is because * it has been observed to generate much faster code with gcc than is * otherwise the case. */ register struct qm_eqcr *eqcr = &portal->eqcr; u32 cfg; u8 pi; eqcr->ring = portal->addr.ce + QM_CL_EQCR; eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); qm_cl_invalidate(EQCR_CI); pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); eqcr->cursor = eqcr->ring + pi; eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? QM_EQCR_VERB_VBIT : 0; eqcr->available = QM_EQCR_SIZE - 1 - qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); eqcr->ithresh = qm_in(EQCR_ITR); #ifdef RTE_LIBRTE_DPAA_HWDEBUG eqcr->busy = 0; eqcr->pmode = pmode; #endif cfg = (qm_in(CFG) & 0x00ffffff) | (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ (eq_stash_prio << 26) | /* QCSP_CFG: EP */ ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ qm_out(CFG, cfg); return 0; } static inline void qm_eqcr_finish(struct qm_portal *portal) { register struct qm_eqcr *eqcr = &portal->eqcr; u8 pi, ci; u32 cfg; /* * Disable EQCI stashing because the QMan only * presents the value it previously stashed to * maintain coherency. Setting the stash threshold * to 1 then 0 ensures that QMan has resyncronized * its internal copy so that the portal is clean * when it is reinitialized in the future */ cfg = (qm_in(CFG) & 0x0fffffff) | (1 << 28); /* QCSP_CFG: EST */ qm_out(CFG, cfg); cfg &= 0x0fffffff; /* stash threshold = 0 */ qm_out(CFG, cfg); pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); /* Refresh EQCR CI cache value */ qm_cl_invalidate(EQCR_CI); eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); #ifdef RTE_LIBRTE_DPAA_HWDEBUG DPAA_ASSERT(!eqcr->busy); #endif if (pi != EQCR_PTR2IDX(eqcr->cursor)) pr_crit("losing uncommitted EQCR entries\n"); if (ci != eqcr->ci) pr_crit("missing existing EQCR completions\n"); if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) pr_crit("EQCR destroyed unquiesced\n"); } static inline int qm_dqrr_init(struct qm_portal *portal, __maybe_unused const struct qm_portal_config *config, enum qm_dqrr_dmode dmode, __maybe_unused enum qm_dqrr_pmode pmode, enum qm_dqrr_cmode cmode, u8 max_fill) { register struct qm_dqrr *dqrr = &portal->dqrr; u32 cfg; /* Make sure the DQRR will be idle when we enable */ qm_out(DQRR_SDQCR, 0); qm_out(DQRR_VDQCR, 0); qm_out(DQRR_PDQCR, 0); dqrr->ring = portal->addr.ce + QM_CL_DQRR; dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); dqrr->cursor = dqrr->ring + dqrr->ci; dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? QM_DQRR_VERB_VBIT : 0; dqrr->ithresh = qm_in(DQRR_ITR); #ifdef RTE_LIBRTE_DPAA_HWDEBUG dqrr->dmode = dmode; dqrr->pmode = pmode; dqrr->cmode = cmode; #endif /* Invalidate every ring entry before beginning */ for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) dccivac(qm_cl(dqrr->ring, cfg)); cfg = (qm_in(CFG) & 0xff000f00) | ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ ((dmode & 1) << 18) | /* DP */ ((cmode & 3) << 16) | /* DCM */ 0xa0 | /* RE+SE */ (0 ? 0x40 : 0) | /* Ignore RP */ (0 ? 0x10 : 0); /* Ignore SP */ qm_out(CFG, cfg); qm_dqrr_set_maxfill(portal, max_fill); return 0; } static inline void qm_dqrr_finish(struct qm_portal *portal) { __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; #ifdef RTE_LIBRTE_DPAA_HWDEBUG if ((dqrr->cmode != qm_dqrr_cdc) && (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) pr_crit("Ignoring completed DQRR entries\n"); #endif } static inline int qm_mr_init(struct qm_portal *portal, __maybe_unused enum qm_mr_pmode pmode, enum qm_mr_cmode cmode) { register struct qm_mr *mr = &portal->mr; u32 cfg; mr->ring = portal->addr.ce + QM_CL_MR; mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); mr->cursor = mr->ring + mr->ci; mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; mr->ithresh = qm_in(MR_ITR); #ifdef RTE_LIBRTE_DPAA_HWDEBUG mr->pmode = pmode; mr->cmode = cmode; #endif cfg = (qm_in(CFG) & 0xfffff0ff) | ((cmode & 1) << 8); /* QCSP_CFG:MM */ qm_out(CFG, cfg); return 0; } static inline void qm_mr_pvb_update(struct qm_portal *portal) { register struct qm_mr *mr = &portal->mr; const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); #ifdef RTE_LIBRTE_DPAA_HWDEBUG DPAA_ASSERT(mr->pmode == qm_mr_pvb); #endif /* when accessing 'verb', use __raw_readb() to ensure that compiler * inlining doesn't try to optimise out "excess reads". */ if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) { mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); if (!mr->pi) mr->vbit ^= QM_MR_VERB_VBIT; mr->fill++; res = MR_INC(res); } dcbit_ro(res); } static inline struct qman_portal *qman_create_portal( struct qman_portal *portal, const struct qm_portal_config *c, const struct qman_cgrs *cgrs) { struct qm_portal *p; char buf[16]; int ret; u32 isdr; p = &portal->p; portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0); /* * prep the low-level portal struct with the mapped addresses from the * config, everything that follows depends on it and "config" is more * for (de)reference */ p->addr.ce = c->addr_virt[DPAA_PORTAL_CE]; p->addr.ci = c->addr_virt[DPAA_PORTAL_CI]; /* * If CI-stashing is used, the current defaults use a threshold of 3, * and stash with high-than-DQRR priority. */ if (qm_eqcr_init(p, qm_eqcr_pvb, portal->use_eqcr_ci_stashing ? 3 : 0, 1)) { pr_err("Qman EQCR initialisation failed\n"); goto fail_eqcr; } if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb, qm_dqrr_cdc, DQRR_MAXFILL)) { pr_err("Qman DQRR initialisation failed\n"); goto fail_dqrr; } if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) { pr_err("Qman MR initialisation failed\n"); goto fail_mr; } if (qm_mc_init(p)) { pr_err("Qman MC initialisation failed\n"); goto fail_mc; } /* static interrupt-gating controls */ qm_dqrr_set_ithresh(p, 0); qm_mr_set_ithresh(p, 0); qm_isr_set_iperiod(p, 0); portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); if (!portal->cgrs) goto fail_cgrs; /* initial snapshot is no-depletion */ qman_cgrs_init(&portal->cgrs[1]); if (cgrs) portal->cgrs[0] = *cgrs; else /* if the given mask is NULL, assume all CGRs can be seen */ qman_cgrs_fill(&portal->cgrs[0]); INIT_LIST_HEAD(&portal->cgr_cbs); spin_lock_init(&portal->cgr_lock); portal->bits = 0; portal->slowpoll = 0; portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; portal->dqrr_disable_ref = 0; portal->cb_dc_ern = NULL; sprintf(buf, "qportal-%d", c->channel); dpa_rbtree_init(&portal->retire_table); isdr = 0xffffffff; qm_isr_disable_write(p, isdr); portal->irq_sources = 0; qm_isr_enable_write(p, portal->irq_sources); qm_isr_status_clear(p, 0xffffffff); snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) { pr_err("request_irq() failed\n"); goto fail_irq; } /* Need EQCR to be empty before continuing */ isdr &= ~QM_PIRQ_EQCI; qm_isr_disable_write(p, isdr); ret = qm_eqcr_get_fill(p); if (ret) { pr_err("Qman EQCR unclean\n"); goto fail_eqcr_empty; } isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI); qm_isr_disable_write(p, isdr); if (qm_dqrr_current(p)) { pr_err("Qman DQRR unclean\n"); qm_dqrr_cdc_consume_n(p, 0xffff); } if (qm_mr_current(p) && drain_mr_fqrni(p)) { /* special handling, drain just in case it's a few FQRNIs */ if (drain_mr_fqrni(p)) goto fail_dqrr_mr_empty; } /* Success */ portal->config = c; qm_isr_disable_write(p, 0); qm_isr_uninhibit(p); /* Write a sane SDQCR */ qm_dqrr_sdqcr_set(p, portal->sdqcr); return portal; fail_dqrr_mr_empty: fail_eqcr_empty: free_irq(c->irq, portal); fail_irq: kfree(portal->cgrs); spin_lock_destroy(&portal->cgr_lock); fail_cgrs: qm_mc_finish(p); fail_mc: qm_mr_finish(p); fail_mr: qm_dqrr_finish(p); fail_dqrr: qm_eqcr_finish(p); fail_eqcr: return NULL; } struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c, const struct qman_cgrs *cgrs) { struct qman_portal *res; struct qman_portal *portal = get_affine_portal(); /* A criteria for calling this function (from qman_driver.c) is that * we're already affine to the cpu and won't schedule onto another cpu. */ res = qman_create_portal(portal, c, cgrs); if (res) { spin_lock(&affine_mask_lock); CPU_SET(c->cpu, &affine_mask); affine_channels[c->cpu] = c->channel; spin_unlock(&affine_mask_lock); } return res; } static inline void qman_destroy_portal(struct qman_portal *qm) { const struct qm_portal_config *pcfg; /* Stop dequeues on the portal */ qm_dqrr_sdqcr_set(&qm->p, 0); /* * NB we do this to "quiesce" EQCR. If we add enqueue-completions or * something related to QM_PIRQ_EQCI, this may need fixing. * Also, due to the prefetching model used for CI updates in the enqueue * path, this update will only invalidate the CI cacheline *after* * working on it, so we need to call this twice to ensure a full update * irrespective of where the enqueue processing was at when the teardown * began. */ qm_eqcr_cce_update(&qm->p); qm_eqcr_cce_update(&qm->p); pcfg = qm->config; free_irq(pcfg->irq, qm); kfree(qm->cgrs); qm_mc_finish(&qm->p); qm_mr_finish(&qm->p); qm_dqrr_finish(&qm->p); qm_eqcr_finish(&qm->p); qm->config = NULL; spin_lock_destroy(&qm->cgr_lock); } const struct qm_portal_config *qman_destroy_affine_portal(void) { /* We don't want to redirect if we're a slave, use "raw" */ struct qman_portal *qm = get_affine_portal(); const struct qm_portal_config *pcfg; int cpu; pcfg = qm->config; cpu = pcfg->cpu; qman_destroy_portal(qm); spin_lock(&affine_mask_lock); CPU_CLR(cpu, &affine_mask); spin_unlock(&affine_mask_lock); return pcfg; } int qman_get_portal_index(void) { struct qman_portal *p = get_affine_portal(); return p->config->index; } /* Inline helper to reduce nesting in __poll_portal_slow() */ static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, const struct qm_mr_entry *msg, u8 verb) { FQLOCK(fq); switch (verb) { case QM_MR_VERB_FQRL: DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); fq_clear(fq, QMAN_FQ_STATE_ORL); table_del_fq(p, fq); break; case QM_MR_VERB_FQRN: DPAA_ASSERT((fq->state == qman_fq_state_parked) || (fq->state == qman_fq_state_sched)); DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); fq_clear(fq, QMAN_FQ_STATE_CHANGING); if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) fq_set(fq, QMAN_FQ_STATE_NE); if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) fq_set(fq, QMAN_FQ_STATE_ORL); else table_del_fq(p, fq); fq->state = qman_fq_state_retired; break; case QM_MR_VERB_FQPN: DPAA_ASSERT(fq->state == qman_fq_state_sched); DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); fq->state = qman_fq_state_parked; } FQUNLOCK(fq); } static u32 __poll_portal_slow(struct qman_portal *p, u32 is) { const struct qm_mr_entry *msg; struct qm_mr_entry swapped_msg; if (is & QM_PIRQ_CSCI) { struct qman_cgrs rr, c; struct qm_mc_result *mcr; struct qman_cgr *cgr; spin_lock(&p->cgr_lock); /* * The CSCI bit must be cleared _before_ issuing the * Query Congestion State command, to ensure that a long * CGR State Change callback cannot miss an intervening * state change. */ qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); qm_mc_start(&p->p); qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); /* mask out the ones I'm not interested in */ qman_cgrs_and(&rr, (const struct qman_cgrs *) &mcr->querycongestion.state, &p->cgrs[0]); /* check previous snapshot for delta, enter/exit congestion */ qman_cgrs_xor(&c, &rr, &p->cgrs[1]); /* update snapshot */ qman_cgrs_cp(&p->cgrs[1], &rr); /* Invoke callback */ list_for_each_entry(cgr, &p->cgr_cbs, node) if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); spin_unlock(&p->cgr_lock); } if (is & QM_PIRQ_EQRI) { qm_eqcr_cce_update(&p->p); qm_eqcr_set_ithresh(&p->p, 0); wake_up(&affine_queue); } if (is & QM_PIRQ_MRI) { struct qman_fq *fq; u8 verb, num = 0; mr_loop: qm_mr_pvb_update(&p->p); msg = qm_mr_current(&p->p); if (!msg) goto mr_done; swapped_msg = *msg; hw_fd_to_cpu(&swapped_msg.ern.fd); verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK; /* The message is a software ERN iff the 0x20 bit is set */ if (verb & 0x20) { switch (verb) { case QM_MR_VERB_FQRNI: /* nada, we drop FQRNIs on the floor */ break; case QM_MR_VERB_FQRN: case QM_MR_VERB_FQRL: /* Lookup in the retirement table */ fq = table_find_fq(p, be32_to_cpu(msg->fq.fqid)); DPAA_BUG_ON(!fq); fq_state_change(p, fq, &swapped_msg, verb); if (fq->cb.fqs) fq->cb.fqs(p, fq, &swapped_msg); break; case QM_MR_VERB_FQPN: /* Parked */ #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP fq = get_fq_table_entry( be32_to_cpu(msg->fq.contextB)); #else fq = (void *)(uintptr_t) be32_to_cpu(msg->fq.contextB); #endif fq_state_change(p, fq, msg, verb); if (fq->cb.fqs) fq->cb.fqs(p, fq, &swapped_msg); break; case QM_MR_VERB_DC_ERN: /* DCP ERN */ if (p->cb_dc_ern) p->cb_dc_ern(p, msg); else if (cb_dc_ern) cb_dc_ern(p, msg); else { static int warn_once; if (!warn_once) { pr_crit("Leaking DCP ERNs!\n"); warn_once = 1; } } break; default: pr_crit("Invalid MR verb 0x%02x\n", verb); } } else { /* Its a software ERN */ #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); #else fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); #endif fq->cb.ern(p, fq, &swapped_msg); } num++; qm_mr_next(&p->p); goto mr_loop; mr_done: qm_mr_cci_consume(&p->p, num); } /* * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific * processing. If that interrupt source has meanwhile been re-asserted, * we mustn't clear it here (or in the top-level interrupt handler). */ return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); } /* * remove some slowish-path stuff from the "fast path" and make sure it isn't * inlined. */ static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) { p->vdqcr_owned = NULL; FQLOCK(fq); fq_clear(fq, QMAN_FQ_STATE_VDQCR); FQUNLOCK(fq); wake_up(&affine_queue); } /* * The only states that would conflict with other things if they ran at the * same time on the same cpu are: * * (i) setting/clearing vdqcr_owned, and * (ii) clearing the NE (Not Empty) flag. * * Both are safe. Because; * * (i) this clearing can only occur after qman_set_vdq() has set the * vdqcr_owned field (which it does before setting VDQCR), and * qman_volatile_dequeue() blocks interrupts and preemption while this is * done so that we can't interfere. * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as * with (i) that API prevents us from interfering until it's safe. * * The good thing is that qman_set_vdq() and qman_retire_fq() run far * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett * advantage comes from this function not having to "lock" anything at all. * * Note also that the callbacks are invoked at points which are safe against the * above potential conflicts, but that this function itself is not re-entrant * (this is because the function tracks one end of each FIFO in the portal and * we do *not* want to lock that). So the consequence is that it is safe for * user callbacks to call into any QMan API. */ static inline unsigned int __poll_portal_fast(struct qman_portal *p, unsigned int poll_limit) { const struct qm_dqrr_entry *dq; struct qman_fq *fq; enum qman_cb_dqrr_result res; unsigned int limit = 0; #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ struct qm_dqrr_entry *shadow; #endif do { qm_dqrr_pvb_update(&p->p); dq = qm_dqrr_current(&p->p); if (!dq) break; #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ /* If running on an LE system the fields of the * dequeue entry must be swapper. Because the * QMan HW will ignore writes the DQRR entry is * copied and the index stored within the copy */ shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; *shadow = *dq; dq = shadow; shadow->fqid = be32_to_cpu(shadow->fqid); shadow->contextB = be32_to_cpu(shadow->contextB); shadow->seqnum = be16_to_cpu(shadow->seqnum); hw_fd_to_cpu(&shadow->fd); #endif if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { /* * VDQCR: don't trust context_b as the FQ may have * been configured for h/w consumption and we're * draining it post-retirement. */ fq = p->vdqcr_owned; /* * We only set QMAN_FQ_STATE_NE when retiring, so we * only need to check for clearing it when doing * volatile dequeues. It's one less thing to check * in the critical path (SDQCR). */ if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) fq_clear(fq, QMAN_FQ_STATE_NE); /* * This is duplicated from the SDQCR code, but we * have stuff to do before *and* after this callback, * and we don't want multiple if()s in the critical * path (SDQCR). */ res = fq->cb.dqrr(p, fq, dq); if (res == qman_cb_dqrr_stop) break; /* Check for VDQCR completion */ if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) clear_vdqcr(p, fq); } else { /* SDQCR: context_b points to the FQ */ #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP fq = get_fq_table_entry(dq->contextB); #else fq = (void *)(uintptr_t)dq->contextB; #endif /* Now let the callback do its stuff */ res = fq->cb.dqrr(p, fq, dq); /* * The callback can request that we exit without * consuming this entry nor advancing; */ if (res == qman_cb_dqrr_stop) break; } /* Interpret 'dq' from a driver perspective. */ /* * Parking isn't possible unless HELDACTIVE was set. NB, * FORCEELIGIBLE implies HELDACTIVE, so we only need to * check for HELDACTIVE to cover both. */ DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || (res != qman_cb_dqrr_park)); /* just means "skip it, I'll consume it myself later on" */ if (res != qman_cb_dqrr_defer) qm_dqrr_cdc_consume_1ptr(&p->p, dq, res == qman_cb_dqrr_park); /* Move forward */ qm_dqrr_next(&p->p); /* * Entry processed and consumed, increment our counter. The * callback can request that we exit after consuming the * entry, and we also exit if we reach our processing limit, * so loop back only if neither of these conditions is met. */ } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop); return limit; } u16 qman_affine_channel(int cpu) { if (cpu < 0) { struct qman_portal *portal = get_affine_portal(); cpu = portal->config->cpu; } DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask)); return affine_channels[cpu]; } struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq) { struct qman_portal *p = get_affine_portal(); const struct qm_dqrr_entry *dq; #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ struct qm_dqrr_entry *shadow; #endif qm_dqrr_pvb_update(&p->p); dq = qm_dqrr_current(&p->p); if (!dq) return NULL; if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) { /* Invalid DQRR - put the portal and consume the DQRR. * Return NULL to user as no packet is seen. */ qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq); return NULL; } #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; *shadow = *dq; dq = shadow; shadow->fqid = be32_to_cpu(shadow->fqid); shadow->contextB = be32_to_cpu(shadow->contextB); shadow->seqnum = be16_to_cpu(shadow->seqnum); hw_fd_to_cpu(&shadow->fd); #endif if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) fq_clear(fq, QMAN_FQ_STATE_NE); return (struct qm_dqrr_entry *)dq; } void qman_dqrr_consume(struct qman_fq *fq, struct qm_dqrr_entry *dq) { struct qman_portal *p = get_affine_portal(); if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) clear_vdqcr(p, fq); qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0); qm_dqrr_next(&p->p); } int qman_poll_dqrr(unsigned int limit) { struct qman_portal *p = get_affine_portal(); int ret; ret = __poll_portal_fast(p, limit); return ret; } void qman_poll(void) { struct qman_portal *p = get_affine_portal(); if ((~p->irq_sources) & QM_PIRQ_SLOW) { if (!(p->slowpoll--)) { u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; u32 active = __poll_portal_slow(p, is); if (active) { qm_isr_status_clear(&p->p, active); p->slowpoll = SLOW_POLL_BUSY; } else p->slowpoll = SLOW_POLL_IDLE; } } if ((~p->irq_sources) & QM_PIRQ_DQRI) __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); } void qman_stop_dequeues(void) { struct qman_portal *p = get_affine_portal(); qman_stop_dequeues_ex(p); } void qman_start_dequeues(void) { struct qman_portal *p = get_affine_portal(); DPAA_ASSERT(p->dqrr_disable_ref > 0); if (!(--p->dqrr_disable_ref)) qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); } void qman_static_dequeue_add(u32 pools) { struct qman_portal *p = get_affine_portal(); pools &= p->config->pools; p->sdqcr |= pools; qm_dqrr_sdqcr_set(&p->p, p->sdqcr); } void qman_static_dequeue_del(u32 pools) { struct qman_portal *p = get_affine_portal(); pools &= p->config->pools; p->sdqcr &= ~pools; qm_dqrr_sdqcr_set(&p->p, p->sdqcr); } u32 qman_static_dequeue_get(void) { struct qman_portal *p = get_affine_portal(); return p->sdqcr; } void qman_dca(struct qm_dqrr_entry *dq, int park_request) { struct qman_portal *p = get_affine_portal(); qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); } /* Frame queue API */ static const char *mcr_result_str(u8 result) { switch (result) { case QM_MCR_RESULT_NULL: return "QM_MCR_RESULT_NULL"; case QM_MCR_RESULT_OK: return "QM_MCR_RESULT_OK"; case QM_MCR_RESULT_ERR_FQID: return "QM_MCR_RESULT_ERR_FQID"; case QM_MCR_RESULT_ERR_FQSTATE: return "QM_MCR_RESULT_ERR_FQSTATE"; case QM_MCR_RESULT_ERR_NOTEMPTY: return "QM_MCR_RESULT_ERR_NOTEMPTY"; case QM_MCR_RESULT_PENDING: return "QM_MCR_RESULT_PENDING"; case QM_MCR_RESULT_ERR_BADCOMMAND: return "QM_MCR_RESULT_ERR_BADCOMMAND"; } return ""; } int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) { struct qm_fqd fqd; struct qm_mcr_queryfq_np np; struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p; if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { int ret = qman_alloc_fqid(&fqid); if (ret) return ret; } spin_lock_init(&fq->fqlock); fq->fqid = fqid; fq->flags = flags; fq->state = qman_fq_state_oos; fq->cgr_groupid = 0; #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) { pr_info("Find empty table entry failed\n"); return -ENOMEM; } #endif if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) return 0; /* Everything else is AS_IS support */ p = get_affine_portal(); mcc = qm_mc_start(&p->p); mcc->queryfq.fqid = cpu_to_be32(fqid); qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); if (mcr->result != QM_MCR_RESULT_OK) { pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); goto err; } fqd = mcr->queryfq.fqd; hw_fqd_to_cpu(&fqd); mcc = qm_mc_start(&p->p); mcc->queryfq_np.fqid = cpu_to_be32(fqid); qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); if (mcr->result != QM_MCR_RESULT_OK) { pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); goto err; } np = mcr->queryfq_np; /* Phew, have queryfq and queryfq_np results, stitch together * the FQ object from those. */ fq->cgr_groupid = fqd.cgid; switch (np.state & QM_MCR_NP_STATE_MASK) { case QM_MCR_NP_STATE_OOS: break; case QM_MCR_NP_STATE_RETIRED: fq->state = qman_fq_state_retired; if (np.frm_cnt) fq_set(fq, QMAN_FQ_STATE_NE); break; case QM_MCR_NP_STATE_TEN_SCHED: case QM_MCR_NP_STATE_TRU_SCHED: case QM_MCR_NP_STATE_ACTIVE: fq->state = qman_fq_state_sched; if (np.state & QM_MCR_NP_STATE_R) fq_set(fq, QMAN_FQ_STATE_CHANGING); break; case QM_MCR_NP_STATE_PARKED: fq->state = qman_fq_state_parked; break; default: DPAA_ASSERT(NULL == "invalid FQ state"); } if (fqd.fq_ctrl & QM_FQCTRL_CGE) fq->state |= QMAN_FQ_STATE_CGR_EN; return 0; err: if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) qman_release_fqid(fqid); return -EIO; } void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) { /* * We don't need to lock the FQ as it is a pre-condition that the FQ be * quiesced. Instead, run some checks. */ switch (fq->state) { case qman_fq_state_parked: DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); /* Fallthrough */ case qman_fq_state_oos: if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) qman_release_fqid(fq->fqid); #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP clear_fq_table_entry(fq->key); #endif return; default: break; } DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); } u32 qman_fq_fqid(struct qman_fq *fq) { return fq->fqid; } void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) { if (state) *state = fq->state; if (flags) *flags = fq->flags; } int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p; u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; if ((fq->state != qman_fq_state_oos) && (fq->state != qman_fq_state_parked)) return -EINVAL; #ifdef RTE_LIBRTE_DPAA_HWDEBUG if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) return -EINVAL; #endif if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { /* And can't be set at the same time as TDTHRESH */ if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) return -EINVAL; } /* Issue an INITFQ_[PARKED|SCHED] management command */ p = get_affine_portal(); FQLOCK(fq); if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || ((fq->state != qman_fq_state_oos) && (fq->state != qman_fq_state_parked)))) { FQUNLOCK(fq); return -EBUSY; } mcc = qm_mc_start(&p->p); if (opts) mcc->initfq = *opts; mcc->initfq.fqid = cpu_to_be32(fq->fqid); mcc->initfq.count = 0; /* * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a * demux pointer. Otherwise, the caller-provided value is allowed to * stand, don't overwrite it. */ if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { dma_addr_t phys_fq; mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP mcc->initfq.fqd.context_b = fq->key; #else mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; #endif /* * and the physical address - NB, if the user wasn't trying to * set CONTEXTA, clear the stashing settings. */ if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; memset(&mcc->initfq.fqd.context_a, 0, sizeof(mcc->initfq.fqd.context_a)); } else { phys_fq = rte_mem_virt2iova(fq); qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); } } if (flags & QMAN_INITFQ_FLAG_LOCAL) { mcc->initfq.fqd.dest.channel = p->config->channel; if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; mcc->initfq.fqd.dest.wq = 4; } } mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); cpu_to_hw_fqd(&mcc->initfq.fqd); qm_mc_commit(&p->p, myverb); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); res = mcr->result; if (res != QM_MCR_RESULT_OK) { FQUNLOCK(fq); return -EIO; } if (opts) { if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) fq_set(fq, QMAN_FQ_STATE_CGR_EN); else fq_clear(fq, QMAN_FQ_STATE_CGR_EN); } if (opts->we_mask & QM_INITFQ_WE_CGID) fq->cgr_groupid = opts->fqd.cgid; } fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? qman_fq_state_sched : qman_fq_state_parked; FQUNLOCK(fq); return 0; } int qman_schedule_fq(struct qman_fq *fq) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p; int ret = 0; u8 res; if (fq->state != qman_fq_state_parked) return -EINVAL; #ifdef RTE_LIBRTE_DPAA_HWDEBUG if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) return -EINVAL; #endif /* Issue a ALTERFQ_SCHED management command */ p = get_affine_portal(); FQLOCK(fq); if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || (fq->state != qman_fq_state_parked))) { ret = -EBUSY; goto out; } mcc = qm_mc_start(&p->p); mcc->alterfq.fqid = cpu_to_be32(fq->fqid); qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); res = mcr->result; if (res != QM_MCR_RESULT_OK) { ret = -EIO; goto out; } fq->state = qman_fq_state_sched; out: FQUNLOCK(fq); return ret; } int qman_retire_fq(struct qman_fq *fq, u32 *flags) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p; int rval; u8 res; if ((fq->state != qman_fq_state_parked) && (fq->state != qman_fq_state_sched)) return -EINVAL; #ifdef RTE_LIBRTE_DPAA_HWDEBUG if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) return -EINVAL; #endif p = get_affine_portal(); FQLOCK(fq); if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || (fq->state == qman_fq_state_retired) || (fq->state == qman_fq_state_oos))) { rval = -EBUSY; goto out; } rval = table_push_fq(p, fq); if (rval) goto out; mcc = qm_mc_start(&p->p); mcc->alterfq.fqid = cpu_to_be32(fq->fqid); qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); res = mcr->result; /* * "Elegant" would be to treat OK/PENDING the same way; set CHANGING, * and defer the flags until FQRNI or FQRN (respectively) show up. But * "Friendly" is to process OK immediately, and not set CHANGING. We do * friendly, otherwise the caller doesn't necessarily have a fully * "retired" FQ on return even if the retirement was immediate. However * this does mean some code duplication between here and * fq_state_change(). */ if (likely(res == QM_MCR_RESULT_OK)) { rval = 0; /* Process 'fq' right away, we'll ignore FQRNI */ if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) fq_set(fq, QMAN_FQ_STATE_NE); if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) fq_set(fq, QMAN_FQ_STATE_ORL); else table_del_fq(p, fq); if (flags) *flags = fq->flags; fq->state = qman_fq_state_retired; if (fq->cb.fqs) { /* * Another issue with supporting "immediate" retirement * is that we're forced to drop FQRNIs, because by the * time they're seen it may already be "too late" (the * fq may have been OOS'd and free()'d already). But if * the upper layer wants a callback whether it's * immediate or not, we have to fake a "MR" entry to * look like an FQRNI... */ struct qm_mr_entry msg; msg.ern.verb = QM_MR_VERB_FQRNI; msg.fq.fqs = mcr->alterfq.fqs; msg.fq.fqid = fq->fqid; #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP msg.fq.contextB = fq->key; #else msg.fq.contextB = (u32)(uintptr_t)fq; #endif fq->cb.fqs(p, fq, &msg); } } else if (res == QM_MCR_RESULT_PENDING) { rval = 1; fq_set(fq, QMAN_FQ_STATE_CHANGING); } else { rval = -EIO; table_del_fq(p, fq); } out: FQUNLOCK(fq); return rval; } int qman_oos_fq(struct qman_fq *fq) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p; int ret = 0; u8 res; if (fq->state != qman_fq_state_retired) return -EINVAL; #ifdef RTE_LIBRTE_DPAA_HWDEBUG if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) return -EINVAL; #endif p = get_affine_portal(); FQLOCK(fq); if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || (fq->state != qman_fq_state_retired))) { ret = -EBUSY; goto out; } mcc = qm_mc_start(&p->p); mcc->alterfq.fqid = cpu_to_be32(fq->fqid); qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); res = mcr->result; if (res != QM_MCR_RESULT_OK) { ret = -EIO; goto out; } fq->state = qman_fq_state_oos; out: FQUNLOCK(fq); return ret; } int qman_fq_flow_control(struct qman_fq *fq, int xon) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p; int ret = 0; u8 res; u8 myverb; if ((fq->state == qman_fq_state_oos) || (fq->state == qman_fq_state_retired) || (fq->state == qman_fq_state_parked)) return -EINVAL; #ifdef RTE_LIBRTE_DPAA_HWDEBUG if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) return -EINVAL; #endif /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ p = get_affine_portal(); FQLOCK(fq); if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || (fq->state == qman_fq_state_parked) || (fq->state == qman_fq_state_oos) || (fq->state == qman_fq_state_retired))) { ret = -EBUSY; goto out; } mcc = qm_mc_start(&p->p); mcc->alterfq.fqid = fq->fqid; mcc->alterfq.count = 0; myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; qm_mc_commit(&p->p, myverb); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); res = mcr->result; if (res != QM_MCR_RESULT_OK) { ret = -EIO; goto out; } out: FQUNLOCK(fq); return ret; } int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); u8 res; mcc = qm_mc_start(&p->p); mcc->queryfq.fqid = cpu_to_be32(fq->fqid); qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); res = mcr->result; if (res == QM_MCR_RESULT_OK) *fqd = mcr->queryfq.fqd; hw_fqd_to_cpu(fqd); if (res != QM_MCR_RESULT_OK) return -EIO; return 0; } int qman_query_fq_has_pkts(struct qman_fq *fq) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); int ret = 0; u8 res; mcc = qm_mc_start(&p->p); mcc->queryfq.fqid = cpu_to_be32(fq->fqid); qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); res = mcr->result; if (res == QM_MCR_RESULT_OK) ret = !!mcr->queryfq_np.frm_cnt; return ret; } int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); u8 res; mcc = qm_mc_start(&p->p); mcc->queryfq.fqid = cpu_to_be32(fq->fqid); qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); res = mcr->result; if (res == QM_MCR_RESULT_OK) { *np = mcr->queryfq_np; np->fqd_link = be24_to_cpu(np->fqd_link); np->odp_seq = be16_to_cpu(np->odp_seq); np->orp_nesn = be16_to_cpu(np->orp_nesn); np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); np->ics_surp = be16_to_cpu(np->ics_surp); np->byte_cnt = be32_to_cpu(np->byte_cnt); np->frm_cnt = be24_to_cpu(np->frm_cnt); np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); np->od1_sfdr = be16_to_cpu(np->od1_sfdr); np->od2_sfdr = be16_to_cpu(np->od2_sfdr); np->od3_sfdr = be16_to_cpu(np->od3_sfdr); } if (res == QM_MCR_RESULT_ERR_FQID) return -ERANGE; else if (res != QM_MCR_RESULT_OK) return -EIO; return 0; } int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); u8 res, myverb; myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : QM_MCR_VERB_QUERYWQ; mcc = qm_mc_start(&p->p); mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); qm_mc_commit(&p->p, myverb); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); res = mcr->result; if (res == QM_MCR_RESULT_OK) { int i, array_len; wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); array_len = ARRAY_SIZE(mcr->querywq.wq_len); for (i = 0; i < array_len; i++) wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); } if (res != QM_MCR_RESULT_OK) { pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); return -EIO; } return 0; } int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, struct qm_mcr_cgrtestwrite *result) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); u8 res; mcc = qm_mc_start(&p->p); mcc->cgrtestwrite.cgid = cgr->cgrid; mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); res = mcr->result; if (res == QM_MCR_RESULT_OK) *result = mcr->cgrtestwrite; if (res != QM_MCR_RESULT_OK) { pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); return -EIO; } return 0; } int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); u8 res; unsigned int i; mcc = qm_mc_start(&p->p); mcc->querycgr.cgid = cgr->cgrid; qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); res = mcr->result; if (res == QM_MCR_RESULT_OK) *cgrd = mcr->querycgr; if (res != QM_MCR_RESULT_OK) { pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); return -EIO; } cgrd->cgr.wr_parm_g.word = be32_to_cpu(cgrd->cgr.wr_parm_g.word); cgrd->cgr.wr_parm_y.word = be32_to_cpu(cgrd->cgr.wr_parm_y.word); cgrd->cgr.wr_parm_r.word = be32_to_cpu(cgrd->cgr.wr_parm_r.word); cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) cgrd->cscn_targ_swp[i] = be32_to_cpu(cgrd->cscn_targ_swp[i]); return 0; } int qman_query_congestion(struct qm_mcr_querycongestion *congestion) { struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); u8 res; unsigned int i; qm_mc_start(&p->p); qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCONGESTION); res = mcr->result; if (res == QM_MCR_RESULT_OK) *congestion = mcr->querycongestion; if (res != QM_MCR_RESULT_OK) { pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); return -EIO; } for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++) congestion->state.state[i] = be32_to_cpu(congestion->state.state[i]); return 0; } int qman_set_vdq(struct qman_fq *fq, u16 num) { struct qman_portal *p = get_affine_portal(); uint32_t vdqcr; int ret = -EBUSY; vdqcr = QM_VDQCR_EXACT; vdqcr |= QM_VDQCR_NUMFRAMES_SET(num); if ((fq->state != qman_fq_state_parked) && (fq->state != qman_fq_state_retired)) { ret = -EINVAL; goto out; } if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { ret = -EBUSY; goto out; } vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; if (!p->vdqcr_owned) { FQLOCK(fq); if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) goto escape; fq_set(fq, QMAN_FQ_STATE_VDQCR); FQUNLOCK(fq); p->vdqcr_owned = fq; ret = 0; } escape: if (!ret) qm_dqrr_vdqcr_set(&p->p, vdqcr); out: return ret; } int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, u32 vdqcr) { struct qman_portal *p; int ret = -EBUSY; if ((fq->state != qman_fq_state_parked) && (fq->state != qman_fq_state_retired)) return -EINVAL; if (vdqcr & QM_VDQCR_FQID_MASK) return -EINVAL; if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) return -EBUSY; vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; p = get_affine_portal(); if (!p->vdqcr_owned) { FQLOCK(fq); if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) goto escape; fq_set(fq, QMAN_FQ_STATE_VDQCR); FQUNLOCK(fq); p->vdqcr_owned = fq; ret = 0; } escape: if (ret) return ret; /* VDQCR is set */ qm_dqrr_vdqcr_set(&p->p, vdqcr); return 0; } static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) { if (avail) qm_eqcr_cce_prefetch(&p->p); else qm_eqcr_cce_update(&p->p); } int qman_eqcr_is_empty(void) { struct qman_portal *p = get_affine_portal(); u8 avail; update_eqcr_ci(p, 0); avail = qm_eqcr_get_fill(&p->p); return (avail == 0); } void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) { if (affine) { struct qman_portal *p = get_affine_portal(); p->cb_dc_ern = handler; } else cb_dc_ern = handler; } static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, struct qman_fq *fq, const struct qm_fd *fd, u32 flags) { struct qm_eqcr_entry *eq; u8 avail; if (p->use_eqcr_ci_stashing) { /* * The stashing case is easy, only update if we need to in * order to try and liberate ring entries. */ eq = qm_eqcr_start_stash(&p->p); } else { /* * The non-stashing case is harder, need to prefetch ahead of * time. */ avail = qm_eqcr_get_avail(&p->p); if (avail < 2) update_eqcr_ci(p, avail); eq = qm_eqcr_start_no_stash(&p->p); } if (unlikely(!eq)) return NULL; if (flags & QMAN_ENQUEUE_FLAG_DCA) eq->dca = QM_EQCR_DCA_ENABLE | ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? QM_EQCR_DCA_PARK : 0) | ((flags >> 8) & QM_EQCR_DCA_IDXMASK); eq->fqid = cpu_to_be32(fq->fqid); #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP eq->tag = cpu_to_be32(fq->key); #else eq->tag = cpu_to_be32((u32)(uintptr_t)fq); #endif eq->fd = *fd; cpu_to_hw_fd(&eq->fd); return eq; } int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) { struct qman_portal *p = get_affine_portal(); struct qm_eqcr_entry *eq; eq = try_p_eq_start(p, fq, fd, flags); if (!eq) return -EBUSY; /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); /* Factor the below out, it's used from qman_enqueue_orp() too */ return 0; } int qman_enqueue_multi(struct qman_fq *fq, const struct qm_fd *fd, int frames_to_send) { struct qman_portal *p = get_affine_portal(); struct qm_portal *portal = &p->p; register struct qm_eqcr *eqcr = &portal->eqcr; struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; u8 i, diff, old_ci, sent = 0; /* Update the available entries if no entry is free */ if (!eqcr->available) { old_ci = eqcr->ci; eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); eqcr->available += diff; if (!diff) return 0; } /* try to send as many frames as possible */ while (eqcr->available && frames_to_send--) { eq->fqid = cpu_to_be32(fq->fqid); #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP eq->tag = cpu_to_be32(fq->key); #else eq->tag = cpu_to_be32((u32)(uintptr_t)fq); #endif eq->fd.opaque_addr = fd->opaque_addr; eq->fd.addr = cpu_to_be40(fd->addr); eq->fd.status = cpu_to_be32(fd->status); eq->fd.opaque = cpu_to_be32(fd->opaque); eq = (void *)((unsigned long)(eq + 1) & (~(unsigned long)(QM_EQCR_SIZE << 6))); eqcr->available--; sent++; fd++; } lwsync(); /* In order for flushes to complete faster, all lines are recorded in * 32 bit word. */ eq = eqcr->cursor; for (i = 0; i < sent; i++) { eq->__dont_write_directly__verb = QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; prev_eq = eq; eq = (void *)((unsigned long)(eq + 1) & (~(unsigned long)(QM_EQCR_SIZE << 6))); if (unlikely((prev_eq + 1) != eq)) eqcr->vbit ^= QM_EQCR_VERB_VBIT; } /* We need to flush all the lines but without load/store operations * between them */ eq = eqcr->cursor; for (i = 0; i < sent; i++) { dcbf(eq); eq = (void *)((unsigned long)(eq + 1) & (~(unsigned long)(QM_EQCR_SIZE << 6))); } /* Update cursor for the next call */ eqcr->cursor = eq; return sent; } int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, struct qman_fq *orp, u16 orp_seqnum) { struct qman_portal *p = get_affine_portal(); struct qm_eqcr_entry *eq; eq = try_p_eq_start(p, fq, fd, flags); if (!eq) return -EBUSY; /* Process ORP-specifics here */ if (flags & QMAN_ENQUEUE_FLAG_NLIS) orp_seqnum |= QM_EQCR_SEQNUM_NLIS; else { orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; if (flags & QMAN_ENQUEUE_FLAG_NESN) orp_seqnum |= QM_EQCR_SEQNUM_NESN; else /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; } eq->seqnum = cpu_to_be16(orp_seqnum); eq->orp = cpu_to_be32(orp->fqid); /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? 0 : QM_EQCR_VERB_CMD_ENQUEUE) | (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); return 0; } int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, struct qm_mcc_initcgr *opts) { struct qm_mc_command *mcc; struct qm_mc_result *mcr; struct qman_portal *p = get_affine_portal(); u8 res; u8 verb = QM_MCC_VERB_MODIFYCGR; mcc = qm_mc_start(&p->p); if (opts) mcc->initcgr = *opts; mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); mcc->initcgr.cgr.wr_parm_g.word = cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); mcc->initcgr.cgr.wr_parm_y.word = cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); mcc->initcgr.cgr.wr_parm_r.word = cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); mcc->initcgr.cgid = cgr->cgrid; if (flags & QMAN_CGR_FLAG_USE_INIT) verb = QM_MCC_VERB_INITCGR; qm_mc_commit(&p->p, verb); while (!(mcr = qm_mc_result(&p->p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); res = mcr->result; return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; } #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \ QM_CHANNEL_SWPORTAL0)) #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0) int qman_create_cgr(struct qman_cgr *cgr, u32 flags, struct qm_mcc_initcgr *opts) { struct qm_mcr_querycgr cgr_state; struct qm_mcc_initcgr local_opts; int ret; struct qman_portal *p; /* We have to check that the provided CGRID is within the limits of the * data-structures, for obvious reasons. However we'll let h/w take * care of determining whether it's within the limits of what exists on * the SoC. */ if (cgr->cgrid >= __CGR_NUM) return -EINVAL; p = get_affine_portal(); memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); cgr->chan = p->config->channel; spin_lock(&p->cgr_lock); /* if no opts specified, just add it to the list */ if (!opts) goto add_list; ret = qman_query_cgr(cgr, &cgr_state); if (ret) goto release_lock; if (opts) local_opts = *opts; if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) local_opts.cgr.cscn_targ_upd_ctrl = QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); else /* Overwrite TARG */ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | TARG_MASK(p); local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; /* send init if flags indicate so */ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); else ret = qman_modify_cgr(cgr, 0, &local_opts); if (ret) goto release_lock; add_list: list_add(&cgr->node, &p->cgr_cbs); /* Determine if newly added object requires its callback to be called */ ret = qman_query_cgr(cgr, &cgr_state); if (ret) { /* we can't go back, so proceed and return success, but screen * and wail to the log file. */ pr_crit("CGR HW state partially modified\n"); ret = 0; goto release_lock; } if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], cgr->cgrid)) cgr->cb(p, cgr, 1); release_lock: spin_unlock(&p->cgr_lock); return ret; } int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, struct qm_mcc_initcgr *opts) { struct qm_mcc_initcgr local_opts; struct qm_mcr_querycgr cgr_state; int ret; if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { pr_warn("QMan version doesn't support CSCN => DCP portal\n"); return -EINVAL; } /* We have to check that the provided CGRID is within the limits of the * data-structures, for obvious reasons. However we'll let h/w take * care of determining whether it's within the limits of what exists on * the SoC. */ if (cgr->cgrid >= __CGR_NUM) return -EINVAL; ret = qman_query_cgr(cgr, &cgr_state); if (ret) return ret; memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); if (opts) local_opts = *opts; if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) local_opts.cgr.cscn_targ_upd_ctrl = QM_CGR_TARG_UDP_CTRL_WRITE_BIT | QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; else local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | TARG_DCP_MASK(dcp_portal); local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; /* send init if flags indicate so */ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); else ret = qman_modify_cgr(cgr, 0, &local_opts); return ret; } int qman_delete_cgr(struct qman_cgr *cgr) { struct qm_mcr_querycgr cgr_state; struct qm_mcc_initcgr local_opts; int ret = 0; struct qman_cgr *i; struct qman_portal *p = get_affine_portal(); if (cgr->chan != p->config->channel) { pr_crit("Attempting to delete cgr from different portal than" " it was create: create 0x%x, delete 0x%x\n", cgr->chan, p->config->channel); ret = -EINVAL; goto put_portal; } memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); spin_lock(&p->cgr_lock); list_del(&cgr->node); /* * If there are no other CGR objects for this CGRID in the list, * update CSCN_TARG accordingly */ list_for_each_entry(i, &p->cgr_cbs, node) if ((i->cgrid == cgr->cgrid) && i->cb) goto release_lock; ret = qman_query_cgr(cgr, &cgr_state); if (ret) { /* add back to the list */ list_add(&cgr->node, &p->cgr_cbs); goto release_lock; } /* Overwrite TARG */ local_opts.we_mask = QM_CGR_WE_CSCN_TARG; if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); else local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & ~(TARG_MASK(p)); ret = qman_modify_cgr(cgr, 0, &local_opts); if (ret) /* add back to the list */ list_add(&cgr->node, &p->cgr_cbs); release_lock: spin_unlock(&p->cgr_lock); put_portal: return ret; } int qman_shutdown_fq(u32 fqid) { struct qman_portal *p; struct qm_portal *low_p; struct qm_mc_command *mcc; struct qm_mc_result *mcr; u8 state; int orl_empty, fq_empty, drain = 0; u32 result; u32 channel, wq; u16 dest_wq; p = get_affine_portal(); low_p = &p->p; /* Determine the state of the FQID */ mcc = qm_mc_start(low_p); mcc->queryfq_np.fqid = cpu_to_be32(fqid); qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP); while (!(mcr = qm_mc_result(low_p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; if (state == QM_MCR_NP_STATE_OOS) return 0; /* Already OOS, no need to do anymore checks */ /* Query which channel the FQ is using */ mcc = qm_mc_start(low_p); mcc->queryfq.fqid = cpu_to_be32(fqid); qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ); while (!(mcr = qm_mc_result(low_p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); /* Need to store these since the MCR gets reused */ dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); channel = dest_wq & 0x7; wq = dest_wq >> 3; switch (state) { case QM_MCR_NP_STATE_TEN_SCHED: case QM_MCR_NP_STATE_TRU_SCHED: case QM_MCR_NP_STATE_ACTIVE: case QM_MCR_NP_STATE_PARKED: orl_empty = 0; mcc = qm_mc_start(low_p); mcc->alterfq.fqid = cpu_to_be32(fqid); qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE); while (!(mcr = qm_mc_result(low_p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); result = mcr->result; /* Make a copy as we reuse MCR below */ if (result == QM_MCR_RESULT_PENDING) { /* Need to wait for the FQRN in the message ring, which * will only occur once the FQ has been drained. In * order for the FQ to drain the portal needs to be set * to dequeue from the channel the FQ is scheduled on */ const struct qm_mr_entry *msg; const struct qm_dqrr_entry *dqrr = NULL; int found_fqrn = 0; __maybe_unused u16 dequeue_wq = 0; /* Flag that we need to drain FQ */ drain = 1; if (channel >= qm_channel_pool1 && channel < (u16)(qm_channel_pool1 + 15)) { /* Pool channel, enable the bit in the portal */ dequeue_wq = (channel - qm_channel_pool1 + 1) << 4 | wq; } else if (channel < qm_channel_pool1) { /* Dedicated channel */ dequeue_wq = wq; } else { pr_info("Cannot recover FQ 0x%x," " it is scheduled on channel 0x%x", fqid, channel); return -EBUSY; } /* Set the sdqcr to drain this channel */ if (channel < qm_channel_pool1) qm_dqrr_sdqcr_set(low_p, QM_SDQCR_TYPE_ACTIVE | QM_SDQCR_CHANNELS_DEDICATED); else qm_dqrr_sdqcr_set(low_p, QM_SDQCR_TYPE_ACTIVE | QM_SDQCR_CHANNELS_POOL_CONV (channel)); while (!found_fqrn) { /* Keep draining DQRR while checking the MR*/ qm_dqrr_pvb_update(low_p); dqrr = qm_dqrr_current(low_p); while (dqrr) { qm_dqrr_cdc_consume_1ptr( low_p, dqrr, 0); qm_dqrr_pvb_update(low_p); qm_dqrr_next(low_p); dqrr = qm_dqrr_current(low_p); } /* Process message ring too */ qm_mr_pvb_update(low_p); msg = qm_mr_current(low_p); while (msg) { if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) == QM_MR_VERB_FQRN) found_fqrn = 1; qm_mr_next(low_p); qm_mr_cci_consume_to_current(low_p); qm_mr_pvb_update(low_p); msg = qm_mr_current(low_p); } cpu_relax(); } } if (result != QM_MCR_RESULT_OK && result != QM_MCR_RESULT_PENDING) { /* error */ pr_err("qman_retire_fq failed on FQ 0x%x," " result=0x%x\n", fqid, result); return -1; } if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { /* ORL had no entries, no need to wait until the * ERNs come in. */ orl_empty = 1; } /* Retirement succeeded, check to see if FQ needs * to be drained. */ if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { /* FQ is Not Empty, drain using volatile DQ commands */ fq_empty = 0; do { const struct qm_dqrr_entry *dqrr = NULL; u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); qm_dqrr_vdqcr_set(low_p, vdqcr); /* Wait for a dequeue to occur */ while (dqrr == NULL) { qm_dqrr_pvb_update(low_p); dqrr = qm_dqrr_current(low_p); if (!dqrr) cpu_relax(); } /* Process the dequeues, making sure to * empty the ring completely. */ while (dqrr) { if (dqrr->fqid == fqid && dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) fq_empty = 1; qm_dqrr_cdc_consume_1ptr(low_p, dqrr, 0); qm_dqrr_pvb_update(low_p); qm_dqrr_next(low_p); dqrr = qm_dqrr_current(low_p); } } while (fq_empty == 0); } qm_dqrr_sdqcr_set(low_p, 0); /* Wait for the ORL to have been completely drained */ while (orl_empty == 0) { const struct qm_mr_entry *msg; qm_mr_pvb_update(low_p); msg = qm_mr_current(low_p); while (msg) { if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) == QM_MR_VERB_FQRL) orl_empty = 1; qm_mr_next(low_p); qm_mr_cci_consume_to_current(low_p); qm_mr_pvb_update(low_p); msg = qm_mr_current(low_p); } cpu_relax(); } mcc = qm_mc_start(low_p); mcc->alterfq.fqid = cpu_to_be32(fqid); qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); while (!(mcr = qm_mc_result(low_p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); if (mcr->result != QM_MCR_RESULT_OK) { pr_err( "OOS after drain Failed on FQID 0x%x, result 0x%x\n", fqid, mcr->result); return -1; } return 0; case QM_MCR_NP_STATE_RETIRED: /* Send OOS Command */ mcc = qm_mc_start(low_p); mcc->alterfq.fqid = cpu_to_be32(fqid); qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); while (!(mcr = qm_mc_result(low_p))) cpu_relax(); DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); if (mcr->result) { pr_err("OOS Failed on FQID 0x%x\n", fqid); return -1; } return 0; } return -1; }