aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/bus
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/bus')
-rw-r--r--drivers/bus/dpaa/base/qbman/bman.h32
-rw-r--r--drivers/bus/dpaa/base/qbman/qman.c5
-rw-r--r--drivers/bus/dpaa/base/qbman/qman.h64
-rw-r--r--drivers/bus/dpaa/dpaa_bus.c4
-rw-r--r--drivers/bus/dpaa/include/fsl_qman.h4
-rw-r--r--drivers/bus/dpaa/rte_dpaa_bus.h6
-rw-r--r--drivers/bus/fslmc/fslmc_vfio.c2
-rw-r--r--drivers/bus/fslmc/mc/fsl_mc_sys.h1
-rw-r--r--drivers/bus/fslmc/portal/dpaa2_hw_pvt.h8
-rw-r--r--drivers/bus/fslmc/rte_fslmc.h8
-rw-r--r--drivers/bus/pci/linux/pci.c91
-rw-r--r--drivers/bus/pci/linux/pci_vfio.c2
-rw-r--r--drivers/bus/pci/pci_common_uio.c1
-rw-r--r--drivers/bus/vdev/vdev.c5
14 files changed, 196 insertions, 37 deletions
diff --git a/drivers/bus/dpaa/base/qbman/bman.h b/drivers/bus/dpaa/base/qbman/bman.h
index 4b088da9..ef0a896e 100644
--- a/drivers/bus/dpaa/base/qbman/bman.h
+++ b/drivers/bus/dpaa/base/qbman/bman.h
@@ -228,7 +228,9 @@ static inline void bm_rcr_finish(struct bm_portal *portal)
u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(!rcr->busy);
+#endif
if (pi != RCR_PTR2IDX(rcr->cursor))
pr_crit("losing uncommitted RCR entries\n");
if (ci != rcr->ci)
@@ -241,7 +243,9 @@ static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
{
register struct bm_rcr *rcr = &portal->rcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(!rcr->busy);
+#endif
if (!rcr->available)
return NULL;
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
@@ -255,8 +259,8 @@ static inline void bm_rcr_abort(struct bm_portal *portal)
{
__maybe_unused register struct bm_rcr *rcr = &portal->rcr;
- DPAA_ASSERT(rcr->busy);
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
+ DPAA_ASSERT(rcr->busy);
rcr->busy = 0;
#endif
}
@@ -266,8 +270,10 @@ static inline struct bm_rcr_entry *bm_rcr_pend_and_next(
{
register struct bm_rcr *rcr = &portal->rcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->busy);
DPAA_ASSERT(rcr->pmode != bm_rcr_pvb);
+#endif
if (rcr->available == 1)
return NULL;
rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
@@ -282,8 +288,10 @@ static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb)
{
register struct bm_rcr *rcr = &portal->rcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->busy);
DPAA_ASSERT(rcr->pmode == bm_rcr_pci);
+#endif
rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
RCR_INC(rcr);
rcr->available--;
@@ -298,7 +306,9 @@ static inline void bm_rcr_pce_prefetch(struct bm_portal *portal)
{
__maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
+#endif
bm_cl_invalidate(RCR_PI);
bm_cl_touch_rw(RCR_PI);
}
@@ -307,8 +317,10 @@ static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb)
{
register struct bm_rcr *rcr = &portal->rcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->busy);
DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
+#endif
rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
RCR_INC(rcr);
rcr->available--;
@@ -324,8 +336,10 @@ static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
register struct bm_rcr *rcr = &portal->rcr;
struct bm_rcr_entry *rcursor;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->busy);
DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
+#endif
lwsync();
rcursor = rcr->cursor;
rcursor->__dont_write_directly__verb = myverb | rcr->vbit;
@@ -342,7 +356,9 @@ static inline u8 bm_rcr_cci_update(struct bm_portal *portal)
register struct bm_rcr *rcr = &portal->rcr;
u8 diff, old_ci = rcr->ci;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->cmode == bm_rcr_cci);
+#endif
rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
rcr->available += diff;
@@ -353,7 +369,9 @@ static inline void bm_rcr_cce_prefetch(struct bm_portal *portal)
{
__maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
+#endif
bm_cl_touch_ro(RCR_CI);
}
@@ -362,7 +380,9 @@ static inline u8 bm_rcr_cce_update(struct bm_portal *portal)
register struct bm_rcr *rcr = &portal->rcr;
u8 diff, old_ci = rcr->ci;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
+#endif
rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1);
bm_cl_invalidate(RCR_CI);
diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
@@ -420,8 +440,8 @@ static inline void bm_mc_finish(struct bm_portal *portal)
{
__maybe_unused register struct bm_mc *mc = &portal->mc;
- DPAA_ASSERT(mc->state == mc_idle);
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
+ DPAA_ASSERT(mc->state == mc_idle);
if (mc->state != mc_idle)
pr_crit("Losing incomplete MC command\n");
#endif
@@ -431,8 +451,8 @@ static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
{
register struct bm_mc *mc = &portal->mc;
- DPAA_ASSERT(mc->state == mc_idle);
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
+ DPAA_ASSERT(mc->state == mc_idle);
mc->state = mc_user;
#endif
dcbz_64(mc->cr);
@@ -443,8 +463,8 @@ static inline void bm_mc_abort(struct bm_portal *portal)
{
__maybe_unused register struct bm_mc *mc = &portal->mc;
- DPAA_ASSERT(mc->state == mc_user);
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
+ DPAA_ASSERT(mc->state == mc_user);
mc->state = mc_idle;
#endif
}
@@ -454,7 +474,9 @@ static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
register struct bm_mc *mc = &portal->mc;
struct bm_mc_result *rr = mc->rr + mc->rridx;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(mc->state == mc_user);
+#endif
lwsync();
mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
dcbf(mc->cr);
@@ -469,7 +491,9 @@ static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
register struct bm_mc *mc = &portal->mc;
struct bm_mc_result *rr = mc->rr + mc->rridx;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(mc->state == mc_hw);
+#endif
/* The inactive response register's verb byte always returns zero until
* its command is submitted and completed. This includes the valid-bit,
* in case you were wondering.
diff --git a/drivers/bus/dpaa/base/qbman/qman.c b/drivers/bus/dpaa/base/qbman/qman.c
index 87fec60d..b8511103 100644
--- a/drivers/bus/dpaa/base/qbman/qman.c
+++ b/drivers/bus/dpaa/base/qbman/qman.c
@@ -416,7 +416,9 @@ static inline void qm_eqcr_finish(struct qm_portal *portal)
qm_cl_invalidate(EQCR_CI);
eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(!eqcr->busy);
+#endif
if (pi != EQCR_PTR2IDX(eqcr->cursor))
pr_crit("losing uncommitted EQCR entries\n");
if (ci != eqcr->ci)
@@ -505,7 +507,9 @@ static inline void qm_mr_pvb_update(struct qm_portal *portal)
register struct qm_mr *mr = &portal->mr;
const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(mr->pmode == qm_mr_pvb);
+#endif
/* when accessing 'verb', use __raw_readb() to ensure that compiler
* inlining doesn't try to optimise out "excess reads".
*/
@@ -1267,6 +1271,7 @@ void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
switch (fq->state) {
case qman_fq_state_parked:
DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
+ /* Fallthrough */
case qman_fq_state_oos:
if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
qman_release_fqid(fq->fqid);
diff --git a/drivers/bus/dpaa/base/qbman/qman.h b/drivers/bus/dpaa/base/qbman/qman.h
index 2c0f694c..283cd31f 100644
--- a/drivers/bus/dpaa/base/qbman/qman.h
+++ b/drivers/bus/dpaa/base/qbman/qman.h
@@ -267,7 +267,9 @@ static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
{
register struct qm_eqcr *eqcr = &portal->eqcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(!eqcr->busy);
+#endif
if (!eqcr->available)
return NULL;
@@ -284,7 +286,9 @@ static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
register struct qm_eqcr *eqcr = &portal->eqcr;
u8 diff, old_ci;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(!eqcr->busy);
+#endif
if (!eqcr->available) {
old_ci = eqcr->ci;
eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
@@ -303,8 +307,8 @@ static inline void qm_eqcr_abort(struct qm_portal *portal)
{
__maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
- DPAA_ASSERT(eqcr->busy);
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
+ DPAA_ASSERT(eqcr->busy);
eqcr->busy = 0;
#endif
}
@@ -314,8 +318,10 @@ static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next(
{
register struct qm_eqcr *eqcr = &portal->eqcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(eqcr->busy);
DPAA_ASSERT(eqcr->pmode != qm_eqcr_pvb);
+#endif
if (eqcr->available == 1)
return NULL;
eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
@@ -336,8 +342,10 @@ static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb)
{
register struct qm_eqcr *eqcr = &portal->eqcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
EQCR_COMMIT_CHECKS(eqcr);
DPAA_ASSERT(eqcr->pmode == qm_eqcr_pci);
+#endif
eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
EQCR_INC(eqcr);
eqcr->available--;
@@ -353,7 +361,9 @@ static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal)
{
__maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce);
+#endif
qm_cl_invalidate(EQCR_PI);
qm_cl_touch_rw(EQCR_PI);
}
@@ -362,8 +372,10 @@ static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb)
{
register struct qm_eqcr *eqcr = &portal->eqcr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
EQCR_COMMIT_CHECKS(eqcr);
DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce);
+#endif
eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
EQCR_INC(eqcr);
eqcr->available--;
@@ -380,8 +392,10 @@ static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
register struct qm_eqcr *eqcr = &portal->eqcr;
struct qm_eqcr_entry *eqcursor;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
EQCR_COMMIT_CHECKS(eqcr);
DPAA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
+#endif
lwsync();
eqcursor = eqcr->cursor;
eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit;
@@ -503,7 +517,9 @@ static inline u8 qm_dqrr_pci_update(struct qm_portal *portal)
register struct qm_dqrr *dqrr = &portal->dqrr;
u8 diff, old_pi = dqrr->pi;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->pmode == qm_dqrr_pci);
+#endif
dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
dqrr->fill += diff;
@@ -514,7 +530,9 @@ static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal)
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce);
+#endif
qm_cl_invalidate(DQRR_PI);
qm_cl_touch_ro(DQRR_PI);
}
@@ -524,7 +542,9 @@ static inline u8 qm_dqrr_pce_update(struct qm_portal *portal)
register struct qm_dqrr *dqrr = &portal->dqrr;
u8 diff, old_pi = dqrr->pi;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce);
+#endif
dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1);
diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
dqrr->fill += diff;
@@ -536,7 +556,9 @@ static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
register struct qm_dqrr *dqrr = &portal->dqrr;
const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
+#endif
/* when accessing 'verb', use __raw_readb() to ensure that compiler
* inlining doesn't try to optimise out "excess reads".
*/
@@ -552,7 +574,9 @@ static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num)
{
register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci);
+#endif
dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
qm_out(DQRR_CI_CINH, dqrr->ci);
}
@@ -561,7 +585,9 @@ static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal)
{
register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci);
+#endif
dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
qm_out(DQRR_CI_CINH, dqrr->ci);
}
@@ -570,7 +596,9 @@ static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal)
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
+#endif
qm_cl_invalidate(DQRR_CI);
qm_cl_touch_rw(DQRR_CI);
}
@@ -579,7 +607,9 @@ static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num)
{
register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
+#endif
dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
qm_cl_out(DQRR_CI, dqrr->ci);
}
@@ -588,7 +618,9 @@ static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal)
{
register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
+#endif
dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
qm_cl_out(DQRR_CI, dqrr->ci);
}
@@ -598,7 +630,9 @@ static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx,
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+#endif
DPAA_ASSERT(idx < QM_DQRR_SIZE);
qm_out(DQRR_DCAP, (0 << 8) | /* S */
((park ? 1 : 0) << 6) | /* PK */
@@ -612,7 +646,9 @@ static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
u8 idx = DQRR_PTR2IDX(dq);
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+#endif
DPAA_ASSERT(idx < QM_DQRR_SIZE);
qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */
@@ -623,7 +659,9 @@ static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask)
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+#endif
qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */
dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
@@ -634,7 +672,9 @@ static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal)
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+#endif
return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
}
@@ -642,7 +682,9 @@ static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal)
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+#endif
qm_cl_invalidate(DQRR_CI);
qm_cl_touch_ro(DQRR_CI);
}
@@ -651,7 +693,9 @@ static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal)
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+#endif
return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1);
}
@@ -659,7 +703,9 @@ static inline u8 qm_dqrr_get_ci(struct qm_portal *portal)
{
register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
+#endif
return dqrr->ci;
}
@@ -667,7 +713,9 @@ static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx)
{
__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
+#endif
qm_out(DQRR_DCAP, (0 << 8) | /* S */
(1 << 6) | /* PK */
(idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */
@@ -677,7 +725,9 @@ static inline void qm_dqrr_park_current(struct qm_portal *portal)
{
register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
+#endif
qm_out(DQRR_DCAP, (0 << 8) | /* S */
(1 << 6) | /* PK */
DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */
@@ -766,7 +816,9 @@ static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
{
register struct qm_mr *mr = &portal->mr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(mr->cmode == qm_mr_cci);
+#endif
mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
qm_out(MR_CI_CINH, mr->ci);
}
@@ -775,7 +827,9 @@ static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
{
register struct qm_mr *mr = &portal->mr;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(mr->cmode == qm_mr_cci);
+#endif
mr->ci = MR_PTR2IDX(mr->cursor);
qm_out(MR_CI_CINH, mr->ci);
}
@@ -806,8 +860,8 @@ static inline void qm_mc_finish(struct qm_portal *portal)
{
__maybe_unused register struct qm_mc *mc = &portal->mc;
- DPAA_ASSERT(mc->state == qman_mc_idle);
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
+ DPAA_ASSERT(mc->state == qman_mc_idle);
if (mc->state != qman_mc_idle)
pr_crit("Losing incomplete MC command\n");
#endif
@@ -817,8 +871,8 @@ static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal)
{
register struct qm_mc *mc = &portal->mc;
- DPAA_ASSERT(mc->state == qman_mc_idle);
#ifdef RTE_LIBRTE_DPAA_HWDEBUG
+ DPAA_ASSERT(mc->state == qman_mc_idle);
mc->state = qman_mc_user;
#endif
dcbz_64(mc->cr);
@@ -830,7 +884,9 @@ static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
register struct qm_mc *mc = &portal->mc;
struct qm_mc_result *rr = mc->rr + mc->rridx;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(mc->state == qman_mc_user);
+#endif
lwsync();
mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
dcbf(mc->cr);
@@ -845,7 +901,9 @@ static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal)
register struct qm_mc *mc = &portal->mc;
struct qm_mc_result *rr = mc->rr + mc->rridx;
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
DPAA_ASSERT(mc->state == qman_mc_hw);
+#endif
/* The inactive response register's verb byte always returns zero until
* its command is submitted and completed. This includes the valid-bit,
* in case you were wondering.
diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c
index 1cc8c893..3a0b2553 100644
--- a/drivers/bus/dpaa/dpaa_bus.c
+++ b/drivers/bus/dpaa/dpaa_bus.c
@@ -490,6 +490,10 @@ rte_dpaa_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
static enum rte_iova_mode
rte_dpaa_get_iommu_class(void)
{
+ if ((access(DPAA_DEV_PATH1, F_OK) != 0) &&
+ (access(DPAA_DEV_PATH2, F_OK) != 0)) {
+ return RTE_IOVA_DC;
+ }
return RTE_IOVA_PA;
}
diff --git a/drivers/bus/dpaa/include/fsl_qman.h b/drivers/bus/dpaa/include/fsl_qman.h
index eedfd7ea..72556dc1 100644
--- a/drivers/bus/dpaa/include/fsl_qman.h
+++ b/drivers/bus/dpaa/include/fsl_qman.h
@@ -1993,8 +1993,8 @@ static inline int qman_poll_fq_for_init(struct qman_fq *fq)
}
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-#define cpu_to_hw_sg(x) (x)
-#define hw_sg_to_cpu(x) (x)
+#define cpu_to_hw_sg(x)
+#define hw_sg_to_cpu(x)
#else
#define cpu_to_hw_sg(x) __cpu_to_hw_sg(x)
#define hw_sg_to_cpu(x) __hw_sg_to_cpu(x)
diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h
index eafc944d..bc933afd 100644
--- a/drivers/bus/dpaa/rte_dpaa_bus.h
+++ b/drivers/bus/dpaa/rte_dpaa_bus.h
@@ -113,10 +113,10 @@ static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr)
int i;
for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr != NULL; i++) {
- if (paddr >= memseg[i].phys_addr && paddr <
- memseg[i].phys_addr + memseg[i].len)
+ if (paddr >= memseg[i].iova && paddr <
+ memseg[i].iova + memseg[i].len)
return (uint8_t *)(memseg[i].addr) +
- (paddr - memseg[i].phys_addr);
+ (paddr - memseg[i].iova);
}
return NULL;
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 7831201a..a9363218 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -249,7 +249,7 @@ int rte_fslmc_vfio_dmamap(void)
dma_map.size = memseg[i].len;
dma_map.vaddr = memseg[i].addr_64;
#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
- dma_map.iova = memseg[i].phys_addr;
+ dma_map.iova = memseg[i].iova;
#else
dma_map.iova = dma_map.vaddr;
#endif
diff --git a/drivers/bus/fslmc/mc/fsl_mc_sys.h b/drivers/bus/fslmc/mc/fsl_mc_sys.h
index d803205e..e4b2404e 100644
--- a/drivers/bus/fslmc/mc/fsl_mc_sys.h
+++ b/drivers/bus/fslmc/mc/fsl_mc_sys.h
@@ -60,7 +60,6 @@ struct fsl_mc_io {
#else /* __linux_driver__ */
#include <stdio.h>
-#include <libio.h>
#include <stdint.h>
#include <errno.h>
#include <sys/uio.h>
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
index c1b842f3..ece1a7d4 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
@@ -284,10 +284,10 @@ static void *dpaa2_mem_ptov(phys_addr_t paddr)
int i;
for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
- if (paddr >= memseg[i].phys_addr &&
- (char *)paddr < (char *)memseg[i].phys_addr + memseg[i].len)
+ if (paddr >= memseg[i].iova &&
+ (char *)paddr < (char *)memseg[i].iova + memseg[i].len)
return (void *)(memseg[i].addr_64
- + (paddr - memseg[i].phys_addr));
+ + (paddr - memseg[i].iova));
}
return NULL;
}
@@ -301,7 +301,7 @@ static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr)
for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
if (vaddr >= memseg[i].addr_64 &&
vaddr < memseg[i].addr_64 + memseg[i].len)
- return memseg[i].phys_addr
+ return memseg[i].iova
+ (vaddr - memseg[i].addr_64);
}
return (phys_addr_t)(NULL);
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 4c32db62..0814e69b 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -175,10 +175,6 @@ static void dpaa2initfn_ ##nm(void) \
} \
RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
-#ifdef __cplusplus
-}
-#endif
-
/**
* Register a DPAA2 MC Object driver.
*
@@ -198,4 +194,8 @@ static void dpaa2objinitfn_ ##nm(void) \
} \
RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+#ifdef __cplusplus
+}
+#endif
+
#endif /* _RTE_FSLMC_H_ */
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index 5da6728f..74deef3a 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -576,6 +576,82 @@ pci_one_device_has_iova_va(void)
return 0;
}
+#if defined(RTE_ARCH_X86)
+static bool
+pci_one_device_iommu_support_va(struct rte_pci_device *dev)
+{
+#define VTD_CAP_MGAW_SHIFT 16
+#define VTD_CAP_MGAW_MASK (0x3fULL << VTD_CAP_MGAW_SHIFT)
+#define X86_VA_WIDTH 47 /* From Documentation/x86/x86_64/mm.txt */
+ struct rte_pci_addr *addr = &dev->addr;
+ char filename[PATH_MAX];
+ FILE *fp;
+ uint64_t mgaw, vtd_cap_reg = 0;
+
+ snprintf(filename, sizeof(filename),
+ "%s/" PCI_PRI_FMT "/iommu/intel-iommu/cap",
+ rte_pci_get_sysfs_path(), addr->domain, addr->bus, addr->devid,
+ addr->function);
+ if (access(filename, F_OK) == -1) {
+ /* We don't have an Intel IOMMU, assume VA supported*/
+ return true;
+ }
+
+ /* We have an intel IOMMU */
+ fp = fopen(filename, "r");
+ if (fp == NULL) {
+ RTE_LOG(ERR, EAL, "%s(): can't open %s\n", __func__, filename);
+ return false;
+ }
+
+ if (fscanf(fp, "%" PRIx64, &vtd_cap_reg) != 1) {
+ RTE_LOG(ERR, EAL, "%s(): can't read %s\n", __func__, filename);
+ fclose(fp);
+ return false;
+ }
+
+ fclose(fp);
+
+ mgaw = ((vtd_cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1;
+ if (mgaw < X86_VA_WIDTH)
+ return false;
+
+ return true;
+}
+#elif defined(RTE_ARCH_PPC_64)
+static bool
+pci_one_device_iommu_support_va(__rte_unused struct rte_pci_device *dev)
+{
+ return false;
+}
+#else
+static bool
+pci_one_device_iommu_support_va(__rte_unused struct rte_pci_device *dev)
+{
+ return true;
+}
+#endif
+
+/*
+ * All devices IOMMUs support VA as IOVA
+ */
+static bool
+pci_devices_iommu_support_va(void)
+{
+ struct rte_pci_device *dev = NULL;
+ struct rte_pci_driver *drv = NULL;
+
+ FOREACH_DRIVER_ON_PCIBUS(drv) {
+ FOREACH_DEVICE_ON_PCIBUS(dev) {
+ if (!rte_pci_match(drv, dev))
+ continue;
+ if (!pci_one_device_iommu_support_va(dev))
+ return false;
+ }
+ }
+ return true;
+}
+
/*
* Get iommu class of PCI devices on the bus.
*/
@@ -586,12 +662,7 @@ rte_pci_get_iommu_class(void)
bool is_vfio_noiommu_enabled = true;
bool has_iova_va;
bool is_bound_uio;
- bool spapr_iommu =
-#if defined(RTE_ARCH_PPC_64)
- true;
-#else
- false;
-#endif
+ bool iommu_no_va;
is_bound = pci_one_device_is_bound();
if (!is_bound)
@@ -599,13 +670,14 @@ rte_pci_get_iommu_class(void)
has_iova_va = pci_one_device_has_iova_va();
is_bound_uio = pci_one_device_bound_uio();
+ iommu_no_va = !pci_devices_iommu_support_va();
#ifdef VFIO_PRESENT
is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ?
true : false;
#endif
if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled &&
- !spapr_iommu)
+ !iommu_no_va)
return RTE_IOVA_VA;
if (has_iova_va) {
@@ -614,8 +686,8 @@ rte_pci_get_iommu_class(void)
RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n");
if (is_bound_uio)
RTE_LOG(WARNING, EAL, "few device bound to UIO\n");
- if (spapr_iommu)
- RTE_LOG(WARNING, EAL, "sPAPR IOMMU does not support IOVA as VA\n");
+ if (iommu_no_va)
+ RTE_LOG(WARNING, EAL, "IOMMU does not support IOVA as VA\n");
}
return RTE_IOVA_PA;
@@ -723,7 +795,6 @@ pci_ioport_map(struct rte_pci_device *dev, int bar __rte_unused,
if (!found)
return -1;
- dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
p->base = start;
RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%x\n", start);
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 1f93fa4d..e7d7f5dc 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -459,7 +459,6 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
struct pci_map *maps;
dev->intr_handle.fd = -1;
- dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
/* store PCI address string */
snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
@@ -576,7 +575,6 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
struct pci_map *maps;
dev->intr_handle.fd = -1;
- dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
/* store PCI address string */
snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
index 06711316..b75988de 100644
--- a/drivers/bus/pci/pci_common_uio.c
+++ b/drivers/bus/pci/pci_common_uio.c
@@ -119,7 +119,6 @@ pci_uio_map_resource(struct rte_pci_device *dev)
dev->intr_handle.fd = -1;
dev->intr_handle.uio_cfg_fd = -1;
- dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
/* secondary processes - use already recorded details */
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c
index fd7736d6..ba0ed7a2 100644
--- a/drivers/bus/vdev/vdev.c
+++ b/drivers/bus/vdev/vdev.c
@@ -289,6 +289,7 @@ static int
vdev_probe(void)
{
struct rte_vdev_device *dev;
+ int ret = 0;
/* call the init function for each virtual device */
TAILQ_FOREACH(dev, &vdev_device_list, next) {
@@ -299,11 +300,11 @@ vdev_probe(void)
if (vdev_probe_all_drivers(dev)) {
VDEV_LOG(ERR, "failed to initialize %s device\n",
rte_vdev_device_name(dev));
- return -1;
+ ret = -1;
}
}
- return 0;
+ return ret;
}
static struct rte_device *