diff options
Diffstat (limited to 'drivers/bus/fslmc/qbman')
| -rw-r--r-- | drivers/bus/fslmc/qbman/include/compat.h | 322 | ||||
| -rw-r--r-- | drivers/bus/fslmc/qbman/include/fsl_qbman_base.h | 4 | ||||
| -rw-r--r-- | drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h | 183 | ||||
| -rw-r--r-- | drivers/bus/fslmc/qbman/qbman_portal.c | 973 | ||||
| -rw-r--r-- | drivers/bus/fslmc/qbman/qbman_portal.h | 140 | ||||
| -rw-r--r-- | drivers/bus/fslmc/qbman/qbman_private.h | 174 | ||||
| -rw-r--r-- | drivers/bus/fslmc/qbman/qbman_sys.h | 151 | ||||
| -rw-r--r-- | drivers/bus/fslmc/qbman/qbman_sys_decl.h | 25 |
8 files changed, 629 insertions, 1343 deletions
diff --git a/drivers/bus/fslmc/qbman/include/compat.h b/drivers/bus/fslmc/qbman/include/compat.h index 529f1ea3..423087cb 100644 --- a/drivers/bus/fslmc/qbman/include/compat.h +++ b/drivers/bus/fslmc/qbman/include/compat.h @@ -30,32 +30,17 @@ #ifndef HEADER_COMPAT_H #define HEADER_COMPAT_H -#include <sched.h> - #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include <stdint.h> #include <stdlib.h> -#include <stddef.h> #include <errno.h> #include <string.h> -#include <pthread.h> -#include <net/ethernet.h> -#include <stdio.h> -#include <stdbool.h> -#include <ctype.h> #include <malloc.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> #include <unistd.h> -#include <sys/mman.h> -#include <limits.h> -#include <assert.h> -#include <dirent.h> -#include <inttypes.h> #include <error.h> +#include <linux/types.h> #include <rte_atomic.h> /* The following definitions are primarily to allow the single-source driver @@ -65,53 +50,11 @@ */ /* Required compiler attributes */ -#define __user #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) -#define ____cacheline_aligned __attribute__((aligned(L1_CACHE_BYTES))) -#undef container_of -#define container_of(ptr, type, member) ({ \ - typeof(((type *)0)->member)(*__mptr) = (ptr); \ - (type *)((char *)__mptr - offsetof(type, member)); }) -#define __stringify_1(x) #x -#define __stringify(x) __stringify_1(x) - -#ifdef ARRAY_SIZE -#undef ARRAY_SIZE -#endif -#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) /* Required types */ -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; typedef uint64_t dma_addr_t; -typedef cpu_set_t cpumask_t; -typedef u32 compat_uptr_t; - -static inline void __user *compat_ptr(compat_uptr_t uptr) -{ - return (void __user *)(unsigned long)uptr; -} - -static inline compat_uptr_t ptr_to_compat(void __user *uptr) -{ - return (u32)(unsigned long)uptr; -} - -/* I/O operations */ -static inline u32 in_be32(volatile void *__p) -{ - volatile u32 *p = __p; - return *p; -} - -static inline void out_be32(volatile void *__p, u32 val) -{ - volatile u32 *p = __p; - *p = val; -} /* Debugging */ #define prflush(fmt, args...) \ @@ -124,275 +67,46 @@ static inline void out_be32(volatile void *__p, u32 val) #define pr_warn(fmt, args...) prflush("WARN:" fmt, ##args) #define pr_info(fmt, args...) prflush(fmt, ##args) +#ifdef RTE_LIBRTE_DPAA2_DEBUG_BUS + +/* Trace the 3 different classes of read/write access to QBMan. #undef as + * required. + */ +#define QBMAN_CCSR_TRACE +#define QBMAN_CINH_TRACE +#define QBMAN_CENA_TRACE + +#define QBMAN_CHECKING + #ifdef pr_debug #undef pr_debug #endif -#define pr_debug(fmt, args...) {} -#define might_sleep_if(c) {} -#define msleep(x) {} -#define WARN_ON(c, str) \ +#define pr_debug(fmt, args...) printf(fmt, ##args) +#define QBMAN_BUG_ON(c) \ do { \ static int warned_##__LINE__; \ if ((c) && !warned_##__LINE__) { \ - pr_warn("%s\n", str); \ pr_warn("(%s:%d)\n", __FILE__, __LINE__); \ warned_##__LINE__ = 1; \ } \ } while (0) -#ifdef CONFIG_BUGON -#define QBMAN_BUG_ON(c) WARN_ON(c, "BUG") #else #define QBMAN_BUG_ON(c) {} +#define pr_debug(fmt, args...) {} #endif -#define ALIGN(x, a) (((x) + ((typeof(x))(a) - 1)) & ~((typeof(x))(a) - 1)) - -/****************/ -/* Linked-lists */ -/****************/ - -struct list_head { - struct list_head *prev; - struct list_head *next; -}; - -#define LIST_HEAD(n) \ -struct list_head n = { \ - .prev = &n, \ - .next = &n \ -} - -#define INIT_LIST_HEAD(p) \ -do { \ - struct list_head *__p298 = (p); \ - __p298->next = __p298; \ - __p298->prev = __p298->next; \ -} while (0) -#define list_entry(node, type, member) \ - (type *)((void *)node - offsetof(type, member)) -#define list_empty(p) \ -({ \ - const struct list_head *__p298 = (p); \ - ((__p298->next == __p298) && (__p298->prev == __p298)); \ -}) -#define list_add(p, l) \ -do { \ - struct list_head *__p298 = (p); \ - struct list_head *__l298 = (l); \ - __p298->next = __l298->next; \ - __p298->prev = __l298; \ - __l298->next->prev = __p298; \ - __l298->next = __p298; \ -} while (0) -#define list_add_tail(p, l) \ -do { \ - struct list_head *__p298 = (p); \ - struct list_head *__l298 = (l); \ - __p298->prev = __l298->prev; \ - __p298->next = __l298; \ - __l298->prev->next = __p298; \ - __l298->prev = __p298; \ -} while (0) -#define list_for_each(i, l) \ - for (i = (l)->next; i != (l); i = i->next) -#define list_for_each_safe(i, j, l) \ - for (i = (l)->next, j = i->next; i != (l); \ - i = j, j = i->next) -#define list_for_each_entry(i, l, name) \ - for (i = list_entry((l)->next, typeof(*i), name); &i->name != (l); \ - i = list_entry(i->name.next, typeof(*i), name)) -#define list_for_each_entry_safe(i, j, l, name) \ - for (i = list_entry((l)->next, typeof(*i), name), \ - j = list_entry(i->name.next, typeof(*j), name); \ - &i->name != (l); \ - i = j, j = list_entry(j->name.next, typeof(*j), name)) -#define list_del(i) \ -do { \ - (i)->next->prev = (i)->prev; \ - (i)->prev->next = (i)->next; \ -} while (0) - /* Other miscellaneous interfaces our APIs depend on; */ -#define lower_32_bits(x) ((u32)(x)) -#define upper_32_bits(x) ((u32)(((x) >> 16) >> 16)) +#define lower_32_bits(x) ((uint32_t)(x)) +#define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) -/* Compiler/type stuff */ -typedef unsigned int gfp_t; -typedef uint32_t phandle; #define __iomem -#define EINTR 4 -#define ENODEV 19 -#define GFP_KERNEL 0 + #define __raw_readb(p) (*(const volatile unsigned char *)(p)) #define __raw_readl(p) (*(const volatile unsigned int *)(p)) #define __raw_writel(v, p) {*(volatile unsigned int *)(p) = (v); } -/* memcpy() stuff - when you know alignments in advance */ -#ifdef CONFIG_TRY_BETTER_MEMCPY -static inline void copy_words(void *dest, const void *src, size_t sz) -{ - u32 *__dest = dest; - const u32 *__src = src; - size_t __sz = sz >> 2; - - QBMAN_BUG_ON((unsigned long)dest & 0x3); - QBMAN_BUG_ON((unsigned long)src & 0x3); - QBMAN_BUG_ON(sz & 0x3); - while (__sz--) - *(__dest++) = *(__src++); -} - -static inline void copy_shorts(void *dest, const void *src, size_t sz) -{ - u16 *__dest = dest; - const u16 *__src = src; - size_t __sz = sz >> 1; - - QBMAN_BUG_ON((unsigned long)dest & 0x1); - QBMAN_BUG_ON((unsigned long)src & 0x1); - QBMAN_BUG_ON(sz & 0x1); - while (__sz--) - *(__dest++) = *(__src++); -} - -static inline void copy_bytes(void *dest, const void *src, size_t sz) -{ - u8 *__dest = dest; - const u8 *__src = src; - - while (sz--) - *(__dest++) = *(__src++); -} -#else -#define copy_words memcpy -#define copy_shorts memcpy -#define copy_bytes memcpy -#endif - -/* Completion stuff */ -#define DECLARE_COMPLETION(n) int n = 0 -#define complete(n) { *n = 1; } -#define wait_for_completion(n) \ -do { \ - while (!*n) { \ - bman_poll(); \ - qman_poll(); \ - } \ - *n = 0; \ -} while (0) - -/* Allocator stuff */ -#define kmalloc(sz, t) malloc(sz) -#define vmalloc(sz) malloc(sz) -#define kfree(p) { if (p) free(p); } -static inline void *kzalloc(size_t sz, gfp_t __foo __rte_unused) -{ - void *ptr = malloc(sz); - - if (ptr) - memset(ptr, 0, sz); - return ptr; -} - -static inline unsigned long get_zeroed_page(gfp_t __foo __rte_unused) -{ - void *p; - - if (posix_memalign(&p, 4096, 4096)) - return 0; - memset(p, 0, 4096); - return (unsigned long)p; -} - -static inline void free_page(unsigned long p) -{ - free((void *)p); -} - -/* Bitfield stuff. */ -#define BITS_PER_ULONG (sizeof(unsigned long) << 3) -#define SHIFT_PER_ULONG (((1 << 5) == BITS_PER_ULONG) ? 5 : 6) -#define BITS_MASK(idx) ((unsigned long)1 << ((idx) & (BITS_PER_ULONG - 1))) -#define BITS_IDX(idx) ((idx) >> SHIFT_PER_ULONG) -static inline unsigned long test_bits(unsigned long mask, - volatile unsigned long *p) -{ - return *p & mask; -} - -static inline int test_bit(int idx, volatile unsigned long *bits) -{ - return test_bits(BITS_MASK(idx), bits + BITS_IDX(idx)); -} - -static inline void set_bits(unsigned long mask, volatile unsigned long *p) -{ - *p |= mask; -} - -static inline void set_bit(int idx, volatile unsigned long *bits) -{ - set_bits(BITS_MASK(idx), bits + BITS_IDX(idx)); -} - -static inline void clear_bits(unsigned long mask, volatile unsigned long *p) -{ - *p &= ~mask; -} - -static inline void clear_bit(int idx, volatile unsigned long *bits) -{ - clear_bits(BITS_MASK(idx), bits + BITS_IDX(idx)); -} - -static inline unsigned long test_and_set_bits(unsigned long mask, - volatile unsigned long *p) -{ - unsigned long ret = test_bits(mask, p); - - set_bits(mask, p); - return ret; -} - -static inline int test_and_set_bit(int idx, volatile unsigned long *bits) -{ - int ret = test_bit(idx, bits); - - set_bit(idx, bits); - return ret; -} - -static inline int test_and_clear_bit(int idx, volatile unsigned long *bits) -{ - int ret = test_bit(idx, bits); - - clear_bit(idx, bits); - return ret; -} - -static inline int find_next_zero_bit(unsigned long *bits, int limit, int idx) -{ - while ((++idx < limit) && test_bit(idx, bits)) - ; - return idx; -} - -static inline int find_first_zero_bit(unsigned long *bits, int limit) -{ - int idx = 0; - - while (test_bit(idx, bits) && (++idx < limit)) - ; - return idx; -} - -static inline u64 div64_u64(u64 n, u64 d) -{ - return n / d; -} - #define atomic_t rte_atomic32_t #define atomic_read(v) rte_atomic32_read(v) #define atomic_set(v, i) rte_atomic32_set(v, i) diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h index ee4b772c..14159609 100644 --- a/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h +++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h @@ -38,10 +38,6 @@ typedef uint64_t dma_addr_t; * */ -#define QMAN_REV_4000 0x04000000 -#define QMAN_REV_4100 0x04010000 -#define QMAN_REV_4101 0x04010001 - /** * struct qbman_block_desc - qbman block descriptor structure * @ccsr_reg_bar: CCSR register map. diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h index 9e9047e2..1e656602 100644 --- a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h +++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h @@ -194,11 +194,38 @@ void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit); /** * struct qbman_result - structure for qbman dequeue response and/or * notification. - * @dont_manipulate_directly: the 16 32bit data to represent the whole + * @donot_manipulate_directly: the 16 32bit data to represent the whole * possible qbman dequeue result. */ struct qbman_result { - uint32_t dont_manipulate_directly[16]; + union { + struct common { + uint8_t verb; + uint8_t reserved[63]; + } common; + struct dq { + uint8_t verb; + uint8_t stat; + __le16 seqnum; + __le16 oprid; + uint8_t reserved; + uint8_t tok; + __le32 fqid; + uint32_t reserved2; + __le32 fq_byte_cnt; + __le32 fq_frm_cnt; + __le64 fqd_ctx; + uint8_t fd[32]; + } dq; + struct scn { + uint8_t verb; + uint8_t stat; + uint8_t state; + uint8_t reserved; + __le32 rid_tok; + __le64 ctx; + } scn; + }; }; /* TODO: @@ -254,11 +281,21 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable); /** * struct qbman_pull_desc - the structure for pull dequeue descriptor - * @dont_manipulate_directly: the 6 32bit data to represent the whole - * possible settings for pull dequeue descriptor. */ struct qbman_pull_desc { - uint32_t dont_manipulate_directly[6]; + union { + uint32_t donot_manipulate_directly[16]; + struct pull { + uint8_t verb; + uint8_t numf; + uint8_t tok; + uint8_t reserved; + uint32_t dq_src; + uint64_t rsp_addr; + uint64_t rsp_addr_virt; + uint8_t padding[40]; + } pull; + }; }; enum qbman_pull_type_e { @@ -292,7 +329,7 @@ void qbman_pull_desc_clear(struct qbman_pull_desc *d); */ void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, struct qbman_result *storage, - dma_addr_t storage_phys, + uint64_t storage_phys, int stash); /** * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued. @@ -415,7 +452,20 @@ struct qbman_result *qbman_get_dqrr_from_idx(struct qbman_swp *s, uint8_t idx); * dequeue result. */ int qbman_result_has_new_result(struct qbman_swp *s, - const struct qbman_result *dq); + struct qbman_result *dq); + +/** + * qbman_check_command_complete() - Check if the previous issued dq commnd + * is completed and results are available in memory. + * @s: the software portal object. + * @dq: the dequeue result read from the memory. + * + * Return 1 for getting a valid dequeue result, or 0 for not getting a valid + * dequeue result. + */ +int qbman_check_command_complete(struct qbman_result *dq); + +int qbman_check_new_result(struct qbman_result *dq); /* -------------------------------------------------------- */ /* Parsing dequeue entries (DQRR and user-provided storage) */ @@ -537,7 +587,7 @@ int qbman_result_is_FQPN(const struct qbman_result *dq); * * Return the state field. */ -uint32_t qbman_result_DQ_flags(const struct qbman_result *dq); +uint8_t qbman_result_DQ_flags(const struct qbman_result *dq); /** * qbman_result_DQ_is_pull() - Check whether the dq response is from a pull @@ -648,24 +698,6 @@ uint32_t qbman_result_SCN_rid(const struct qbman_result *scn); */ uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn); -/** - * qbman_result_SCN_state_in_mem() - Get the state in notification written - * in memory - * @scn: the state change notification. - * - * Return the state. - */ -uint8_t qbman_result_SCN_state_in_mem(const struct qbman_result *scn); - -/** - * qbman_result_SCN_rid_in_mem() - Get the resource id in notification written - * in memory. - * @scn: the state change notification. - * - * Return the resource id. - */ -uint32_t qbman_result_SCN_rid_in_mem(const struct qbman_result *scn); - /* Type-specific "resource IDs". Mainly for illustration purposes, though it * also gives the appropriate type widths. */ @@ -746,22 +778,35 @@ uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn); /* Enqueues */ /************/ -/** - * struct qbman_eq_desc - structure of enqueue descriptor - * @dont_manipulate_directly: the 8 32bit data to represent the whole - * possible qbman enqueue setting in enqueue descriptor. - */ +/* struct qbman_eq_desc - structure of enqueue descriptor */ struct qbman_eq_desc { - uint32_t dont_manipulate_directly[8]; + union { + uint32_t donot_manipulate_directly[8]; + struct eq { + uint8_t verb; + uint8_t dca; + uint16_t seqnum; + uint16_t orpid; + uint16_t reserved1; + uint32_t tgtid; + uint32_t tag; + uint16_t qdbin; + uint8_t qpri; + uint8_t reserved[3]; + uint8_t wae; + uint8_t rspid; + uint64_t rsp_addr; + } eq; + }; }; /** * struct qbman_eq_response - structure of enqueue response - * @dont_manipulate_directly: the 16 32bit data to represent the whole + * @donot_manipulate_directly: the 16 32bit data to represent the whole * enqueue response. */ struct qbman_eq_response { - uint32_t dont_manipulate_directly[16]; + uint32_t donot_manipulate_directly[16]; }; /** @@ -801,7 +846,7 @@ void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success); * sequeue number. */ void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success, - uint32_t opr_id, uint32_t seqnum, int incomplete); + uint16_t opr_id, uint16_t seqnum, int incomplete); /** * qbman_eq_desc_set_orp_hole() - fill a hole in the order-restoration sequence @@ -810,8 +855,8 @@ void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success, * @opr_id: the order point record id. * @seqnum: the order restoration sequence number. */ -void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint32_t opr_id, - uint32_t seqnum); +void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum); /** * qbman_eq_desc_set_orp_nesn() - advance NESN (Next Expected Sequence Number) @@ -820,8 +865,8 @@ void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint32_t opr_id, * @opr_id: the order point record id. * @seqnum: the order restoration sequence number. */ -void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint32_t opr_id, - uint32_t seqnum); +void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum); /** * qbman_eq_desc_set_response() - Set the enqueue response info. * @d: the enqueue descriptor @@ -835,7 +880,7 @@ void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint32_t opr_id, * expresses a cache-warming attribute. */ void qbman_eq_desc_set_response(struct qbman_eq_desc *d, - dma_addr_t storage_phys, + uint64_t storage_phys, int stash); /** @@ -873,7 +918,7 @@ void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid); * @qd_prio: the queuing destination priority. */ void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid, - uint32_t qd_bin, uint32_t qd_prio); + uint16_t qd_bin, uint8_t qd_prio); /** * qbman_eq_desc_set_eqdi() - enable/disable EQDI interrupt @@ -898,7 +943,7 @@ void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable); * being rescheduled.) */ void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, - uint32_t dqrr_idx, int park); + uint8_t dqrr_idx, int park); /** * qbman_swp_enqueue() - Issue an enqueue command. @@ -914,19 +959,33 @@ void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, const struct qbman_fd *fd); /** - * qbman_swp_enqueue_multiple_eqdesc() - Enqueue multiple frames with separte - * enqueue descriptors. + * qbman_swp_enqueue_multiple() - Enqueue multiple frames with same + eq descriptor * @s: the software portal used for enqueue. - * @d: the enqueue descriptors + * @d: the enqueue descriptor. * @fd: the frame descriptor to be enqueued. * @num_frames: the number of the frames to be enqueued. * * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. */ -int qbman_swp_enqueue_multiple_eqdesc(struct qbman_swp *s, +int qbman_swp_enqueue_multiple(struct qbman_swp *s, const struct qbman_eq_desc *d, const struct qbman_fd *fd, int num_frames); +/** + * qbman_swp_enqueue_multiple_desc() - Enqueue multiple frames with + * individual eq descriptor. + * @s: the software portal used for enqueue. + * @d: the enqueue descriptor. + * @fd: the frame descriptor to be enqueued. + * @num_frames: the number of the frames to be enqueued. + * + * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. + */ +int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); /* TODO: * qbman_swp_enqueue_thresh() - Set threshold for EQRI interrupt. @@ -943,11 +1002,20 @@ int qbman_swp_enqueue_thresh(struct qbman_swp *s, unsigned int thresh); /*******************/ /** * struct qbman_release_desc - The structure for buffer release descriptor - * @dont_manipulate_directly: the 32bit data to represent the whole + * @donot_manipulate_directly: the 32bit data to represent the whole * possible settings of qbman release descriptor. */ struct qbman_release_desc { - uint32_t dont_manipulate_directly[1]; + union { + uint32_t donot_manipulate_directly[16]; + struct br { + uint8_t verb; + uint8_t reserved; + uint16_t bpid; + uint32_t reserved2; + uint64_t buf[7]; + } br; + }; }; /** @@ -961,7 +1029,7 @@ void qbman_release_desc_clear(struct qbman_release_desc *d); * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to * @d: the qbman release descriptor. */ -void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint32_t bpid); +void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid); /** * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI @@ -1004,7 +1072,7 @@ int qbman_swp_release_thresh(struct qbman_swp *s, unsigned int thresh); * Return 0 for success, or negative error code if the acquire command * fails. */ -int qbman_swp_acquire(struct qbman_swp *s, uint32_t bpid, uint64_t *buffers, +int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, unsigned int num_buffers); /*****************/ @@ -1119,19 +1187,4 @@ int qbman_swp_CDAN_disable(struct qbman_swp *s, uint16_t channelid); */ int qbman_swp_CDAN_set_context_enable(struct qbman_swp *s, uint16_t channelid, uint64_t ctx); -int qbman_swp_fill_ring(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - uint8_t burst_index); -int qbman_swp_flush_ring(struct qbman_swp *s); -void qbman_sync(void); -int qbman_swp_send_multiple(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - int frames_to_send); - -int qbman_check_command_complete(struct qbman_swp *s, - const struct qbman_result *dq); - -int qbman_get_version(void); #endif /* !_FSL_QBMAN_PORTAL_H */ diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c index dd62e9af..809770c7 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.c +++ b/drivers/bus/fslmc/qbman/qbman_portal.c @@ -69,21 +69,23 @@ /* Pre-defined attribute codes */ /*******************************/ -struct qb_attr_code code_generic_verb = QB_CODE(0, 0, 7); -struct qb_attr_code code_generic_rslt = QB_CODE(0, 8, 8); +#define QBMAN_RESPONSE_VERB_MASK 0x7f /*************************/ /* SDQCR attribute codes */ /*************************/ +#define QB_SDQCR_FC_SHIFT 29 +#define QB_SDQCR_FC_MASK 0x1 +#define QB_SDQCR_DCT_SHIFT 24 +#define QB_SDQCR_DCT_MASK 0x3 +#define QB_SDQCR_TOK_SHIFT 16 +#define QB_SDQCR_TOK_MASK 0xff +#define QB_SDQCR_SRC_SHIFT 0 +#define QB_SDQCR_SRC_MASK 0xffff + +/* opaque token for static dequeues */ +#define QMAN_SDQCR_TOKEN 0xbb -/* we put these here because at least some of them are required by - * qbman_swp_init() - */ -struct qb_attr_code code_sdqcr_dct = QB_CODE(0, 24, 2); -struct qb_attr_code code_sdqcr_fc = QB_CODE(0, 29, 1); -struct qb_attr_code code_sdqcr_tok = QB_CODE(0, 16, 8); -static struct qb_attr_code code_eq_dca_idx; -#define CODE_SDQCR_DQSRC(n) QB_CODE(0, n, 1) enum qbman_sdqcr_dct { qbman_sdqcr_dct_null = 0, qbman_sdqcr_dct_prio_ics, @@ -96,17 +98,13 @@ enum qbman_sdqcr_fc { qbman_sdqcr_fc_up_to_3 = 1 }; -struct qb_attr_code code_sdqcr_dqsrc = QB_CODE(0, 0, 16); - /* We need to keep track of which SWP triggered a pull command * so keep an array of portal IDs and use the token field to * be able to find the proper portal */ -#define MAX_QBMAN_PORTALS 35 +#define MAX_QBMAN_PORTALS 64 static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS]; -uint32_t qman_version; - /*********************************/ /* Portal constructor/destructor */ /*********************************/ @@ -128,7 +126,7 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) { int ret; uint32_t eqcr_pi; - struct qbman_swp *p = kmalloc(sizeof(*p), GFP_KERNEL); + struct qbman_swp *p = malloc(sizeof(*p)); if (!p) return NULL; @@ -138,9 +136,10 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) #endif p->mc.valid_bit = QB_VALID_BIT; p->sdq = 0; - qb_attr_code_encode(&code_sdqcr_dct, &p->sdq, qbman_sdqcr_dct_prio_ics); - qb_attr_code_encode(&code_sdqcr_fc, &p->sdq, qbman_sdqcr_fc_up_to_3); - qb_attr_code_encode(&code_sdqcr_tok, &p->sdq, 0xbb); + p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; + p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; + p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; + atomic_set(&p->vdq.busy, 1); p->vdq.valid_bit = QB_VALID_BIT; p->dqrr.next_idx = 0; @@ -149,25 +148,21 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) { p->dqrr.dqrr_size = 4; p->dqrr.reset_bug = 1; - /* Set size of DQRR to 4, encoded in 2 bits */ - code_eq_dca_idx = (struct qb_attr_code)QB_CODE(0, 8, 2); } else { p->dqrr.dqrr_size = 8; p->dqrr.reset_bug = 0; - /* Set size of DQRR to 8, encoded in 3 bits */ - code_eq_dca_idx = (struct qb_attr_code)QB_CODE(0, 8, 3); } ret = qbman_swp_sys_init(&p->sys, d, p->dqrr.dqrr_size); if (ret) { - kfree(p); + free(p); pr_err("qbman_swp_sys_init() failed %d\n", ret); return NULL; } /* SDQCR needs to be initialized to 0 when no channels are * being dequeued from or else the QMan HW will indicate an * error. The values that were calculated above will be - * applied when dequeues from a specific channel are enabled + * applied when dequeues from a specific channel are enabled. */ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0); eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI); @@ -188,7 +183,7 @@ void qbman_swp_finish(struct qbman_swp *p) #endif qbman_swp_sys_finish(&p->sys); portal_idx_map[p->desc.idx] = NULL; - kfree(p); + free(p); } const struct qbman_swp_desc *qbman_swp_get_desc(struct qbman_swp *p) @@ -282,9 +277,9 @@ void *qbman_swp_mc_start(struct qbman_swp *p) return ret; } -void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint32_t cmd_verb) +void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb) { - uint32_t *v = cmd; + uint8_t *v = cmd; #ifdef QBMAN_CHECKING QBMAN_BUG_ON(!(p->mc.check != swp_mc_can_submit)); #endif @@ -325,35 +320,22 @@ void *qbman_swp_mc_result(struct qbman_swp *p) /* Enqueue */ /***********/ -/* These should be const, eventually */ -static struct qb_attr_code code_eq_cmd = QB_CODE(0, 0, 2); -static struct qb_attr_code code_eq_eqdi = QB_CODE(0, 3, 1); -static struct qb_attr_code code_eq_dca_en = QB_CODE(0, 15, 1); -static struct qb_attr_code code_eq_dca_pk = QB_CODE(0, 14, 1); -/* Can't set code_eq_dca_idx width. Need qman version. Read at runtime */ -static struct qb_attr_code code_eq_orp_en = QB_CODE(0, 2, 1); -static struct qb_attr_code code_eq_orp_is_nesn = QB_CODE(0, 31, 1); -static struct qb_attr_code code_eq_orp_nlis = QB_CODE(0, 30, 1); -static struct qb_attr_code code_eq_orp_seqnum = QB_CODE(0, 16, 14); -static struct qb_attr_code code_eq_opr_id = QB_CODE(1, 0, 16); -static struct qb_attr_code code_eq_tgt_id = QB_CODE(2, 0, 24); -/* static struct qb_attr_code code_eq_tag = QB_CODE(3, 0, 32); */ -static struct qb_attr_code code_eq_qd_en = QB_CODE(0, 4, 1); -static struct qb_attr_code code_eq_qd_bin = QB_CODE(4, 0, 16); -static struct qb_attr_code code_eq_qd_pri = QB_CODE(4, 16, 4); -static struct qb_attr_code code_eq_rsp_stash = QB_CODE(5, 16, 1); -static struct qb_attr_code code_eq_rsp_id = QB_CODE(5, 24, 8); -static struct qb_attr_code code_eq_rsp_lo = QB_CODE(6, 0, 32); - -enum qbman_eq_cmd_e { - /* No enqueue, primarily for plugging ORP gaps for dropped frames */ - qbman_eq_cmd_empty, - /* DMA an enqueue response once complete */ - qbman_eq_cmd_respond, - /* DMA an enqueue response only if the enqueue fails */ - qbman_eq_cmd_respond_reject +#define QB_ENQUEUE_CMD_OPTIONS_SHIFT 0 +enum qb_enqueue_commands { + enqueue_empty = 0, + enqueue_response_always = 1, + enqueue_rejects_to_fq = 2 }; +#define QB_ENQUEUE_CMD_EC_OPTION_MASK 0x3 +#define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2 +#define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3 +#define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4 +#define QB_ENQUEUE_CMD_DCA_PK_SHIFT 6 +#define QB_ENQUEUE_CMD_DCA_EN_SHIFT 7 +#define QB_ENQUEUE_CMD_NLIS_SHIFT 14 +#define QB_ENQUEUE_CMD_IS_NESN_SHIFT 15 + void qbman_eq_desc_clear(struct qbman_eq_desc *d) { memset(d, 0, sizeof(*d)); @@ -361,115 +343,110 @@ void qbman_eq_desc_clear(struct qbman_eq_desc *d) void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_orp_en, cl, 0); - qb_attr_code_encode(&code_eq_cmd, cl, - respond_success ? qbman_eq_cmd_respond : - qbman_eq_cmd_respond_reject); + d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT); + if (respond_success) + d->eq.verb |= enqueue_response_always; + else + d->eq.verb |= enqueue_rejects_to_fq; } void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success, - uint32_t opr_id, uint32_t seqnum, int incomplete) + uint16_t opr_id, uint16_t seqnum, int incomplete) { - uint32_t *cl = qb_cl(d); + d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; + if (respond_success) + d->eq.verb |= enqueue_response_always; + else + d->eq.verb |= enqueue_rejects_to_fq; - qb_attr_code_encode(&code_eq_orp_en, cl, 1); - qb_attr_code_encode(&code_eq_cmd, cl, - respond_success ? qbman_eq_cmd_respond : - qbman_eq_cmd_respond_reject); - qb_attr_code_encode(&code_eq_opr_id, cl, opr_id); - qb_attr_code_encode(&code_eq_orp_seqnum, cl, seqnum); - qb_attr_code_encode(&code_eq_orp_nlis, cl, !!incomplete); + d->eq.orpid = opr_id; + d->eq.seqnum = seqnum; + if (incomplete) + d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_NLIS_SHIFT; + else + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); } -void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint32_t opr_id, - uint32_t seqnum) +void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_orp_en, cl, 1); - qb_attr_code_encode(&code_eq_cmd, cl, qbman_eq_cmd_empty); - qb_attr_code_encode(&code_eq_opr_id, cl, opr_id); - qb_attr_code_encode(&code_eq_orp_seqnum, cl, seqnum); - qb_attr_code_encode(&code_eq_orp_nlis, cl, 0); - qb_attr_code_encode(&code_eq_orp_is_nesn, cl, 0); + d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; + d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK; + d->eq.orpid = opr_id; + d->eq.seqnum = seqnum; + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT); } -void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint32_t opr_id, - uint32_t seqnum) +void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_orp_en, cl, 1); - qb_attr_code_encode(&code_eq_cmd, cl, qbman_eq_cmd_empty); - qb_attr_code_encode(&code_eq_opr_id, cl, opr_id); - qb_attr_code_encode(&code_eq_orp_seqnum, cl, seqnum); - qb_attr_code_encode(&code_eq_orp_nlis, cl, 0); - qb_attr_code_encode(&code_eq_orp_is_nesn, cl, 1); + d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; + d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK; + d->eq.orpid = opr_id; + d->eq.seqnum = seqnum; + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); + d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT; } void qbman_eq_desc_set_response(struct qbman_eq_desc *d, dma_addr_t storage_phys, int stash) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode_64(&code_eq_rsp_lo, (uint64_t *)cl, storage_phys); - qb_attr_code_encode(&code_eq_rsp_stash, cl, !!stash); + d->eq.rsp_addr = storage_phys; + d->eq.wae = stash; } void qbman_eq_desc_set_token(struct qbman_eq_desc *d, uint8_t token) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_rsp_id, cl, (uint32_t)token); + d->eq.rspid = token; } void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_qd_en, cl, 0); - qb_attr_code_encode(&code_eq_tgt_id, cl, fqid); + d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT); + d->eq.tgtid = fqid; } void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid, - uint32_t qd_bin, uint32_t qd_prio) + uint16_t qd_bin, uint8_t qd_prio) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_qd_en, cl, 1); - qb_attr_code_encode(&code_eq_tgt_id, cl, qdid); - qb_attr_code_encode(&code_eq_qd_bin, cl, qd_bin); - qb_attr_code_encode(&code_eq_qd_pri, cl, qd_prio); + d->eq.verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT; + d->eq.tgtid = qdid; + d->eq.qdbin = qd_bin; + d->eq.qpri = qd_prio; } void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_eqdi, cl, !!enable); + if (enable) + d->eq.verb |= 1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT; + else + d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT); } void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, - uint32_t dqrr_idx, int park) + uint8_t dqrr_idx, int park) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_eq_dca_en, cl, !!enable); if (enable) { - qb_attr_code_encode(&code_eq_dca_pk, cl, !!park); - qb_attr_code_encode(&code_eq_dca_idx, cl, dqrr_idx); + d->eq.dca = dqrr_idx; + if (park) + d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT; + else + d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT); + d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT; + } else { + d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT); } } #define EQAR_IDX(eqar) ((eqar) & 0x7) #define EQAR_VB(eqar) ((eqar) & 0x80) #define EQAR_SUCCESS(eqar) ((eqar) & 0x100) + static int qbman_swp_enqueue_array_mode(struct qbman_swp *s, const struct qbman_eq_desc *d, - const struct qbman_fd *fd) + const struct qbman_fd *fd) { uint32_t *p; const uint32_t *cl = qb_cl(d); @@ -479,20 +456,20 @@ static int qbman_swp_enqueue_array_mode(struct qbman_swp *s, if (!EQAR_SUCCESS(eqar)) return -EBUSY; p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); - word_copy(&p[1], &cl[1], 7); - word_copy(&p[8], fd, sizeof(*fd) >> 2); + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); /* Set the verb byte, have to substitute in the valid-bit */ lwsync(); p[0] = cl[0] | EQAR_VB(eqar); qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); return 0; } static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, const struct qbman_eq_desc *d, - const struct qbman_fd *fd) + const struct qbman_fd *fd) { uint32_t *p; const uint32_t *cl = qb_cl(d); @@ -511,31 +488,44 @@ static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, } p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); - word_copy(&p[1], &cl[1], 7); - word_copy(&p[8], fd, sizeof(*fd) >> 2); + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); lwsync(); + /* Set the verb byte, have to substitute in the valid-bit */ p[0] = cl[0] | s->eqcr.pi_vb; qbman_cena_write_complete_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); s->eqcr.pi++; s->eqcr.pi &= 0xF; s->eqcr.available--; if (!(s->eqcr.pi & 7)) s->eqcr.pi_vb ^= QB_VALID_BIT; + return 0; } -int qbman_swp_fill_ring(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - __attribute__((unused)) uint8_t burst_index) +int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + if (s->sys.eqcr_mode == qman_eqcr_vb_array) + return qbman_swp_enqueue_array_mode(s, d, fd); + else /* Use ring mode by default */ + return qbman_swp_enqueue_ring_mode(s, d, fd); +} + +int qbman_swp_enqueue_multiple(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) { uint32_t *p; const uint32_t *cl = qb_cl(d); - uint32_t eqcr_ci; + uint32_t eqcr_ci, eqcr_pi; uint8_t diff; + int i, num_enqueued = 0; + uint64_t addr_cena; if (!s->eqcr.available) { eqcr_ci = s->eqcr.ci; @@ -545,64 +535,58 @@ int qbman_swp_fill_ring(struct qbman_swp *s, eqcr_ci, s->eqcr.ci); s->eqcr.available += diff; if (!diff) - return -EBUSY; + return 0; } - p = qbman_cena_write_start_wo_shadow(&s->sys, - QBMAN_CENA_SWP_EQCR((s->eqcr.pi/* +burst_index */) & 7)); - /* word_copy(&p[1], &cl[1], 7); */ - memcpy(&p[1], &cl[1], 7 * 4); - /* word_copy(&p[8], fd, sizeof(*fd) >> 2); */ - memcpy(&p[8], fd, sizeof(struct qbman_fd)); - - /* lwsync(); */ - p[0] = cl[0] | s->eqcr.pi_vb; - - s->eqcr.pi++; - s->eqcr.pi &= 0xF; - s->eqcr.available--; - if (!(s->eqcr.pi & 7)) - s->eqcr.pi_vb ^= QB_VALID_BIT; - - return 0; -} -int qbman_swp_flush_ring(struct qbman_swp *s) -{ - void *ptr = s->sys.addr_cena; + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + eqcr_pi &= 0xF; + } - dcbf((uint64_t)ptr); - dcbf((uint64_t)ptr + 0x40); - dcbf((uint64_t)ptr + 0x80); - dcbf((uint64_t)ptr + 0xc0); - dcbf((uint64_t)ptr + 0x100); - dcbf((uint64_t)ptr + 0x140); - dcbf((uint64_t)ptr + 0x180); - dcbf((uint64_t)ptr + 0x1c0); + lwsync(); - return 0; -} + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + p[0] = cl[0] | s->eqcr.pi_vb; + eqcr_pi++; + eqcr_pi &= 0xF; + if (!(eqcr_pi & 7)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + } -void qbman_sync(void) -{ - lwsync(); -} + /* Flush all the cacheline without load/store in between */ + eqcr_pi = s->eqcr.pi; + addr_cena = (uint64_t)s->sys.addr_cena; + for (i = 0; i < num_enqueued; i++) { + dcbf((uint64_t *)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + eqcr_pi++; + eqcr_pi &= 0xF; + } + s->eqcr.pi = eqcr_pi; -int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, - const struct qbman_fd *fd) -{ - if (s->sys.eqcr_mode == qman_eqcr_vb_array) - return qbman_swp_enqueue_array_mode(s, d, fd); - else /* Use ring mode by default */ - return qbman_swp_enqueue_ring_mode(s, d, fd); + return num_enqueued; } -int qbman_swp_enqueue_multiple_eqdesc(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - int num_frames) +int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) { uint32_t *p; - const uint32_t *cl = qb_cl(d); + const uint32_t *cl; uint32_t eqcr_ci, eqcr_pi; uint8_t diff; int i, num_enqueued = 0; @@ -627,29 +611,26 @@ int qbman_swp_enqueue_multiple_eqdesc(struct qbman_swp *s, for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + cl = qb_cl(&d[i]); memcpy(&p[1], &cl[1], 28); memcpy(&p[8], &fd[i], sizeof(*fd)); eqcr_pi++; eqcr_pi &= 0xF; - /*Pointing to the next enqueue descriptor*/ - cl += (sizeof(struct qbman_eq_desc) / sizeof(uint32_t)); } lwsync(); /* Set the verb byte, have to substitute in the valid-bit */ eqcr_pi = s->eqcr.pi; - cl = qb_cl(d); for (i = 0; i < num_enqueued; i++) { p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + cl = qb_cl(&d[i]); p[0] = cl[0] | s->eqcr.pi_vb; eqcr_pi++; eqcr_pi &= 0xF; if (!(eqcr_pi & 7)) s->eqcr.pi_vb ^= QB_VALID_BIT; - /*Pointing to the next enqueue descriptor*/ - cl += (sizeof(struct qbman_eq_desc) / sizeof(uint32_t)); } /* Flush all the cacheline without load/store in between */ @@ -672,23 +653,26 @@ int qbman_swp_enqueue_multiple_eqdesc(struct qbman_swp *s, void qbman_swp_push_get(struct qbman_swp *s, uint8_t channel_idx, int *enabled) { - struct qb_attr_code code = CODE_SDQCR_DQSRC(channel_idx); + uint16_t src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; QBMAN_BUG_ON(channel_idx > 15); - *enabled = (int)qb_attr_code_decode(&code, &s->sdq); + *enabled = src | (1 << channel_idx); } void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable) { uint16_t dqsrc; - struct qb_attr_code code = CODE_SDQCR_DQSRC(channel_idx); QBMAN_BUG_ON(channel_idx > 15); - qb_attr_code_encode(&code, &s->sdq, !!enable); + if (enable) + s->sdq |= 1 << channel_idx; + else + s->sdq &= ~(1 << channel_idx); + /* Read make the complete src map. If no channels are enabled * the SDQCR must be 0 or else QMan will assert errors */ - dqsrc = (uint16_t)qb_attr_code_decode(&code_sdqcr_dqsrc, &s->sdq); + dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; if (dqsrc != 0) qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, s->sdq); else @@ -700,14 +684,10 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable) /***************************/ /* These should be const, eventually */ -static struct qb_attr_code code_pull_dct = QB_CODE(0, 0, 2); -static struct qb_attr_code code_pull_dt = QB_CODE(0, 2, 2); -static struct qb_attr_code code_pull_rls = QB_CODE(0, 4, 1); -static struct qb_attr_code code_pull_stash = QB_CODE(0, 5, 1); -static struct qb_attr_code code_pull_numframes = QB_CODE(0, 8, 4); -static struct qb_attr_code code_pull_token = QB_CODE(0, 16, 8); -static struct qb_attr_code code_pull_dqsource = QB_CODE(1, 0, 24); -static struct qb_attr_code code_pull_rsp_lo = QB_CODE(2, 0, 32); +#define QB_VDQCR_VERB_DCT_SHIFT 0 +#define QB_VDQCR_VERB_DT_SHIFT 2 +#define QB_VDQCR_VERB_RLS_SHIFT 4 +#define QB_VDQCR_VERB_WAE_SHIFT 5 enum qb_pull_dt_e { qb_pull_dt_channel, @@ -725,63 +705,52 @@ void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, dma_addr_t storage_phys, int stash) { - uint32_t *cl = qb_cl(d); - /* Squiggle the pointer 'storage' into the extra 2 words of the - * descriptor (which aren't copied to the hw command) - */ - *(void **)&cl[4] = storage; + d->pull.rsp_addr_virt = (uint64_t)storage; + if (!storage) { - qb_attr_code_encode(&code_pull_rls, cl, 0); + d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT); return; } - qb_attr_code_encode(&code_pull_rls, cl, 1); - qb_attr_code_encode(&code_pull_stash, cl, !!stash); - qb_attr_code_encode_64(&code_pull_rsp_lo, (uint64_t *)cl, storage_phys); + d->pull.verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT; + if (stash) + d->pull.verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT; + else + d->pull.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); + + d->pull.rsp_addr = storage_phys; } void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, uint8_t numframes) { - uint32_t *cl = qb_cl(d); - - QBMAN_BUG_ON(!numframes || (numframes > 16)); - qb_attr_code_encode(&code_pull_numframes, cl, - (uint32_t)(numframes - 1)); + d->pull.numf = numframes - 1; } void qbman_pull_desc_set_token(struct qbman_pull_desc *d, uint8_t token) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_pull_token, cl, token); + d->pull.tok = token; } void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, uint32_t fqid) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_pull_dct, cl, 1); - qb_attr_code_encode(&code_pull_dt, cl, qb_pull_dt_framequeue); - qb_attr_code_encode(&code_pull_dqsource, cl, fqid); + d->pull.verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT; + d->pull.verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT; + d->pull.dq_src = fqid; } void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid, enum qbman_pull_type_e dct) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_pull_dct, cl, dct); - qb_attr_code_encode(&code_pull_dt, cl, qb_pull_dt_workqueue); - qb_attr_code_encode(&code_pull_dqsource, cl, wqid); + d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; + d->pull.verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT; + d->pull.dq_src = wqid; } void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, enum qbman_pull_type_e dct) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_pull_dct, cl, dct); - qb_attr_code_encode(&code_pull_dt, cl, qb_pull_dt_channel); - qb_attr_code_encode(&code_pull_dqsource, cl, chid); + d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; + d->pull.verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT; + d->pull.dq_src = chid; } int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) @@ -793,18 +762,18 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) atomic_inc(&s->vdq.busy); return -EBUSY; } - s->vdq.storage = *(void **)&cl[4]; - /* We use portal index +1 as token so that 0 still indicates - * that the result isn't valid yet. - */ - qb_attr_code_encode(&code_pull_token, cl, s->desc.idx + 1); + + d->pull.tok = s->sys.idx + 1; + s->vdq.storage = (void *)d->pull.rsp_addr_virt; p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR); - word_copy(&p[1], &cl[1], 3); + memcpy(&p[1], &cl[1], 12); + /* Set the verb byte, have to substitute in the valid-bit */ lwsync(); p[0] = cl[0] | s->vdq.valid_bit; s->vdq.valid_bit ^= QB_VALID_BIT; qbman_cena_write_complete_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR); + return 0; } @@ -812,16 +781,7 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) /* Polling DQRR */ /****************/ -static struct qb_attr_code code_dqrr_verb = QB_CODE(0, 0, 8); -static struct qb_attr_code code_dqrr_response = QB_CODE(0, 0, 7); -static struct qb_attr_code code_dqrr_stat = QB_CODE(0, 8, 8); -static struct qb_attr_code code_dqrr_seqnum = QB_CODE(0, 16, 14); -static struct qb_attr_code code_dqrr_odpid = QB_CODE(1, 0, 16); -/* static struct qb_attr_code code_dqrr_tok = QB_CODE(1, 24, 8); */ -static struct qb_attr_code code_dqrr_fqid = QB_CODE(2, 0, 24); -static struct qb_attr_code code_dqrr_byte_count = QB_CODE(4, 0, 32); -static struct qb_attr_code code_dqrr_frame_count = QB_CODE(5, 0, 24); -static struct qb_attr_code code_dqrr_ctx_lo = QB_CODE(6, 0, 32); +#define QMAN_DQRR_PI_MASK 0xf #define QBMAN_RESULT_DQ 0x60 #define QBMAN_RESULT_FQRN 0x21 @@ -834,8 +794,6 @@ static struct qb_attr_code code_dqrr_ctx_lo = QB_CODE(6, 0, 32); #define QBMAN_RESULT_BPSCN 0x29 #define QBMAN_RESULT_CSCN_WQ 0x2a -static struct qb_attr_code code_dqpi_pi = QB_CODE(0, 0, 4); - /* NULL return if there are no unconsumed DQRR entries. Returns a DQRR entry * only once, so repeated calls can return a sequence of DQRR entries, without * requiring they be consumed immediately or in any particular order. @@ -845,8 +803,7 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) uint32_t verb; uint32_t response_verb; uint32_t flags; - const struct qbman_result *dq; - const uint32_t *p; + const struct qbman_result *p; /* Before using valid-bit to detect if something is there, we have to * handle the case of the DQRR reset bug... @@ -859,11 +816,13 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) * will be much less efficient than all subsequent trips around * it... */ - uint32_t dqpi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI); - uint32_t pi = qb_attr_code_decode(&code_dqpi_pi, &dqpi); + uint8_t pi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI) & + QMAN_DQRR_PI_MASK; + /* there are new entries if pi != next_idx */ if (pi == s->dqrr.next_idx) return NULL; + /* if next_idx is/was the last ring index, and 'pi' is * different, we can disable the workaround as all the ring * entries have now been DMA'd to so valid-bit checking is @@ -878,12 +837,12 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) s->dqrr.reset_bug = 0; } qbman_cena_invalidate_prefetch(&s->sys, - QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); } - dq = qbman_cena_read_wo_shadow(&s->sys, - QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); - p = qb_cl(dq); - verb = qb_attr_code_decode(&code_dqrr_verb, p); + p = qbman_cena_read_wo_shadow(&s->sys, + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + verb = p->dq.verb; + /* If the valid-bit isn't of the expected polarity, nothing there. Note, * in the DQRR reset bug workaround, we shouldn't need to skip these * check, because we've already determined that a new entry is available @@ -903,16 +862,16 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) s->dqrr.valid_bit ^= QB_VALID_BIT; } /* If this is the final response to a volatile dequeue command - * indicate that the vdq is no longer busy. + * indicate that the vdq is no longer busy */ - flags = qbman_result_DQ_flags(dq); - response_verb = qb_attr_code_decode(&code_dqrr_response, &verb); + flags = p->dq.stat; + response_verb = verb & QBMAN_RESPONSE_VERB_MASK; if ((response_verb == QBMAN_RESULT_DQ) && (flags & QBMAN_DQ_STAT_VOLATILE) && (flags & QBMAN_DQ_STAT_EXPIRED)) atomic_inc(&s->vdq.busy); - return dq; + return p; } /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ @@ -925,80 +884,69 @@ void qbman_swp_dqrr_consume(struct qbman_swp *s, /*********************************/ /* Polling user-provided storage */ /*********************************/ - -int qbman_result_has_new_result(__attribute__((unused)) struct qbman_swp *s, - const struct qbman_result *dq) +int qbman_result_has_new_result(struct qbman_swp *s, + struct qbman_result *dq) { - /* To avoid converting the little-endian DQ entry to host-endian prior - * to us knowing whether there is a valid entry or not (and run the - * risk of corrupting the incoming hardware LE write), we detect in - * hardware endianness rather than host. This means we need a different - * "code" depending on whether we are BE or LE in software, which is - * where DQRR_TOK_OFFSET comes in... + if (dq->dq.tok == 0) + return 0; + + /* + * Set token to be 0 so we will detect change back to 1 + * next time the looping is traversed. Const is cast away here + * as we want users to treat the dequeue responses as read only. */ - static struct qb_attr_code code_dqrr_tok_detect = - QB_CODE(0, DQRR_TOK_OFFSET, 8); - /* The user trying to poll for a result treats "dq" as const. It is - * however the same address that was provided to us non-const in the - * first place, for directing hardware DMA to. So we can cast away the - * const because it is mutable from our perspective. + ((struct qbman_result *)dq)->dq.tok = 0; + + /* + * VDQCR "no longer busy" hook - not quite the same as DQRR, because the + * fact "VDQCR" shows busy doesn't mean that we hold the result that + * makes it available. Eg. we may be looking at our 10th dequeue result, + * having released VDQCR after the 1st result and it is now busy due to + * some other command! */ - uint32_t *p = (uint32_t *)(unsigned long)qb_cl(dq); - uint32_t token; + if (s->vdq.storage == dq) { + s->vdq.storage = NULL; + atomic_inc(&s->vdq.busy); + } + + return 1; +} - token = qb_attr_code_decode(&code_dqrr_tok_detect, &p[1]); - if (token == 0) +int qbman_check_new_result(struct qbman_result *dq) +{ + if (dq->dq.tok == 0) return 0; - /* Entry is valid - overwrite token back to 0 so - * a) If this memory is reused tokesn will be 0 - * b) If someone calls "has_new_result()" again on this entry it - * will not appear to be new - */ - qb_attr_code_encode(&code_dqrr_tok_detect, &p[1], 0); - /* Only now do we convert from hardware to host endianness. Also, as we - * are returning success, the user has promised not to call us again, so - * there's no risk of us converting the endianness twice... + /* + * Set token to be 0 so we will detect change back to 1 + * next time the looping is traversed. Const is cast away here + * as we want users to treat the dequeue responses as read only. */ - make_le32_n(p, 16); + ((struct qbman_result *)dq)->dq.tok = 0; + return 1; } -int qbman_check_command_complete(struct qbman_swp *s, - const struct qbman_result *dq) +int qbman_check_command_complete(struct qbman_result *dq) { - /* To avoid converting the little-endian DQ entry to host-endian prior - * to us knowing whether there is a valid entry or not (and run the - * risk of corrupting the incoming hardware LE write), we detect in - * hardware endianness rather than host. This means we need a different - * "code" depending on whether we are BE or LE in software, which is - * where DQRR_TOK_OFFSET comes in... - */ - static struct qb_attr_code code_dqrr_tok_detect = - QB_CODE(0, DQRR_TOK_OFFSET, 8); - /* The user trying to poll for a result treats "dq" as const. It is - * however the same address that was provided to us non-const in the - * first place, for directing hardware DMA to. So we can cast away the - * const because it is mutable from our perspective. - */ - uint32_t *p = (uint32_t *)(unsigned long)qb_cl(dq); - uint32_t token; + struct qbman_swp *s; - token = qb_attr_code_decode(&code_dqrr_tok_detect, &p[1]); - if (token == 0) + if (dq->dq.tok == 0) return 0; - /* TODO: Remove qbman_swp from parameters and make it a local - * once we've tested the reserve portal map change - */ - s = portal_idx_map[token - 1]; - /* When token is set it indicates that VDQ command has been fetched - * by qbman and is working on it. It is safe for software to issue - * another VDQ command, so incrementing the busy variable. + + s = portal_idx_map[dq->dq.tok - 1]; + /* + * VDQCR "no longer busy" hook - not quite the same as DQRR, because the + * fact "VDQCR" shows busy doesn't mean that we hold the result that + * makes it available. Eg. we may be looking at our 10th dequeue result, + * having released VDQCR after the 1st result and it is now busy due to + * some other command! */ if (s->vdq.storage == dq) { s->vdq.storage = NULL; atomic_inc(&s->vdq.busy); } + return 1; } @@ -1006,23 +954,10 @@ int qbman_check_command_complete(struct qbman_swp *s, /* Categorising qbman results */ /********************************/ -static struct qb_attr_code code_result_in_mem = - QB_CODE(0, QBMAN_RESULT_VERB_OFFSET_IN_MEM, 7); - static inline int __qbman_result_is_x(const struct qbman_result *dq, - uint32_t x) -{ - const uint32_t *p = qb_cl(dq); - uint32_t response_verb = qb_attr_code_decode(&code_dqrr_response, p); - - return (response_verb == x); -} - -static inline int __qbman_result_is_x_in_mem(const struct qbman_result *dq, - uint32_t x) + uint8_t x) { - const uint32_t *p = qb_cl(dq); - uint32_t response_verb = qb_attr_code_decode(&code_result_in_mem, p); + uint8_t response_verb = dq->dq.verb & QBMAN_RESPONSE_VERB_MASK; return (response_verb == x); } @@ -1044,28 +979,28 @@ int qbman_result_is_CDAN(const struct qbman_result *dq) int qbman_result_is_CSCN(const struct qbman_result *dq) { - return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_CSCN_MEM) || + return __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_MEM) || __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_WQ); } int qbman_result_is_BPSCN(const struct qbman_result *dq) { - return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_BPSCN); + return __qbman_result_is_x(dq, QBMAN_RESULT_BPSCN); } int qbman_result_is_CGCU(const struct qbman_result *dq) { - return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_CGCU); + return __qbman_result_is_x(dq, QBMAN_RESULT_CGCU); } int qbman_result_is_FQRN(const struct qbman_result *dq) { - return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_FQRN); + return __qbman_result_is_x(dq, QBMAN_RESULT_FQRN); } int qbman_result_is_FQRNI(const struct qbman_result *dq) { - return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_FQRNI); + return __qbman_result_is_x(dq, QBMAN_RESULT_FQRNI); } int qbman_result_is_FQPN(const struct qbman_result *dq) @@ -1079,109 +1014,62 @@ int qbman_result_is_FQPN(const struct qbman_result *dq) /* These APIs assume qbman_result_is_DQ() is TRUE */ -uint32_t qbman_result_DQ_flags(const struct qbman_result *dq) +uint8_t qbman_result_DQ_flags(const struct qbman_result *dq) { - const uint32_t *p = qb_cl(dq); - - return qb_attr_code_decode(&code_dqrr_stat, p); + return dq->dq.stat; } uint16_t qbman_result_DQ_seqnum(const struct qbman_result *dq) { - const uint32_t *p = qb_cl(dq); - - return (uint16_t)qb_attr_code_decode(&code_dqrr_seqnum, p); + return dq->dq.seqnum; } uint16_t qbman_result_DQ_odpid(const struct qbman_result *dq) { - const uint32_t *p = qb_cl(dq); - - return (uint16_t)qb_attr_code_decode(&code_dqrr_odpid, p); + return dq->dq.oprid; } uint32_t qbman_result_DQ_fqid(const struct qbman_result *dq) { - const uint32_t *p = qb_cl(dq); - - return qb_attr_code_decode(&code_dqrr_fqid, p); + return dq->dq.fqid; } uint32_t qbman_result_DQ_byte_count(const struct qbman_result *dq) { - const uint32_t *p = qb_cl(dq); - - return qb_attr_code_decode(&code_dqrr_byte_count, p); + return dq->dq.fq_byte_cnt; } uint32_t qbman_result_DQ_frame_count(const struct qbman_result *dq) { - const uint32_t *p = qb_cl(dq); - - return qb_attr_code_decode(&code_dqrr_frame_count, p); + return dq->dq.fq_frm_cnt; } uint64_t qbman_result_DQ_fqd_ctx(const struct qbman_result *dq) { - const uint64_t *p = (const uint64_t *)qb_cl(dq); - - return qb_attr_code_decode_64(&code_dqrr_ctx_lo, p); + return dq->dq.fqd_ctx; } const struct qbman_fd *qbman_result_DQ_fd(const struct qbman_result *dq) { - const uint32_t *p = qb_cl(dq); - - return (const struct qbman_fd *)&p[8]; + return (const struct qbman_fd *)&dq->dq.fd[0]; } /**************************************/ /* Parsing state-change notifications */ /**************************************/ - -static struct qb_attr_code code_scn_state = QB_CODE(0, 16, 8); -static struct qb_attr_code code_scn_rid = QB_CODE(1, 0, 24); -static struct qb_attr_code code_scn_state_in_mem = - QB_CODE(0, SCN_STATE_OFFSET_IN_MEM, 8); -static struct qb_attr_code code_scn_rid_in_mem = - QB_CODE(1, SCN_RID_OFFSET_IN_MEM, 24); -static struct qb_attr_code code_scn_ctx_lo = QB_CODE(2, 0, 32); - uint8_t qbman_result_SCN_state(const struct qbman_result *scn) { - const uint32_t *p = qb_cl(scn); - - return (uint8_t)qb_attr_code_decode(&code_scn_state, p); + return scn->scn.state; } uint32_t qbman_result_SCN_rid(const struct qbman_result *scn) { - const uint32_t *p = qb_cl(scn); - - return qb_attr_code_decode(&code_scn_rid, p); + return scn->scn.rid_tok; } uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn) { - const uint64_t *p = (const uint64_t *)qb_cl(scn); - - return qb_attr_code_decode_64(&code_scn_ctx_lo, p); -} - -uint8_t qbman_result_SCN_state_in_mem(const struct qbman_result *scn) -{ - const uint32_t *p = qb_cl(scn); - - return (uint8_t)qb_attr_code_decode(&code_scn_state_in_mem, p); -} - -uint32_t qbman_result_SCN_rid_in_mem(const struct qbman_result *scn) -{ - const uint32_t *p = qb_cl(scn); - uint32_t result_rid; - - result_rid = qb_attr_code_decode(&code_scn_rid_in_mem, p); - return make_le24(result_rid); + return scn->scn.ctx; } /*****************/ @@ -1189,34 +1077,27 @@ uint32_t qbman_result_SCN_rid_in_mem(const struct qbman_result *scn) /*****************/ uint16_t qbman_result_bpscn_bpid(const struct qbman_result *scn) { - return (uint16_t)qbman_result_SCN_rid_in_mem(scn) & 0x3FFF; + return (uint16_t)qbman_result_SCN_rid(scn) & 0x3FFF; } int qbman_result_bpscn_has_free_bufs(const struct qbman_result *scn) { - return !(int)(qbman_result_SCN_state_in_mem(scn) & 0x1); + return !(int)(qbman_result_SCN_state(scn) & 0x1); } int qbman_result_bpscn_is_depleted(const struct qbman_result *scn) { - return (int)(qbman_result_SCN_state_in_mem(scn) & 0x2); + return (int)(qbman_result_SCN_state(scn) & 0x2); } int qbman_result_bpscn_is_surplus(const struct qbman_result *scn) { - return (int)(qbman_result_SCN_state_in_mem(scn) & 0x4); + return (int)(qbman_result_SCN_state(scn) & 0x4); } uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn) { - uint64_t ctx; - uint32_t ctx_hi, ctx_lo; - - ctx = qbman_result_SCN_ctx(scn); - ctx_hi = upper32(ctx); - ctx_lo = lower32(ctx); - return ((uint64_t)make_le32(ctx_hi) << 32 | - (uint64_t)make_le32(ctx_lo)); + return qbman_result_SCN_ctx(scn); } /*****************/ @@ -1224,52 +1105,37 @@ uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn) /*****************/ uint16_t qbman_result_cgcu_cgid(const struct qbman_result *scn) { - return (uint16_t)qbman_result_SCN_rid_in_mem(scn) & 0xFFFF; + return (uint16_t)qbman_result_SCN_rid(scn) & 0xFFFF; } uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn) { - uint64_t ctx; - uint32_t ctx_hi, ctx_lo; - - ctx = qbman_result_SCN_ctx(scn); - ctx_hi = upper32(ctx); - ctx_lo = lower32(ctx); - return ((uint64_t)(make_le32(ctx_hi) & 0xFF) << 32) | - (uint64_t)make_le32(ctx_lo); + return qbman_result_SCN_ctx(scn); } /******************/ /* Buffer release */ /******************/ - -/* These should be const, eventually */ -/* static struct qb_attr_code code_release_num = QB_CODE(0, 0, 3); */ -static struct qb_attr_code code_release_set_me = QB_CODE(0, 5, 1); -static struct qb_attr_code code_release_rcdi = QB_CODE(0, 6, 1); -static struct qb_attr_code code_release_bpid = QB_CODE(0, 16, 16); +#define QB_BR_RC_VALID_SHIFT 5 +#define QB_BR_RCDI_SHIFT 6 void qbman_release_desc_clear(struct qbman_release_desc *d) { - uint32_t *cl; - memset(d, 0, sizeof(*d)); - cl = qb_cl(d); - qb_attr_code_encode(&code_release_set_me, cl, 1); + d->br.verb = 1 << QB_BR_RC_VALID_SHIFT; } -void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint32_t bpid) +void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_release_bpid, cl, bpid); + d->br.bpid = bpid; } void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable) { - uint32_t *cl = qb_cl(d); - - qb_attr_code_encode(&code_release_rcdi, cl, !!enable); + if (enable) + d->br.verb |= 1 << QB_BR_RCDI_SHIFT; + else + d->br.verb &= ~(1 << QB_BR_RCDI_SHIFT); } #define RAR_IDX(rar) ((rar) & 0x7) @@ -1286,12 +1152,16 @@ int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, pr_debug("RAR=%08x\n", rar); if (!RAR_SUCCESS(rar)) return -EBUSY; + QBMAN_BUG_ON(!num_buffers || (num_buffers > 7)); + /* Start the release command */ p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + /* Copy the caller's buffer pointers to the command */ u64_to_le32_copy(&p[2], buffers, num_buffers); + /* Set the verb byte, have to substitute in the valid-bit and the number * of buffers. */ @@ -1299,25 +1169,38 @@ int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, p[0] = cl[0] | RAR_VB(rar) | num_buffers; qbman_cena_write_complete_wo_shadow(&s->sys, QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + return 0; } /*******************/ /* Buffer acquires */ /*******************/ +struct qbman_acquire_desc { + uint8_t verb; + uint8_t reserved; + uint16_t bpid; + uint8_t num; + uint8_t reserved2[59]; +}; -/* These should be const, eventually */ -static struct qb_attr_code code_acquire_bpid = QB_CODE(0, 16, 16); -static struct qb_attr_code code_acquire_num = QB_CODE(1, 0, 3); -static struct qb_attr_code code_acquire_r_num = QB_CODE(1, 0, 3); +struct qbman_acquire_rslt { + uint8_t verb; + uint8_t rslt; + uint16_t reserved; + uint8_t num; + uint8_t reserved2[3]; + uint64_t buf[7]; +}; -int qbman_swp_acquire(struct qbman_swp *s, uint32_t bpid, uint64_t *buffers, +int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, unsigned int num_buffers) { - uint32_t *p; - uint32_t rslt, num; + struct qbman_acquire_desc *p; + struct qbman_acquire_rslt *r; - QBMAN_BUG_ON(!num_buffers || (num_buffers > 7)); + if (!num_buffers || (num_buffers > 7)) + return -EINVAL; /* Start the management command */ p = qbman_swp_mc_start(s); @@ -1326,59 +1209,81 @@ int qbman_swp_acquire(struct qbman_swp *s, uint32_t bpid, uint64_t *buffers, return -EBUSY; /* Encode the caller-provided attributes */ - qb_attr_code_encode(&code_acquire_bpid, p, bpid); - qb_attr_code_encode(&code_acquire_num, p, num_buffers); + p->bpid = bpid; + p->num = num_buffers; /* Complete the management command */ - p = qbman_swp_mc_complete(s, p, p[0] | QBMAN_MC_ACQUIRE); + r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE); + if (unlikely(!r)) { + pr_err("qbman: acquire from BPID %d failed, no response\n", + bpid); + return -EIO; + } /* Decode the outcome */ - rslt = qb_attr_code_decode(&code_generic_rslt, p); - num = qb_attr_code_decode(&code_acquire_r_num, p); - QBMAN_BUG_ON(qb_attr_code_decode(&code_generic_verb, p) != - QBMAN_MC_ACQUIRE); + QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE); /* Determine success or failure */ - if (unlikely(rslt != QBMAN_MC_RSLT_OK)) { + if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n", - bpid, rslt); + bpid, r->rslt); return -EIO; } - QBMAN_BUG_ON(num > num_buffers); + + QBMAN_BUG_ON(r->num > num_buffers); + /* Copy the acquired buffers to the caller's array */ - u64_from_le32_copy(buffers, &p[2], num); - return (int)num; + u64_from_le32_copy(buffers, &r->buf[0], r->num); + + return (int)r->num; } /*****************/ /* FQ management */ /*****************/ +struct qbman_alt_fq_state_desc { + uint8_t verb; + uint8_t reserved[3]; + uint32_t fqid; + uint8_t reserved2[56]; +}; -static struct qb_attr_code code_fqalt_fqid = QB_CODE(1, 0, 32); +struct qbman_alt_fq_state_rslt { + uint8_t verb; + uint8_t rslt; + uint8_t reserved[62]; +}; + +#define ALT_FQ_FQID_MASK 0x00FFFFFF static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid, uint8_t alt_fq_verb) { - uint32_t *p; - uint32_t rslt; + struct qbman_alt_fq_state_desc *p; + struct qbman_alt_fq_state_rslt *r; /* Start the management command */ p = qbman_swp_mc_start(s); if (!p) return -EBUSY; - qb_attr_code_encode(&code_fqalt_fqid, p, fqid); + p->fqid = fqid & ALT_FQ_FQID_MASK; + /* Complete the management command */ - p = qbman_swp_mc_complete(s, p, p[0] | alt_fq_verb); + r = qbman_swp_mc_complete(s, p, alt_fq_verb); + if (unlikely(!r)) { + pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n", + alt_fq_verb); + return -EIO; + } /* Decode the outcome */ - rslt = qb_attr_code_decode(&code_generic_rslt, p); - QBMAN_BUG_ON(qb_attr_code_decode(&code_generic_verb, p) != alt_fq_verb); + QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb); /* Determine success or failure */ - if (unlikely(rslt != QBMAN_MC_RSLT_OK)) { + if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n", - fqid, alt_fq_verb, rslt); + fqid, alt_fq_verb, r->rslt); return -EIO; } @@ -1409,10 +1314,24 @@ int qbman_swp_fq_xoff(struct qbman_swp *s, uint32_t fqid) /* Channel management */ /**********************/ -static struct qb_attr_code code_cdan_cid = QB_CODE(0, 16, 12); -static struct qb_attr_code code_cdan_we = QB_CODE(1, 0, 8); -static struct qb_attr_code code_cdan_en = QB_CODE(1, 8, 1); -static struct qb_attr_code code_cdan_ctx_lo = QB_CODE(2, 0, 32); +struct qbman_cdan_ctrl_desc { + uint8_t verb; + uint8_t reserved; + uint16_t ch; + uint8_t we; + uint8_t ctrl; + uint16_t reserved2; + uint64_t cdan_ctx; + uint8_t reserved3[48]; + +}; + +struct qbman_cdan_ctrl_rslt { + uint8_t verb; + uint8_t rslt; + uint16_t ch; + uint8_t reserved[60]; +}; /* Hide "ICD" for now as we don't use it, don't set it, and don't test it, so it * would be irresponsible to expose it. @@ -1424,8 +1343,8 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, uint8_t we_mask, uint8_t cdan_en, uint64_t ctx) { - uint32_t *p; - uint32_t rslt; + struct qbman_cdan_ctrl_desc *p; + struct qbman_cdan_ctrl_rslt *r; /* Start the management command */ p = qbman_swp_mc_start(s); @@ -1433,22 +1352,29 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, return -EBUSY; /* Encode the caller-provided attributes */ - qb_attr_code_encode(&code_cdan_cid, p, channelid); - qb_attr_code_encode(&code_cdan_we, p, we_mask); - qb_attr_code_encode(&code_cdan_en, p, cdan_en); - qb_attr_code_encode_64(&code_cdan_ctx_lo, (uint64_t *)p, ctx); + p->ch = channelid; + p->we = we_mask; + if (cdan_en) + p->ctrl = 1; + else + p->ctrl = 0; + p->cdan_ctx = ctx; + /* Complete the management command */ - p = qbman_swp_mc_complete(s, p, p[0] | QBMAN_WQCHAN_CONFIGURE); + r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE); + if (unlikely(!r)) { + pr_err("qbman: wqchan config failed, no response\n"); + return -EIO; + } /* Decode the outcome */ - rslt = qb_attr_code_decode(&code_generic_rslt, p); - QBMAN_BUG_ON(qb_attr_code_decode(&code_generic_verb, p) - != QBMAN_WQCHAN_CONFIGURE); + QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) + != QBMAN_WQCHAN_CONFIGURE); /* Determine success or failure */ - if (unlikely(rslt != QBMAN_MC_RSLT_OK)) { + if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { pr_err("CDAN cQID %d failed: code = 0x%02x\n", - channelid, rslt); + channelid, r->rslt); return -EIO; } @@ -1497,92 +1423,3 @@ struct qbman_result *qbman_get_dqrr_from_idx(struct qbman_swp *s, uint8_t idx) dq = qbman_cena_read(&s->sys, QBMAN_CENA_SWP_DQRR(idx)); return dq; } - -int qbman_swp_send_multiple(struct qbman_swp *s, - const struct qbman_eq_desc *d, - const struct qbman_fd *fd, - int frames_to_send) -{ - uint32_t *p; - const uint32_t *cl = qb_cl(d); - uint32_t eqcr_ci; - uint8_t diff; - int sent = 0; - int i; - int initial_pi = s->eqcr.pi; - uint64_t start_pointer; - - if (!s->eqcr.available) { - eqcr_ci = s->eqcr.ci; - s->eqcr.ci = qbman_cena_read_reg(&s->sys, - QBMAN_CENA_SWP_EQCR_CI) & 0xF; - diff = qm_cyc_diff(QBMAN_EQCR_SIZE, - eqcr_ci, s->eqcr.ci); - if (!diff) - goto done; - s->eqcr.available += diff; - } - - /* we are trying to send frames_to_send, - * if we have enough space in the ring - */ - while (s->eqcr.available && frames_to_send--) { - p = qbman_cena_write_start_wo_shadow_fast(&s->sys, - QBMAN_CENA_SWP_EQCR((initial_pi) & 7)); - /* Write command (except of first byte) and FD */ - memcpy(&p[1], &cl[1], 7 * 4); - memcpy(&p[8], &fd[sent], sizeof(struct qbman_fd)); - - initial_pi++; - initial_pi &= 0xF; - s->eqcr.available--; - sent++; - } - -done: - initial_pi = s->eqcr.pi; - lwsync(); - - /* in order for flushes to complete faster: - * we use a following trick: we record all lines in 32 bit word - */ - - initial_pi = s->eqcr.pi; - for (i = 0; i < sent; i++) { - p = qbman_cena_write_start_wo_shadow_fast(&s->sys, - QBMAN_CENA_SWP_EQCR((initial_pi) & 7)); - - p[0] = cl[0] | s->eqcr.pi_vb; - initial_pi++; - initial_pi &= 0xF; - - if (!(initial_pi & 7)) - s->eqcr.pi_vb ^= QB_VALID_BIT; - } - - initial_pi = s->eqcr.pi; - - /* We need to flush all the lines but without - * load/store operations between them. - * We assign start_pointer before we start loop so that - * in loop we do not read it from memory - */ - start_pointer = (uint64_t)s->sys.addr_cena; - for (i = 0; i < sent; i++) { - p = (uint32_t *)(start_pointer - + QBMAN_CENA_SWP_EQCR(initial_pi & 7)); - dcbf((uint64_t)p); - initial_pi++; - initial_pi &= 0xF; - } - - /* Update producer index for the next call */ - s->eqcr.pi = initial_pi; - - return sent; -} - -int qbman_get_version(void) -{ - return qman_version; -} diff --git a/drivers/bus/fslmc/qbman/qbman_portal.h b/drivers/bus/fslmc/qbman/qbman_portal.h index 7aa1d4f6..d9f3ed7e 100644 --- a/drivers/bus/fslmc/qbman/qbman_portal.h +++ b/drivers/bus/fslmc/qbman/qbman_portal.h @@ -26,9 +26,14 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "qbman_private.h" +#include "qbman_sys.h" #include <fsl_qbman_portal.h> +uint32_t qman_version; +#define QMAN_REV_4000 0x04000000 +#define QMAN_REV_4100 0x04010000 +#define QMAN_REV_4101 0x04010001 + /* All QBMan command and result structures use this "valid bit" encoding */ #define QB_VALID_BIT ((uint32_t)0x80) @@ -39,7 +44,8 @@ #define QBMAN_EQCR_SIZE 8 -static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last) +static inline uint8_t qm_cyc_diff(uint8_t ringsize, uint8_t first, + uint8_t last) { /* 'first' is included, 'last' is excluded */ if (first <= last) @@ -122,138 +128,22 @@ struct qbman_swp { * non-NULL if only if the response is complete). */ void *qbman_swp_mc_start(struct qbman_swp *p); -void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint32_t cmd_verb); +void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb); void *qbman_swp_mc_result(struct qbman_swp *p); /* Wraps up submit + poll-for-result */ static inline void *qbman_swp_mc_complete(struct qbman_swp *swp, void *cmd, - uint32_t cmd_verb) + uint8_t cmd_verb) { - int loopvar; + int loopvar = 1000; qbman_swp_mc_submit(swp, cmd, cmd_verb); - DBG_POLL_START(loopvar); do { - DBG_POLL_CHECK(loopvar); cmd = qbman_swp_mc_result(swp); - } while (!cmd); - return cmd; -} - -/* ------------ */ -/* qb_attr_code */ -/* ------------ */ - -/* This struct locates a sub-field within a QBMan portal (CENA) cacheline which - * is either serving as a configuration command or a query result. The - * representation is inherently little-endian, as the indexing of the words is - * itself little-endian in nature and DPAA2 QBMan is little endian for anything - * that crosses a word boundary too (64-bit fields are the obvious examples). - */ -struct qb_attr_code { - unsigned int word; /* which uint32_t[] array member encodes the field */ - unsigned int lsoffset; /* encoding offset from ls-bit */ - unsigned int width; /* encoding width. (bool must be 1.) */ -}; - -/* Some pre-defined codes */ -extern struct qb_attr_code code_generic_verb; -extern struct qb_attr_code code_generic_rslt; - -/* Macros to define codes */ -#define QB_CODE(a, b, c) { a, b, c} -#define QB_CODE_NULL \ - QB_CODE((unsigned int)-1, (unsigned int)-1, (unsigned int)-1) - -/* Rotate a code "ms", meaning that it moves from less-significant bytes to - * more-significant, from less-significant words to more-significant, etc. The - * "ls" version does the inverse, from more-significant towards - * less-significant. - */ -static inline void qb_attr_code_rotate_ms(struct qb_attr_code *code, - unsigned int bits) -{ - code->lsoffset += bits; - while (code->lsoffset > 31) { - code->word++; - code->lsoffset -= 32; - } -} - -static inline void qb_attr_code_rotate_ls(struct qb_attr_code *code, - unsigned int bits) -{ - /* Don't be fooled, this trick should work because the types are - * unsigned. So the case that interests the while loop (the rotate has - * gone too far and the word count needs to compensate for it), is - * manifested when lsoffset is negative. But that equates to a really - * large unsigned value, starting with lots of "F"s. As such, we can - * continue adding 32 back to it until it wraps back round above zero, - * to a value of 31 or less... - */ - code->lsoffset -= bits; - while (code->lsoffset > 31) { - code->word--; - code->lsoffset += 32; - } -} + } while (!cmd && loopvar--); + QBMAN_BUG_ON(!loopvar); -/* Implement a loop of code rotations until 'expr' evaluates to FALSE (0). */ -#define qb_attr_code_for_ms(code, bits, expr) \ - for (; expr; qb_attr_code_rotate_ms(code, bits)) -#define qb_attr_code_for_ls(code, bits, expr) \ - for (; expr; qb_attr_code_rotate_ls(code, bits)) - -/* decode a field from a cacheline */ -static inline uint32_t qb_attr_code_decode(const struct qb_attr_code *code, - const uint32_t *cacheline) -{ - return d32_uint32_t(code->lsoffset, code->width, cacheline[code->word]); -} - -static inline uint64_t qb_attr_code_decode_64(const struct qb_attr_code *code, - const uint64_t *cacheline) -{ - return cacheline[code->word / 2]; -} - -/* encode a field to a cacheline */ -static inline void qb_attr_code_encode(const struct qb_attr_code *code, - uint32_t *cacheline, uint32_t val) -{ - cacheline[code->word] = - r32_uint32_t(code->lsoffset, code->width, cacheline[code->word]) - | e32_uint32_t(code->lsoffset, code->width, val); -} - -static inline void qb_attr_code_encode_64(const struct qb_attr_code *code, - uint64_t *cacheline, uint64_t val) -{ - cacheline[code->word / 2] = val; -} - -/* Small-width signed values (two's-complement) will decode into medium-width - * positives. (Eg. for an 8-bit signed field, which stores values from -128 to - * +127, a setting of -7 would appear to decode to the 32-bit unsigned value - * 249. Likewise -120 would decode as 136.) This function allows the caller to - * "re-sign" such fields to 32-bit signed. (Eg. -7, which was 249 with an 8-bit - * encoding, will become 0xfffffff9 if you cast the return value to uint32_t). - */ -static inline int32_t qb_attr_code_makesigned(const struct qb_attr_code *code, - uint32_t val) -{ - QBMAN_BUG_ON(val >= (1u << code->width)); - /* code->width should never exceed the width of val. If it does then a - * different function with larger val size must be used to translate - * from unsigned to signed - */ - QBMAN_BUG_ON(code->width > sizeof(val) * CHAR_BIT); - /* If the high bit was set, it was encoding a negative */ - if (val >= 1u << (code->width - 1)) - return (int32_t)0 - (int32_t)(((uint32_t)1 << code->width) - - val); - /* Otherwise, it was encoding a positive */ - return (int32_t)val; + return cmd; } /* ---------------------- */ @@ -274,4 +164,4 @@ static inline int32_t qb_attr_code_makesigned(const struct qb_attr_code *code, * an inline) is necessary to work with different descriptor types and to work * correctly with const and non-const inputs (and similarly-qualified outputs). */ -#define qb_cl(d) (&(d)->dont_manipulate_directly[0]) +#define qb_cl(d) (&(d)->donot_manipulate_directly[0]) diff --git a/drivers/bus/fslmc/qbman/qbman_private.h b/drivers/bus/fslmc/qbman/qbman_private.h deleted file mode 100644 index 292ec6a9..00000000 --- a/drivers/bus/fslmc/qbman/qbman_private.h +++ /dev/null @@ -1,174 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* Perform extra checking */ -#define QBMAN_CHECKING - -/* To maximise the amount of logic that is common between the Linux driver and - * other targets (such as the embedded MC firmware), we pivot here between the - * inclusion of two platform-specific headers. - * - * The first, qbman_sys_decl.h, includes any and all required system headers as - * well as providing any definitions for the purposes of compatibility. The - * second, qbman_sys.h, is where platform-specific routines go. - * - * The point of the split is that the platform-independent code (including this - * header) may depend on platform-specific declarations, yet other - * platform-specific routines may depend on platform-independent definitions. - */ - -#include "qbman_sys_decl.h" - -/* When things go wrong, it is a convenient trick to insert a few FOO() - * statements in the code to trace progress. TODO: remove this once we are - * hacking the code less actively. - */ -#define FOO() fsl_os_print("FOO: %s:%d\n", __FILE__, __LINE__) - -/* Any time there is a register interface which we poll on, this provides a - * "break after x iterations" scheme for it. It's handy for debugging, eg. - * where you don't want millions of lines of log output from a polling loop - * that won't, because such things tend to drown out the earlier log output - * that might explain what caused the problem. (NB: put ";" after each macro!) - * TODO: we should probably remove this once we're done sanitising the - * simulator... - */ -#define DBG_POLL_START(loopvar) (loopvar = 10) -#define DBG_POLL_CHECK(loopvar) \ -do { \ - if (!(loopvar--)) \ - QBMAN_BUG_ON(NULL == "DBG_POLL_CHECK"); \ -} while (0) - -/* For CCSR or portal-CINH registers that contain fields at arbitrary offsets - * and widths, these macro-generated encode/decode/isolate/remove inlines can - * be used. - * - * Eg. to "d"ecode a 14-bit field out of a register (into a "uint16_t" type), - * where the field is located 3 bits "up" from the least-significant bit of the - * register (ie. the field location within the 32-bit register corresponds to a - * mask of 0x0001fff8), you would do; - * uint16_t field = d32_uint16_t(3, 14, reg_value); - * - * Or to "e"ncode a 1-bit boolean value (input type is "int", zero is FALSE, - * non-zero is TRUE, so must convert all non-zero inputs to 1, hence the "!!" - * operator) into a register at bit location 0x00080000 (19 bits "in" from the - * LS bit), do; - * reg_value |= e32_int(19, 1, !!field); - * - * If you wish to read-modify-write a register, such that you leave the 14-bit - * field as-is but have all other fields set to zero, then "i"solate the 14-bit - * value using; - * reg_value = i32_uint16_t(3, 14, reg_value); - * - * Alternatively, you could "r"emove the 1-bit boolean field (setting it to - * zero) but leaving all other fields as-is; - * reg_val = r32_int(19, 1, reg_value); - * - */ -#ifdef __LP64__ -#define MAKE_MASK32(width) ((uint32_t)(( 1ULL << width) - 1)) -#else -#define MAKE_MASK32(width) (width == 32 ? 0xffffffff : \ - (uint32_t)((1 << width) - 1)) -#endif -#define DECLARE_CODEC32(t) \ -static inline uint32_t e32_##t(uint32_t lsoffset, uint32_t width, t val) \ -{ \ - QBMAN_BUG_ON(width > (sizeof(t) * 8)); \ - return ((uint32_t)val & MAKE_MASK32(width)) << lsoffset; \ -} \ -static inline t d32_##t(uint32_t lsoffset, uint32_t width, uint32_t val) \ -{ \ - QBMAN_BUG_ON(width > (sizeof(t) * 8)); \ - return (t)((val >> lsoffset) & MAKE_MASK32(width)); \ -} \ -static inline uint32_t i32_##t(uint32_t lsoffset, uint32_t width, \ - uint32_t val) \ -{ \ - QBMAN_BUG_ON(width > (sizeof(t) * 8)); \ - return e32_##t(lsoffset, width, d32_##t(lsoffset, width, val)); \ -} \ -static inline uint32_t r32_##t(uint32_t lsoffset, uint32_t width, \ - uint32_t val) \ -{ \ - QBMAN_BUG_ON(width > (sizeof(t) * 8)); \ - return ~(MAKE_MASK32(width) << lsoffset) & val; \ -} -DECLARE_CODEC32(uint32_t) -DECLARE_CODEC32(uint16_t) -DECLARE_CODEC32(uint8_t) -DECLARE_CODEC32(int) - - /*********************/ - /* Debugging assists */ - /*********************/ - -static inline void __hexdump(unsigned long start, unsigned long end, - unsigned long p, size_t sz, const unsigned char *c) -{ - while (start < end) { - unsigned int pos = 0; - char buf[64]; - int nl = 0; - - pos += sprintf(buf + pos, "%08lx: ", start); - do { - if ((start < p) || (start >= (p + sz))) - pos += sprintf(buf + pos, ".."); - else - pos += sprintf(buf + pos, "%02x", *(c++)); - if (!(++start & 15)) { - buf[pos++] = '\n'; - nl = 1; - } else { - nl = 0; - if (!(start & 1)) - buf[pos++] = ' '; - if (!(start & 3)) - buf[pos++] = ' '; - } - } while (start & 15); - if (!nl) - buf[pos++] = '\n'; - buf[pos] = '\0'; - pr_info("%s", buf); - } -} - -static inline void hexdump(const void *ptr, size_t sz) -{ - unsigned long p = (unsigned long)ptr; - unsigned long start = p & ~(unsigned long)15; - unsigned long end = (p + sz + 15) & ~(unsigned long)15; - const unsigned char *c = ptr; - - __hexdump(start, end, p, sz, c); -} - -#include "qbman_sys.h" diff --git a/drivers/bus/fslmc/qbman/qbman_sys.h b/drivers/bus/fslmc/qbman/qbman_sys.h index 5dbcaa57..c216e9cf 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys.h +++ b/drivers/bus/fslmc/qbman/qbman_sys.h @@ -40,20 +40,49 @@ * *not* to provide linux compatibility. */ -/* Trace the 3 different classes of read/write access to QBMan. #undef as - * required. - */ -#undef QBMAN_CCSR_TRACE -#undef QBMAN_CINH_TRACE -#undef QBMAN_CENA_TRACE +#include "qbman_sys_decl.h" -static inline void word_copy(void *d, const void *s, unsigned int cnt) +/* Debugging assists */ +static inline void __hexdump(unsigned long start, unsigned long end, + unsigned long p, size_t sz, const unsigned char *c) { - uint32_t *dd = d; - const uint32_t *ss = s; + while (start < end) { + unsigned int pos = 0; + char buf[64]; + int nl = 0; + + pos += sprintf(buf + pos, "%08lx: ", start); + do { + if ((start < p) || (start >= (p + sz))) + pos += sprintf(buf + pos, ".."); + else + pos += sprintf(buf + pos, "%02x", *(c++)); + if (!(++start & 15)) { + buf[pos++] = '\n'; + nl = 1; + } else { + nl = 0; + if (!(start & 1)) + buf[pos++] = ' '; + if (!(start & 3)) + buf[pos++] = ' '; + } + } while (start & 15); + if (!nl) + buf[pos++] = '\n'; + buf[pos] = '\0'; + pr_info("%s", buf); + } +} - while (cnt--) - *(dd++) = *(ss++); +static inline void hexdump(const void *ptr, size_t sz) +{ + unsigned long p = (unsigned long)ptr; + unsigned long start = p & ~15; + unsigned long end = (p + sz + 15) & ~15; + const unsigned char *c = ptr; + + __hexdump(start, end, p, sz, c); } /* Currently, the CENA support code expects each 32-bit word to be written in @@ -103,34 +132,6 @@ static inline void u64_from_le32_copy(uint64_t *d, const void *s, } } -/* Convert a host-native 32bit value into little endian */ -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -static inline uint32_t make_le32(uint32_t val) -{ - return ((val & 0xff) << 24) | ((val & 0xff00) << 8) | - ((val & 0xff0000) >> 8) | ((val & 0xff000000) >> 24); -} - -static inline uint32_t make_le24(uint32_t val) -{ - return (((val & 0xff) << 16) | (val & 0xff00) | - ((val & 0xff0000) >> 16)); -} - -static inline void make_le32_n(uint32_t *val, unsigned int num) -{ - while (num--) { - *val = make_le32(*val); - val++; - } -} - -#else -#define make_le32(val) (val) -#define make_le24(val) (val) -#define make_le32_n(val, len) do {} while (0) -#endif - /******************/ /* Portal access */ /******************/ @@ -226,7 +227,6 @@ static inline void qbman_cena_write_complete_wo_shadow(struct qbman_swp_sys *s, #ifdef QBMAN_CENA_TRACE pr_info("qbman_cena_write_complete(%p:%d:0x%03x)\n", s->addr_cena, s->idx, offset); - hexdump(cmd, 64); #endif dcbf(s->addr_cena + offset); } @@ -259,12 +259,8 @@ static inline void *qbman_cena_read_wo_shadow(struct qbman_swp_sys *s, uint32_t offset) { #ifdef QBMAN_CENA_TRACE - pr_info("qbman_cena_read(%p:%d:0x%03x) %p\n", - s->addr_cena, s->idx, offset, shadow); -#endif - -#ifdef QBMAN_CENA_TRACE - hexdump(shadow, 64); + pr_info("qbman_cena_read(%p:%d:0x%03x)\n", + s->addr_cena, s->idx, offset); #endif return s->addr_cena + offset; } @@ -297,20 +293,20 @@ static inline void qbman_cena_prefetch(struct qbman_swp_sys *s, * qbman_portal.c. So use of it is declared locally here. */ #define QBMAN_CINH_SWP_CFG 0xd00 +#define QBMAN_CINH_SWP_CFG 0xd00 +#define SWP_CFG_DQRR_MF_SHIFT 20 +#define SWP_CFG_EST_SHIFT 16 +#define SWP_CFG_WN_SHIFT 14 +#define SWP_CFG_RPM_SHIFT 12 +#define SWP_CFG_DCM_SHIFT 10 +#define SWP_CFG_EPM_SHIFT 8 +#define SWP_CFG_SD_SHIFT 5 +#define SWP_CFG_SP_SHIFT 4 +#define SWP_CFG_SE_SHIFT 3 +#define SWP_CFG_DP_SHIFT 2 +#define SWP_CFG_DE_SHIFT 1 +#define SWP_CFG_EP_SHIFT 0 -/* For MC portal use, we always configure with - * DQRR_MF is (SWP_CFG,20,3) - DQRR max fill (<- 0x4) - * EST is (SWP_CFG,16,3) - EQCR_CI stashing threshold (<- 0x2) - * RPM is (SWP_CFG,12,2) - RCR production notification mode (<- 0x3) - * DCM is (SWP_CFG,10,2) - DQRR consumption notification mode (<- 0x2) - * EPM is (SWP_CFG,8,2) - EQCR production notification mode (<- 0x2) - * SD is (SWP_CFG,5,1) - memory stashing drop enable (<- TRUE) - * SP is (SWP_CFG,4,1) - memory stashing priority (<- TRUE) - * SE is (SWP_CFG,3,1) - memory stashing enable (<- TRUE) - * DP is (SWP_CFG,2,1) - dequeue stashing priority (<- TRUE) - * DE is (SWP_CFG,1,1) - dequeue stashing enable (<- TRUE) - * EP is (SWP_CFG,0,1) - EQCR_CI stashing priority (<- TRUE) - */ static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn, uint8_t est, uint8_t rpm, uint8_t dcm, uint8_t epm, int sd, int sp, int se, @@ -318,12 +314,19 @@ static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn, { uint32_t reg; - reg = e32_uint8_t(20, (uint32_t)(3 + (max_fill >> 3)), max_fill) | - e32_uint8_t(16, 3, est) | - e32_uint8_t(12, 2, rpm) | e32_uint8_t(10, 2, dcm) | - e32_uint8_t(8, 2, epm) | e32_int(5, 1, sd) | - e32_int(4, 1, sp) | e32_int(3, 1, se) | e32_int(2, 1, dp) | - e32_int(1, 1, de) | e32_int(0, 1, ep) | e32_uint8_t(14, 1, wn); + reg = (max_fill << SWP_CFG_DQRR_MF_SHIFT | + est << SWP_CFG_EST_SHIFT | + wn << SWP_CFG_WN_SHIFT | + rpm << SWP_CFG_RPM_SHIFT | + dcm << SWP_CFG_DCM_SHIFT | + epm << SWP_CFG_EPM_SHIFT | + sd << SWP_CFG_SD_SHIFT | + sp << SWP_CFG_SP_SHIFT | + se << SWP_CFG_SE_SHIFT | + dp << SWP_CFG_DP_SHIFT | + de << SWP_CFG_DE_SHIFT | + ep << SWP_CFG_EP_SHIFT); + return reg; } @@ -336,7 +339,7 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, s->addr_cena = d->cena_bar; s->addr_cinh = d->cinh_bar; s->idx = (uint32_t)d->idx; - s->cena = (void *)get_zeroed_page(GFP_KERNEL); + s->cena = malloc(4096); if (!s->cena) { pr_err("Could not allocate page for cena shadow\n"); return -1; @@ -361,7 +364,7 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); if (!reg) { pr_err("The portal %d is not enabled!\n", s->idx); - kfree(s->cena); + free(s->cena); return -1; } return 0; @@ -369,17 +372,5 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, static inline void qbman_swp_sys_finish(struct qbman_swp_sys *s) { - free_page((unsigned long)s->cena); -} - -static inline void * -qbman_cena_write_start_wo_shadow_fast(struct qbman_swp_sys *s, - uint32_t offset) -{ -#ifdef QBMAN_CENA_TRACE - pr_info("qbman_cena_write_start(%p:%d:0x%03x)\n", - s->addr_cena, s->idx, offset); -#endif - QBMAN_BUG_ON(offset & 63); - return (s->addr_cena + offset); + free(s->cena); } diff --git a/drivers/bus/fslmc/qbman/qbman_sys_decl.h b/drivers/bus/fslmc/qbman/qbman_sys_decl.h index e52f5ed2..e1125cfe 100644 --- a/drivers/bus/fslmc/qbman/qbman_sys_decl.h +++ b/drivers/bus/fslmc/qbman/qbman_sys_decl.h @@ -34,27 +34,6 @@ #error "Unknown endianness!" #endif -/* The platform-independent code shouldn't need endianness, except for - * weird/fast-path cases like qbman_result_has_token(), which needs to - * perform a passive and endianness-specific test on a read-only data structure - * very quickly. It's an exception, and this symbol is used for that case. - */ -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#define DQRR_TOK_OFFSET 0 -#define QBMAN_RESULT_VERB_OFFSET_IN_MEM 24 -#define SCN_STATE_OFFSET_IN_MEM 8 -#define SCN_RID_OFFSET_IN_MEM 8 -#else -#define DQRR_TOK_OFFSET 24 -#define QBMAN_RESULT_VERB_OFFSET_IN_MEM 0 -#define SCN_STATE_OFFSET_IN_MEM 16 -#define SCN_RID_OFFSET_IN_MEM 0 -#endif - -/* Similarly-named functions */ -#define upper32(a) upper_32_bits(a) -#define lower32(a) lower_32_bits(a) - /****************/ /* arch assists */ /****************/ @@ -64,10 +43,10 @@ #define dccivac(p) { asm volatile("dc civac, %0" : : "r"(p) : "memory"); } static inline void prefetch_for_load(void *p) { - asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); + asm volatile("prfm pldl1keep, [%0, #0]" : : "r" (p)); } static inline void prefetch_for_store(void *p) { - asm volatile("prfm pstl1keep, [%0, #64]" : : "r" (p)); + asm volatile("prfm pstl1keep, [%0, #0]" : : "r" (p)); } |