/* *------------------------------------------------------------------ * Copyright (c) 2023 Intel and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. *------------------------------------------------------------------ */ #include /** * idpf_ctlq_alloc_desc_ring - Allocate Control Queue (CQ) rings * @hw: pointer to hw struct * @cq: pointer to the specific Control queue */ static int idpf_ctlq_alloc_desc_ring (vlib_main_t *vm, idpf_device_t *id, struct idpf_ctlq_info *cq) { size_t size = cq->ring_size * sizeof (idpf_ctlq_desc_t); /* Fixme: alloc dma va */ cq->desc_ring.va = idpf_alloc_dma_mem (vm, id, &cq->desc_ring, size); if (!cq->desc_ring.va) return IDPF_ERR_NO_MEMORY; return IDPF_SUCCESS; } /** * idpf_ctlq_alloc_bufs - Allocate Control Queue (CQ) buffers * @hw: pointer to hw struct * @cq: pointer to the specific Control queue * * Allocate the buffer head for all control queues, and if it's a receive * queue, allocate DMA buffers */ static int idpf_ctlq_alloc_bufs (vlib_main_t *vm, idpf_device_t *id, struct idpf_ctlq_info *cq) { int i = 0; u16 len; /* Do not allocate DMA buffers for transmit queues */ if (cq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_TX) return IDPF_SUCCESS; /* We'll be allocating the buffer info memory first, then we can * allocate the mapped buffers for the event processing */ len = cq->ring_size * sizeof (idpf_dma_mem_t *); cq->bi.rx_buff = (idpf_dma_mem_t **) clib_mem_alloc (len); if (!cq->bi.rx_buff) return IDPF_ERR_NO_MEMORY; clib_memset (cq->bi.rx_buff, 0, len); /* allocate the mapped buffers (except for the last one) */ for (i = 0; i < cq->ring_size - 1; i++) { idpf_dma_mem_t *bi; int num = 1; /* number of idpf_dma_mem to be allocated */ cq->bi.rx_buff[i] = (idpf_dma_mem_t *) clib_mem_alloc (num * sizeof (idpf_dma_mem_t)); if (!cq->bi.rx_buff[i]) goto unwind_alloc_cq_bufs; bi = cq->bi.rx_buff[i]; bi->va = idpf_alloc_dma_mem (vm, id, bi, cq->buf_size); if (!bi->va) { /* unwind will not free the failed entry */ clib_mem_free (cq->bi.rx_buff[i]); goto unwind_alloc_cq_bufs; } } return IDPF_SUCCESS; unwind_alloc_cq_bufs: /* don't try to free the one that failed... */ i--; for (; i >= 0; i--) { idpf_free_dma_mem (id, cq->bi.rx_buff[i]); clib_mem_free (cq->bi.rx_buff[i]); } clib_mem_free (cq->bi.rx_buff); return IDPF_ERR_NO_MEMORY; } /** * idpf_ctlq_free_desc_ring - Free Control Queue (CQ) rings * @hw: pointer to hw struct * @cq: pointer to the specific Control queue * * This assumes the posted send buffers have already been cleaned * and de-allocated */ static void idpf_ctlq_free_desc_ring (idpf_device_t *id, struct idpf_ctlq_info *cq) { idpf_free_dma_mem (id, &cq->desc_ring); } /** * idpf_ctlq_free_bufs - Free CQ buffer info elements * @hw: pointer to hw struct * @cq: pointer to the specific Control queue * * Free the DMA buffers for RX queues, and DMA buffer header for both RX and TX * queues. The upper layers are expected to manage freeing of TX DMA buffers */ static void idpf_ctlq_free_bufs (idpf_device_t *id, struct idpf_ctlq_info *cq) { void *bi; if (cq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_RX) { int i; /* free DMA buffers for rx queues*/ for (i = 0; i < cq->ring_size; i++) { if (cq->bi.rx_buff[i]) { idpf_free_dma_mem (id, cq->bi.rx_buff[i]); /* Attention */ clib_mem_free (cq->bi.rx_buff[i]); } } bi = (void *) cq->bi.rx_buff; } else { bi = (void *) cq->bi.tx_msg; } /* free the buffer header */ clib_mem_free (bi); } /** * idpf_ctlq_dealloc_ring_res - Free memory allocated for control queue * @hw: pointer to hw struct * @cq: pointer to the specific Control queue * * Free the memory used by the ring, buffers and other related structures */ void idpf_ctlq_dealloc_ring_res (idpf_device_t *id, struct idpf_ctlq_info *cq) { /* free ring buffers and the ring itself */ idpf_ctlq_free_bufs (id, cq); idpf_ctlq_free_desc_ring (id, cq); } /** * idpf_ctlq_alloc_ring_res - allocate memory for descriptor ring and bufs * @hw: pointer to hw struct * @cq: pointer to control queue struct * * Do *NOT* hold the lock when calling this as the memory allocation routines * called are not going to be atomic context safe */ int idpf_ctlq_alloc_ring_res (vlib_main_t *vm, idpf_device_t *id, struct idpf_ctlq_info *cq) { int ret_code; /* verify input for valid configuration */ if (!cq->ring_size || !cq->buf_size) return IDPF_ERR_CFG; /* allocate the ring memory */ ret_code = idpf_ctlq_alloc_desc_ring (vm, id, cq); if (ret_code) return ret_code; /* allocate buffers in the rings */ ret_code = idpf_ctlq_alloc_bufs (vm, id, cq); if (ret_code) goto idpf_init_cq_free_ring; /* success! */ return IDPF_SUCCESS; idpf_init_cq_free_ring: idpf_free_dma_mem (id, &cq->desc_ring); return ret_code; } /** * idpf_ctlq_setup_regs - initialize control queue registers * @cq: pointer to the specific control queue * @q_create_info: structs containing info for each queue to be initialized */ static void idpf_ctlq_setup_regs (struct idpf_ctlq_info *cq, idpf_ctlq_create_info_t *q_create_info) { /* set head and tail registers in our local struct */ cq->reg.head = q_create_info->reg.head; cq->reg.tail = q_create_info->reg.tail; cq->reg.len = q_create_info->reg.len; cq->reg.bah = q_create_info->reg.bah; cq->reg.bal = q_create_info->reg.bal; cq->reg.len_mask = q_create_info->reg.len_mask; cq->reg.len_ena_mask = q_create_info->reg.len_ena_mask; cq->reg.head_mask = q_create_info->reg.head_mask; } /** * idpf_ctlq_init_regs - Initialize control queue registers * @hw: pointer to hw struct * @cq: pointer to the specific Control queue * @is_rxq: true if receive control queue, false otherwise * * Initialize registers. The caller is expected to have already initialized the * descriptor ring memory and buffer memory */ static void idpf_ctlq_init_regs (vlib_main_t *vm, idpf_device_t *id, struct idpf_ctlq_info *cq, bool is_rxq) { /* Update tail to post pre-allocated buffers for rx queues */ if (is_rxq) idpf_reg_write (id, cq->reg.tail, (u32) (cq->ring_size - 1)); /* For non-Mailbox control queues only TAIL need to be set */ if (cq->q_id != -1) return; /* Clear Head for both send or receive */ idpf_reg_write (id, cq->reg.head, 0); /* set starting point */ idpf_reg_write (id, cq->reg.bal, IDPF_LO_DWORD (cq->desc_ring.pa)); idpf_reg_write (id, cq->reg.bah, IDPF_HI_DWORD (cq->desc_ring.pa)); idpf_reg_write (id, cq->reg.len, (cq->ring_size | cq->reg.len_ena_mask)); } /** * idpf_ctlq_init_rxq_bufs - populate receive queue descriptors with buf * @cq: pointer to the specific Control queue * * Record the address of the receive queue DMA buffers in the descriptors. * The buffers must have been previously allocated. */ static void idpf_ctlq_init_rxq_bufs (struct idpf_ctlq_info *cq) { int i = 0; for (i = 0; i < cq->ring_size; i++) { idpf_ctlq_desc_t *desc = IDPF_CTLQ_DESC (cq, i); idpf_dma_mem_t *bi = cq->bi.rx_buff[i]; /* No buffer to post to descriptor, continue */ if (!bi) continue; desc->flags = IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD; desc->opcode = 0; desc->datalen = (u16) bi->size; desc->ret_val = 0; desc->cookie_high = 0; desc->cookie_low = 0; desc->params.indirect.addr_high = IDPF_HI_DWORD (bi->pa); desc->params.indirect.addr_low = IDPF_LO_DWORD (bi->pa); desc->params.indirect.param0 = 0; desc->params.indirect.param1 = 0; } } /** * idpf_ctlq_shutdown - shutdown the CQ * @hw: pointer to hw struct * @cq: pointer to the specific Control queue * * The main shutdown routine for any controq queue */ static void idpf_ctlq_shutdown (idpf_device_t *id, struct idpf_ctlq_info *cq) { clib_spinlock_init (&cq->cq_lock); if (!cq->ring_size) goto shutdown_sq_out; /* free ring buffers and the ring itself */ idpf_ctlq_dealloc_ring_res (id, cq); /* Set ring_size to 0 to indicate uninitialized queue */ cq->ring_size = 0; shutdown_sq_out: clib_spinlock_unlock (&cq->cq_lock); clib_spinlock_free (&cq->cq_lock); } /** * idpf_ctlq_add - add one control queue * @hw: pointer to hardware struct * @qinfo: info for queue to be created * @cq_out: (output) double pointer to control queue to be created * * Allocate and initialize a control queue and add it to the control queue * list. The cq parameter will be allocated/initialized and passed back to the * caller if no errors occur. * * Note: idpf_ctlq_init must be called prior to any calls to idpf_ctlq_add */ int idpf_ctlq_add (vlib_main_t *vm, idpf_device_t *id, idpf_ctlq_create_info_t *qinfo, struct idpf_ctlq_info **cq_out) { bool is_rxq = false; int status = IDPF_SUCCESS; if (!qinfo->len || !qinfo->buf_size || qinfo->len > IDPF_CTLQ_MAX_RING_SIZE || qinfo->buf_size > IDPF_CTLQ_MAX_BUF_LEN) return IDPF_ERR_CFG; /* Fixme: memory allocation */ *cq_out = vlib_physmem_alloc_aligned_on_numa ( vm, sizeof (struct idpf_ctlq_info), CLIB_CACHE_LINE_BYTES, id->numa_node); if (!(*cq_out)) return IDPF_ERR_NO_MEMORY; if ((vlib_pci_map_dma (vm, id->pci_dev_handle, *cq_out))) { status = IDPF_ERR_NO_MEMORY; goto init_free_q; } (*cq_out)->cq_type = qinfo->type; (*cq_out)->q_id = qinfo->id; (*cq_out)->buf_size = qinfo->buf_size; (*cq_out)->ring_size = qinfo->len; (*cq_out)->next_to_use = 0; (*cq_out)->next_to_clean = 0; (*cq_out)->next_to_post = (*cq_out)->ring_size - 1; switch (qinfo->type) { case IDPF_CTLQ_TYPE_MAILBOX_RX: is_rxq = true; case IDPF_CTLQ_TYPE_MAILBOX_TX: status = idpf_ctlq_alloc_ring_res (vm, id, *cq_out); break; default: status = IDPF_ERR_PARAM; break; } if (status) goto init_free_q; if (is_rxq) { idpf_ctlq_init_rxq_bufs (*cq_out); } else { /* Allocate the array of msg pointers for TX queues */ (*cq_out)->bi.tx_msg = (idpf_ctlq_msg_t **) clib_mem_alloc ( qinfo->len * sizeof (idpf_ctlq_msg_t *)); if (!(*cq_out)->bi.tx_msg) { status = IDPF_ERR_NO_MEMORY; goto init_dealloc_q_mem; } } idpf_ctlq_setup_regs (*cq_out, qinfo); idpf_ctlq_init_regs (vm, id, *cq_out, is_rxq); /* Fixeme: lock issue */ clib_spinlock_init (&(*cq_out)->cq_lock); LIST_INSERT_HEAD (&id->cq_list_head, (*cq_out), cq_list); return status; init_dealloc_q_mem: /* free ring buffers and the ring itself */ idpf_ctlq_dealloc_ring_res (id, *cq_out); init_free_q: clib_mem_free (*cq_out); return status; } /** * idpf_ctlq_remove - deallocate and remove specified control queue * @hw: pointer to hardware struct * @cq: pointer to control queue to be removed */ void idpf_ctlq_remove (idpf_device_t *id, struct idpf_ctlq_info *cq) { LIST_REMOVE (cq, cq_list); idpf_ctlq_shutdown (id, cq); clib_mem_free (cq); } /** * idpf_ctlq_init - main initialization routine for all control queues * @hw: pointer to hardware struct * @num_q: number of queues to initialize * @q_info: array of structs containing info for each queue to be initialized * * This initializes any number and any type of control queues. This is an all * or nothing routine; if one fails, all previously allocated queues will be * destroyed. This must be called prior to using the individual add/remove * APIs. */ int idpf_ctlq_init (vlib_main_t *vm, idpf_device_t *id, u8 num_q, idpf_ctlq_create_info_t *q_info) { struct idpf_ctlq_info *cq = NULL; int ret_code = IDPF_SUCCESS; int i = 0; LIST_INIT (&id->cq_list_head); for (i = 0; i < num_q; i++) { idpf_ctlq_create_info_t *qinfo = q_info + i; ret_code = idpf_ctlq_add (vm, id, qinfo, &cq); if (ret_code) goto init_destroy_qs; } return ret_code; init_destroy_qs: LIST_FOR_EACH_ENTRY_SAFE (cq, NULL, &id->cq_list_head, struct idpf_ctlq_info, cq_list) { idpf_ctlq_remove (id, cq); } return ret_code; } /** * idpf_ctlq_deinit - destroy all control queues * @hw: pointer to hw struct */ void idpf_ctlq_deinit (idpf_device_t *id) { struct idpf_ctlq_info *cq = NULL; LIST_FOR_EACH_ENTRY_SAFE (cq, NULL, &id->cq_list_head, struct idpf_ctlq_info, cq_list) { idpf_ctlq_remove (id, cq); } return; } /** * idpf_ctlq_send - send command to Control Queue (CTQ) * @id: pointer to device struct * @cq: handle to control queue struct to send on * @num_q_msg: number of messages to send on control queue * @q_msg: pointer to array of queue messages to be sent * * The caller is expected to allocate DMAable buffers and pass them to the * send routine via the q_msg struct / control queue specific data struct. * The control queue will hold a reference to each send message until * the completion for that message has been cleaned. */ int idpf_ctlq_send (idpf_device_t *id, struct idpf_ctlq_info *cq, u16 num_q_msg, idpf_ctlq_msg_t q_msg[]) { idpf_ctlq_desc_t *desc; int num_desc_avail = 0; int status = IDPF_SUCCESS; int i = 0; if (!cq || !cq->ring_size) return -ENOBUFS; clib_spinlock_lock (&cq->cq_lock); /* Ensure there are enough descriptors to send all messages */ num_desc_avail = IDPF_CTLQ_DESC_UNUSED (cq); if (num_desc_avail == 0 || num_desc_avail < num_q_msg) { status = -ENOSPC; goto sq_send_command_out; } for (i = 0; i < num_q_msg; i++) { idpf_ctlq_msg_t *msg = &q_msg[i]; u64 msg_cookie; desc = IDPF_CTLQ_DESC (cq, cq->next_to_use); /* Pay attention to CPU_TO_LE16 */ desc->opcode = msg->opcode; desc->pfid_vfid = msg->func_id; msg_cookie = msg->cookie.cookie; desc->cookie_high = IDPF_HI_DWORD (msg_cookie); desc->cookie_low = IDPF_LO_DWORD (msg_cookie); desc->flags = (msg->host_id & IDPF_HOST_ID_MASK) << IDPF_CTLQ_FLAG_HOST_ID_S; if (msg->data_len) { idpf_dma_mem_t *buff = msg->ctx.indirect.payload; desc->datalen |= msg->data_len; desc->flags |= IDPF_CTLQ_FLAG_BUF; desc->flags |= IDPF_CTLQ_FLAG_RD; /* Update the address values in the desc with the pa * value for respective buffer */ desc->params.indirect.addr_high = IDPF_HI_DWORD (buff->pa); desc->params.indirect.addr_low = IDPF_LO_DWORD (buff->pa); clib_memcpy (&desc->params, msg->ctx.indirect.context, IDPF_INDIRECT_CTX_SIZE); } else { clib_memcpy (&desc->params, msg->ctx.direct, IDPF_DIRECT_CTX_SIZE); } /* Store buffer info */ cq->bi.tx_msg[cq->next_to_use] = msg; (cq->next_to_use)++; if (cq->next_to_use == cq->ring_size) cq->next_to_use = 0; } /* Force memory write to complete before letting hardware * know that there are new descriptors to fetch. */ CLIB_MEMORY_BARRIER (); idpf_reg_write (id, cq->reg.tail, cq->next_to_use); sq_send_command_out: clib_spinlock_unlock (&cq->cq_lock); return status; } /** * idpf_ctlq_clean_sq - reclaim send descriptors on HW write back for the * requested queue * @cq: pointer to the specific Control queue * @clean_count: (input|output) number of descriptors to clean as input, and * number of descriptors actually cleaned as output * @msg_status: (output) pointer to msg pointer array to be populated; needs * to be allocated by caller * * Returns an array of message pointers associated with the cleaned * descriptors. The pointers are to the original ctlq_msgs sent on the cleaned * descriptors. The status will be returned for each; any messages that failed * to send will have a non-zero status. The caller is expected to free original * ctlq_msgs and free or reuse the DMA buffers. */ int idpf_ctlq_clean_sq (struct idpf_ctlq_info *cq, u16 *clean_count, idpf_ctlq_msg_t *msg_status[]) { idpf_ctlq_desc_t *desc; u16 i = 0, num_to_clean; u16 ntc, desc_err; int ret = IDPF_SUCCESS; if (!cq || !cq->ring_size) return IDPF_ERR_CTLQ_EMPTY; if (*clean_count == 0) return IDPF_SUCCESS; if (*clean_count > cq->ring_size) return IDPF_ERR_PARAM; /* Fixme rte func */ clib_spinlock_lock (&cq->cq_lock); ntc = cq->next_to_clean; num_to_clean = *clean_count; for (i = 0; i < num_to_clean; i++) { /* Fetch next descriptor and check if marked as done */ desc = IDPF_CTLQ_DESC (cq, ntc); if (!(desc->flags & IDPF_CTLQ_FLAG_DD)) break; desc_err = desc->ret_val; if (desc_err) { /* strip off FW internal code */ desc_err &= 0xff; } msg_status[i] = cq->bi.tx_msg[ntc]; msg_status[i]->status = desc_err; cq->bi.tx_msg[ntc] = NULL; /* Zero out any stale data */ clib_memset (desc, 0, sizeof (*desc)); ntc++; if (ntc == cq->ring_size) ntc = 0; } cq->next_to_clean = ntc; clib_spinlock_unlock (&cq->cq_lock); /* Return number of descriptors actually cleaned */ *clean_count = i; return ret; } /** * idpf_ctlq_post_rx_buffs - post buffers to descriptor ring * @hw: pointer to hw struct * @cq: pointer to control queue handle * @buff_count: (input|output) input is number of buffers caller is trying to * return; output is number of buffers that were not posted * @buffs: array of pointers to dma mem structs to be given to hardware * * Caller uses this function to return DMA buffers to the descriptor ring after * consuming them; buff_count will be the number of buffers. * * Note: this function needs to be called after a receive call even * if there are no DMA buffers to be returned, i.e. buff_count = 0, * buffs = NULL to support direct commands */ int idpf_ctlq_post_rx_buffs (idpf_device_t *id, struct idpf_ctlq_info *cq, u16 *buff_count, idpf_dma_mem_t **buffs) { idpf_ctlq_desc_t *desc; u16 ntp = cq->next_to_post; bool buffs_avail = false; u16 tbp = ntp + 1; int status = IDPF_SUCCESS; int i = 0; if (*buff_count > cq->ring_size) return IDPF_ERR_PARAM; if (*buff_count > 0) buffs_avail = true; clib_spinlock_lock (&cq->cq_lock); if (tbp >= cq->ring_size) tbp = 0; if (tbp == cq->next_to_clean) /* Nothing to do */ goto post_buffs_out; /* Post buffers for as many as provided or up until the last one used */ while (ntp != cq->next_to_clean) { desc = IDPF_CTLQ_DESC (cq, ntp); if (cq->bi.rx_buff[ntp]) goto fill_desc; if (!buffs_avail) { /* If the caller hasn't given us any buffers or * there are none left, search the ring itself * for an available buffer to move to this * entry starting at the next entry in the ring */ tbp = ntp + 1; /* Wrap ring if necessary */ if (tbp >= cq->ring_size) tbp = 0; while (tbp != cq->next_to_clean) { if (cq->bi.rx_buff[tbp]) { cq->bi.rx_buff[ntp] = cq->bi.rx_buff[tbp]; cq->bi.rx_buff[tbp] = NULL; /* Found a buffer, no need to * search anymore */ break; } /* Wrap ring if necessary */ tbp++; if (tbp >= cq->ring_size) tbp = 0; } if (tbp == cq->next_to_clean) goto post_buffs_out; } else { /* Give back pointer to DMA buffer */ cq->bi.rx_buff[ntp] = buffs[i]; i++; if (i >= *buff_count) buffs_avail = false; } fill_desc: desc->flags = IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD; /* Post buffers to descriptor */ desc->datalen = cq->bi.rx_buff[ntp]->size; desc->params.indirect.addr_high = IDPF_HI_DWORD (cq->bi.rx_buff[ntp]->pa); desc->params.indirect.addr_low = IDPF_LO_DWORD (cq->bi.rx_buff[ntp]->pa); ntp++; if (ntp == cq->ring_size) ntp = 0; } post_buffs_out: /* Only update tail if buffers were actually posted */ if (cq->next_to_post != ntp) { if (ntp) /* Update next_to_post to ntp - 1 since current ntp * will not have a buffer */ cq->next_to_post = ntp - 1; else /* Wrap to end of end ring since current ntp is 0 */ cq->next_to_post = cq->ring_size - 1; idpf_reg_write (id, cq->reg.tail, cq->next_to_post); } clib_spinlock_unlock (&cq->cq_lock); /* return the number of buffers that were not posted */ *buff_count = *buff_count - i; return status; } /** * idpf_ctlq_recv - receive control queue message call back * @cq: pointer to control queue handle to receive on * @num_q_msg: (input|output) input number of messages that should be received; * output number of messages actually received * @q_msg: (output) array of received control queue messages on this q; * needs to be pre-allocated by caller for as many messages as requested * * Called by interrupt handler or polling mechanism. Caller is expected * to free buffers */ int idpf_ctlq_recv (struct idpf_ctlq_info *cq, u16 *num_q_msg, idpf_ctlq_msg_t *q_msg) { u16 num_to_clean, ntc, ret_val, flags; idpf_ctlq_desc_t *desc; int ret_code = 0; u16 i = 0; if (!cq || !cq->ring_size) return -ENOBUFS; if (*num_q_msg == 0) return 0; else if (*num_q_msg > cq->ring_size) return -EINVAL; /* Fixme: take the lock before we start messing with the ring */ clib_spinlock_lock (&cq->cq_lock); ntc = cq->next_to_clean; num_to_clean = *num_q_msg; for (i = 0; i < num_to_clean; i++) { u64 msg_cookie; /* Fetch next descriptor and check if marked as done */ desc = IDPF_CTLQ_DESC (cq, ntc); flags = desc->flags; if (!(flags & IDPF_CTLQ_FLAG_DD)) break; ret_val = desc->ret_val; q_msg[i].vmvf_type = (flags & (IDPF_CTLQ_FLAG_FTYPE_VM | IDPF_CTLQ_FLAG_FTYPE_PF)) >> IDPF_CTLQ_FLAG_FTYPE_S; if (flags & IDPF_CTLQ_FLAG_ERR) ret_code = IDPF_ERR_CTLQ_ERROR; msg_cookie = (u64) desc->cookie_high << 32; msg_cookie |= (u64) desc->cookie_low; clib_memcpy_fast (&q_msg[i].cookie, &msg_cookie, sizeof (u64)); q_msg[i].opcode = desc->opcode; q_msg[i].data_len = desc->datalen; q_msg[i].status = ret_val; if (desc->datalen) { clib_memcpy_fast (q_msg[i].ctx.indirect.context, &desc->params.indirect, IDPF_INDIRECT_CTX_SIZE); /* Assign pointer to dma buffer to ctlq_msg array * to be given to upper layer */ q_msg[i].ctx.indirect.payload = cq->bi.rx_buff[ntc]; /* Zero out pointer to DMA buffer info; * will be repopulated by post buffers API */ cq->bi.rx_buff[ntc] = NULL; } else { clib_memcpy_fast (q_msg[i].ctx.direct, desc->params.raw, IDPF_DIRECT_CTX_SIZE); } /* Zero out stale data in descriptor */ clib_memset (desc, 0, sizeof (idpf_ctlq_desc_t)); ntc++; if (ntc == cq->ring_size) ntc = 0; }; cq->next_to_clean = ntc; /* Fixme */ clib_spinlock_unlock (&cq->cq_lock); *num_q_msg = i; if (*num_q_msg == 0) ret_code = -ENOMSG; return ret_code; }