summaryrefslogtreecommitdiffstats
path: root/lib/librte_vhost
diff options
context:
space:
mode:
Diffstat (limited to 'lib/librte_vhost')
-rw-r--r--lib/librte_vhost/fd_man.c34
-rw-r--r--lib/librte_vhost/fd_man.h1
-rw-r--r--lib/librte_vhost/socket.c19
-rw-r--r--lib/librte_vhost/vhost.h20
-rw-r--r--lib/librte_vhost/vhost_user.c41
-rw-r--r--lib/librte_vhost/virtio_net.c154
6 files changed, 224 insertions, 45 deletions
diff --git a/lib/librte_vhost/fd_man.c b/lib/librte_vhost/fd_man.c
index 8a075da2..b76b8bce 100644
--- a/lib/librte_vhost/fd_man.c
+++ b/lib/librte_vhost/fd_man.c
@@ -192,6 +192,38 @@ fdset_del(struct fdset *pfdset, int fd)
return dat;
}
+/**
+ * Unregister the fd from the fdset.
+ *
+ * If parameters are invalid, return directly -2.
+ * And check whether fd is busy, if yes, return -1.
+ * Otherwise, try to delete the fd from fdset and
+ * return true.
+ */
+int
+fdset_try_del(struct fdset *pfdset, int fd)
+{
+ int i;
+
+ if (pfdset == NULL || fd == -1)
+ return -2;
+
+ pthread_mutex_lock(&pfdset->fd_mutex);
+ i = fdset_find_fd(pfdset, fd);
+ if (i != -1 && pfdset->fd[i].busy) {
+ pthread_mutex_unlock(&pfdset->fd_mutex);
+ return -1;
+ }
+
+ if (i != -1) {
+ pfdset->fd[i].fd = -1;
+ pfdset->fd[i].rcb = pfdset->fd[i].wcb = NULL;
+ pfdset->fd[i].dat = NULL;
+ }
+
+ pthread_mutex_unlock(&pfdset->fd_mutex);
+ return 0;
+}
/**
* This functions runs in infinite blocking loop until there is no fd in
@@ -275,7 +307,7 @@ fdset_event_dispatch(struct fdset *pfdset)
* because the fd is closed in the cb,
* the old fd val could be reused by when creates new
* listen fd in another thread, we couldn't call
- * fd_set_del.
+ * fdset_del.
*/
if (remove1 || remove2) {
pfdentry->fd = -1;
diff --git a/lib/librte_vhost/fd_man.h b/lib/librte_vhost/fd_man.h
index d319cac6..20781c02 100644
--- a/lib/librte_vhost/fd_man.h
+++ b/lib/librte_vhost/fd_man.h
@@ -63,6 +63,7 @@ int fdset_add(struct fdset *pfdset, int fd,
fd_cb rcb, fd_cb wcb, void *dat);
void *fdset_del(struct fdset *pfdset, int fd);
+int fdset_try_del(struct fdset *pfdset, int fd);
void fdset_event_dispatch(struct fdset *pfdset);
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index a95fbfbb..805b2e5b 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -166,6 +166,11 @@ send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num)
msgh.msg_control = control;
msgh.msg_controllen = sizeof(control);
cmsg = CMSG_FIRSTHDR(&msgh);
+ if (cmsg == NULL) {
+ RTE_LOG(ERR, VHOST_CONFIG, "cmsg == NULL\n");
+ errno = EINVAL;
+ return -1;
+ }
cmsg->cmsg_len = CMSG_LEN(fdsize);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
@@ -598,13 +603,25 @@ rte_vhost_driver_unregister(const char *path)
vhost_user_remove_reconnect(vsocket);
}
+again:
pthread_mutex_lock(&vsocket->conn_mutex);
for (conn = TAILQ_FIRST(&vsocket->conn_list);
conn != NULL;
conn = next) {
next = TAILQ_NEXT(conn, next);
- fdset_del(&vhost_user.fdset, conn->connfd);
+ /*
+ * If r/wcb is executing, release the
+ * conn_mutex lock, and try again since
+ * the r/wcb may use the conn_mutex lock.
+ */
+ if (fdset_try_del(&vhost_user.fdset,
+ conn->connfd) == -1) {
+ pthread_mutex_unlock(
+ &vsocket->conn_mutex);
+ goto again;
+ }
+
RTE_LOG(INFO, VHOST_CONFIG,
"free connfd = %d for device '%s'\n",
conn->connfd, path);
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index c49db0c0..abc5a908 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -52,6 +52,8 @@
#define BUF_VECTOR_MAX 256
+#define VHOST_LOG_CACHE_NR 32
+
/**
* Structure contains buffer address, length and descriptor index
* from vring to do scatter RX.
@@ -75,6 +77,14 @@ struct zcopy_mbuf {
};
TAILQ_HEAD(zcopy_mbuf_list, zcopy_mbuf);
+/*
+ * Structure that contains the info for batched dirty logging.
+ */
+struct log_cache_entry {
+ uint32_t offset;
+ unsigned long val;
+};
+
/**
* Structure contains variables relevant to RX/TX virtqueues.
*/
@@ -110,6 +120,9 @@ struct vhost_virtqueue {
struct vring_used_elem *shadow_used_ring;
uint16_t shadow_used_idx;
+
+ struct log_cache_entry log_cache[VHOST_LOG_CACHE_NR];
+ uint16_t log_cache_nb_elem;
} __rte_cache_aligned;
/* Old kernels have no such macros defined */
@@ -195,7 +208,8 @@ struct virtio_memory {
#ifdef RTE_LIBRTE_VHOST_DEBUG
#define VHOST_MAX_PRINT_BUFF 6072
#define LOG_LEVEL RTE_LOG_DEBUG
-#define LOG_DEBUG(log_type, fmt, args...) RTE_LOG(DEBUG, log_type, fmt, ##args)
+#define VHOST_LOG_DEBUG(log_type, fmt, args...) \
+ RTE_LOG(DEBUG, log_type, fmt, ##args)
#define PRINT_PACKET(device, addr, size, header) do { \
char *pkt_addr = (char *)(addr); \
unsigned int index; \
@@ -211,11 +225,11 @@ struct virtio_memory {
} \
snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), "\n"); \
\
- LOG_DEBUG(VHOST_DATA, "%s", packet); \
+ VHOST_LOG_DEBUG(VHOST_DATA, "%s", packet); \
} while (0)
#else
#define LOG_LEVEL RTE_LOG_INFO
-#define LOG_DEBUG(log_type, fmt, args...) do {} while (0)
+#define VHOST_LOG_DEBUG(log_type, fmt, args...) do {} while (0)
#define PRINT_PACKET(device, addr, size, header) do {} while (0)
#endif
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 550a1329..04c92ceb 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -169,7 +169,7 @@ vhost_user_set_features(struct virtio_net *dev, uint64_t features)
} else {
dev->vhost_hlen = sizeof(struct virtio_net_hdr);
}
- LOG_DEBUG(VHOST_CONFIG,
+ VHOST_LOG_DEBUG(VHOST_CONFIG,
"(%d) mergeable RX buffers %s, virtio 1 %s\n",
dev->vid,
(dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
@@ -396,13 +396,13 @@ vhost_user_set_vring_addr(struct virtio_net **pdev,
vq->log_guest_addr = addr->log_guest_addr;
- LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
+ VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
dev->vid, vq->desc);
- LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n",
+ VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n",
dev->vid, vq->avail);
- LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n",
+ VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n",
dev->vid, vq->used);
- LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n",
+ VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n",
dev->vid, vq->log_guest_addr);
return 0;
@@ -421,7 +421,7 @@ vhost_user_set_vring_base(struct virtio_net *dev,
return 0;
}
-static void
+static int
add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
uint64_t host_phys_addr, uint64_t size)
{
@@ -431,6 +431,10 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
dev->max_guest_pages *= 2;
dev->guest_pages = realloc(dev->guest_pages,
dev->max_guest_pages * sizeof(*page));
+ if (!dev->guest_pages) {
+ RTE_LOG(ERR, VHOST_CONFIG, "cannot realloc guest_pages\n");
+ return -1;
+ }
}
if (dev->nr_guest_pages > 0) {
@@ -439,7 +443,7 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
if (host_phys_addr == last_page->host_phys_addr +
last_page->size) {
last_page->size += size;
- return;
+ return 0;
}
}
@@ -447,9 +451,11 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
page->guest_phys_addr = guest_phys_addr;
page->host_phys_addr = host_phys_addr;
page->size = size;
+
+ return 0;
}
-static void
+static int
add_guest_pages(struct virtio_net *dev, struct virtio_memory_region *reg,
uint64_t page_size)
{
@@ -463,7 +469,9 @@ add_guest_pages(struct virtio_net *dev, struct virtio_memory_region *reg,
size = page_size - (guest_phys_addr & (page_size - 1));
size = RTE_MIN(size, reg_size);
- add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size);
+ if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size) < 0)
+ return -1;
+
host_user_addr += size;
guest_phys_addr += size;
reg_size -= size;
@@ -472,12 +480,16 @@ add_guest_pages(struct virtio_net *dev, struct virtio_memory_region *reg,
size = RTE_MIN(reg_size, page_size);
host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t)
host_user_addr);
- add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size);
+ if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr,
+ size) < 0)
+ return -1;
host_user_addr += size;
guest_phys_addr += size;
reg_size -= size;
}
+
+ return 0;
}
#ifdef RTE_LIBRTE_VHOST_DEBUG
@@ -624,7 +636,12 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg)
mmap_offset;
if (dev->dequeue_zero_copy)
- add_guest_pages(dev, reg, alignment);
+ if (add_guest_pages(dev, reg, alignment) < 0) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "adding guest pages to region %u failed.\n",
+ i);
+ goto err_mmap;
+ }
RTE_LOG(INFO, VHOST_CONFIG,
"guest memory region %u, size: 0x%" PRIx64 "\n"
@@ -869,7 +886,7 @@ vhost_user_set_log_base(struct virtio_net *dev, struct VhostUserMsg *msg)
* mmap from 0 to workaround a hugepage mmap bug: mmap will
* fail when offset is not page size aligned.
*/
- addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
close(fd);
if (addr == MAP_FAILED) {
RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n");
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 745cc53f..2932bade 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -58,7 +58,7 @@
static inline void __attribute__((always_inline))
vhost_set_bit(unsigned int nr, volatile uint8_t *addr)
{
- __sync_fetch_and_or_8(addr, (1U << nr));
+ __sync_fetch_and_or_1(addr, (1U << nr));
}
static inline void __attribute__((always_inline))
@@ -90,6 +90,93 @@ vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
}
static inline void __attribute__((always_inline))
+vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
+{
+ unsigned long *log_base;
+ int i;
+
+ if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) ||
+ !dev->log_base))
+ return;
+
+ log_base = (unsigned long *)(uintptr_t)dev->log_base;
+
+ /*
+ * It is expected a write memory barrier has been issued
+ * before this function is called.
+ */
+
+ for (i = 0; i < vq->log_cache_nb_elem; i++) {
+ struct log_cache_entry *elem = vq->log_cache + i;
+
+ __sync_fetch_and_or(log_base + elem->offset, elem->val);
+ }
+
+ rte_smp_wmb();
+
+ vq->log_cache_nb_elem = 0;
+}
+
+static inline void __attribute__((always_inline))
+vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
+ uint64_t page)
+{
+ uint32_t bit_nr = page % (sizeof(unsigned long) << 3);
+ uint32_t offset = page / (sizeof(unsigned long) << 3);
+ int i;
+
+ for (i = 0; i < vq->log_cache_nb_elem; i++) {
+ struct log_cache_entry *elem = vq->log_cache + i;
+
+ if (elem->offset == offset) {
+ elem->val |= (1UL << bit_nr);
+ return;
+ }
+ }
+
+ if (unlikely(i >= VHOST_LOG_CACHE_NR)) {
+ /*
+ * No more room for a new log cache entry,
+ * so write the dirty log map directly.
+ */
+ rte_smp_wmb();
+ vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
+
+ return;
+ }
+
+ vq->log_cache[i].offset = offset;
+ vq->log_cache[i].val = (1UL << bit_nr);
+}
+
+static inline void __attribute__((always_inline))
+vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
+ uint64_t addr, uint64_t len)
+{
+ uint64_t page;
+
+ if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) ||
+ !dev->log_base || !len))
+ return;
+
+ if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
+ return;
+
+ page = addr / VHOST_LOG_PAGE;
+ while (page * VHOST_LOG_PAGE < addr + len) {
+ vhost_log_cache_page(dev, vq, page);
+ page += 1;
+ }
+}
+
+static inline void __attribute__((always_inline))
+vhost_log_cache_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq,
+ uint64_t offset, uint64_t len)
+{
+ vhost_log_cache_write(dev, vq, vq->log_guest_addr + offset, len);
+}
+
+static inline void __attribute__((always_inline))
vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq,
uint64_t offset, uint64_t len)
{
@@ -147,7 +234,7 @@ do_flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
rte_memcpy(&vq->used->ring[to],
&vq->shadow_used_ring[from],
size * sizeof(struct vring_used_elem));
- vhost_log_used_vring(dev, vq,
+ vhost_log_cache_used_vring(dev, vq,
offsetof(struct vring_used, ring[to]),
size * sizeof(struct vring_used_elem));
}
@@ -175,6 +262,8 @@ flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq)
rte_smp_wmb();
+ vhost_log_cache_sync(dev, vq);
+
*(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx;
vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
sizeof(vq->used->idx));
@@ -249,8 +338,9 @@ copy_virtio_net_hdr(struct virtio_net *dev, uint64_t desc_addr,
}
static inline int __attribute__((always_inline))
-copy_mbuf_to_desc(struct virtio_net *dev, struct vring_desc *descs,
- struct rte_mbuf *m, uint16_t desc_idx, uint32_t size)
+copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
+ struct vring_desc *descs, struct rte_mbuf *m,
+ uint16_t desc_idx, uint32_t size)
{
uint32_t desc_avail, desc_offset;
uint32_t mbuf_avail, mbuf_offset;
@@ -298,14 +388,14 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vring_desc *descs,
rte_memcpy((void *)(uintptr_t)dst,
(void *)(uintptr_t)src, len);
- PRINT_PACKET(dev, (uintptr_t)dst, len, 0);
+ PRINT_PACKET(dev, (uintptr_t)dst, (uint32_t)len, 0);
remain -= len;
guest_addr += len;
dst += len;
}
}
- vhost_log_write(dev, desc_gaddr, dev->vhost_hlen);
+ vhost_log_cache_write(dev, vq, desc_gaddr, dev->vhost_hlen);
desc_avail = desc->len - dev->vhost_hlen;
if (unlikely(desc_chunck_len < dev->vhost_hlen)) {
@@ -368,7 +458,8 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vring_desc *descs,
rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),
rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
cpy_len);
- vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len);
+ vhost_log_cache_write(dev, vq, desc_gaddr + desc_offset,
+ cpy_len);
PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
cpy_len, 0);
@@ -401,7 +492,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
uint32_t i, sz;
uint64_t dlen;
- LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
+ VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) {
RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
dev->vid, __func__, queue_id);
@@ -423,7 +514,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
if (count == 0)
goto out_access_unlock;
- LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n",
+ VHOST_LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n",
dev->vid, start_idx, start_idx + count);
/* Retrieve all of the desc indexes first to avoid caching issues. */
@@ -434,7 +525,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
vq->used->ring[used_idx].id = desc_indexes[i];
vq->used->ring[used_idx].len = pkts[i]->pkt_len +
dev->vhost_hlen;
- vhost_log_used_vring(dev, vq,
+ vhost_log_cache_used_vring(dev, vq,
offsetof(struct vring_used, ring[used_idx]),
sizeof(vq->used->ring[used_idx]));
}
@@ -474,11 +565,11 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
sz = vq->size;
}
- err = copy_mbuf_to_desc(dev, descs, pkts[i], desc_idx, sz);
+ err = copy_mbuf_to_desc(dev, vq, descs, pkts[i], desc_idx, sz);
if (unlikely(err)) {
used_idx = (start_idx + i) & (vq->size - 1);
vq->used->ring[used_idx].len = dev->vhost_hlen;
- vhost_log_used_vring(dev, vq,
+ vhost_log_cache_used_vring(dev, vq,
offsetof(struct vring_used, ring[used_idx]),
sizeof(vq->used->ring[used_idx]));
}
@@ -492,6 +583,8 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
rte_smp_wmb();
+ vhost_log_cache_sync(dev, vq);
+
*(volatile uint16_t *)&vq->used->idx += count;
vq->last_used_idx += count;
vhost_log_used_vring(dev, vq,
@@ -623,8 +716,9 @@ reserve_avail_buf_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
}
static inline int __attribute__((always_inline))
-copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct rte_mbuf *m,
- struct buf_vector *buf_vec, uint16_t num_buffers)
+copy_mbuf_to_desc_mergeable(struct virtio_net *dev,
+ struct vhost_virtqueue *vq, struct rte_mbuf *m,
+ struct buf_vector *buf_vec, uint16_t num_buffers)
{
struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0};
struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -657,7 +751,7 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct rte_mbuf *m,
rte_prefetch0((void *)(uintptr_t)hdr_addr);
virtio_hdr.num_buffers = num_buffers;
- LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
+ VHOST_LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
dev->vid, num_buffers);
desc_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
@@ -736,14 +830,15 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct rte_mbuf *m,
len);
PRINT_PACKET(dev, (uintptr_t)dst,
- len, 0);
+ (uint32_t)len, 0);
remain -= len;
guest_addr += len;
dst += len;
}
}
- vhost_log_write(dev, hdr_phys_addr, dev->vhost_hlen);
+ vhost_log_cache_write(dev, vq, hdr_phys_addr,
+ dev->vhost_hlen);
PRINT_PACKET(dev, (uintptr_t)hdr_addr,
dev->vhost_hlen, 0);
@@ -754,7 +849,8 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct rte_mbuf *m,
rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),
rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
cpy_len);
- vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len);
+ vhost_log_cache_write(dev, vq, desc_gaddr + desc_offset,
+ cpy_len);
PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
cpy_len, 0);
@@ -778,7 +874,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
struct buf_vector buf_vec[BUF_VECTOR_MAX];
uint16_t avail_head;
- LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
+ VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) {
RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
dev->vid, __func__, queue_id);
@@ -806,18 +902,18 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
if (unlikely(reserve_avail_buf_mergeable(dev, vq,
pkt_len, buf_vec, &num_buffers,
avail_head) < 0)) {
- LOG_DEBUG(VHOST_DATA,
+ VHOST_LOG_DEBUG(VHOST_DATA,
"(%d) failed to get enough desc from vring\n",
dev->vid);
vq->shadow_used_idx -= num_buffers;
break;
}
- LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
+ VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
dev->vid, vq->last_avail_idx,
vq->last_avail_idx + num_buffers);
- if (copy_mbuf_to_desc_mergeable(dev, pkts[pkt_idx],
+ if (copy_mbuf_to_desc_mergeable(dev, vq, pkts[pkt_idx],
buf_vec, num_buffers) < 0) {
vq->shadow_used_idx -= num_buffers;
break;
@@ -1116,7 +1212,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vring_desc *descs,
rte_prefetch0((void *)(uintptr_t)(desc_addr + desc_offset));
PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
- desc_chunck_len, 0);
+ (uint32_t)desc_chunck_len, 0);
mbuf_offset = 0;
mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM;
@@ -1181,7 +1277,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vring_desc *descs,
desc_avail = desc->len;
PRINT_PACKET(dev, (uintptr_t)desc_addr,
- desc_chunck_len, 0);
+ (uint32_t)desc_chunck_len, 0);
} else if (unlikely(desc_chunck_len == 0)) {
desc_chunck_len = desc_avail;
desc_gaddr += desc_offset;
@@ -1194,7 +1290,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vring_desc *descs,
desc_offset = 0;
PRINT_PACKET(dev, (uintptr_t)desc_addr,
- desc_chunck_len, 0);
+ (uint32_t)desc_chunck_len, 0);
}
/*
@@ -1237,7 +1333,7 @@ update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
{
vq->used->ring[used_idx].id = desc_idx;
vq->used->ring[used_idx].len = 0;
- vhost_log_used_vring(dev, vq,
+ vhost_log_cache_used_vring(dev, vq,
offsetof(struct vring_used, ring[used_idx]),
sizeof(vq->used->ring[used_idx]));
}
@@ -1252,6 +1348,8 @@ update_used_idx(struct virtio_net *dev, struct vhost_virtqueue *vq,
rte_smp_wmb();
rte_smp_rmb();
+ vhost_log_cache_sync(dev, vq);
+
vq->used->idx += count;
vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
sizeof(vq->used->idx));
@@ -1417,7 +1515,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
if (free_entries == 0)
goto out_access_unlock;
- LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
+ VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
/* Prefetch available and used ring */
avail_idx = vq->last_avail_idx & (vq->size - 1);
@@ -1427,7 +1525,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
count = RTE_MIN(count, MAX_PKT_BURST);
count = RTE_MIN(count, free_entries);
- LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
+ VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
dev->vid, count);
/* Retrieve all of the head indexes first to avoid caching issues. */