diff options
Diffstat (limited to 'lib/librte_vhost')
-rw-r--r-- | lib/librte_vhost/rte_vhost.h | 4 | ||||
-rw-r--r-- | lib/librte_vhost/socket.c | 22 | ||||
-rw-r--r-- | lib/librte_vhost/vdpa.c | 7 | ||||
-rw-r--r-- | lib/librte_vhost/vhost.c | 16 | ||||
-rw-r--r-- | lib/librte_vhost/vhost.h | 46 | ||||
-rw-r--r-- | lib/librte_vhost/vhost_crypto.c | 4 | ||||
-rw-r--r-- | lib/librte_vhost/vhost_user.c | 54 | ||||
-rw-r--r-- | lib/librte_vhost/virtio_net.c | 34 |
8 files changed, 121 insertions, 66 deletions
diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h index d280ac42..5905e240 100644 --- a/lib/librte_vhost/rte_vhost.h +++ b/lib/librte_vhost/rte_vhost.h @@ -488,7 +488,7 @@ int rte_vhost_get_ifname(int vid, char *buf, size_t len); * virtio queue index * * @return - * num of avail entires left + * num of avail entries left */ uint16_t rte_vhost_avail_entries(int vid, uint16_t queue_id); @@ -536,7 +536,7 @@ uint16_t rte_vhost_dequeue_burst(int vid, uint16_t queue_id, /** * Get guest mem table: a list of memory regions. * - * An rte_vhost_vhost_memory object will be allocated internaly, to hold the + * An rte_vhost_vhost_memory object will be allocated internally, to hold the * guest memory regions. Application should free it at destroy_device() * callback. * diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c index 9883b049..09799516 100644 --- a/lib/librte_vhost/socket.c +++ b/lib/librte_vhost/socket.c @@ -240,7 +240,7 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) RTE_LOG(ERR, VHOST_CONFIG, "failed to add vhost user connection with fd %d\n", fd); - goto err; + goto err_cleanup; } } @@ -257,7 +257,7 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) if (vsocket->notify_ops->destroy_connection) vsocket->notify_ops->destroy_connection(conn->vid); - goto err; + goto err_cleanup; } pthread_mutex_lock(&vsocket->conn_mutex); @@ -267,6 +267,8 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) fdset_pipe_notify(&vhost_user.fdset); return; +err_cleanup: + vhost_destroy_device(vid); err: free(conn); close(fd); @@ -295,13 +297,19 @@ vhost_user_read_cb(int connfd, void *dat, int *remove) ret = vhost_user_msg_handler(conn->vid, connfd); if (ret < 0) { + struct virtio_net *dev = get_device(conn->vid); + close(connfd); *remove = 1; - vhost_destroy_device(conn->vid); + + if (dev) + vhost_destroy_device_notify(dev); if (vsocket->notify_ops->destroy_connection) vsocket->notify_ops->destroy_connection(conn->vid); + vhost_destroy_device(conn->vid); + pthread_mutex_lock(&vsocket->conn_mutex); TAILQ_REMOVE(&vsocket->conn_list, conn, next); pthread_mutex_unlock(&vsocket->conn_mutex); @@ -547,6 +555,9 @@ find_vhost_user_socket(const char *path) { int i; + if (path == NULL) + return NULL; + for (i = 0; i < vhost_user.vsocket_cnt; i++) { struct vhost_user_socket *vsocket = vhost_user.vsockets[i]; @@ -562,7 +573,7 @@ rte_vhost_driver_attach_vdpa_device(const char *path, int did) { struct vhost_user_socket *vsocket; - if (rte_vdpa_get_device(did) == NULL) + if (rte_vdpa_get_device(did) == NULL || path == NULL) return -1; pthread_mutex_lock(&vhost_user.mutex); @@ -961,6 +972,9 @@ rte_vhost_driver_unregister(const char *path) int count; struct vhost_user_connection *conn, *next; + if (path == NULL) + return -1; + again: pthread_mutex_lock(&vhost_user.mutex); diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c index e7d849ee..f560419b 100644 --- a/lib/librte_vhost/vdpa.c +++ b/lib/librte_vhost/vdpa.c @@ -49,7 +49,7 @@ rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr, char device_name[MAX_VDPA_NAME_LEN]; int i; - if (vdpa_device_num >= MAX_VHOST_DEVICE) + if (vdpa_device_num >= MAX_VHOST_DEVICE || addr == NULL || ops == NULL) return -1; for (i = 0; i < MAX_VHOST_DEVICE; i++) { @@ -66,7 +66,7 @@ rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr, if (i == MAX_VHOST_DEVICE) return -1; - sprintf(device_name, "vdpa-dev-%d", i); + snprintf(device_name, sizeof(device_name), "vdpa-dev-%d", i); dev = rte_zmalloc(device_name, sizeof(struct rte_vdpa_device), RTE_CACHE_LINE_SIZE); if (!dev) @@ -99,6 +99,9 @@ rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr) struct rte_vdpa_device *dev; int i; + if (addr == NULL) + return -1; + for (i = 0; i < MAX_VHOST_DEVICE; ++i) { dev = vdpa_devices[i]; if (dev && is_same_vdpa_device(&dev->addr, addr)) diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 70ac6bc9..488cf169 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -460,7 +460,7 @@ rte_vhost_get_mtu(int vid, uint16_t *mtu) { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL || mtu == NULL) return -ENODEV; if (!(dev->flags & VIRTIO_DEV_READY)) @@ -528,7 +528,7 @@ rte_vhost_get_ifname(int vid, char *buf, size_t len) { struct virtio_net *dev = get_device(vid); - if (dev == NULL) + if (dev == NULL || buf == NULL) return -1; len = RTE_MIN(len, sizeof(dev->ifname)); @@ -545,7 +545,7 @@ rte_vhost_get_negotiated_features(int vid, uint64_t *features) struct virtio_net *dev; dev = get_device(vid); - if (!dev) + if (dev == NULL || features == NULL) return -1; *features = dev->features; @@ -560,7 +560,7 @@ rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) size_t size; dev = get_device(vid); - if (!dev) + if (dev == NULL || mem == NULL) return -1; size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); @@ -583,7 +583,7 @@ rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, struct vhost_virtqueue *vq; dev = get_device(vid); - if (!dev) + if (dev == NULL || vring == NULL) return -1; if (vring_idx >= VHOST_MAX_VRING) @@ -776,7 +776,7 @@ int rte_vhost_get_log_base(int vid, uint64_t *log_base, { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL || log_base == NULL || log_size == NULL) return -1; if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { @@ -797,7 +797,7 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id, { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) return -1; if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { @@ -818,7 +818,7 @@ int rte_vhost_set_vring_base(int vid, uint16_t queue_id, { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL) return -1; if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index 552b9298..bb9cff9f 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -686,16 +686,20 @@ vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq) if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { uint16_t old = vq->signalled_used; uint16_t new = vq->last_used_idx; + bool signalled_used_valid = vq->signalled_used_valid; + + vq->signalled_used = new; + vq->signalled_used_valid = true; VHOST_LOG_DEBUG(VHOST_DATA, "%s: used_event_idx=%d, old=%d, new=%d\n", __func__, vhost_used_event(vq), old, new); - if (vhost_need_event(vhost_used_event(vq), new, old) - && (vq->callfd >= 0)) { - vq->signalled_used = vq->last_used_idx; + + if ((vhost_need_event(vhost_used_event(vq), new, old) && + (vq->callfd >= 0)) || + unlikely(!signalled_used_valid)) eventfd_write(vq->callfd, (eventfd_t) 1); - } } else { /* Kick the guest if necessary. */ if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) @@ -755,4 +759,38 @@ kick: eventfd_write(vq->callfd, (eventfd_t)1); } +static __rte_always_inline void +restore_mbuf(struct rte_mbuf *m) +{ + uint32_t mbuf_size, priv_size; + + while (m) { + priv_size = rte_pktmbuf_priv_size(m->pool); + mbuf_size = sizeof(struct rte_mbuf) + priv_size; + /* start of buffer is after mbuf structure and priv data */ + + m->buf_addr = (char *)m + mbuf_size; + m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; + m = m->next; + } +} + +static __rte_always_inline bool +mbuf_is_consumed(struct rte_mbuf *m) +{ + while (m) { + if (rte_mbuf_refcnt_read(m) > 1) + return false; + m = m->next; + } + + return true; +} + +static __rte_always_inline void +put_zmbuf(struct zcopy_mbuf *zmbuf) +{ + zmbuf->in_use = 0; +} + #endif /* _VHOST_NET_CDEV_H_ */ diff --git a/lib/librte_vhost/vhost_crypto.c b/lib/librte_vhost/vhost_crypto.c index 0694c0a7..fc362ba9 100644 --- a/lib/librte_vhost/vhost_crypto.c +++ b/lib/librte_vhost/vhost_crypto.c @@ -1102,7 +1102,7 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, if (unlikely(copy_data(digest_addr, vc_req, &digest_desc, chain->para.hash_result_len, - nb_descs, vq_size)) < 0) { + nb_descs, vq_size) < 0)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } @@ -1617,7 +1617,7 @@ rte_vhost_crypto_fetch_requests(int vid, uint32_t qid, op->sym->m_src->data_off = 0; if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, - op, head, desc_idx)) < 0) + op, head, desc_idx) < 0)) break; } diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index 19e04c95..5552f8bb 100644 --- a/lib/librte_vhost/vhost_user.c +++ b/lib/librte_vhost/vhost_user.c @@ -93,15 +93,47 @@ get_blk_size(int fd) return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize; } +/* + * Reclaim all the outstanding zmbufs for a virtqueue. + */ +static void +drain_zmbuf_list(struct vhost_virtqueue *vq) +{ + struct zcopy_mbuf *zmbuf, *next; + + for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); + zmbuf != NULL; zmbuf = next) { + next = TAILQ_NEXT(zmbuf, next); + + while (!mbuf_is_consumed(zmbuf->mbuf)) + usleep(1000); + + TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); + restore_mbuf(zmbuf->mbuf); + rte_pktmbuf_free(zmbuf->mbuf); + put_zmbuf(zmbuf); + vq->nr_zmbuf -= 1; + } +} + static void free_mem_region(struct virtio_net *dev) { uint32_t i; struct rte_vhost_mem_region *reg; + struct vhost_virtqueue *vq; if (!dev || !dev->mem) return; + if (dev->dequeue_zero_copy) { + for (i = 0; i < dev->nr_vring; i++) { + vq = dev->virtqueue[i]; + if (vq) + drain_zmbuf_list(vq); + } + } + for (i = 0; i < dev->mem->nregions; i++) { reg = &dev->mem->regions[i]; if (reg->host_user_addr) { @@ -1199,8 +1231,12 @@ vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg, * the ring starts already enabled. Otherwise, it is enabled via * the SET_VRING_ENABLE message. */ - if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) + if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) { vq->enabled = 1; + if (dev->notify_ops->vring_state_changed) + dev->notify_ops->vring_state_changed( + dev->vid, file.index, 1); + } if (vq->kickfd >= 0) close(vq->kickfd); @@ -1212,15 +1248,7 @@ vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg, static void free_zmbufs(struct vhost_virtqueue *vq) { - struct zcopy_mbuf *zmbuf, *next; - - for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); - zmbuf != NULL; zmbuf = next) { - next = TAILQ_NEXT(zmbuf, next); - - rte_pktmbuf_free(zmbuf->mbuf); - TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); - } + drain_zmbuf_list(vq); rte_free(vq->zmbufs); } @@ -1274,6 +1302,8 @@ vhost_user_get_vring_base(struct virtio_net **pdev, vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; + vq->signalled_used_valid = false; + if (dev->dequeue_zero_copy) free_zmbufs(vq); if (vq_is_packed(dev)) { @@ -1321,6 +1351,10 @@ vhost_user_set_vring_enable(struct virtio_net **pdev, dev->notify_ops->vring_state_changed(dev->vid, index, enable); + /* On disable, rings have to be stopped being processed. */ + if (!enable && dev->dequeue_zero_copy) + drain_zmbuf_list(dev->virtqueue[index]); + dev->virtqueue[index]->enabled = enable; return VH_RESULT_OK; diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 15d682c3..a6576891 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -1088,12 +1088,6 @@ vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m) } } -static __rte_always_inline void -put_zmbuf(struct zcopy_mbuf *zmbuf) -{ - zmbuf->in_use = 0; -} - static __rte_always_inline int copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct buf_vector *buf_vec, uint16_t nr_vec, @@ -1331,34 +1325,6 @@ again: return NULL; } -static __rte_always_inline bool -mbuf_is_consumed(struct rte_mbuf *m) -{ - while (m) { - if (rte_mbuf_refcnt_read(m) > 1) - return false; - m = m->next; - } - - return true; -} - -static __rte_always_inline void -restore_mbuf(struct rte_mbuf *m) -{ - uint32_t mbuf_size, priv_size; - - while (m) { - priv_size = rte_pktmbuf_priv_size(m->pool); - mbuf_size = sizeof(struct rte_mbuf) + priv_size; - /* start of buffer is after mbuf structure and priv data */ - - m->buf_addr = (char *)m + mbuf_size; - m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; - m = m->next; - } -} - static __rte_always_inline uint16_t virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) |