vhost: convert vhost device driver to a plugin

convert vhost device driver to a plugin as described in
https://jira.fd.io/browse/VPP-2065

Type: improvement

Signed-off-by: Steven Luong <sluong@cisco.com>
Change-Id: Ibfe2f351bcaed36a04b136d082ae414145dd37b5

14 files changed, 347 insertions, 25 deletions

diff --git a/MAINTAINERS b/MAINTAINERS
index b18e48b40a8..9bc8be134be 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -115,9 +115,9 @@ F:	src/vnet/devices/tap/
 
 VNET Vhost User Driver
 I:	vhost
-Y:	src/vnet/devices/virtio/FEATURE.yaml
+Y:	src/plugins/vhost/FEATURE.yaml
 M:	Steven Luong <sluong@cisco.com>
-F:	src/vnet/devices/virtio/vhost_user*
+F:	src/plugins/vhost
 
 VNET Native Virtio Drivers
 I:	virtio
diff --git a/src/plugins/vhost/CMakeLists.txt b/src/plugins/vhost/CMakeLists.txt
new file mode 100644
index 00000000000..f72d9f20346
--- /dev/null
+++ b/src/plugins/vhost/CMakeLists.txt
@@ -0,0 +1,32 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_vpp_plugin(vhost
+  SOURCES
+  plugin.c
+  vhost_user.c
+  vhost_user_api.c
+  vhost_user_input.c
+  vhost_user_output.c
+  vhost_std.h
+  vhost_user.h
+  vhost_user_inline.h
+  virtio_std.h
+
+  MULTIARCH_SOURCES
+  vhost_user_input.c
+  vhost_user_output.c
+
+  API_FILES
+  vhost_user.api
+)
diff --git a/src/plugins/vhost/FEATURE.yaml b/src/plugins/vhost/FEATURE.yaml
new file mode 100644
index 00000000000..7104dda1dc5
--- /dev/null
+++ b/src/plugins/vhost/FEATURE.yaml
@@ -0,0 +1,13 @@
+---
+name: Vhost-user Device Driver
+maintainer: sluong@cisco.com
+features:
+  - Device mode to emulate vhost-user interface presented to VPP from the
+    guest VM.
+  - Support virtio 1.0 in virtio
+  - Support virtio 1.1 packed ring in virtio [experimental]
+  - Support multi-queue, GSO, checksum offload, indirect descriptor,
+    jumbo frame, and packed ring.
+description: "Vhost-user implementation"
+state: production
+properties: [API, CLI, STATS, MULTITHREAD]
diff --git a/src/plugins/vhost/plugin.c b/src/plugins/vhost/plugin.c
new file mode 100644
index 00000000000..dc5de1c9ed8
--- /dev/null
+++ b/src/plugins/vhost/plugin.c
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2022 Cisco Systems, Inc.
+ * License: Cisco Proprietary Closed Source License - Cisco Internal.
+ * The software, documentation and any fonts accompanying this License whether
+ * on disk, in read only memory, on any other media or in any other form (col-
+ * lectively the “Software”) are licensed, not sold, to you by Cisco, Inc.
+ * (“Cisco”) for use only under the terms of this License, and Cisco reserves
+ * all rights not expressly granted to you. The rights granted herein are
+ * limited to Cisco’s intel- lectual property rights in the Cisco Software and
+ * do not include any other patents or intellectual property rights. You own
+ * the media on which the Cisco Software is recorded but Cisco and/or Cisco’s
+ * licensor(s) retain ownership of the Software itself.
+ */
+
+#include <vlib/vlib.h>
+#include <vnet/plugin/plugin.h>
+#include <vpp/app/version.h>
+
+VLIB_PLUGIN_REGISTER () = {
+  .version = VPP_BUILD_VER,
+  .description = "Vhost-User",
+};
diff --git a/src/plugins/vhost/vhost_std.h b/src/plugins/vhost/vhost_std.h
new file mode 100644
index 00000000000..7799093bac3
--- /dev/null
+++ b/src/plugins/vhost/vhost_std.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __VHOST_STD_H__
+#define __VHOST_STD_H__
+
+typedef struct
+{
+  u64 guest_phys_addr;
+  u64 memory_size;
+  u64 userspace_addr;
+  u64 mmap_offset;
+} vhost_memory_region_t;
+
+typedef struct
+{
+  u32 nregions;
+  u32 padding;
+  vhost_memory_region_t regions[0];
+} vhost_memory_t;
+
+typedef struct
+{
+  u32 index;
+  u32 num;
+} vhost_vring_state_t;
+
+typedef struct
+{
+  u32 index;
+  int fd;
+} vhost_vring_file_t;
+
+typedef struct
+{
+  u32 index;
+  u32 flags;
+  u64 desc_user_addr;
+  u64 used_user_addr;
+  u64 avail_user_addr;
+  u64 log_guest_addr;
+} vhost_vring_addr_t;
+
+typedef struct
+{
+  u64 size;
+  u64 offset;
+} vhost_user_log_t;
+
+#endif
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/devices/virtio/vhost_user.api b/src/plugins/vhost/vhost_user.api
index b026ba768a9..b026ba768a9 100644
--- a/src/vnet/devices/virtio/vhost_user.api
+++ b/src/plugins/vhost/vhost_user.api
diff --git a/src/vnet/devices/virtio/vhost_user.c b/src/plugins/vhost/vhost_user.c
index b6e0806db90..fac20800848 100644
--- a/src/vnet/devices/virtio/vhost_user.c
+++ b/src/plugins/vhost/vhost_user.c
@@ -39,8 +39,8 @@
 #include <vnet/interface/rx_queue_funcs.h>
 #include <vnet/interface/tx_queue_funcs.h>
 
-#include <vnet/devices/virtio/vhost_user.h>
-#include <vnet/devices/virtio/vhost_user_inline.h>
+#include <vhost/vhost_user.h>
+#include <vhost/vhost_user_inline.h>
 
 /**
  * @file
diff --git a/src/vnet/devices/virtio/vhost_user.h b/src/plugins/vhost/vhost_user.h
index f44951e030a..3479d65cbc6 100644
--- a/src/vnet/devices/virtio/vhost_user.h
+++ b/src/plugins/vhost/vhost_user.h
@@ -15,8 +15,8 @@
 #ifndef __VIRTIO_VHOST_USER_H__
 #define __VIRTIO_VHOST_USER_H__
 
-#include <vnet/devices/virtio/virtio_std.h>
-#include <vnet/devices/virtio/vhost_std.h>
+#include <vhost/virtio_std.h>
+#include <vhost/vhost_std.h>
 
 /* vhost-user data structures */
 
diff --git a/src/vnet/devices/virtio/vhost_user_api.c b/src/plugins/vhost/vhost_user_api.c
index cc1896b108a..d522803a011 100644
--- a/src/vnet/devices/virtio/vhost_user_api.c
+++ b/src/plugins/vhost/vhost_user_api.c
@@ -22,14 +22,13 @@
 
 #include <vnet/interface.h>
 #include <vnet/api_errno.h>
-#include <vnet/devices/virtio/vhost_user.h>
+#include <vhost/vhost_user.h>
 #include <vnet/ethernet/ethernet.h>
 #include <vnet/ethernet/ethernet_types_api.h>
-#include <vnet/devices/virtio/virtio_types_api.h>
 
 #include <vnet/format_fns.h>
-#include <vnet/devices/virtio/vhost_user.api_enum.h>
-#include <vnet/devices/virtio/vhost_user.api_types.h>
+#include <vhost/vhost_user.api_enum.h>
+#include <vhost/vhost_user.api_types.h>
 
 #define REPLY_MSG_ID_BASE msg_id_base
 #include <vlibapi/api_helper_macros.h>
@@ -261,6 +260,13 @@ vl_api_delete_vhost_user_if_t_handler (vl_api_delete_vhost_user_if_t * mp)
 }
 
 static void
+vhost_user_features_encode (u64 features, u32 *first, u32 *last)
+{
+  *first = clib_net_to_host_u32 (features);
+  *last = clib_net_to_host_u32 (features >> 32);
+}
+
+static void
 send_sw_interface_vhost_user_details (vpe_api_main_t * am,
 				      vl_api_registration_t * reg,
 				      vhost_user_intf_details_t * vui,
@@ -274,8 +280,8 @@ send_sw_interface_vhost_user_details (vpe_api_main_t * am,
     ntohs (REPLY_MSG_ID_BASE + VL_API_SW_INTERFACE_VHOST_USER_DETAILS);
   mp->sw_if_index = ntohl (vui->sw_if_index);
   mp->virtio_net_hdr_sz = ntohl (vui->virtio_net_hdr_sz);
-  virtio_features_encode (vui->features, (u32 *) & mp->features_first_32,
-			  (u32 *) & mp->features_last_32);
+  vhost_user_features_encode (vui->features, (u32 *) &mp->features_first_32,
+			      (u32 *) &mp->features_last_32);
   mp->is_server = vui->is_server;
   mp->num_regions = ntohl (vui->num_regions);
   mp->sock_errno = ntohl (vui->sock_errno);
@@ -324,7 +330,7 @@ static void
   vec_free (ifaces);
 }
 
-#include <vnet/devices/virtio/vhost_user.api.c>
+#include <vhost/vhost_user.api.c>
 static clib_error_t *
 vhost_user_api_hookup (vlib_main_t * vm)
 {
diff --git a/src/vnet/devices/virtio/vhost_user_inline.h b/src/plugins/vhost/vhost_user_inline.h
index 8bdff3733a7..8bdff3733a7 100644
--- a/src/vnet/devices/virtio/vhost_user_inline.h
+++ b/src/plugins/vhost/vhost_user_inline.h
diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/plugins/vhost/vhost_user_input.c
index 841a9798212..c083f43ba7c 100644
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/plugins/vhost/vhost_user_input.c
@@ -40,8 +40,8 @@
 #include <vnet/tcp/tcp_packet.h>
 #include <vnet/interface/rx_queue_funcs.h>
 
-#include <vnet/devices/virtio/vhost_user.h>
-#include <vnet/devices/virtio/vhost_user_inline.h>
+#include <vhost/vhost_user.h>
+#include <vhost/vhost_user_inline.h>
 
 #include <vnet/ip/ip4_packet.h>
 #include <vnet/ip/ip6_packet.h>
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/plugins/vhost/vhost_user_output.c
index 3b7bf97c3f8..9b52fc6265c 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/plugins/vhost/vhost_user_output.c
@@ -39,8 +39,8 @@
 #include <vnet/feature/feature.h>
 #include <vnet/ip/ip_psh_cksum.h>
 
-#include <vnet/devices/virtio/vhost_user.h>
-#include <vnet/devices/virtio/vhost_user_inline.h>
+#include <vhost/vhost_user.h>
+#include <vhost/vhost_user_inline.h>
 
 #include <vnet/gso/hdr_offset_parser.h>
 /*
diff --git a/src/plugins/vhost/virtio_std.h b/src/plugins/vhost/virtio_std.h
new file mode 100644
index 00000000000..fa826933a9c
--- /dev/null
+++ b/src/plugins/vhost/virtio_std.h
@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __VIRTIO_STD_H__
+#define __VIRTIO_STD_H__
+
+#define foreach_virtio_net_features                                           \
+  _ (VIRTIO_NET_F_CSUM, 0)	 /* Host handles pkts w/ partial csum */      \
+  _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */     \
+  _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,                                        \
+     2)				     /* Dynamic offload configuration. */     \
+  _ (VIRTIO_NET_F_MTU, 3)	     /* Initial MTU advice. */                \
+  _ (VIRTIO_NET_F_MAC, 5)	     /* Host has given MAC address. */        \
+  _ (VIRTIO_NET_F_GSO, 6)	     /* Host handles pkts w/ any GSO. */      \
+  _ (VIRTIO_NET_F_GUEST_TSO4, 7)     /* Guest can handle TSOv4 in. */         \
+  _ (VIRTIO_NET_F_GUEST_TSO6, 8)     /* Guest can handle TSOv6 in. */         \
+  _ (VIRTIO_NET_F_GUEST_ECN, 9)	     /* Guest can handle TSO[6] w/ ECN in. */ \
+  _ (VIRTIO_NET_F_GUEST_UFO, 10)     /* Guest can handle UFO in. */           \
+  _ (VIRTIO_NET_F_HOST_TSO4, 11)     /* Host can handle TSOv4 in. */          \
+  _ (VIRTIO_NET_F_HOST_TSO6, 12)     /* Host can handle TSOv6 in. */          \
+  _ (VIRTIO_NET_F_HOST_ECN, 13)	     /* Host can handle TSO[6] w/ ECN in. */  \
+  _ (VIRTIO_NET_F_HOST_UFO, 14)	     /* Host can handle UFO in. */            \
+  _ (VIRTIO_NET_F_MRG_RXBUF, 15)     /* Host can merge receive buffers. */    \
+  _ (VIRTIO_NET_F_STATUS, 16)	     /* virtio_net_config.status available */ \
+  _ (VIRTIO_NET_F_CTRL_VQ, 17)	     /* Control channel available */          \
+  _ (VIRTIO_NET_F_CTRL_RX, 18)	     /* Control channel RX mode support */    \
+  _ (VIRTIO_NET_F_CTRL_VLAN, 19)     /* Control channel VLAN filtering */     \
+  _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */      \
+  _ (VIRTIO_NET_F_GUEST_ANNOUNCE,                                             \
+     21)		  /* Guest can announce device on the network */      \
+  _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */         \
+  _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */                    \
+  _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24)                                            \
+  _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */                     \
+  _ (VIRTIO_F_ANY_LAYOUT,                                                     \
+     27) /* Can the device handle any descriptor layout */                    \
+  _ (VIRTIO_RING_F_INDIRECT_DESC,                                             \
+     28) /* Support indirect buffer descriptors */                            \
+  _ (VIRTIO_RING_F_EVENT_IDX,                                                 \
+     29) /* The Guest publishes the used index for which it expects an        \
+	  * interrupt at the end of the avail ring. Host should ignore the    \
+	  * avail->flags field. */                                            \
+  /* The Host publishes the avail index for which it expects a kick           \
+   * at the end of the used ring. Guest should ignore the used->flags field.  \
+   */                                                                         \
+  _ (VHOST_USER_F_PROTOCOL_FEATURES, 30)                                      \
+  _ (VIRTIO_F_VERSION_1, 32) /* v1.0 compliant. */                            \
+  _ (VIRTIO_F_IOMMU_PLATFORM, 33)                                             \
+  _ (VIRTIO_F_RING_PACKED, 34)                                                \
+  _ (VIRTIO_F_IN_ORDER, 35) /* all buffers are used by the device in the */   \
+  /* same order in which they have been made available */                     \
+  _ (VIRTIO_F_ORDER_PLATFORM, 36) /* memory accesses by the driver and the */ \
+  /* device are ordered in a way described by the platfor */                  \
+  _ (VIRTIO_F_NOTIFICATION_DATA,                                              \
+     38) /* the driver passes extra data (besides */                          \
+  /* identifying the virtqueue) in its device notifications. */               \
+  _ (VIRTIO_NET_F_SPEED_DUPLEX, 63) /* Device set linkspeed and duplex */
+
+typedef enum
+{
+#define _(f, n) f = n,
+  foreach_virtio_net_features
+#undef _
+} vnet_virtio_net_feature_t;
+
+#define VIRTIO_FEATURE(X) (1ULL << X)
+
+#define VRING_MAX_SIZE 32768
+
+#define VRING_DESC_F_NEXT     1
+#define VRING_DESC_F_WRITE    2
+#define VRING_DESC_F_INDIRECT 4
+
+#define VRING_DESC_F_AVAIL (1 << 7)
+#define VRING_DESC_F_USED  (1 << 15)
+
+#define foreach_virtio_event_idx_flags                                        \
+  _ (VRING_EVENT_F_ENABLE, 0)                                                 \
+  _ (VRING_EVENT_F_DISABLE, 1)                                                \
+  _ (VRING_EVENT_F_DESC, 2)
+
+typedef enum
+{
+#define _(f, n) f = n,
+  foreach_virtio_event_idx_flags
+#undef _
+} vnet_virtio_event_idx_flags_t;
+
+#define VRING_USED_F_NO_NOTIFY	   1
+#define VRING_AVAIL_F_NO_INTERRUPT 1
+
+typedef struct
+{
+  u64 addr;
+  u32 len;
+  u16 flags;
+  u16 next;
+} vnet_virtio_vring_desc_t;
+
+typedef struct
+{
+  u16 flags;
+  u16 idx;
+  u16 ring[0];
+  /*  u16 used_event; */
+} vnet_virtio_vring_avail_t;
+
+typedef struct
+{
+  u32 id;
+  u32 len;
+} vnet_virtio_vring_used_elem_t;
+
+typedef struct
+{
+  u16 flags;
+  u16 idx;
+  vnet_virtio_vring_used_elem_t ring[0];
+  /* u16 avail_event; */
+} vnet_virtio_vring_used_t;
+
+typedef CLIB_PACKED (struct {
+  u64 addr;  // packet data buffer address
+  u32 len;   // packet data buffer size
+  u16 id;    // buffer id
+  u16 flags; // flags
+}) vnet_virtio_vring_packed_desc_t;
+
+STATIC_ASSERT_SIZEOF (vnet_virtio_vring_packed_desc_t, 16);
+
+typedef CLIB_PACKED (struct {
+  u16 off_wrap;
+  u16 flags;
+}) vnet_virtio_vring_desc_event_t;
+
+#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */
+#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */
+
+#define VIRTIO_NET_HDR_GSO_NONE	 0    /* Not a GSO frame */
+#define VIRTIO_NET_HDR_GSO_TCPV4 1    /* GSO frame, IPv4 TCP (TSO) */
+#define VIRTIO_NET_HDR_GSO_UDP	 3    /* GSO frame, IPv4 UDP (UFO) */
+#define VIRTIO_NET_HDR_GSO_TCPV6 4    /* GSO frame, IPv6 TCP */
+#define VIRTIO_NET_HDR_GSO_ECN	 0x80 /* TCP has ECN set */
+
+typedef CLIB_PACKED (struct {
+  u8 flags;
+  u8 gso_type;
+  u16 hdr_len;	   /* Ethernet + IP + tcp/udp hdrs */
+  u16 gso_size;	   /* Bytes to append to hdr_len per frame */
+  u16 csum_start;  /* Position to start checksumming from */
+  u16 csum_offset; /* Offset after that to place checksum */
+  u16 num_buffers; /* Number of merged rx buffers */
+}) vnet_virtio_net_hdr_v1_t;
+
+typedef CLIB_PACKED (struct {
+  u8 flags;
+  u8 gso_type;
+  u16 hdr_len;
+  u16 gso_size;
+  u16 csum_start;
+  u16 csum_offset;
+}) vnet_virtio_net_hdr_t;
+
+typedef CLIB_PACKED (struct {
+  vnet_virtio_net_hdr_t hdr;
+  u16 num_buffers;
+}) vnet_virtio_net_hdr_mrg_rxbuf_t;
+
+#endif
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt
index fd62217dbf9..7006bd50e67 100644
--- a/src/vnet/CMakeLists.txt
+++ b/src/vnet/CMakeLists.txt
@@ -986,10 +986,6 @@ list(APPEND VNET_SOURCES
   devices/virtio/format.c
   devices/virtio/node.c
   devices/virtio/pci.c
-  devices/virtio/vhost_user.c
-  devices/virtio/vhost_user_input.c
-  devices/virtio/vhost_user_output.c
-  devices/virtio/vhost_user_api.c
   devices/virtio/virtio.c
   devices/virtio/virtio_api.c
   devices/virtio/virtio_pci_legacy.c
@@ -1006,20 +1002,16 @@ list(APPEND VNET_HEADERS
   devices/virtio/virtio_pci_legacy.h
   devices/virtio/virtio_pci_modern.h
   devices/virtio/vhost_std.h
-  devices/virtio/vhost_user.h
   devices/virtio/virtio_types_api.h
 )
 
 list(APPEND VNET_MULTIARCH_SOURCES
-  devices/virtio/vhost_user_input.c
-  devices/virtio/vhost_user_output.c
   devices/virtio/node.c
   devices/af_packet/node.c
   devices/virtio/device.c
 )
 
 list(APPEND VNET_API_FILES
-  devices/virtio/vhost_user.api
   devices/virtio/virtio.api
   devices/virtio/virtio_types.api
 )