virtio: add modern device support

Type: feature

Change-Id: I205f7c146a213d603d9d1e46fcf5195a876608dc
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>

7 files changed, 678 insertions, 47 deletions

diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt
index 168a35ebe5c..fd490d6167e 100644
--- a/src/vnet/CMakeLists.txt
+++ b/src/vnet/CMakeLists.txt
@@ -1098,6 +1098,7 @@ list(APPEND VNET_SOURCES
   devices/virtio/virtio.c
   devices/virtio/virtio_api.c
   devices/virtio/virtio_pci_legacy.c
+  devices/virtio/virtio_pci_modern.c
   devices/virtio/virtio_types_api.c
 )
 
@@ -1105,6 +1106,7 @@ list(APPEND VNET_HEADERS
   devices/virtio/pci.h
   devices/virtio/virtio.h
   devices/virtio/virtio_pci_legacy.h
+  devices/virtio/virtio_pci_modern.h
   devices/virtio/vhost_user.h
   devices/virtio/virtio_types_api.h
 )
diff --git a/src/vnet/devices/virtio/FEATURE.yaml b/src/vnet/devices/virtio/FEATURE.yaml
index c1fc9fd35ce..ac7133bae7f 100644
--- a/src/vnet/devices/virtio/FEATURE.yaml
+++ b/src/vnet/devices/virtio/FEATURE.yaml
@@ -6,6 +6,7 @@ features:
     the host interface.
   - Device mode to emulate vhost-user interface presented to VPP from the
     guest VM.
+  - Support virtio 1.0 in virtio
   - Support multi-queue, GSO, checksum offload, indirect descriptor,
     jumbo frame, and packed ring.
   - Support virtio 1.1 packed ring in vhost
diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c
index 16c7f6d5ed5..494a3f383fa 100644
--- a/src/vnet/devices/virtio/pci.c
+++ b/src/vnet/devices/virtio/pci.c
@@ -600,6 +600,9 @@ virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif,
     | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
     | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
 
+  if (vif->is_modern)
+    supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1);
+
   if (req_features == 0)
     {
       req_features = supported_features;
@@ -650,21 +653,21 @@ virtio_pci_reset_device (vlib_main_t * vm, virtio_if_t * vif)
    * Read the status and verify it
    */
   status = vif->virtio_pci_func->get_status (vm, vif);
-  if (!
-      ((status & VIRTIO_CONFIG_STATUS_ACK)
-       && (status & VIRTIO_CONFIG_STATUS_DRIVER)))
+  if ((status & VIRTIO_CONFIG_STATUS_ACK)
+      && (status & VIRTIO_CONFIG_STATUS_DRIVER))
+    vif->status = status;
+  else
     return -1;
-  vif->status = status;
 
   return 0;
 }
 
 clib_error_t *
-virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif)
+virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif, void **bar)
 {
   clib_error_t *error = 0;
   struct virtio_pci_cap cap;
-  u8 pos, common_cfg = 0, notify_base = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
+  u8 pos, common_cfg = 0, notify = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
   vlib_pci_dev_handle_t h = vif->pci_dev_handle;
 
   if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
@@ -723,18 +726,40 @@ virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif)
       virtio_log_debug (vif,
 			"[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
 			pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
+
+      vif->bar = bar[cap.bar];
+      vif->bar_id = cap.bar;
+
       switch (cap.cfg_type)
 	{
 	case VIRTIO_PCI_CAP_COMMON_CFG:
+	  vif->common_offset = cap.offset;
 	  common_cfg = 1;
 	  break;
 	case VIRTIO_PCI_CAP_NOTIFY_CFG:
-	  notify_base = 1;
+	  if ((error =
+	       vlib_pci_read_write_config (vm, h, VLIB_READ,
+					   pos + sizeof (cap),
+					   &vif->notify_off_multiplier,
+					   sizeof
+					   (vif->notify_off_multiplier))))
+	    {
+	      virtio_log_error (vif, "notify off multiplier is not given");
+	    }
+	  else
+	    {
+	      virtio_log_debug (vif, "notify off multiplier is %u",
+				vif->notify_off_multiplier);
+	      vif->notify_offset = cap.offset;
+	      notify = 1;
+	    }
 	  break;
 	case VIRTIO_PCI_CAP_DEVICE_CFG:
+	  vif->device_offset = cap.offset;
 	  dev_cfg = 1;
 	  break;
 	case VIRTIO_PCI_CAP_ISR_CFG:
+	  vif->isr_offset = cap.offset;
 	  isr = 1;
 	  break;
 	case VIRTIO_PCI_CAP_PCI_CFG:
@@ -746,16 +771,18 @@ virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif)
       pos = cap.cap_next;
     }
 
-  vif->virtio_pci_func = &virtio_pci_legacy_func;
-
-  if (common_cfg == 0 || notify_base == 0 || dev_cfg == 0 || isr == 0)
+  if (common_cfg == 0 || notify == 0 || dev_cfg == 0 || isr == 0)
     {
+      vif->virtio_pci_func = &virtio_pci_legacy_func;
       virtio_log_debug (vif, "legacy virtio pci device found");
       return error;
     }
 
+  vif->is_modern = 1;
+  vif->virtio_pci_func = &virtio_pci_modern_func;
+
   if (!pci_cfg)
-    clib_error_return (error, "modern virtio pci device found");
+    virtio_log_debug (vif, "modern virtio pci device found");
 
   virtio_log_debug (vif, "transitional virtio pci device found");
   return error;
@@ -763,13 +790,13 @@ virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif)
 
 static clib_error_t *
 virtio_pci_device_init (vlib_main_t * vm, virtio_if_t * vif,
-			virtio_pci_create_if_args_t * args)
+			virtio_pci_create_if_args_t * args, void **bar)
 {
   clib_error_t *error = 0;
   vlib_thread_main_t *vtm = vlib_get_thread_main ();
   u8 status = 0;
 
-  if ((error = virtio_pci_read_caps (vm, vif)))
+  if ((error = virtio_pci_read_caps (vm, vif, bar)))
     {
       args->rv = VNET_API_ERROR_UNSUPPORTED;
       virtio_log_error (vif, "Device is not supported");
@@ -1017,6 +1044,21 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args)
       goto error;
     }
 
+  void *bar[6];
+  for (u32 i = 0; i <= 5; i++)
+    {
+
+      if ((error = vlib_pci_map_region (vm, h, i, &bar[i])))
+	{
+	  virtio_log_debug (vif, "no pci map region for bar %u", i);
+	}
+      else
+	{
+	  virtio_log_debug (vif, "pci map region for bar %u at %p", i,
+			    bar[i]);
+	}
+    }
+
   if ((error = vlib_pci_io_region (vm, h, 0)))
     {
       virtio_log_error (vif, "error encountered on pci io region");
@@ -1084,7 +1126,7 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args)
       goto error;
     }
 
-  if ((error = virtio_pci_device_init (vm, vif, args)))
+  if ((error = virtio_pci_device_init (vm, vif, args, bar)))
     {
       virtio_log_error (vif, "error encountered on device init");
       goto error;
diff --git a/src/vnet/devices/virtio/pci.h b/src/vnet/devices/virtio/pci.h
index dfc0711305a..f9a58280aae 100644
--- a/src/vnet/devices/virtio/pci.h
+++ b/src/vnet/devices/virtio/pci.h
@@ -82,7 +82,8 @@ typedef enum
  * at the end of the avail ring. Host should ignore the avail->flags field. */ \
 /* The Host publishes the avail index for which it expects a kick \
  * at the end of the used ring. Guest should ignore the used->flags field. */ \
-  _ (VHOST_USER_F_PROTOCOL_FEATURES, 30)
+  _ (VHOST_USER_F_PROTOCOL_FEATURES, 30)                                      \
+  _ (VIRTIO_F_VERSION_1, 32)                                                  \
 
 #define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 2
 #define VIRTIO_NET_F_MTU 3
@@ -147,8 +148,8 @@ typedef struct
   /* About the whole device. */
   u32 device_feature_select;	/* read-write */
   u32 device_feature;		/* read-only */
-  u32 guest_feature_select;	/* read-write */
-  u32 guest_feature;		/* read-write */
+  u32 driver_feature_select;	/* read-write */
+  u32 driver_feature;		/* read-write */
   u16 msix_config;		/* read-write */
   u16 num_queues;		/* read-only */
   u8 device_status;		/* read-write */
@@ -160,12 +161,9 @@ typedef struct
   u16 queue_msix_vector;	/* read-write */
   u16 queue_enable;		/* read-write */
   u16 queue_notify_off;		/* read-only */
-  u32 queue_desc_lo;		/* read-write */
-  u32 queue_desc_hi;		/* read-write */
-  u32 queue_avail_lo;		/* read-write */
-  u32 queue_avail_hi;		/* read-write */
-  u32 queue_used_lo;		/* read-write */
-  u32 queue_used_hi;		/* read-write */
+  u64 queue_desc;		/* read-write */
+  u64 queue_driver;		/* read-write */
+  u64 queue_device;		/* read-write */
 } virtio_pci_common_cfg_t;
 
 typedef struct
@@ -259,6 +257,7 @@ typedef struct
 } virtio_pci_create_if_args_t;
 
 extern const virtio_pci_func_t virtio_pci_legacy_func;
+extern const virtio_pci_func_t virtio_pci_modern_func;
 
 extern void device_status (vlib_main_t * vm, virtio_if_t * vif);
 void virtio_pci_create_if (vlib_main_t * vm,
diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h
index e1ba71dae2e..7a5dcd83fe3 100644
--- a/src/vnet/devices/virtio/virtio.h
+++ b/src/vnet/devices/virtio/virtio.h
@@ -57,7 +57,7 @@
 /* The Host publishes the avail index for which it expects a kick \
  * at the end of the used ring. Guest should ignore the used->flags field. */ \
   _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \
-  _ (VIRTIO_F_VERSION_1, 32)
+  _ (VIRTIO_F_VERSION_1, 32)  /* v1.0 compliant. */           \
 
 #define foreach_virtio_if_flag		\
   _(0, ADMIN_UP, "admin-up")		\
@@ -130,6 +130,7 @@ typedef union
   u32 as_u32;
 } pci_addr_t;
 
+/* forward declaration */
 typedef struct _virtio_pci_func virtio_pci_func_t;
 
 typedef struct
@@ -161,35 +162,49 @@ typedef struct
 
     CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
   int packet_coalesce;
-  union
-  {
-    u32 id;
-    pci_addr_t pci_addr;
-  };
-  int *vhost_fds;
   u32 dev_instance;
   u32 numa_node;
   u64 remote_features;
 
   /* error */
   clib_error_t *error;
-  u8 support_int_mode;		/* support interrupt mode */
-  u16 max_queue_pairs;
-  u16 msix_table_size;
-  u8 status;
   u8 mac_addr[6];
-  u8 *host_if_name;
-  u8 *net_ns;
-  u8 *host_bridge;
-  u8 host_mac_addr[6];
-  ip4_address_t host_ip4_addr;
-  u8 host_ip4_prefix_len;
-  ip6_address_t host_ip6_addr;
-  u8 host_ip6_prefix_len;
-  u32 host_mtu_size;
-  u32 tap_flags;
-  int ifindex;
-  virtio_vring_t *cxq_vring;
+  union
+  {
+    struct			/* tun/tap interface */
+    {
+      ip6_address_t host_ip6_addr;
+      int *vhost_fds;
+      u8 *host_if_name;
+      u8 *net_ns;
+      u8 *host_bridge;
+      u8 host_mac_addr[6];
+      u32 id;
+      u32 host_mtu_size;
+      u32 tap_flags;
+      int ifindex;
+      ip4_address_t host_ip4_addr;
+      u8 host_ip4_prefix_len;
+      u8 host_ip6_prefix_len;
+    };
+    struct			/* native virtio */
+    {
+      void *bar;
+      virtio_vring_t *cxq_vring;
+      pci_addr_t pci_addr;
+      u32 bar_id;
+      u32 notify_off_multiplier;
+      u32 is_modern;
+      u16 common_offset;
+      u16 notify_offset;
+      u16 device_offset;
+      u16 isr_offset;
+      u16 max_queue_pairs;
+      u16 msix_table_size;
+      u8 support_int_mode;	/* support interrupt mode */
+      u8 status;
+    };
+  };
   const virtio_pci_func_t *virtio_pci_func;
 } virtio_if_t;
 
@@ -221,6 +236,8 @@ extern void virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr,
 extern void virtio_set_packet_coalesce (virtio_if_t * vif);
 extern void virtio_pci_legacy_notify_queue (vlib_main_t * vm,
 					    virtio_if_t * vif, u16 queue_id);
+extern void virtio_pci_modern_notify_queue (vlib_main_t * vm,
+					    virtio_if_t * vif, u16 queue_id);
 format_function_t format_virtio_device_name;
 format_function_t format_virtio_log_name;
 
@@ -228,7 +245,12 @@ static_always_inline void
 virtio_kick (vlib_main_t * vm, virtio_vring_t * vring, virtio_if_t * vif)
 {
   if (vif->type == VIRTIO_IF_TYPE_PCI)
-    virtio_pci_legacy_notify_queue (vm, vif, vring->queue_id);
+    {
+      if (vif->is_modern)
+	virtio_pci_modern_notify_queue (vm, vif, vring->queue_id);
+      else
+	virtio_pci_legacy_notify_queue (vm, vif, vring->queue_id);
+    }
   else
     {
       u64 x = 1;
diff --git a/src/vnet/devices/virtio/virtio_pci_modern.c b/src/vnet/devices/virtio/virtio_pci_modern.c
new file mode 100644
index 00000000000..24e8a4fe404
--- /dev/null
+++ b/src/vnet/devices/virtio/virtio_pci_modern.c
@@ -0,0 +1,440 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <fcntl.h>
+#include <sys/ioctl.h>
+
+#include <vppinfra/types.h>
+#include <vlib/vlib.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/ip/ip4_packet.h>
+#include <vnet/ip/ip6_packet.h>
+#include <vnet/devices/virtio/virtio.h>
+#include <vnet/devices/virtio/virtio_pci_modern.h>
+#include <vnet/devices/virtio/pci.h>
+
+
+static u64
+virtio_pci_modern_get_device_features (vlib_main_t * vm, virtio_if_t * vif)
+{
+  u64 features_lo, features_hi;
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DEVICE_FEATURE_SELECT_OFFSET (vif),
+			    VIRTIO_FEATURE_SELECT_LO);
+  features_lo =
+    virtio_pci_reg_read_u32 (vif, VIRTIO_DEVICE_FEATURE_OFFSET (vif));
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DEVICE_FEATURE_SELECT_OFFSET (vif),
+			    VIRTIO_FEATURE_SELECT_HI);
+  features_hi =
+    virtio_pci_reg_read_u32 (vif, VIRTIO_DEVICE_FEATURE_OFFSET (vif));
+  u64 features = ((features_hi << 32) | features_lo);
+  return features;
+}
+
+static u64
+virtio_pci_modern_get_driver_features (vlib_main_t * vm, virtio_if_t * vif)
+{
+  u64 features_lo, features_hi;
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DRIVER_FEATURE_SELECT_OFFSET (vif),
+			    VIRTIO_FEATURE_SELECT_LO);
+  features_lo =
+    virtio_pci_reg_read_u32 (vif, VIRTIO_DRIVER_FEATURE_OFFSET (vif));
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DRIVER_FEATURE_SELECT_OFFSET (vif),
+			    VIRTIO_FEATURE_SELECT_HI);
+  features_hi =
+    virtio_pci_reg_read_u32 (vif, VIRTIO_DRIVER_FEATURE_OFFSET (vif));
+
+  vif->features = ((features_hi << 32) | features_lo);
+  return vif->features;
+}
+
+static void
+virtio_pci_modern_set_driver_features (vlib_main_t * vm, virtio_if_t * vif,
+				       u64 features)
+{
+  u32 features_lo = (u32) features, features_hi = (u32) (features >> 32);
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DRIVER_FEATURE_SELECT_OFFSET (vif),
+			    VIRTIO_FEATURE_SELECT_LO);
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DRIVER_FEATURE_OFFSET (vif),
+			    features_lo);
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DRIVER_FEATURE_SELECT_OFFSET (vif),
+			    VIRTIO_FEATURE_SELECT_HI);
+  virtio_pci_reg_write_u32 (vif, VIRTIO_DRIVER_FEATURE_OFFSET (vif),
+			    features_hi);
+
+  if (features != virtio_pci_modern_get_driver_features (vm, vif))
+    {
+      clib_warning ("modern set guest features failed!");
+    }
+}
+
+static u16
+virtio_pci_modern_get_msix_config (virtio_if_t * vif)
+{
+  u16 msix_config;
+  msix_config =
+    virtio_pci_reg_read_u16 (vif, VIRTIO_MSIX_CONFIG_VECTOR_OFFSET (vif));
+  return msix_config;
+}
+
+static u16
+virtio_pci_modern_set_msix_config (vlib_main_t * vm, virtio_if_t * vif,
+				   u16 msix_config)
+{
+  virtio_pci_reg_write_u16 (vif, VIRTIO_MSIX_CONFIG_VECTOR_OFFSET (vif),
+			    msix_config);
+  return virtio_pci_modern_get_msix_config (vif);
+}
+
+static u16
+virtio_pci_modern_get_num_queues (virtio_if_t * vif)
+{
+  u16 num_queues = 0;
+  num_queues = virtio_pci_reg_read_u16 (vif, VIRTIO_NUM_QUEUES_OFFSET (vif));
+  return num_queues;
+}
+
+static u8
+virtio_pci_modern_get_status (vlib_main_t * vm, virtio_if_t * vif)
+{
+  u8 status = 0;
+  status = virtio_pci_reg_read_u8 (vif, VIRTIO_DEVICE_STATUS_OFFSET (vif));
+  return status;
+}
+
+static void
+virtio_pci_modern_set_status (vlib_main_t * vm, virtio_if_t * vif, u8 status)
+{
+  if (status != VIRTIO_CONFIG_STATUS_RESET)
+    status |= virtio_pci_modern_get_status (vm, vif);
+  virtio_pci_reg_write_u8 (vif, VIRTIO_DEVICE_STATUS_OFFSET (vif), status);
+}
+
+static u8
+virtio_pci_modern_reset (vlib_main_t * vm, virtio_if_t * vif)
+{
+  virtio_pci_modern_set_status (vm, vif, VIRTIO_CONFIG_STATUS_RESET);
+  return virtio_pci_modern_get_status (vm, vif);
+}
+
+static u8
+virtio_pci_modern_get_config_generation (virtio_if_t * vif)
+{
+  u8 config_generation = 0;
+  config_generation =
+    virtio_pci_reg_read_u8 (vif, VIRTIO_CONFIG_GENERATION_OFFSET (vif));
+  return config_generation;
+}
+
+static void
+virtio_pci_modern_set_queue_select (virtio_if_t * vif, u16 queue_select)
+{
+  virtio_pci_reg_write_u16 (vif, VIRTIO_QUEUE_SELECT_OFFSET (vif),
+			    queue_select);
+}
+
+static u16
+virtio_pci_modern_get_queue_size (vlib_main_t * vm, virtio_if_t * vif,
+				  u16 queue_id)
+{
+  u16 queue_size = 0;
+  virtio_pci_modern_set_queue_select (vif, queue_id);
+  queue_size = virtio_pci_reg_read_u16 (vif, VIRTIO_QUEUE_SIZE_OFFSET (vif));
+  return queue_size;
+}
+
+static void
+virtio_pci_modern_set_queue_size (vlib_main_t * vm, virtio_if_t * vif,
+				  u16 queue_id, u16 queue_size)
+{
+  if (!is_pow2 (queue_size))
+    {
+      return;
+    }
+
+  if (virtio_pci_modern_get_queue_size (vm, vif, queue_id) > queue_size)
+    virtio_pci_reg_write_u16 (vif, VIRTIO_QUEUE_SIZE_OFFSET (vif),
+			      queue_size);
+}
+
+static u16
+virtio_pci_modern_get_queue_msix_vector (virtio_if_t * vif)
+{
+  u16 queue_msix_vector = 0;
+  queue_msix_vector =
+    virtio_pci_reg_read_u16 (vif, VIRTIO_QUEUE_MSIX_VECTOR_OFFSET (vif));
+  return queue_msix_vector;
+}
+
+static u16
+virtio_pci_modern_set_queue_msix_vector (vlib_main_t * vm, virtio_if_t * vif,
+					 u16 queue_msix_vector, u16 queue_id)
+{
+  virtio_pci_modern_set_queue_select (vif, queue_id);
+  virtio_pci_reg_write_u16 (vif, VIRTIO_QUEUE_MSIX_VECTOR_OFFSET (vif),
+			    queue_msix_vector);
+  return virtio_pci_modern_get_queue_msix_vector (vif);
+}
+
+static u16
+virtio_pci_modern_get_queue_enable (virtio_if_t * vif, u16 queue_id)
+{
+  u16 queue_enable = 0;
+  virtio_pci_modern_set_queue_select (vif, queue_id);
+  queue_enable =
+    virtio_pci_reg_read_u16 (vif, VIRTIO_QUEUE_ENABLE_OFFSET (vif));
+  return queue_enable;
+}
+
+static void
+virtio_pci_modern_set_queue_enable (virtio_if_t * vif, u16 queue_id,
+				    u16 queue_enable)
+{
+  virtio_pci_modern_set_queue_select (vif, queue_id);
+  virtio_pci_reg_write_u16 (vif, VIRTIO_QUEUE_ENABLE_OFFSET (vif),
+			    queue_enable);
+}
+
+static u16
+virtio_pci_modern_get_queue_notify_off (virtio_if_t * vif, u16 queue_id)
+{
+  u16 queue_notify_off = 0;
+  virtio_pci_modern_set_queue_select (vif, queue_id);
+  queue_notify_off =
+    virtio_pci_reg_read_u16 (vif, VIRTIO_QUEUE_NOTIFY_OFF_OFFSET (vif));
+  return queue_notify_off;
+}
+
+static u64
+virtio_pci_modern_get_queue_desc (virtio_if_t * vif)
+{
+  u64 queue_desc = 0;
+  queue_desc = virtio_pci_reg_read_u64 (vif, VIRTIO_QUEUE_DESC_OFFSET (vif));
+  return queue_desc;
+}
+
+static void
+virtio_pci_modern_set_queue_desc (virtio_if_t * vif, u64 queue_desc)
+{
+  virtio_pci_reg_write_u64 (vif, VIRTIO_QUEUE_DESC_OFFSET (vif), queue_desc);
+}
+
+static u64
+virtio_pci_modern_get_queue_driver (virtio_if_t * vif)
+{
+  u64 queue_driver = 0;
+  queue_driver =
+    virtio_pci_reg_read_u64 (vif, VIRTIO_QUEUE_DRIVER_OFFSET (vif));
+  return queue_driver;
+}
+
+static void
+virtio_pci_modern_set_queue_driver (virtio_if_t * vif, u64 queue_driver)
+{
+  virtio_pci_reg_write_u64 (vif, VIRTIO_QUEUE_DRIVER_OFFSET (vif),
+			    queue_driver);
+}
+
+static u64
+virtio_pci_modern_get_queue_device (virtio_if_t * vif)
+{
+  u64 queue_device = 0;
+  queue_device =
+    virtio_pci_reg_read_u64 (vif, VIRTIO_QUEUE_DEVICE_OFFSET (vif));
+  return queue_device;
+}
+
+static void
+virtio_pci_modern_set_queue_device (virtio_if_t * vif, u64 queue_device)
+{
+  virtio_pci_reg_write_u64 (vif, VIRTIO_QUEUE_DEVICE_OFFSET (vif),
+			    queue_device);
+}
+
+static u8
+virtio_pci_modern_setup_queue (vlib_main_t * vm, virtio_if_t * vif,
+			       u16 queue_id, void *p)
+{
+  struct vring vr;
+  u16 queue_size = 0;
+
+  virtio_pci_modern_set_queue_select (vif, queue_id);
+  queue_size = virtio_pci_modern_get_queue_size (vm, vif, queue_id);
+  vring_init (&vr, queue_size, p, VIRTIO_PCI_VRING_ALIGN);
+
+  u64 desc = vlib_physmem_get_pa (vm, vr.desc);
+  virtio_pci_modern_set_queue_desc (vif, desc);
+  if (desc != virtio_pci_modern_get_queue_desc (vif))
+    return 1;
+
+  u64 avail = vlib_physmem_get_pa (vm, vr.avail);
+  virtio_pci_modern_set_queue_driver (vif, avail);
+  if (avail != virtio_pci_modern_get_queue_driver (vif))
+    return 1;
+
+  u64 used = vlib_physmem_get_pa (vm, vr.used);
+  virtio_pci_modern_set_queue_device (vif, used);
+  if (used != virtio_pci_modern_get_queue_device (vif))
+    return 1;
+
+  virtio_pci_modern_set_queue_enable (vif, queue_id, 1);
+
+  if (virtio_pci_modern_get_queue_enable (vif, queue_id))
+    return 0;
+
+  return 1;
+}
+
+static void
+virtio_pci_modern_del_queue (vlib_main_t * vm, virtio_if_t * vif,
+			     u16 queue_id)
+{
+  virtio_pci_modern_set_queue_select (vif, queue_id);
+  virtio_pci_modern_set_queue_enable (vif, queue_id, 0);
+  virtio_pci_modern_set_queue_desc (vif, 0);
+  virtio_pci_modern_set_queue_driver (vif, 0);
+  virtio_pci_modern_set_queue_device (vif, 0);
+}
+
+static void
+virtio_pci_modern_get_device_mac (vlib_main_t * vm, virtio_if_t * vif)
+{
+  *((u32 *) vif->mac_addr) =
+    virtio_pci_reg_read_u32 (vif, VIRTIO_MAC_OFFSET (vif));
+  *((u16 *) (vif->mac_addr + 4)) =
+    virtio_pci_reg_read_u16 (vif, VIRTIO_MAC_OFFSET (vif) + 4);
+}
+
+static void
+virtio_pci_modern_set_device_mac (vlib_main_t * vm, virtio_if_t * vif)
+{
+  virtio_pci_reg_write_u32 (vif, VIRTIO_MAC_OFFSET (vif),
+			    *((u32 *) vif->mac_addr));
+  virtio_pci_reg_write_u16 (vif, VIRTIO_MAC_OFFSET (vif) + 4,
+			    *((u16 *) (vif->mac_addr + 4)));
+}
+
+static u16
+virtio_pci_modern_get_device_status (vlib_main_t * vm, virtio_if_t * vif)
+{
+  u16 status = 0;
+  status = virtio_pci_reg_read_u16 (vif, VIRTIO_STATUS_OFFSET (vif));
+  return status;
+}
+
+static u16
+virtio_pci_modern_get_max_virtqueue_pairs (vlib_main_t * vm,
+					   virtio_if_t * vif)
+{
+  u16 max_virtqueue_pairs = 0;
+  max_virtqueue_pairs =
+    virtio_pci_reg_read_u16 (vif, VIRTIO_MAX_VIRTQUEUE_PAIRS_OFFSET (vif));
+  u16 supported_queues = virtio_pci_modern_get_num_queues (vif);
+  virtio_log_debug (vif, "max-virtqueue-pairs %u, supported-queues %u",
+		    max_virtqueue_pairs, supported_queues);
+  return max_virtqueue_pairs;
+}
+
+static u16
+virtio_pci_modern_get_device_mtu (vlib_main_t * vm, virtio_if_t * vif)
+{
+  u16 mtu = 0;
+  mtu = virtio_pci_reg_read_u16 (vif, VIRTIO_MTU_OFFSET (vif));
+  return mtu;
+}
+
+static void
+virtio_pci_modern_read_config (vlib_main_t * vm, virtio_if_t * vif, void *dst,
+			       int len, u32 addr)
+{
+  u8 config_count;
+  do
+    {
+      config_count = virtio_pci_modern_get_config_generation (vif);
+      virtio_pci_modern_get_device_mac (vm, vif);
+      u16 status = virtio_pci_modern_get_device_status (vm, vif);
+      u16 max_queue_pairs =
+	virtio_pci_modern_get_max_virtqueue_pairs (vm, vif);
+      u16 mtu = virtio_pci_modern_get_device_mtu (vm, vif);
+      virtio_log_debug (vif, "status %u, max_queue_pairs %u, mtu %u", status,
+			max_queue_pairs, mtu);
+    }
+  while (config_count != virtio_pci_modern_get_config_generation (vif));
+}
+
+static void
+virtio_pci_modern_write_config (vlib_main_t * vm, virtio_if_t * vif,
+				void *src, int len, u32 addr)
+{
+  // do nothing
+}
+
+static u8
+virtio_pci_modern_get_isr (vlib_main_t * vm, virtio_if_t * vif)
+{
+  return virtio_pci_reg_read_u8 (vif, VIRTIO_ISR_OFFSET (vif));
+}
+
+inline void
+virtio_pci_modern_notify_queue (vlib_main_t * vm, virtio_if_t * vif,
+				u16 queue_id)
+{
+  u16 queue_notify_off =
+    virtio_pci_modern_get_queue_notify_off (vif, queue_id);
+  virtio_pci_reg_write_u16 (vif,
+			    VIRTIO_NOTIFICATION_OFFSET (vif) +
+			    vif->notify_off_multiplier * queue_notify_off,
+			    queue_id);
+}
+
+static void
+virtio_pci_modern_device_debug_config_space (vlib_main_t * vm,
+					     virtio_if_t * vif)
+{
+  // do nothing for now
+}
+
+const virtio_pci_func_t virtio_pci_modern_func = {
+  .read_config = virtio_pci_modern_read_config,
+  .write_config = virtio_pci_modern_write_config,
+  .get_device_features = virtio_pci_modern_get_device_features,
+  .get_driver_features = virtio_pci_modern_get_driver_features,
+  .set_driver_features = virtio_pci_modern_set_driver_features,
+  .get_status = virtio_pci_modern_get_status,
+  .set_status = virtio_pci_modern_set_status,
+  .device_reset = virtio_pci_modern_reset,
+  .get_isr = virtio_pci_modern_get_isr,
+  .get_queue_size = virtio_pci_modern_get_queue_size,
+  .set_queue_size = virtio_pci_modern_set_queue_size,
+  .setup_queue = virtio_pci_modern_setup_queue,
+  .del_queue = virtio_pci_modern_del_queue,
+  .notify_queue = virtio_pci_modern_notify_queue,
+  .set_config_irq = virtio_pci_modern_set_msix_config,
+  .set_queue_irq = virtio_pci_modern_set_queue_msix_vector,
+  .get_mac = virtio_pci_modern_get_device_mac,
+  .set_mac = virtio_pci_modern_set_device_mac,
+  .get_device_status = virtio_pci_modern_get_device_status,
+  .get_max_queue_pairs = virtio_pci_modern_get_max_virtqueue_pairs,
+  .get_mtu = virtio_pci_modern_get_device_mtu,
+  .device_debug_config_space = virtio_pci_modern_device_debug_config_space,
+};
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/devices/virtio/virtio_pci_modern.h b/src/vnet/devices/virtio/virtio_pci_modern.h
new file mode 100644
index 00000000000..f6a8713984f
--- /dev/null
+++ b/src/vnet/devices/virtio/virtio_pci_modern.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/devices/virtio/virtio.h>
+#include <vnet/devices/virtio/pci.h>
+
+/* common configuration */
+#define VIRTIO_FEATURE_SELECT_HI 1
+#define VIRTIO_FEATURE_SELECT_LO 0
+
+#define VIRTIO_DEVICE_FEATURE_SELECT_OFFSET(v)                    \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					 device_feature_select))
+#define VIRTIO_DEVICE_FEATURE_OFFSET(v)                           \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					        device_feature))
+#define VIRTIO_DRIVER_FEATURE_SELECT_OFFSET(v)                    \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					 driver_feature_select))
+#define VIRTIO_DRIVER_FEATURE_OFFSET(v)                           \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					        driver_feature))
+#define VIRTIO_MSIX_CONFIG_VECTOR_OFFSET(v)                       \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					           msix_config))
+#define VIRTIO_NUM_QUEUES_OFFSET(v)                               \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					            num_queues))
+#define VIRTIO_DEVICE_STATUS_OFFSET(v)                            \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					         device_status))
+#define VIRTIO_CONFIG_GENERATION_OFFSET(v)                        \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					     config_generation))
+#define VIRTIO_QUEUE_SELECT_OFFSET(v)                             \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					          queue_select))
+#define VIRTIO_QUEUE_SIZE_OFFSET(v)                               \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					            queue_size))
+#define VIRTIO_QUEUE_MSIX_VECTOR_OFFSET(v)                        \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					     queue_msix_vector))
+#define VIRTIO_QUEUE_ENABLE_OFFSET(v)                             \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					          queue_enable))
+#define VIRTIO_QUEUE_NOTIFY_OFF_OFFSET(v)                         \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					      queue_notify_off))
+#define VIRTIO_QUEUE_DESC_OFFSET(v)                               \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					            queue_desc))
+#define VIRTIO_QUEUE_DRIVER_OFFSET(v)                             \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					          queue_driver))
+#define VIRTIO_QUEUE_DEVICE_OFFSET(v)                             \
+   (v->common_offset + STRUCT_OFFSET_OF (virtio_pci_common_cfg_t, \
+					          queue_device))
+/* device configuration */
+#define VIRTIO_MAC_OFFSET(v)                                      \
+   (v->device_offset + STRUCT_OFFSET_OF (virtio_net_config_t,     \
+					                   mac))
+#define VIRTIO_STATUS_OFFSET(v)                                   \
+   (v->device_offset + STRUCT_OFFSET_OF (virtio_net_config_t,     \
+					                status))
+#define VIRTIO_MAX_VIRTQUEUE_PAIRS_OFFSET(v)                      \
+   (v->device_offset + STRUCT_OFFSET_OF (virtio_net_config_t,     \
+					   max_virtqueue_pairs))
+#define VIRTIO_MTU_OFFSET(v)                                      \
+   (v->device_offset + STRUCT_OFFSET_OF (virtio_net_config_t,     \
+					                   mtu))
+/* interrupt service routine */
+#define VIRTIO_ISR_OFFSET(v) (v->isr_offset)
+/* notification */
+#define VIRTIO_NOTIFICATION_OFFSET(v) (v->notify_offset)
+
+#define _(t)                                                         \
+static_always_inline t                                               \
+virtio_pci_reg_read_##t (virtio_if_t * vif, u32 offset)              \
+{                                                                    \
+  t val;                                                             \
+  val = *(volatile t *) (vif->bar + offset);		             \
+  return val;                                                        \
+}
+
+_(u64);
+_(u32);
+_(u16);
+_(u8);
+
+#undef _
+
+#define _(t)                                                         \
+static_always_inline void                                            \
+virtio_pci_reg_write_##t (virtio_if_t * vif, u32 offset, t val)      \
+{                                                                    \
+  *(volatile t *) ((u8 *) vif->bar + offset) = val;	             \
+}
+
+_(u64);
+_(u32);
+_(u16);
+_(u8);
+
+#undef _
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */