virtio: RSS support

Add RSS support to make use of multiple queues.

With 4 RX queues and RSS enabled

ping from host	to guest vm ip	queue use
192.168.2.3	192.168.2.1	2
''		192.168.2.10	0
''		192.168.2.5	1
''		192.168.2.105	3

With 4 RX queues and RSS disabled, queue 0 is always used for all of the above cases

Type: improvement

Change-Id: I3ca78fd83fce26cbe8f23fee0a9034cb572bacb7
Signed-off-by: Steven Luong <sluong@cisco.com>

9 files changed, 190 insertions, 56 deletions

diff --git a/src/vnet/devices/virtio/FEATURE.yaml b/src/vnet/devices/virtio/FEATURE.yaml
index 446a45b61a3..870023861de 100644
--- a/src/vnet/devices/virtio/FEATURE.yaml
+++ b/src/vnet/devices/virtio/FEATURE.yaml
@@ -13,6 +13,7 @@ features:
   - Support virtio 1.1 packed ring in vhost
   - Support for tx queue size configuration (tested on host kernel 5.15
     and qemu version 6.2.0)
+  - Support RSS (tested on ubuntu 23.10)
 description: "Virtio implementation"
 missing:
   - API dump filtering by sw_if_index
diff --git a/src/vnet/devices/virtio/cli.c b/src/vnet/devices/virtio/cli.c
index c1b6c8be065..c4364600722 100644
--- a/src/vnet/devices/virtio/cli.c
+++ b/src/vnet/devices/virtio/cli.c
@@ -62,6 +62,8 @@ virtio_pci_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	args.bind = VIRTIO_BIND_FORCE;
       else if (unformat (line_input, "bind"))
 	args.bind = VIRTIO_BIND_DEFAULT;
+      else if (unformat (line_input, "rss-enabled"))
+	args.rss_enabled = 1;
       else
 	return clib_error_return (0, "unknown input `%U'",
 				  format_unformat_error, input);
@@ -77,7 +79,7 @@ VLIB_CLI_COMMAND (virtio_pci_create_command, static) = {
   .path = "create interface virtio",
   .short_help = "create interface virtio <pci-address> "
 		"[feature-mask <hex-mask>] [tx-queue-size <size>] "
-		"[gso-enabled] [csum-enabled] "
+		"[gso-enabled] [csum-enabled] [rss-enabled] "
 		"[buffering [size <buffering-szie>]] [packed] [bind [force]]",
   .function = virtio_pci_create_command_fn,
 };
diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c
index 6234f64fcfb..140cdb94153 100644
--- a/src/vnet/devices/virtio/pci.c
+++ b/src/vnet/devices/virtio/pci.c
@@ -37,6 +37,13 @@
 
 #define PCI_MSIX_ENABLE 0x8000
 
+static const u8 virtio_rss_key[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {
+  0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
+  0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
+  0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
+  0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
+};
+
 static pci_device_id_t virtio_pci_device_ids[] = {
   {
    .vendor_id = PCI_VENDOR_ID_VIRTIO,
@@ -584,6 +591,35 @@ virtio_pci_enable_multiqueue (vlib_main_t * vm, virtio_if_t * vif,
   return status;
 }
 
+static int
+virtio_pci_enable_multiqueue_rss (vlib_main_t *vm, virtio_if_t *vif,
+				  u16 num_queues)
+{
+  virtio_ctrl_msg_t mq_hdr;
+  virtio_net_rss_config *rss = (virtio_net_rss_config *) mq_hdr.data;
+  virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
+
+  STATIC_ASSERT (sizeof (*rss) <= sizeof (mq_hdr.data),
+		 "virtio_net_rss_config size too big");
+  mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
+  mq_hdr.ctrl.cmd = VIRTIO_NET_CTRL_MQ_RSS_CONFIG;
+  mq_hdr.status = VIRTIO_NET_ERR;
+
+  rss->hash_types = VIRTIO_NET_HASH_TYPE_SUPPORTED;
+  rss->indirection_table_mask = VIRTIO_NET_RSS_MAX_TABLE_LEN - 1;
+  rss->unclassified_queue = 0;
+  for (int i = 0; i < VIRTIO_NET_RSS_MAX_TABLE_LEN; i++)
+    rss->indirection_table[i] = i % num_queues;
+  rss->max_tx_vq = num_queues;
+  rss->hash_key_length = VIRTIO_NET_RSS_MAX_KEY_SIZE;
+  clib_memcpy (rss->hash_key_data, virtio_rss_key,
+	       VIRTIO_NET_RSS_MAX_KEY_SIZE);
+
+  status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (*rss));
+  virtio_log_debug (vif, "multi-queue with rss enable %u queues", num_queues);
+  return status;
+}
+
 static u8
 virtio_pci_queue_size_valid (u16 qsz)
 {
@@ -933,6 +969,9 @@ virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif,
     | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
     | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
 
+  if (vif->rss_enabled)
+    supported_features |= VIRTIO_FEATURE (VIRTIO_NET_F_RSS);
+
   if (vif->is_modern)
     supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1);
 
@@ -1375,6 +1414,7 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args)
   vif->dev_instance = vif - vim->interfaces;
   vif->per_interface_next_index = ~0;
   vif->pci_addr.as_u32 = args->addr;
+  vif->rss_enabled = args->rss_enabled;
   if (args->virtio_flags & VIRTIO_FLAG_PACKED)
     vif->is_packed = 1;
 
@@ -1536,8 +1576,16 @@ virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args)
   if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
       (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)))
     {
-      if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs))
-	virtio_log_warning (vif, "multiqueue is not set");
+      if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_RSS))
+	{
+	  if (virtio_pci_enable_multiqueue_rss (vm, vif, vif->max_queue_pairs))
+	    virtio_log_warning (vif, "multiqueue with rss is not set");
+	}
+      else
+	{
+	  if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs))
+	    virtio_log_warning (vif, "multiqueue is not set");
+	}
     }
   return;
 
diff --git a/src/vnet/devices/virtio/pci.h b/src/vnet/devices/virtio/pci.h
index 5eb80f823be..745ad6fce87 100644
--- a/src/vnet/devices/virtio/pci.h
+++ b/src/vnet/devices/virtio/pci.h
@@ -60,6 +60,9 @@ typedef enum
 /* If multiqueue is provided by host, then we support it. */
 #define VIRTIO_NET_CTRL_MQ   4
 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET        0
+#define VIRTIO_NET_CTRL_MQ_RSS_CONFIG	       1
+#define VIRTIO_NET_CTRL_MQ_HASH_CONFIG	       2
+
 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN        1
 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX        0x8000
 
@@ -139,14 +142,77 @@ typedef struct
   u64 queue_device;		/* read-write */
 } virtio_pci_common_cfg_t;
 
+#define foreach_virtio_net_hash_report_type                                   \
+  _ (NONE, 0)                                                                 \
+  _ (IPV4, 1)                                                                 \
+  _ (TCPV4, 2)                                                                \
+  _ (UDPV4, 3)                                                                \
+  _ (IPV6, 4)                                                                 \
+  _ (TCPV6, 5)                                                                \
+  _ (UDPV6, 6)                                                                \
+  _ (IPV6_EX, 7)                                                              \
+  _ (TCPV6_EX, 8)                                                             \
+  _ (UDPV6_EX, 9)
+
+typedef enum
+{
+#define _(n, i) VIRTIO_NET_HASH_REPORT_##n = i,
+  foreach_virtio_net_hash_report_type
+#undef _
+} virtio_net_hash_report_type_t;
+
 typedef struct
 {
   u8 mac[6];
   u16 status;
   u16 max_virtqueue_pairs;
   u16 mtu;
+  u32 speed;
+  u8 duplex;
+  u8 rss_max_key_size;
+  u16 rss_max_indirection_table_length;
+  u32 supported_hash_types;
 } virtio_net_config_t;
 
+#define VIRTIO_NET_RSS_MAX_TABLE_LEN 128
+#define VIRTIO_NET_RSS_MAX_KEY_SIZE  40
+
+#define foreach_virtio_net_hash_type                                          \
+  _ (IPV4, 0)                                                                 \
+  _ (TCPV4, 1)                                                                \
+  _ (UDPV4, 2)                                                                \
+  _ (IPV6, 3)                                                                 \
+  _ (TCPV6, 4)                                                                \
+  _ (UDPV6, 5)                                                                \
+  _ (IPV6_EX, 6)                                                              \
+  _ (TCPV6_EX, 7)                                                             \
+  _ (UDPV6_EX, 8)
+
+typedef enum
+{
+#define _(n, i) VIRTIO_NET_HASH_TYPE_##n = (1 << i),
+  foreach_virtio_net_hash_type
+#undef _
+} virtio_net_hash_type_t;
+
+#define VIRTIO_NET_HASH_TYPE_SUPPORTED                                        \
+  (VIRTIO_NET_HASH_TYPE_IPV4 | VIRTIO_NET_HASH_TYPE_TCPV4 |                   \
+   VIRTIO_NET_HASH_TYPE_UDPV4 | VIRTIO_NET_HASH_TYPE_IPV6 |                   \
+   VIRTIO_NET_HASH_TYPE_TCPV6 | VIRTIO_NET_HASH_TYPE_UDPV6 |                  \
+   VIRTIO_NET_HASH_TYPE_IPV6_EX | VIRTIO_NET_HASH_TYPE_TCPV6_EX |             \
+   VIRTIO_NET_HASH_TYPE_UDPV6_EX)
+
+typedef struct
+{
+  u32 hash_types;
+  u16 indirection_table_mask;
+  u16 unclassified_queue;
+  u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
+  u16 max_tx_vq;
+  u8 hash_key_length;
+  u8 hash_key_data[VIRTIO_NET_RSS_MAX_KEY_SIZE];
+} virtio_net_rss_config;
+
 /*
  * Control virtqueue data structures
  *
@@ -210,13 +276,14 @@ typedef struct _virtio_pci_func
   void (*device_debug_config_space) (vlib_main_t * vm, virtio_if_t * vif);
 } virtio_pci_func_t;
 
-#define foreach_virtio_flags  \
-  _ (GSO, 0)                  \
-  _ (CSUM_OFFLOAD, 1)         \
-  _ (GRO_COALESCE, 2)         \
-  _ (PACKED, 3)               \
-  _ (IN_ORDER, 4)	      \
-  _ (BUFFERING, 5)
+#define foreach_virtio_flags                                                  \
+  _ (GSO, 0)                                                                  \
+  _ (CSUM_OFFLOAD, 1)                                                         \
+  _ (GRO_COALESCE, 2)                                                         \
+  _ (PACKED, 3)                                                               \
+  _ (IN_ORDER, 4)                                                             \
+  _ (BUFFERING, 5)                                                            \
+  _ (RSS, 6)
 
 typedef enum
 {
@@ -243,6 +310,7 @@ typedef struct
   u64 features;
   u8 gso_enabled;
   u8 checksum_offload_enabled;
+  u8 rss_enabled;
   u32 tx_queue_size;
   virtio_bind_t bind;
   u32 buffering_size;
diff --git a/src/vnet/devices/virtio/virtio.api b/src/vnet/devices/virtio/virtio.api
index a11492ec258..14a58491282 100644
--- a/src/vnet/devices/virtio/virtio.api
+++ b/src/vnet/devices/virtio/virtio.api
@@ -63,6 +63,7 @@ enumflag virtio_flags {
         VIRTIO_API_FLAG_PACKED = 8, /* enable packed ring support, provided it is available from backend */
         VIRTIO_API_FLAG_IN_ORDER = 16, /* enable in order support, provided it is available from backend */
         VIRTIO_API_FLAG_BUFFERING = 32 [backwards_compatible], /* enable buffering to handle backend jitter/delays */
+        VIRTIO_API_FLAG_RSS = 64 [backwards_compatible], /* enable rss support */
 };
 
 /** \brief Initialize a new virtio pci interface with the given parameters
diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c
index d2302fa1dc4..840936a43ff 100644
--- a/src/vnet/devices/virtio/virtio.c
+++ b/src/vnet/devices/virtio/virtio.c
@@ -404,6 +404,7 @@ virtio_show (vlib_main_t *vm, u32 *hw_if_indices, u8 show_descr,
       vlib_cli_output (vm, "  csum-enabled %d", vif->csum_offload_enabled);
       vlib_cli_output (vm, "  packet-coalesce %d", vif->packet_coalesce);
       vlib_cli_output (vm, "  packet-buffering %d", vif->packet_buffering);
+      vlib_cli_output (vm, "  rss-enabled %d", vif->rss_enabled);
       if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI))
 	vlib_cli_output (vm, "  Mac Address: %U", format_ethernet_address,
 			 vif->mac_addr);
diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h
index 431b1d25c26..a8e258884a4 100644
--- a/src/vnet/devices/virtio/virtio.h
+++ b/src/vnet/devices/virtio/virtio.h
@@ -140,6 +140,7 @@ typedef struct
   vnet_virtio_vring_t *txq_vrings;
   int gso_enabled;
   int csum_offload_enabled;
+  int rss_enabled;
   union
   {
     int *tap_fds;
diff --git a/src/vnet/devices/virtio/virtio_api.c b/src/vnet/devices/virtio/virtio_api.c
index 3197a2fab6d..2e25efff29e 100644
--- a/src/vnet/devices/virtio/virtio_api.c
+++ b/src/vnet/devices/virtio/virtio_api.c
@@ -111,18 +111,18 @@ vl_api_virtio_pci_create_v2_t_handler (vl_api_virtio_pci_create_v2_t * mp)
   STATIC_ASSERT (((int) VIRTIO_API_FLAG_BUFFERING ==
 		  (int) VIRTIO_FLAG_BUFFERING),
 		 "virtio buffering api flag mismatch");
+  STATIC_ASSERT (((int) VIRTIO_API_FLAG_RSS == (int) VIRTIO_FLAG_RSS),
+		 "virtio rss api flag mismatch");
 
   ap->virtio_flags = clib_net_to_host_u32 (mp->virtio_flags);
   ap->features = clib_net_to_host_u64 (mp->features);
 
   if (ap->virtio_flags & VIRTIO_API_FLAG_GSO)
     ap->gso_enabled = 1;
-  else
-    ap->gso_enabled = 0;
   if (ap->virtio_flags & VIRTIO_API_FLAG_CSUM_OFFLOAD)
     ap->checksum_offload_enabled = 1;
-  else
-    ap->checksum_offload_enabled = 0;
+  if (ap->virtio_flags & VIRTIO_API_FLAG_RSS)
+    ap->rss_enabled = 1;
 
   virtio_pci_create_if (vm, ap);
 
diff --git a/src/vnet/devices/virtio/virtio_std.h b/src/vnet/devices/virtio/virtio_std.h
index ec988c08dbb..66b8bac092c 100644
--- a/src/vnet/devices/virtio/virtio_std.h
+++ b/src/vnet/devices/virtio/virtio_std.h
@@ -15,49 +15,61 @@
 #ifndef __VIRTIO_STD_H__
 #define __VIRTIO_STD_H__
 
-#define foreach_virtio_net_features      \
-  _ (VIRTIO_NET_F_CSUM, 0)      /* Host handles pkts w/ partial csum */ \
-  _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */ \
-  _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, 2) /* Dynamic offload configuration. */ \
-  _ (VIRTIO_NET_F_MTU, 3)       /* Initial MTU advice. */ \
-  _ (VIRTIO_NET_F_MAC, 5)       /* Host has given MAC address. */ \
-  _ (VIRTIO_NET_F_GSO, 6)       /* Host handles pkts w/ any GSO. */ \
-  _ (VIRTIO_NET_F_GUEST_TSO4, 7)        /* Guest can handle TSOv4 in. */ \
-  _ (VIRTIO_NET_F_GUEST_TSO6, 8)        /* Guest can handle TSOv6 in. */ \
-  _ (VIRTIO_NET_F_GUEST_ECN, 9) /* Guest can handle TSO[6] w/ ECN in. */ \
-  _ (VIRTIO_NET_F_GUEST_UFO, 10)        /* Guest can handle UFO in. */ \
-  _ (VIRTIO_NET_F_HOST_TSO4, 11)        /* Host can handle TSOv4 in. */ \
-  _ (VIRTIO_NET_F_HOST_TSO6, 12)        /* Host can handle TSOv6 in. */ \
-  _ (VIRTIO_NET_F_HOST_ECN, 13) /* Host can handle TSO[6] w/ ECN in. */ \
-  _ (VIRTIO_NET_F_HOST_UFO, 14) /* Host can handle UFO in. */ \
-  _ (VIRTIO_NET_F_MRG_RXBUF, 15)        /* Host can merge receive buffers. */ \
-  _ (VIRTIO_NET_F_STATUS, 16)   /* virtio_net_config.status available */ \
-  _ (VIRTIO_NET_F_CTRL_VQ, 17)  /* Control channel available */ \
-  _ (VIRTIO_NET_F_CTRL_RX, 18)  /* Control channel RX mode support */ \
-  _ (VIRTIO_NET_F_CTRL_VLAN, 19)        /* Control channel VLAN filtering */ \
-  _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20)    /* Extra RX mode control support */ \
-  _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21)   /* Guest can announce device on the network */ \
-  _ (VIRTIO_NET_F_MQ, 22)               /* Device supports Receive Flow Steering */ \
-  _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23)    /* Set MAC address */ \
-  _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24) \
-  _ (VHOST_F_LOG_ALL, 26)      /* Log all write descriptors */ \
-  _ (VIRTIO_F_ANY_LAYOUT, 27)  /* Can the device handle any descriptor layout */ \
-  _ (VIRTIO_RING_F_INDIRECT_DESC, 28)   /* Support indirect buffer descriptors */ \
-  _ (VIRTIO_RING_F_EVENT_IDX, 29)       /* The Guest publishes the used index for which it expects an interrupt \
- * at the end of the avail ring. Host should ignore the avail->flags field. */ \
-/* The Host publishes the avail index for which it expects a kick \
- * at the end of the used ring. Guest should ignore the used->flags field. */ \
-  _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \
-  _ (VIRTIO_F_VERSION_1, 32)  /* v1.0 compliant. */           \
-  _ (VIRTIO_F_IOMMU_PLATFORM, 33) \
-  _ (VIRTIO_F_RING_PACKED, 34) \
-  _ (VIRTIO_F_IN_ORDER, 35)  /* all buffers are used by the device in the */ \
-                         /* same order in which they have been made available */ \
+#define foreach_virtio_net_features                                           \
+  _ (VIRTIO_NET_F_CSUM, 0)	 /* Host handles pkts w/ partial csum */      \
+  _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */     \
+  _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,                                        \
+     2)				     /* Dynamic offload configuration. */     \
+  _ (VIRTIO_NET_F_MTU, 3)	     /* Initial MTU advice. */                \
+  _ (VIRTIO_NET_F_MAC, 5)	     /* Host has given MAC address. */        \
+  _ (VIRTIO_NET_F_GSO, 6)	     /* Host handles pkts w/ any GSO. */      \
+  _ (VIRTIO_NET_F_GUEST_TSO4, 7)     /* Guest can handle TSOv4 in. */         \
+  _ (VIRTIO_NET_F_GUEST_TSO6, 8)     /* Guest can handle TSOv6 in. */         \
+  _ (VIRTIO_NET_F_GUEST_ECN, 9)	     /* Guest can handle TSO[6] w/ ECN in. */ \
+  _ (VIRTIO_NET_F_GUEST_UFO, 10)     /* Guest can handle UFO in. */           \
+  _ (VIRTIO_NET_F_HOST_TSO4, 11)     /* Host can handle TSOv4 in. */          \
+  _ (VIRTIO_NET_F_HOST_TSO6, 12)     /* Host can handle TSOv6 in. */          \
+  _ (VIRTIO_NET_F_HOST_ECN, 13)	     /* Host can handle TSO[6] w/ ECN in. */  \
+  _ (VIRTIO_NET_F_HOST_UFO, 14)	     /* Host can handle UFO in. */            \
+  _ (VIRTIO_NET_F_MRG_RXBUF, 15)     /* Host can merge receive buffers. */    \
+  _ (VIRTIO_NET_F_STATUS, 16)	     /* virtio_net_config.status available */ \
+  _ (VIRTIO_NET_F_CTRL_VQ, 17)	     /* Control channel available */          \
+  _ (VIRTIO_NET_F_CTRL_RX, 18)	     /* Control channel RX mode support */    \
+  _ (VIRTIO_NET_F_CTRL_VLAN, 19)     /* Control channel VLAN filtering */     \
+  _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */      \
+  _ (VIRTIO_NET_F_GUEST_ANNOUNCE,                                             \
+     21)		  /* Guest can announce device on the network */      \
+  _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */         \
+  _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */                    \
+  _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24)                                            \
+  _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */                     \
+  _ (VIRTIO_F_ANY_LAYOUT,                                                     \
+     27) /* Can the device handle any descriptor layout */                    \
+  _ (VIRTIO_RING_F_INDIRECT_DESC,                                             \
+     28) /* Support indirect buffer descriptors */                            \
+  _ (VIRTIO_RING_F_EVENT_IDX,                                                 \
+     29) /* The Guest publishes the used index for which it expects an        \
+	  * interrupt at the end of the avail ring. Host should ignore the    \
+	  * avail->flags field. */                                            \
+  /* The Host publishes the avail index for which it expects a kick           \
+   * at the end of the used ring. Guest should ignore the used->flags field.  \
+   */                                                                         \
+  _ (VHOST_USER_F_PROTOCOL_FEATURES, 30)                                      \
+  _ (VIRTIO_F_VERSION_1, 32) /* v1.0 compliant. */                            \
+  _ (VIRTIO_F_IOMMU_PLATFORM, 33)                                             \
+  _ (VIRTIO_F_RING_PACKED, 34)                                                \
+  _ (VIRTIO_F_IN_ORDER, 35) /* all buffers are used by the device in the */   \
+  /* same order in which they have been made available */                     \
   _ (VIRTIO_F_ORDER_PLATFORM, 36) /* memory accesses by the driver and the */ \
-                      /* device are ordered in a way described by the platfor */ \
-  _ (VIRTIO_F_NOTIFICATION_DATA, 38) /* the driver passes extra data (besides */ \
-                      /* identifying the virtqueue) in its device notifications. */ \
-  _ (VIRTIO_NET_F_SPEED_DUPLEX, 63)	/* Device set linkspeed and duplex */
+  /* device are ordered in a way described by the platfor */                  \
+  _ (VIRTIO_F_NOTIFICATION_DATA,                                              \
+     38) /* the driver passes extra data (besides */                          \
+  /* identifying the virtqueue) in its device notifications. */               \
+  _ (VIRTIO_NET_F_RING_RESET, 40) /* Device supports individual ring reset */ \
+  _ (VIRTIO_NET_F_HASH_REPORT,                                                \
+     57)		   /* Device supports per packet hash value */        \
+  _ (VIRTIO_NET_F_RSS, 60) /* Device supports RSS */                          \
+  _ (VIRTIO_NET_F_SPEED_DUPLEX, 63) /* Device set linkspeed and duplex */
 
 typedef enum
 {