vmxnet3 device driver

Implemented vmxnet3 deivice driver for VMWare ESXi. Tested with Ubuntu 18.04
connected to ESXi 6.0

Ubuntu-18.04 (VPP) --- ESXi-6.0

Change-Id: I85fbc86f2d8532b017bc4271612d17e24e498e4d
Signed-off-by: Steven Luong  <sluong@cisco.com>

14 files changed, 3173 insertions, 0 deletions

diff --git a/src/plugins/vmxnet3/CMakeLists.txt b/src/plugins/vmxnet3/CMakeLists.txt
new file mode 100644
index 00000000000..a81d8d59387
--- /dev/null
+++ b/src/plugins/vmxnet3/CMakeLists.txt
@@ -0,0 +1,37 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_vpp_plugin(vmxnet3
+  SOURCES
+  cli.c
+  format.c
+  input.c
+  output.c
+  plugin.c
+  vmxnet3.c
+  vmxnet3_api.c
+
+  MULTIARCH_SOURCES
+  input.c
+  output.c
+
+  API_FILES
+  vmxnet3.api
+
+  API_TEST_SOURCES
+  vmxnet3_test.c
+
+  INSTALL_HEADERS
+  vmxnet3_all_api_h.h
+  vmxnet3_msg_enum.h
+)
diff --git a/src/plugins/vmxnet3/README.md b/src/plugins/vmxnet3/README.md
new file mode 100644
index 00000000000..a49671325c9
--- /dev/null
+++ b/src/plugins/vmxnet3/README.md
@@ -0,0 +1,59 @@
+# VMWARE vmxnet3 device driver plugin
+
+##Overview
+This plugin provides native PCI driver support for VMWare vmxnet3.
+
+##Prerequisites
+ * This code is tested with vfio-pci driver installed with Ubuntu 18.04 which
+has kernel version 4.15.0-33-generic.
+
+ * This code is tested with ESXi vSwitch version 6.0, release build 3620759.
+
+ * Driver requires MSI-X interrupt support, which is not supported by
+uio_pci_generic driver, so vfio-pci needs to be used. On systems without IOMMU
+vfio driver can still be used with recent kernels which support no-iommu mode.
+
+##Known issues
+
+* NUMA support
+* TSO
+* VLAN filter
+
+## Usage
+### System setup
+
+1. load VFIO driver
+```
+sudo modprobe vfio-pci
+```
+
+2. (systems without IOMMU only) enable unsafe NOIOMMU mode
+```
+echo Y | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
+```
+
+3. Bind interface to vfio-pci
+```
+dpdk-devbind.py --bind vfio-pci 0b:00.0
+```
+
+### Interface Creation
+Interface can be dynamically created with following CLI:
+```
+create interface vmxnet3 0000:0b:00.0
+set int state vmxnet3-0/b/0/0 up
+```
+
+### Interface Deletion
+Interface can be deleted with following CLI:
+```
+delete interface vmxnet3 <if-name>
+```
+
+### Interface Statistics
+Interface statistics can be displayed with `show hardware-interface <if-name>`
+command.
+
+### Show Interface CLI
+Interface and ring information can be obtained with
+`show vmxnet3 [if-name] [desc]`
diff --git a/src/plugins/vmxnet3/cli.c b/src/plugins/vmxnet3/cli.c
new file mode 100644
index 00000000000..3e3ab9d74f7
--- /dev/null
+++ b/src/plugins/vmxnet3/cli.c
@@ -0,0 +1,371 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+#include <stdint.h>
+#include <net/if.h>
+#include <sys/ioctl.h>
+#include <inttypes.h>
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+
+#include <vmxnet3/vmxnet3.h>
+
+static clib_error_t *
+vmxnet3_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			   vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vmxnet3_create_if_args_t args = { 0 };
+  u32 tmp;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U", unformat_vlib_pci_addr, &args.addr))
+	;
+      else if (unformat (line_input, "elog"))
+	args.enable_elog = 1;
+      else if (unformat (line_input, "rx-queue-size %u", &tmp))
+	args.rxq_size = tmp;
+      else if (unformat (line_input, "tx-queue-size %u", &tmp))
+	args.txq_size = tmp;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+
+  vmxnet3_create_if (vm, &args);
+
+  return args.error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (vmxnet3_create_command, static) = {
+  .path = "create interface vmxnet3",
+  .short_help = "create interface vmxnet3 <pci-address>"
+                "[rx-queue-size <size>] [tx-queue-size <size>]",
+  .function = vmxnet3_create_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+vmxnet3_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			   vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 sw_if_index = ~0;
+  vnet_hw_interface_t *hw;
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_device_t *vd;
+  vnet_main_t *vnm = vnet_get_main ();
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (line_input, "%U", unformat_vnet_sw_interface,
+			 vnm, &sw_if_index))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+  if (sw_if_index == ~0)
+    return clib_error_return (0,
+			      "please specify interface name or sw_if_index");
+
+  hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  if (hw == NULL || vmxnet3_device_class.index != hw->dev_class_index)
+    return clib_error_return (0, "not a vmxnet3 interface");
+
+  vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
+
+  vmxnet3_delete_if (vm, vd);
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (vmxnet3_delete_command, static) = {
+  .path = "delete interface vmxnet3",
+  .short_help = "delete interface vmxnet3 "
+    "{<interface> | sw_if_index <sw_idx>}",
+  .function = vmxnet3_delete_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+vmxnet3_test_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			 vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 sw_if_index = ~0;
+  vnet_hw_interface_t *hw;
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_device_t *vd;
+  vnet_main_t *vnm = vnet_get_main ();
+  int enable_elog = 0, disable_elog = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (line_input, "elog-on"))
+	enable_elog = 1;
+      else if (unformat (line_input, "elog-off"))
+	disable_elog = 1;
+      else if (unformat (line_input, "%U", unformat_vnet_sw_interface,
+			 vnm, &sw_if_index))
+	;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+  if (sw_if_index == ~0)
+    return clib_error_return (0,
+			      "please specify interface name or sw_if_index");
+
+  hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  if (hw == NULL || vmxnet3_device_class.index != hw->dev_class_index)
+    return clib_error_return (0, "not a vmxnet3 interface");
+
+  vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
+
+  if (enable_elog)
+    vd->flags |= VMXNET3_DEVICE_F_ELOG;
+
+  if (disable_elog)
+    vd->flags &= ~VMXNET3_DEVICE_F_ELOG;
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (vmxnet3_test_command, static) = {
+  .path = "test vmxnet3",
+  .short_help = "test vmxnet3 <interface> | sw_if_index <sw_idx> [irq] "
+    "[elog-on] [elog-off]",
+  .function = vmxnet3_test_command_fn,
+};
+/* *INDENT-ON* */
+
+static void
+show_vmxnet3 (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr)
+{
+  u32 i, desc_idx;
+  vmxnet3_device_t *vd;
+  vnet_main_t *vnm = &vnet_main;
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vnet_hw_interface_t *hi;
+  vmxnet3_rxq_t *rxq;
+  vmxnet3_rx_desc *rxd;
+  vmxnet3_rx_comp *rx_comp;
+  vmxnet3_txq_t *txq;
+  vmxnet3_tx_desc *txd;
+  vmxnet3_tx_comp *tx_comp;
+  u16 qid;
+
+  if (!hw_if_indices)
+    return;
+
+  for (i = 0; i < vec_len (hw_if_indices); i++)
+    {
+      hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
+      vd = vec_elt_at_index (vmxm->devices, hi->dev_instance);
+      vlib_cli_output (vm, "Interface: %s (ifindex %d)",
+		       hi->name, hw_if_indices[i]);
+      vlib_cli_output (vm, "  Version: %u", vd->version);
+      vlib_cli_output (vm, "  PCI Address: %U", format_vlib_pci_addr,
+		       &vd->pci_addr);
+      vlib_cli_output (vm, "  Mac Address: %U", format_ethernet_address,
+		       vd->mac_addr);
+      vlib_cli_output (vm, "  hw if index: %u", vd->hw_if_index);
+      vlib_cli_output (vm, "  Device instance: %u", vd->dev_instance);
+      vlib_cli_output (vm, "  Number of interrupts: %u", vd->num_intrs);
+
+      vec_foreach_index (qid, vd->rxqs)
+      {
+	rxq = vec_elt_at_index (vd->rxqs, qid);
+	u16 rid;
+
+	vlib_cli_output (vm, "  Queue %u (RX)", qid);
+	vlib_cli_output (vm, "    RX completion next index %u",
+			 rxq->rx_comp_ring.next);
+	for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
+	  {
+	    vmxnet3_rx_ring *ring = &rxq->rx_ring[rid];
+
+	    vlib_cli_output (vm,
+			     "    ring %u size %u fill %u "
+			     "consume %u produce %u", rid,
+			     rxq->size, ring->fill, ring->consume,
+			     ring->produce);
+	    if (show_descr)
+	      {
+		vlib_cli_output (vm, "RX descriptors table");
+		vlib_cli_output (vm, "  %5s  %18s  %10s",
+				 "slot", "address", "flags");
+		for (desc_idx = 0; desc_idx < rxq->size; desc_idx++)
+		  {
+		    rxd = &rxq->rx_desc[rid][desc_idx];
+		    vlib_cli_output (vm, "  %5u  0x%016llx  0x%08x",
+				     desc_idx, rxd->address, rxd->flags);
+		  }
+		vlib_cli_output (vm, "RX completion descriptors table");
+		vlib_cli_output (vm, "  %5s  %10s  %10s  %10s  %10s",
+				 "slot", "index", "rss", "len", "flags");
+		for (desc_idx = 0; desc_idx < rxq->size; desc_idx++)
+		  {
+		    rx_comp = &rxq->rx_comp[desc_idx];
+		    vlib_cli_output (vm, "  %5u  0x%08x  %10u  %10u  0x%08x",
+				     desc_idx, rx_comp->index, rx_comp->rss,
+				     rx_comp->len, rx_comp->flags);
+		  }
+	      }
+	  }
+      }
+
+      vec_foreach_index (qid, vd->rxqs)
+      {
+	txq = vec_elt_at_index (vd->txqs, 0);
+	vlib_cli_output (vm, "  Queue %u (TX)", qid);
+	vlib_cli_output (vm, "    TX completion next index %u",
+			 txq->tx_comp_ring.next);
+	vlib_cli_output (vm, "    size %u consume %u produce %u",
+			 txq->size, txq->tx_ring.consume,
+			 txq->tx_ring.produce);
+	if (show_descr)
+	  {
+	    vlib_cli_output (vm, "TX descriptors table");
+	    vlib_cli_output (vm, "  %5s  %18s  %10s  %10s",
+			     "slot", "address", "flags0", "flags1");
+	    for (desc_idx = 0; desc_idx < txq->size; desc_idx++)
+	      {
+		txd = &txq->tx_desc[desc_idx];
+		vlib_cli_output (vm, "  %5u  0x%016llx  0x%08x  0x%08x",
+				 desc_idx, txd->address, txd->flags[0],
+				 txd->flags[1]);
+	      }
+	    vlib_cli_output (vm, "TX completion descriptors table");
+	    vlib_cli_output (vm, "  %5s  %10s  %10s",
+			     "slot", "index", "flags");
+	    for (desc_idx = 0; desc_idx < txq->size; desc_idx++)
+	      {
+		tx_comp = &txq->tx_comp[desc_idx];
+		vlib_cli_output (vm, "  %5u  0x%08x  0x%08x",
+				 desc_idx, tx_comp->index, tx_comp->flags);
+	      }
+	  }
+      }
+    }
+}
+
+static clib_error_t *
+show_vmxnet3_fn (vlib_main_t * vm, unformat_input_t * input,
+		 vlib_cli_command_t * cmd)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vnet_main_t *vnm = &vnet_main;
+  vmxnet3_device_t *vd;
+  clib_error_t *error = 0;
+  u32 hw_if_index, *hw_if_indices = 0;
+  vnet_hw_interface_t *hi;
+  u8 show_descr = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (input, "%U", unformat_vnet_hw_interface, vnm, &hw_if_index))
+	{
+	  hi = vnet_get_hw_interface (vnm, hw_if_index);
+	  if (vmxnet3_device_class.index != hi->dev_class_index)
+	    {
+	      error = clib_error_return (0, "unknown input `%U'",
+					 format_unformat_error, input);
+	      goto done;
+	    }
+	  vec_add1 (hw_if_indices, hw_if_index);
+	}
+      else if (unformat (input, "descriptors") || unformat (input, "desc"))
+	show_descr = 1;
+      else
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, input);
+	  goto done;
+	}
+    }
+
+  if (vec_len (hw_if_indices) == 0)
+    {
+      pool_foreach (vd, vmxm->devices,
+		    vec_add1 (hw_if_indices, vd->hw_if_index);
+	);
+    }
+
+  show_vmxnet3 (vm, hw_if_indices, show_descr);
+
+done:
+  vec_free (hw_if_indices);
+  return error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_vmxnet3_command, static) = {
+  .path = "show vmxnet3",
+  .short_help = "show vmxnet3 [<interface>]",
+  .function = show_vmxnet3_fn,
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+vmxnet3_cli_init (vlib_main_t * vm)
+{
+  /* initialize binary API */
+  vmxnet3_plugin_api_hookup (vm);
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (vmxnet3_cli_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/format.c b/src/plugins/vmxnet3/format.c
new file mode 100644
index 00000000000..8ee812ecc06
--- /dev/null
+++ b/src/plugins/vmxnet3/format.c
@@ -0,0 +1,143 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+
+#include <vmxnet3/vmxnet3.h>
+
+u8 *
+format_vmxnet3_device_name (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_device_t *vd = vec_elt_at_index (vmxm->devices, i);
+  vlib_pci_addr_t *addr = vlib_pci_get_addr (vd->pci_dev_handle);
+
+  s = format (s, "vmxnet3-%x/%x/%x/%x",
+	      addr->domain, addr->bus, addr->slot, addr->function);
+  return s;
+}
+
+u8 *
+format_vmxnet3_device_flags (u8 * s, va_list * args)
+{
+  vmxnet3_device_t *vd = va_arg (*args, vmxnet3_device_t *);
+  u8 *t = 0;
+
+#define _(a, b, c) if (vd->flags & (1 << a)) \
+    t = format (t, "%s%s", t ? " ":"", c);
+  foreach_vmxnet3_device_flags
+#undef _
+    s = format (s, "%v", t);
+  vec_free (t);
+  return s;
+}
+
+u8 *
+format_vmxnet3_device (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_device_t *vd = vec_elt_at_index (vmxm->devices, i);
+  u32 indent = format_get_indent (s);
+  vmxnet3_queues *q = &vd->dma->queues;
+  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, 0);
+  vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, 0);
+
+  s = format (s, "flags: %U", format_vmxnet3_device_flags, vd);
+  s = format (s, "\n%Urx queues %u, rx desc %u, tx queues %u, tx desc %u",
+	      format_white_space, indent,
+	      vd->num_rx_queues, rxq->size, vd->num_tx_queues, txq->size);
+  if (vd->error)
+    s = format (s, "\n%Uerror %U", format_white_space, indent,
+		format_clib_error, vd->error);
+
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
+
+  s = format (s, "\n%UTX:", format_white_space, indent);
+  s = format (s, "\n%U  TSO packets                         %llu",
+	      format_white_space, indent, q->tx.stats.tso_pkts);
+  s = format (s, "\n%U  TSO bytes                           %llu",
+	      format_white_space, indent, q->tx.stats.tso_bytes);
+  s = format (s, "\n%U  ucast packets                       %llu",
+	      format_white_space, indent, q->tx.stats.ucast_pkts);
+  s = format (s, "\n%U  ucast bytes                         %llu",
+	      format_white_space, indent, q->tx.stats.ucast_bytes);
+  s = format (s, "\n%U  mcast packets                       %llu",
+	      format_white_space, indent, q->tx.stats.mcast_pkts);
+  s = format (s, "\n%U  mcast bytes                         %llu",
+	      format_white_space, indent, q->tx.stats.mcast_bytes);
+  s = format (s, "\n%U  bcast packets                       %llu",
+	      format_white_space, indent, q->tx.stats.bcast_pkts);
+  s = format (s, "\n%U  bcast bytes                         %llu",
+	      format_white_space, indent, q->tx.stats.bcast_bytes);
+  s = format (s, "\n%U  Errors packets                      %llu",
+	      format_white_space, indent, q->tx.stats.error_pkts);
+  s = format (s, "\n%U  Discard packets                     %llu",
+	      format_white_space, indent, q->tx.stats.discard_pkts);
+
+  s = format (s, "\n%URX:", format_white_space, indent);
+  s = format (s, "\n%U  LRO packets                         %llu",
+	      format_white_space, indent, q->rx.stats.lro_pkts);
+  s = format (s, "\n%U  LRO bytes                           %llu",
+	      format_white_space, indent, q->rx.stats.lro_bytes);
+  s = format (s, "\n%U  ucast packets                       %llu",
+	      format_white_space, indent, q->rx.stats.ucast_pkts);
+  s = format (s, "\n%U  ucast bytes                         %llu",
+	      format_white_space, indent, q->rx.stats.ucast_bytes);
+  s = format (s, "\n%U  mcast packets                       %llu",
+	      format_white_space, indent, q->rx.stats.mcast_pkts);
+  s = format (s, "\n%U  mcast bytes                         %llu",
+	      format_white_space, indent, q->rx.stats.mcast_bytes);
+  s = format (s, "\n%U  bcast packets                       %llu",
+	      format_white_space, indent, q->rx.stats.bcast_pkts);
+  s = format (s, "\n%U  bcast bytes                         %llu",
+	      format_white_space, indent, q->rx.stats.bcast_bytes);
+  s = format (s, "\n%U  No Bufs                             %llu",
+	      format_white_space, indent, q->rx.stats.nobuf_pkts);
+  s = format (s, "\n%U  Error packets                       %llu",
+	      format_white_space, indent, q->rx.stats.error_pkts);
+  return s;
+}
+
+u8 *
+format_vmxnet3_input_trace (u8 * s, va_list * args)
+{
+  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
+  vlib_node_t *node = va_arg (*args, vlib_node_t *);
+  vmxnet3_input_trace_t *t = va_arg (*args, vmxnet3_input_trace_t *);
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, t->hw_if_index);
+
+  s = format (s, "vmxnet3: %v (%d) next-node %U",
+	      hi->name, t->hw_if_index, format_vlib_next_node_name, vm,
+	      node->index, t->next_index);
+  s = format (s, "\n  buffer %U", format_vlib_buffer, &t->buffer);
+
+  return s;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/input.c b/src/plugins/vmxnet3/input.c
new file mode 100644
index 00000000000..a5a5d2f43b8
--- /dev/null
+++ b/src/plugins/vmxnet3/input.c
@@ -0,0 +1,295 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/devices/devices.h>
+
+#include <vmxnet3/vmxnet3.h>
+
+#define foreach_vmxnet3_input_error \
+  _(BUFFER_ALLOC, "buffer alloc error") \
+  _(RX_PACKET, "Rx packet error") \
+  _(NO_BUFFER, "Rx no buffer error")
+
+typedef enum
+{
+#define _(f,s) VMXNET3_INPUT_ERROR_##f,
+  foreach_vmxnet3_input_error
+#undef _
+    VMXNET3_INPUT_N_ERROR,
+} vmxnet3_input_error_t;
+
+static __clib_unused char *vmxnet3_input_error_strings[] = {
+#define _(n,s) s,
+  foreach_vmxnet3_input_error
+#undef _
+};
+
+static_always_inline u16
+vmxnet3_find_rid (vmxnet3_device_t * vd, vmxnet3_rx_comp * rx_comp)
+{
+  u32 rid;
+
+  // rid is bits 16-25 (10 bits number)
+  rid = rx_comp->index & (0xffffffff >> 6);
+  rid >>= 16;
+  if ((rid >= vd->num_rx_queues) && (rid < (vd->num_rx_queues << 1)))
+    return 1;
+  else
+    return 0;
+}
+
+static_always_inline void
+vmxnet3_rx_comp_ring_advance_next (vmxnet3_rxq_t * rxq)
+{
+  vmxnet3_rx_comp_ring *comp_ring = &rxq->rx_comp_ring;
+
+  comp_ring->next++;
+  if (PREDICT_FALSE (comp_ring->next == rxq->size))
+    {
+      comp_ring->next = 0;
+      comp_ring->gen ^= VMXNET3_RXCF_GEN;
+    }
+}
+
+static_always_inline uword
+vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+			     vlib_frame_t * frame, vmxnet3_device_t * vd,
+			     u16 qid)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  uword n_trace = vlib_get_trace_count (vm, node);
+  u32 n_rx_packets = 0, n_rx_bytes = 0;
+  vmxnet3_rx_comp *rx_comp;
+  u32 comp_idx;
+  u32 desc_idx;
+  vmxnet3_rxq_t *rxq;
+  u32 thread_index = vm->thread_index;
+  u32 buffer_indices[VLIB_FRAME_SIZE], *bi;
+  u16 nexts[VLIB_FRAME_SIZE], *next;
+  vmxnet3_rx_ring *ring;
+  vmxnet3_rx_comp_ring *comp_ring;
+  u16 rid;
+  vlib_buffer_t *prev_b0 = 0, *hb = 0;
+
+  rxq = vec_elt_at_index (vd->rxqs, qid);
+  comp_ring = &rxq->rx_comp_ring;
+  bi = buffer_indices;
+  next = nexts;
+  while (comp_ring->gen ==
+	 (rxq->rx_comp[comp_ring->next].flags & VMXNET3_RXCF_GEN))
+    {
+      vlib_buffer_t *b0;
+
+      comp_idx = comp_ring->next;
+      rx_comp = &rxq->rx_comp[comp_idx];
+
+      rid = vmxnet3_find_rid (vd, rx_comp);
+      ring = &rxq->rx_ring[rid];
+
+      if (PREDICT_TRUE (ring->fill >= 1))
+	ring->fill--;
+      else
+	{
+	  vlib_error_count (vm, node->node_index,
+			    VMXNET3_INPUT_ERROR_NO_BUFFER, 1);
+	  break;
+	}
+
+      vmxnet3_rx_comp_ring_advance_next (rxq);
+      desc_idx = rx_comp->index & VMXNET3_RXC_INDEX;
+      ring->consume = desc_idx;
+
+      bi[0] = ring->bufs[desc_idx];
+      ring->bufs[desc_idx] = ~0;
+
+      b0 = vlib_get_buffer (vm, bi[0]);
+      vnet_buffer (b0)->sw_if_index[VLIB_RX] = vd->sw_if_index;
+      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+      b0->current_length = rx_comp->len & VMXNET3_RXCL_LEN_MASK;
+      b0->current_data = 0;
+      b0->total_length_not_including_first_buffer = 0;
+      b0->next_buffer = 0;
+      b0->flags = 0;
+      b0->error = 0;
+      ASSERT (b0->current_length != 0);
+
+      if (rx_comp->index & VMXNET3_RXCI_SOP)
+	{
+	  /* start segment */
+	  hb = b0;
+	  if (!(rx_comp->index & VMXNET3_RXCI_EOP))
+	    {
+	      hb->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
+	      b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
+	      prev_b0 = b0;
+	    }
+	  else
+	    {
+	      /*
+	       * Both start and end of packet is set. It is a complete packet
+	       */
+	      prev_b0 = 0;
+	    }
+
+	}
+      else if (rx_comp->index & VMXNET3_RXCI_EOP)
+	{
+	  /* end of segment */
+	  if (prev_b0)
+	    {
+	      prev_b0->next_buffer = bi[0];
+	      prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
+	      hb->total_length_not_including_first_buffer +=
+		b0->current_length;
+	      prev_b0 = 0;	// Get next packet
+	    }
+	  else
+	    {
+	      /* EOP without SOP, error */
+	      hb = 0;
+	      bi++;
+	      vlib_error_count (vm, node->node_index,
+				VMXNET3_INPUT_ERROR_RX_PACKET, 1);
+	      vlib_buffer_free_one (vm, bi[0]);
+	      continue;
+	    }
+	}
+      else if (prev_b0)		// !sop && !eop
+	{
+	  /* mid chain */
+	  b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
+	  prev_b0->next_buffer = bi[0];
+	  prev_b0 = b0;
+	  hb->total_length_not_including_first_buffer += b0->current_length;
+	}
+      else
+	{
+	  ASSERT (0);
+	}
+
+      bi++;
+      n_rx_bytes += b0->current_length;
+
+      if (!prev_b0)
+	{
+	  next[0] = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+	  n_rx_packets++;
+	  next++;
+	  hb = 0;
+	}
+    }
+
+  if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
+    {
+      u32 n_left = n_rx_packets;
+
+      bi = buffer_indices;
+      next = nexts;
+      while (n_trace && n_left)
+	{
+	  vlib_buffer_t *b;
+	  vmxnet3_input_trace_t *tr;
+
+	  b = vlib_get_buffer (vm, bi[0]);
+	  vlib_trace_buffer (vm, node, next[0], b, /* follow_chain */ 0);
+	  tr = vlib_add_trace (vm, node, b, sizeof (*tr));
+	  tr->next_index = next[0];
+	  tr->hw_if_index = vd->hw_if_index;
+	  tr->buffer = *b;
+
+	  n_trace--;
+	  n_left--;
+	  bi++;
+	  next++;
+	}
+      vlib_set_trace_count (vm, node, n_trace);
+    }
+
+  if (PREDICT_TRUE (n_rx_packets))
+    {
+      clib_error_t *error;
+
+      vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts,
+				   n_rx_packets);
+      vlib_increment_combined_counter
+	(vnm->interface_main.combined_sw_if_counters +
+	 VNET_INTERFACE_COUNTER_RX, thread_index,
+	 vd->hw_if_index, n_rx_packets, n_rx_bytes);
+
+      error = vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
+      if (PREDICT_FALSE (error != 0))
+	{
+	  vlib_error_count (vm, node->node_index,
+			    VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
+	}
+      error = vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
+      if (PREDICT_FALSE (error != 0))
+	{
+	  vlib_error_count (vm, node->node_index,
+			    VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
+	}
+    }
+
+  return n_rx_packets;
+}
+
+VLIB_NODE_FN (vmxnet3_input_node) (vlib_main_t * vm,
+				   vlib_node_runtime_t * node,
+				   vlib_frame_t * frame)
+{
+  u32 n_rx = 0;
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
+  vnet_device_and_queue_t *dq;
+
+  foreach_device_and_queue (dq, rt->devices_and_queues)
+  {
+    vmxnet3_device_t *vd;
+    vd = vec_elt_at_index (vmxm->devices, dq->dev_instance);
+    if ((vd->flags & VMXNET3_DEVICE_F_ADMIN_UP) == 0)
+      continue;
+    n_rx += vmxnet3_device_input_inline (vm, node, frame, vd, dq->queue_id);
+  }
+  return n_rx;
+}
+
+#ifndef CLIB_MARCH_VARIANT
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (vmxnet3_input_node) = {
+  .name = "vmxnet3-input",
+  .sibling_of = "device-input",
+  .format_trace = format_vmxnet3_input_trace,
+  .type = VLIB_NODE_TYPE_INPUT,
+  .state = VLIB_NODE_STATE_DISABLED,
+  .n_errors = VMXNET3_INPUT_N_ERROR,
+  .error_strings = vmxnet3_input_error_strings,
+};
+#endif
+
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/output.c b/src/plugins/vmxnet3/output.c
new file mode 100644
index 00000000000..2ca1cc9f107
--- /dev/null
+++ b/src/plugins/vmxnet3/output.c
@@ -0,0 +1,209 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/devices/devices.h>
+
+#include <vmxnet3/vmxnet3.h>
+
+static_always_inline void
+vmxnet3_tx_comp_ring_advance_next (vmxnet3_txq_t * txq)
+{
+  vmxnet3_tx_comp_ring *comp_ring = &txq->tx_comp_ring;
+
+  comp_ring->next++;
+  if (PREDICT_FALSE (comp_ring->next == txq->size))
+    {
+      comp_ring->next = 0;
+      comp_ring->gen ^= VMXNET3_TXCF_GEN;
+    }
+}
+
+static_always_inline void
+vmxnet3_tx_ring_advance_produce (vmxnet3_txq_t * txq)
+{
+  txq->tx_ring.produce++;
+  if (PREDICT_FALSE (txq->tx_ring.produce == txq->size))
+    {
+      txq->tx_ring.produce = 0;
+      txq->tx_ring.gen ^= VMXNET3_TXF_GEN;
+    }
+}
+
+static_always_inline void
+vmxnet3_tx_ring_advance_consume (vmxnet3_txq_t * txq)
+{
+  txq->tx_ring.consume++;
+  txq->tx_ring.consume &= txq->size - 1;
+}
+
+static_always_inline void
+vmxnet3_txq_release (vlib_main_t * vm, vmxnet3_device_t * vd,
+		     vmxnet3_txq_t * txq)
+{
+  vmxnet3_tx_comp *tx_comp;
+  u32 bi0;
+  vmxnet3_tx_comp_ring *comp_ring;
+  u16 eop_idx, desc_idx;
+
+  comp_ring = &txq->tx_comp_ring;
+  tx_comp = &txq->tx_comp[comp_ring->next];
+
+  while ((tx_comp->flags & VMXNET3_TXCF_GEN) == comp_ring->gen)
+    {
+      eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
+      do
+	{
+	  desc_idx = txq->tx_ring.consume;
+	  bi0 = txq->tx_ring.bufs[desc_idx];
+	  txq->tx_ring.bufs[desc_idx] = ~0;
+	  vlib_buffer_free_no_next (vm, &bi0, 1);
+	  vmxnet3_tx_ring_advance_consume (txq);
+	}
+      while (desc_idx != eop_idx);
+
+      vmxnet3_tx_comp_ring_advance_next (txq);
+      tx_comp = &txq->tx_comp[comp_ring->next];
+    }
+}
+
+static_always_inline u16
+vmxnet3_tx_ring_space_left (vmxnet3_txq_t * txq)
+{
+  u16 count;
+
+  count = (txq->tx_ring.consume - txq->tx_ring.produce - 1);
+  /* Wrapped? */
+  if (txq->tx_ring.produce >= txq->tx_ring.consume)
+    count += txq->size;
+  return count;
+}
+
+VNET_DEVICE_CLASS_TX_FN (vmxnet3_device_class) (vlib_main_t * vm,
+						vlib_node_runtime_t * node,
+						vlib_frame_t * frame)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
+  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, rd->dev_instance);
+  u32 *buffers = vlib_frame_args (frame);
+  u32 bi0;
+  vlib_buffer_t *b0;
+  vmxnet3_tx_desc *txd;
+  u32 desc_idx, generation, first_idx;
+  u16 space_left;
+  u16 n_left = frame->n_vectors;
+  vmxnet3_txq_t *txq;
+  u32 thread_index = vm->thread_index;
+  u16 qid = thread_index;
+  u16 n_retry = 5;
+
+  txq = vec_elt_at_index (vd->txqs, qid % vd->num_tx_queues);
+
+  clib_spinlock_lock_if_init (&txq->lock);
+
+retry:
+  vmxnet3_txq_release (vm, vd, txq);
+
+  while (n_left)
+    {
+      bi0 = buffers[0];
+      txd = 0;
+
+      /*
+       * Toggle the generation bit for SOP fragment to avoid device starts
+       * reading incomplete packet
+       */
+      generation = txq->tx_ring.gen ^ VMXNET3_TXF_GEN;
+      first_idx = txq->tx_ring.produce;
+      while (1)
+	{
+	  b0 = vlib_get_buffer (vm, bi0);
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+
+	  space_left = vmxnet3_tx_ring_space_left (txq);
+	  if (PREDICT_FALSE (space_left == 0))
+	    {
+	      break;
+	    }
+
+	  desc_idx = txq->tx_ring.produce;
+
+	  vmxnet3_tx_ring_advance_produce (txq);
+	  txq->tx_ring.bufs[desc_idx] = bi0;
+
+	  txd = &txq->tx_desc[desc_idx];
+	  txd->address =
+	    vlib_get_buffer_data_physical_address (vm,
+						   bi0) + b0->current_data;
+
+	  txd->flags[0] = generation | b0->current_length;
+
+	  generation = txq->tx_ring.gen;
+	  if (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
+	    {
+	      txd->flags[1] = 0;
+	      bi0 = b0->next_buffer;
+	    }
+	  else
+	    break;
+	}
+
+      if (PREDICT_TRUE (txd != 0))
+	{
+	  txd->flags[1] = VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
+	  asm volatile ("":::"memory");
+	  /*
+	   * Now toggle back the generation bit for the first segment.
+	   * Device can start reading the packet
+	   */
+	  txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
+	  vmxnet3_reg_write (vd, 0, VMXNET3_REG_TXPROD, txq->tx_ring.produce);
+	}
+
+      if (PREDICT_FALSE (space_left == 0))
+	{
+	  break;
+	}
+
+      buffers++;
+      n_left--;
+    }
+
+  if (PREDICT_FALSE (n_left))
+    {
+      if (PREDICT_TRUE (n_retry--))
+	goto retry;
+      vlib_buffer_free (vm, buffers, n_left);
+      vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_NO_FREE_SLOTS,
+			n_left);
+    }
+  clib_spinlock_unlock_if_init (&txq->lock);
+
+  return (frame->n_vectors - n_left);
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/plugin.c b/src/plugins/vmxnet3/plugin.c
new file mode 100644
index 00000000000..8c7112d6064
--- /dev/null
+++ b/src/plugins/vmxnet3/plugin.c
@@ -0,0 +1,35 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vnet/plugin/plugin.h>
+#include <vpp/app/version.h>
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+  .version = VPP_BUILD_VER,
+  .description = "VMWare Vmxnet3 Device Plugin",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/vmxnet3.api b/src/plugins/vmxnet3/vmxnet3.api
new file mode 100644
index 00000000000..68beac030c1
--- /dev/null
+++ b/src/plugins/vmxnet3/vmxnet3.api
@@ -0,0 +1,125 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+option version = "1.0.0";
+
+/** \brief
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param pci_addr - pci address as unsigned 32bit integer:
+		      0-15 domain, 16-23 bus, 24-28 slot, 29-31 function 
+		      ddddddddddddddddbbbbbbbbsssssfff
+    @param enable_elog - turn on elog (optional - default is off)
+    @param rxq_size - receive queue size (optional - default is 1024)
+    @param txq_size - transmit queue size (optional - default is 1024)
+*/
+
+define vmxnet3_create
+{
+  u32 client_index;
+  u32 context;
+
+  u32 pci_addr;
+  i32 enable_elog;
+  u16 rxq_size;
+  u16 txq_size;
+};
+
+/** \brief
+    @param context - sender context, to match reply w/ request
+    @param retval - return value for request
+    @param sw_if_index - software index for the new vmxnet3 interface
+*/
+
+define vmxnet3_create_reply
+{
+  u32 context;
+  i32 retval;
+  u32 sw_if_index;
+};
+
+/** \brief
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - interface index
+*/
+
+autoreply define vmxnet3_delete
+{
+  u32 client_index;
+  u32 context;
+
+  u32 sw_if_index;
+};
+
+/** \brief Memory interface details structure
+    @param context - sender context, to match reply w/ request (memif_dump)
+    @param sw_if_index - index of the interface
+    @param if_name - name of the interface
+    @param hw_addr - interface MAC address
+    @param id - id associated with the interface
+    @param role - role of the interface in the connection (master/slave)
+    @param mode - interface mode
+    @param socket_id - id of the socket filename used by this interface
+           to establish new connections
+    @param ring_size - the number of entries of RX/TX rings
+    @param buffer_size - size of the buffer allocated for each ring entry
+    @param admin_up_down - interface administrative status
+    @param link_up_down - interface link status
+
+*/
+define vmxnet3_details
+{
+  u32 context;
+
+  u32 sw_if_index;
+  u8 if_name[64];
+  u8 hw_addr[6];
+  u32 pci_addr;
+  u8 version;
+
+  u16 rx_qid;
+  u16 rx_qsize;
+  u16 rx_fill[2];
+  u16 rx_next;
+  u16 rx_produce[2];
+  u16 rx_consume[2];
+
+  u16 tx_qid;
+  u16 tx_qsize;
+  u16 tx_next;
+  u16 tx_produce;
+  u16 tx_consume;
+
+  u8 admin_up_down;
+};
+
+/** \brief Dump all vmxnet3 interfaces
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define vmxnet3_dump
+{
+  u32 client_index;
+  u32 context;
+};
+
+/*
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/vmxnet3.c b/src/plugins/vmxnet3/vmxnet3.c
new file mode 100644
index 00000000000..74ef12619d2
--- /dev/null
+++ b/src/plugins/vmxnet3/vmxnet3.c
@@ -0,0 +1,612 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vppinfra/types.h>
+#include <vlib/vlib.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/plugin/plugin.h>
+#include <vpp/app/version.h>
+
+#include <vmxnet3/vmxnet3.h>
+
+#define PCI_VENDOR_ID_VMWARE				0x15ad
+#define PCI_DEVICE_ID_VMWARE_VMXNET3			0x07b0
+
+vmxnet3_main_t vmxnet3_main;
+
+static pci_device_id_t vmxnet3_pci_device_ids[] = {
+  {
+   .vendor_id = PCI_VENDOR_ID_VMWARE,
+   .device_id = PCI_DEVICE_ID_VMWARE_VMXNET3},
+  {0},
+};
+
+static clib_error_t *
+vmxnet3_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index,
+				 u32 flags)
+{
+  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_device_t *vd = vec_elt_at_index (vmxm->devices, hi->dev_instance);
+  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
+
+  if (vd->flags & VMXNET3_DEVICE_F_ERROR)
+    return clib_error_return (0, "device is in error state");
+
+  if (is_up)
+    {
+      vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
+				   VNET_HW_INTERFACE_FLAG_LINK_UP);
+      vd->flags |= VMXNET3_DEVICE_F_ADMIN_UP;
+    }
+  else
+    {
+      vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
+      vd->flags &= ~VMXNET3_DEVICE_F_ADMIN_UP;
+    }
+  return 0;
+}
+
+static clib_error_t *
+vmxnet3_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
+				  vnet_hw_interface_rx_mode mode)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
+  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
+
+  if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
+    rxq->int_mode = 0;
+  else
+    rxq->int_mode = 1;
+
+  return 0;
+}
+
+static char *vmxnet3_tx_func_error_strings[] = {
+#define _(n,s) s,
+  foreach_vmxnet3_tx_func_error
+#undef _
+};
+
+/* *INDENT-OFF* */
+VNET_DEVICE_CLASS (vmxnet3_device_class,) =
+{
+  .name = "VMXNET3 interface",
+  .format_device = format_vmxnet3_device,
+  .format_device_name = format_vmxnet3_device_name,
+  .admin_up_down_function = vmxnet3_interface_admin_up_down,
+  .rx_mode_change_function = vmxnet3_interface_rx_mode_change,
+  .tx_function_n_errors = VMXNET3_TX_N_ERROR,
+  .tx_function_error_strings = vmxnet3_tx_func_error_strings,
+};
+/* *INDENT-ON* */
+
+static u32
+vmxnet3_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
+{
+  return 0;
+}
+
+static void
+vmxnet3_write_mac (vmxnet3_device_t * vd)
+{
+  u32 val;
+
+  memcpy (&val, vd->mac_addr, 4);
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACL, val);
+
+  val = 0;
+  memcpy (&val, vd->mac_addr + 4, 2);
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACH, val);
+}
+
+static clib_error_t *
+vmxnet3_provision_driver_shared (vlib_main_t * vm, vmxnet3_device_t * vd)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_shared *shared;
+  vmxnet3_queues *q;
+  u64 shared_dma;
+  clib_error_t *error;
+  u16 qid = 0, rid;
+  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
+  vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, qid);
+
+  vd->dma = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
+					sizeof (*vd->dma), 512);
+  if (error)
+    return error;
+
+  memset (vd->dma, 0, sizeof (*vd->dma));
+
+  q = &vd->dma->queues;
+  q->tx.cfg.desc_address = vmxnet3_dma_addr (vm, vd, txq->tx_desc);
+  q->tx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, txq->tx_comp);
+  q->tx.cfg.num_desc = txq->size;
+  q->tx.cfg.num_comp = txq->size;
+  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
+    {
+      q->rx.cfg.desc_address[rid] = vmxnet3_dma_addr (vm, vd,
+						      rxq->rx_desc[rid]);
+      q->rx.cfg.num_desc[rid] = rxq->size;
+    }
+  q->rx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, rxq->rx_comp);
+  q->rx.cfg.num_comp = rxq->size;
+
+  shared = &vd->dma->shared;
+  shared->magic = VMXNET3_SHARED_MAGIC;
+  shared->misc.version = VMXNET3_VERSION_MAGIC;
+  if (sizeof (void *) == 4)
+    shared->misc.guest_info = VMXNET3_GOS_BITS_32;
+  else
+    shared->misc.guest_info = VMXNET3_GOS_BITS_64;
+  shared->misc.guest_info |= VMXNET3_GOS_TYPE_LINUX;
+  shared->misc.version_support = VMXNET3_VERSION_SELECT;
+  shared->misc.upt_version_support = VMXNET3_UPT_VERSION_SELECT;
+  shared->misc.queue_desc_address = vmxnet3_dma_addr (vm, vd, q);
+  shared->misc.queue_desc_len = sizeof (*q);
+  shared->misc.mtu = VMXNET3_MTU;
+  shared->misc.num_tx_queues = vd->num_tx_queues;
+  shared->misc.num_rx_queues = vd->num_rx_queues;
+  shared->interrupt.num_intrs = vd->num_intrs;
+  shared->interrupt.control = VMXNET3_IC_DISABLE_ALL;
+  shared->rx_filter.mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST |
+    VMXNET3_RXMODE_ALL_MULTI;
+  shared_dma = vmxnet3_dma_addr (vm, vd, shared);
+
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAL, shared_dma);
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAH, shared_dma >> 32);
+
+  return 0;
+}
+
+static inline void
+vmxnet3_enable_interrupt (vmxnet3_device_t * vd)
+{
+  int i;
+  vmxnet3_shared *shared = &vd->dma->shared;
+
+  shared->interrupt.control &= ~VMXNET3_IC_DISABLE_ALL;
+  for (i = 0; i < vd->num_intrs; i++)
+    vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 0);
+}
+
+static inline void
+vmxnet3_disable_interrupt (vmxnet3_device_t * vd)
+{
+  int i;
+  vmxnet3_shared *shared = &vd->dma->shared;
+
+  shared->interrupt.control |= VMXNET3_IC_DISABLE_ALL;
+  for (i = 0; i < vd->num_intrs; i++)
+    vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 1);
+}
+
+static clib_error_t *
+vmxnet3_rxq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_rxq_t *rxq;
+  clib_error_t *error;
+  u16 rid;
+
+  vec_validate_aligned (vd->rxqs, qid, CLIB_CACHE_LINE_BYTES);
+  rxq = vec_elt_at_index (vd->rxqs, qid);
+  memset (rxq, 0, sizeof (*rxq));
+  rxq->size = qsz;
+  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
+    {
+      rxq->rx_desc[rid] =
+	vlib_physmem_alloc_aligned (vm, vmxm->physmem_region,
+				    &error, qsz * sizeof (*rxq->rx_desc[rid]),
+				    512);
+      if (error)
+	return error;
+      memset (rxq->rx_desc[rid], 0, qsz * sizeof (*rxq->rx_desc[rid]));
+    }
+  rxq->rx_comp = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
+					     qsz * sizeof (*rxq->rx_comp),
+					     512);
+  if (error)
+    return error;
+  memset (rxq->rx_comp, 0, qsz * sizeof (*rxq->rx_comp));
+  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
+    {
+      vmxnet3_rx_ring *ring;
+
+      ring = &rxq->rx_ring[rid];
+      ring->gen = VMXNET3_RXF_GEN;
+      vec_validate_aligned (ring->bufs, rxq->size, CLIB_CACHE_LINE_BYTES);
+    }
+  rxq->rx_comp_ring.gen = VMXNET3_RXCF_GEN;
+
+  return 0;
+}
+
+static clib_error_t *
+vmxnet3_txq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_txq_t *txq;
+  clib_error_t *error;
+
+  if (qid >= vd->num_tx_queues)
+    {
+      qid = qid % vd->num_tx_queues;
+      txq = vec_elt_at_index (vd->txqs, qid);
+      if (txq->lock == 0)
+	clib_spinlock_init (&txq->lock);
+      vd->flags |= VMXNET3_DEVICE_F_SHARED_TXQ_LOCK;
+    }
+
+  vec_validate_aligned (vd->txqs, qid, CLIB_CACHE_LINE_BYTES);
+  txq = vec_elt_at_index (vd->txqs, qid);
+  memset (txq, 0, sizeof (*txq));
+  txq->size = qsz;
+  txq->tx_desc = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
+					     qsz * sizeof (*txq->tx_desc),
+					     512);
+  if (error)
+    return error;
+  memset (txq->tx_desc, 0, qsz * sizeof (*txq->tx_desc));
+  txq->tx_comp = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
+					     qsz * sizeof (*txq->tx_comp),
+					     512);
+  if (error)
+    return error;
+  memset (txq->tx_comp, 0, qsz * sizeof (*txq->tx_comp));
+  vec_validate_aligned (txq->tx_ring.bufs, txq->size, CLIB_CACHE_LINE_BYTES);
+  txq->tx_ring.gen = VMXNET3_TXF_GEN;
+  txq->tx_comp_ring.gen = VMXNET3_TXCF_GEN;
+
+  return 0;
+}
+
+static clib_error_t *
+vmxnet3_device_init (vlib_main_t * vm, vmxnet3_device_t * vd,
+		     vmxnet3_create_if_args_t * args)
+{
+  clib_error_t *error = 0;
+  u32 ret, i;
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+
+  vd->num_tx_queues = 1;
+  vd->num_rx_queues = 1;
+  vd->num_intrs = 1;
+
+  /* Quiesce the device */
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
+  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
+  if (ret != 0)
+    {
+      error = clib_error_return (0, "error on quisecing device rc (%u)", ret);
+      return error;
+    }
+
+  /* Reset the device */
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
+  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
+  if (ret != 0)
+    {
+      error = clib_error_return (0, "error on resetting device rc (%u)", ret);
+      return error;
+    }
+
+  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_VRRS);
+  vd->version = count_leading_zeros (ret);
+  vd->version = uword_bits - vd->version;
+
+  if (vd->version == 0 || vd->version > 3)
+    {
+      error = clib_error_return (0, "unsupported hardware version %u",
+				 vd->version);
+      return error;
+    }
+
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_VRRS, 1 << (vd->version - 1));
+
+  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_UVRS);
+  if (ret & 1)
+    vmxnet3_reg_write (vd, 1, VMXNET3_REG_UVRS, 1);
+  else
+    {
+      error = clib_error_return (0, "unsupported upt version %u", ret);
+      return error;
+    }
+
+  /* Get the mac address */
+  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACL);
+  clib_memcpy (vd->mac_addr, &ret, 4);
+  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACH);
+  clib_memcpy (vd->mac_addr + 4, &ret, 2);
+
+  if (vmxm->physmem_region_alloc == 0)
+    {
+      u32 flags = VLIB_PHYSMEM_F_INIT_MHEAP | VLIB_PHYSMEM_F_HUGETLB;
+      error =
+	vlib_physmem_region_alloc (vm, "vmxnet3 descriptors", 4 << 20, 0,
+				   flags, &vmxm->physmem_region);
+      if (error)
+	return error;
+      vmxm->physmem_region_alloc = 1;
+    }
+
+  error = vmxnet3_rxq_init (vm, vd, 0, args->rxq_size);
+  if (error)
+    return error;
+
+  for (i = 0; i < tm->n_vlib_mains; i++)
+    {
+      error = vmxnet3_txq_init (vm, vd, i, args->txq_size);
+      if (error)
+	return error;
+    }
+
+  error = vmxnet3_provision_driver_shared (vm, vd);
+  if (error)
+    return error;
+
+  vmxnet3_write_mac (vd);
+
+  /* Activate device */
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
+  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
+  if (ret != 0)
+    {
+      error =
+	clib_error_return (0, "error on activating device rc (%u)", ret);
+      return error;
+    }
+
+  /* Disable interrupts */
+  vmxnet3_disable_interrupt (vd);
+
+  vec_foreach_index (i, vd->rxqs)
+  {
+    vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
+
+    vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
+    vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
+  }
+  vd->flags |= VMXNET3_DEVICE_F_INITIALIZED;
+
+  vmxnet3_enable_interrupt (vd);
+
+  return error;
+}
+
+static void
+vmxnet3_irq_handler (vlib_pci_dev_handle_t h, u16 line)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  uword pd = vlib_pci_get_private_data (h);
+  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
+  u16 qid = line;
+
+  if (vec_len (vd->rxqs) > qid && vd->rxqs[qid].int_mode != 0)
+    vnet_device_input_set_interrupt_pending (vnm, vd->hw_if_index, qid);
+}
+
+static u8
+vmxnet3_queue_size_valid (u16 qsz)
+{
+  if (qsz < 64 || qsz > 4096)
+    return 0;
+  if ((qsz % 64) != 0)
+    return 0;
+  return 1;
+}
+
+void
+vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vmxnet3_device_t *vd;
+  vlib_pci_dev_handle_t h;
+  clib_error_t *error = 0;
+
+  if (args->rxq_size == 0)
+    args->rxq_size = VMXNET3_NUM_RX_DESC;
+  if (args->txq_size == 0)
+    args->txq_size = VMXNET3_NUM_TX_DESC;
+
+  if (!vmxnet3_queue_size_valid (args->rxq_size) ||
+      !vmxnet3_queue_size_valid (args->txq_size))
+    {
+      args->rv = VNET_API_ERROR_INVALID_VALUE;
+      args->error =
+	clib_error_return (error,
+			   "queue size must be <= 4096, >= 64, "
+			   "and multiples of 64");
+      return;
+    }
+
+  /* *INDENT-OFF* */
+  pool_foreach (vd, vmxm->devices, ({
+    if (vd->pci_addr.as_u32 == args->addr.as_u32)
+      {
+	args->rv = VNET_API_ERROR_INVALID_VALUE;
+	args->error =
+	  clib_error_return (error, "PCI address in use");
+	return;
+      }
+  }));
+  /* *INDENT-ON* */
+
+  pool_get (vmxm->devices, vd);
+  vd->dev_instance = vd - vmxm->devices;
+  vd->per_interface_next_index = ~0;
+  vd->pci_addr = args->addr;
+
+  if (args->enable_elog)
+    vd->flags |= VMXNET3_DEVICE_F_ELOG;
+
+  if ((error =
+       vlib_pci_device_open (&args->addr, vmxnet3_pci_device_ids, &h)))
+    {
+      pool_put (vmxm->devices, vd);
+      args->rv = VNET_API_ERROR_INVALID_INTERFACE;
+      args->error =
+	clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
+			   &args->addr);
+      return;
+    }
+  vd->pci_dev_handle = h;
+
+  vlib_pci_set_private_data (h, vd->dev_instance);
+
+  if ((error = vlib_pci_bus_master_enable (h)))
+    goto error;
+
+  if ((error = vlib_pci_map_region (h, 0, (void **) &vd->bar[0])))
+    goto error;
+
+  if ((error = vlib_pci_map_region (h, 1, (void **) &vd->bar[1])))
+    goto error;
+
+  if ((error = vlib_pci_register_msix_handler (h, 0, 1,
+					       &vmxnet3_irq_handler)))
+    goto error;
+
+  if ((error = vlib_pci_enable_msix_irq (h, 0, 1)))
+    goto error;
+
+  if ((error = vlib_pci_intr_enable (h)))
+    goto error;
+
+  if ((error = vmxnet3_device_init (vm, vd, args)))
+    goto error;
+
+  /* create interface */
+  error = ethernet_register_interface (vnm, vmxnet3_device_class.index,
+				       vd->dev_instance, vd->mac_addr,
+				       &vd->hw_if_index, vmxnet3_flag_change);
+
+  if (error)
+    goto error;
+
+  vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vd->hw_if_index);
+  vd->sw_if_index = sw->sw_if_index;
+  args->sw_if_index = sw->sw_if_index;
+
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vd->hw_if_index);
+  hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
+  vnet_hw_interface_set_input_node (vnm, vd->hw_if_index,
+				    vmxnet3_input_node.index);
+  vnet_hw_interface_assign_rx_thread (vnm, vd->hw_if_index, 0, ~0);
+  return;
+
+error:
+  vmxnet3_delete_if (vm, vd);
+  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
+  args->error = error;
+}
+
+void
+vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * vd)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  u32 i, bi;
+  u16 desc_idx;
+
+  /* Quiesce the device */
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
+
+  /* Reset the device */
+  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
+
+  if (vd->hw_if_index)
+    {
+      vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
+      vnet_hw_interface_unassign_rx_thread (vnm, vd->hw_if_index, 0);
+      ethernet_delete_interface (vnm, vd->hw_if_index);
+    }
+
+  vlib_pci_device_close (vd->pci_dev_handle);
+
+  /* *INDENT-OFF* */
+  vec_foreach_index (i, vd->rxqs)
+    {
+      vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
+      u16 mask = rxq->size - 1;
+      u16 rid;
+
+      for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
+	{
+	  vmxnet3_rx_ring *ring;
+
+	  ring = &rxq->rx_ring[rid];
+	  desc_idx = ring->consume;
+	  while (ring->fill)
+	    {
+	      desc_idx &= mask;
+	      bi = ring->bufs[desc_idx];
+	      vlib_buffer_free_no_next (vm, &bi, 1);
+	      ring->fill--;
+	      desc_idx++;
+	    }
+	  vec_free (ring->bufs);
+	  vlib_physmem_free (vm, vmxm->physmem_region, rxq->rx_desc[rid]);
+	}
+      vlib_physmem_free (vm, vmxm->physmem_region, rxq->rx_comp);
+    }
+  /* *INDENT-ON* */
+  vec_free (vd->rxqs);
+
+  /* *INDENT-OFF* */
+  vec_foreach_index (i, vd->txqs)
+    {
+      vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, i);
+      u16 mask = txq->size - 1;
+      u16 end_idx;
+
+      desc_idx = txq->tx_ring.consume;
+      end_idx = txq->tx_ring.produce;
+      while (desc_idx != end_idx)
+	{
+	  bi = txq->tx_ring.bufs[desc_idx];
+	  vlib_buffer_free_no_next (vm, &bi, 1);
+	  desc_idx++;
+	  desc_idx &= mask;
+	}
+      clib_spinlock_free (&txq->lock);
+      vec_free (txq->tx_ring.bufs);
+      vlib_physmem_free (vm, vmxm->physmem_region, txq->tx_desc);
+      vlib_physmem_free (vm, vmxm->physmem_region, txq->tx_comp);
+    }
+  /* *INDENT-ON* */
+  vec_free (vd->txqs);
+
+  vlib_physmem_free (vm, vmxm->physmem_region, vd->dma);
+
+  clib_error_free (vd->error);
+  memset (vd, 0, sizeof (*vd));
+  pool_put (vmxm->devices, vd);
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/vmxnet3.h b/src/plugins/vmxnet3/vmxnet3.h
new file mode 100644
index 00000000000..13799407d1e
--- /dev/null
+++ b/src/plugins/vmxnet3/vmxnet3.h
@@ -0,0 +1,639 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __included_vmnet_vmnet_h__
+#define __included_vmnet_vmnet_h__
+
+#define foreach_vmxnet3_tx_func_error	       \
+  _(ERROR_PACKETS, "error packets") \
+  _(NO_FREE_SLOTS, "no free tx slots")
+
+typedef enum
+{
+#define _(f,s) VMXNET3_TX_ERROR_##f,
+  foreach_vmxnet3_tx_func_error
+#undef _
+    VMXNET3_TX_N_ERROR,
+} vmxnet3_tx_func_error_t;
+
+#define foreach_vmxnet3_rxmode_flags \
+  _(0, UCAST, "unicast") \
+  _(1, MCAST, "multicast")		   \
+  _(2, BCAST, "broadcast") \
+  _(3, ALL_MULTI, "all multicast") \
+  _(4, PROMISC, "promiscuous")
+
+enum
+{
+#define _(a, b, c) VMXNET3_RXMODE_##b = (1 << a),
+  foreach_vmxnet3_rxmode_flags
+#undef _
+};
+
+/* BAR 0 */
+#define VMXNET3_REG_IMR     0x0000	/* Interrupt Mask Register */
+#define VMXNET3_REG_TXPROD  0x0600	/* Tx Producer Index */
+#define VMXNET3_REG_RXPROD  0x0800	/* Rx Producer Index for ring 1 */
+#define VMXNET3_REG_RXPROD2 0x0A00	/* Rx Producer Index for ring 2 */
+
+
+/* BAR 1 */
+#define VMXNET3_REG_VRRS 0x0000	/* VMXNET3 Revision Report Selection */
+#define VMXNET3_REG_UVRS 0x0008	/* UPT Version Report Selection */
+#define VMXNET3_REG_DSAL 0x0010	/* Driver Shared Address Low */
+#define VMXNET3_REG_DSAH 0x0018	/* Driver Shared Address High */
+#define VMXNET3_REG_CMD  0x0020	/* Command */
+#define VMXNET3_REG_MACL 0x0028	/* MAC Address Low */
+#define VMXNET3_REG_MACH 0x0030	/* MAC Address High */
+#define VMXNET3_REG_ICR  0x0038	/* Interrupt Cause Register */
+#define VMXNET3_REG_ECR  0x0040	/* Event Cause Register */
+
+#define VMXNET3_VLAN_LEN 4
+#define VMXNET3_FCS_LEN  4
+#define VMXNET3_MTU (1514 + VMXNET3_VLAN_LEN + VMXNET3_FCS_LEN)
+
+#define VMXNET3_RXF_BTYPE (1 << 14)	/* rx body buffer type */
+#define VMXNET3_RXF_GEN   (1 << 31)	/* rx generation */
+#define VMXNET3_RXCF_GEN  (1 << 31)	/* rx completion generation */
+#define VMXNET3_RXC_INDEX (0xFFF)	/* rx completion index mask */
+
+#define VMXNET3_TXF_GEN  (1 << 14)	/* tx generation */
+#define VMXNET3_TXF_EOP  (1 << 12)	/* tx end of packet */
+#define VMXNET3_TXF_CQ   (1 << 13)	/* tx completion request */
+#define VMXNET3_TXCF_GEN (1 << 31)	/* tx completion generation */
+#define VMXNET3_TXC_INDEX (0xFFF)	/* tx completion index mask */
+
+#define VMXNET3_RX_RING_SIZE 2
+#define VMXNET3_INPUT_REFILL_THRESHOLD 32
+#define VMXNET3_NUM_TX_DESC 1024
+#define VMXNET3_NUM_TX_COMP VMXNET3_NUM_TX_DESC
+#define VMXNET3_NUM_RX_DESC 1024
+#define VMXNET3_NUM_RX_COMP VMXNET3_NUM_RX_DESC
+
+#define VMXNET3_VERSION_MAGIC 0x69505845
+#define VMXNET3_SHARED_MAGIC  0xbabefee1
+#define VMXNET3_VERSION_SELECT     1
+#define VMXNET3_UPT_VERSION_SELECT 1
+#define VMXNET3_MAX_INTRS          25
+#define VMXNET3_IC_DISABLE_ALL     0x1
+
+#define VMXNET3_GOS_BITS_32     (1 << 0)
+#define VMXNET3_GOS_BITS_64     (2 << 0)
+#define VMXNET3_GOS_TYPE_LINUX  (1 << 2)
+#define VMXNET3_RXCL_LEN_MASK   (0x3FFF)	// 14 bits
+#define VMXNET3_RXCL_ERROR      (1 << 14)
+#define VMXNET3_RXCI_EOP        (1 << 14)
+#define VMXNET3_RXCI_SOP        (1 << 15)
+
+#define foreach_vmxnet3_device_flags \
+  _(0, INITIALIZED, "initialized") \
+  _(1, ERROR, "error")		   \
+  _(2, ADMIN_UP, "admin-up") \
+  _(3, IOVA, "iova") \
+  _(4, LINK_UP, "link-up") \
+  _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \
+  _(6, ELOG, "elog")
+
+enum
+{
+#define _(a, b, c) VMXNET3_DEVICE_F_##b = (1 << a),
+  foreach_vmxnet3_device_flags
+#undef _
+};
+
+#define foreach_vmxnet3_set_cmds \
+  _(0, ACTIVATE_DEV, "activate device") \
+  _(1, QUIESCE_DEV, "quiesce device") \
+  _(2, RESET_DEV, "reset device") \
+  _(3, UPDATE_RX_MODE, "update rx mode") \
+  _(4, UPDATE_MAC_FILTERS, "update mac filters") \
+  _(5, UPDATE_VLAN_FILTERS, "update vlan filters") \
+  _(6, UPDATE_RSSIDT, "update rss idt") \
+  _(7, UPDATE_IML, "update iml") \
+  _(8, UPDATE_PMCFG, "update pm cfg") \
+  _(9, UPDATE_FEATURE, "update feature") \
+  _(10, STOP_EMULATION, "stop emulation") \
+  _(11, LOAD_PLUGIN, "load plugin") \
+  _(12, ACTIVATE_VF, "activate vf") \
+  _(13, RESERVED3, "reserved 3") \
+  _(14, RESERVED4, "reservced 4") \
+  _(15, REGISTER_MEMREGS, "register mem regs")
+
+enum
+{
+#define _(a, b, c) VMXNET3_CMD_##b = (a + 0xCAFE0000),
+  foreach_vmxnet3_set_cmds
+#undef _
+};
+
+#define foreach_vmxnet3_get_cmds \
+  _(0, GET_QUEUE_STATUS, "get queue status") \
+  _(1, GET_STATS, "get stats") \
+  _(2, GET_LINK, "get link") \
+  _(3, GET_PERM_MAC_LO, "get perm mac lo") \
+  _(4, GET_PERM_MAC_HI, "get perm mac hi") \
+  _(5, GET_DID_LO, "get did lo") \
+  _(6, GET_DID_HI, "get did hi") \
+  _(7, GET_DEV_EXTRA_INFO, "get dev extra info") \
+  _(8, GET_CONF_INTR, "get conf intr") \
+  _(9, GET_ADAPTIVE_RING_INFO, "get adaptive ring info") \
+  _(10, GET_TXDATA_DESC_SIZE, "gte txdata desc size") \
+  _(11, RESERVED5, "reserved5")
+
+enum
+{
+#define _(a, b, c) VMXNET3_CMD_##b = (a + 0xF00D0000),
+  foreach_vmxnet3_get_cmds
+#undef _
+};
+
+typedef CLIB_PACKED (struct
+		     {
+		     u32 version; u32 guest_info; u32 version_support;
+		     u32 upt_version_support; u64 upt_features;
+		     u64 driver_data_address; u64 queue_desc_address;
+		     u32 driver_data_len; u32 queue_desc_len;
+		     u32 mtu;
+		     u16 max_num_rx_sg; u8 num_tx_queues; u8 num_rx_queues;
+		     u32 pad[4];
+		     }) vmxnet3_misc_config;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u8 mask_mode;
+		     u8 num_intrs;
+		     u8 event_intr_index;
+		     u8 moderation_level[VMXNET3_MAX_INTRS]; u32 control;
+		     u32 pad[2];
+		     }) vmxnet3_interrupt_config;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u32 mode;
+		     u16 multicast_len;
+		     u16 pad; u64 multicast_address; u8 vlan_filter[512];
+		     }) vmxnet3_rx_filter_config;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u32 version; u32 length;
+		     u64 address;
+		     }) vmxnet3_variable_config;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u32 magic;
+		     u32 pad;
+		     vmxnet3_misc_config misc;
+		     vmxnet3_interrupt_config interrupt;
+		     vmxnet3_rx_filter_config rx_filter;
+		     vmxnet3_variable_config rss;
+		     vmxnet3_variable_config pattern;
+		     vmxnet3_variable_config plugin; u32 ecr;
+		     u32 pad1[5];
+		     }) vmxnet3_shared;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u8 stopped;
+		     u8 pad[3];
+		     u32 error;
+		     }) vmxnet3_queue_status;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u32 num_deferred; u32 threshold;
+		     u64 pad;
+		     }) vmxnet3_tx_queue_control;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u64 desc_address;
+		     u64 data_address;
+		     u64 comp_address; u64 driver_data_address; u64 pad;
+		     u32 num_desc;
+		     u32 num_data;
+		     u32 num_comp; u32 driver_data_len; u8 intr_index;
+		     u8 pad1[7];
+		     }) vmxnet3_tx_queue_config;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u64 tso_pkts;
+		     u64 tso_bytes;
+		     u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
+		     u64 mcast_bytes;
+		     u64 bcast_pkts; u64 bcast_bytes; u64 error_pkts;
+		     u64 discard_pkts;
+		     }) vmxnet3_tx_stats;
+
+typedef CLIB_PACKED (struct
+		     {
+		     vmxnet3_tx_queue_control ctrl;
+		     vmxnet3_tx_queue_config cfg;
+		     vmxnet3_queue_status status; vmxnet3_tx_stats stats;
+		     u8 pad[88];
+		     }) vmxnet3_tx_queue;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u8 update_prod; u8 pad[7];
+		     u64 pad1;
+		     }) vmxnet3_rx_queue_control;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u64 desc_address[2];
+		     u64 comp_address; u64 driver_data_address; u64 pad;
+		     u32 num_desc[2];
+		     u32 num_comp; u32 driver_data_len; u8 intr_index;
+		     u8 pad1[7];
+		     }) vmxnet3_rx_queue_config;
+
+typedef CLIB_PACKED (struct
+		     {
+		     u64 lro_pkts;
+		     u64 lro_bytes;
+		     u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
+		     u64 mcast_bytes;
+		     u64 bcast_pkts; u64 bcast_bytes; u64 nobuf_pkts;
+		     u64 error_pkts;
+		     }) vmxnet3_rx_stats;
+
+typedef CLIB_PACKED (struct
+		     {
+		     vmxnet3_rx_queue_control ctrl;
+		     vmxnet3_rx_queue_config cfg;
+		     vmxnet3_queue_status status; vmxnet3_rx_stats stats;
+		     u8 pad[88];
+		     }) vmxnet3_rx_queue;
+
+typedef CLIB_PACKED (struct
+		     {
+		     vmxnet3_tx_queue tx; vmxnet3_rx_queue rx;
+		     }) vmxnet3_queues;
+
+/*
+ * flags:
+ *   buffer length   -- bits 0-13
+ *   buffer type     -- bit  14
+ *   descriptor type -- bit  15
+ *   reserved        -- bits 16-30
+ *   generation      -- bit  31
+ */
+typedef CLIB_PACKED (struct
+		     {
+		     u64 address;
+		     u32 flags;
+		     u32 pad;
+		     }) vmxnet3_rx_desc;
+
+/*
+ * index:
+ *   RX desc index           -- bits 0-11
+ *   ext1                    -- bits 12-13
+ *   end of packet           -- bit  14
+ *   start of packet         -- bit  15
+ *   ring ID                 -- bits 16-25
+ *   RSS hash type           -- bits 26-29
+ *   checksum not calculated -- bit  30
+ *   ext2                    -- bit  31
+ *
+ * rss: RSS hash value
+ *
+ * len:
+ *   data length             -- bits 0-13
+ *   error                   -- bit  14
+ *   tag is stripped         -- bit  15
+ *   tag stripped            -- bits 16-31
+ *
+ * flags:
+ *   checksum                -- bits 0 - 15
+ *   tcp/udp checksum correct-- bit  16
+ *   udp packet              -- bit  17
+ *   tcp packet              -- bit  18
+ *   ip checksum correct     -- bit  19
+ *   ipv6                    -- bit  20
+ *   ipv4                    -- bit  21
+ *   ip fragment             -- bit  22
+ *   frame crc correct       -- bit  23
+ *   completion type         -- bits 24-30
+ *   generation              -- bit  31
+ */
+typedef CLIB_PACKED (struct
+		     {
+		     u32 index; u32 rss;
+		     u32 len;
+		     u32 flags;
+		     }) vmxnet3_rx_comp;
+
+/*
+ * index:
+ *   TX desc index           -- bits 0-11
+ *   ext1                    -- bits 12-31
+ *
+ * flags:
+ *   reserved                -- bits 0-23
+ *   completion type         -- bits 24-30
+ *   generation              -- bit  31
+ */
+typedef CLIB_PACKED (struct
+		     {
+		     u32 index;
+		     u32 pad[2];
+		     u32 flags;
+		     }) vmxnet3_tx_comp;
+
+/*
+ * flags[0]:
+ *   length                  -- bits 0-13
+ *   generation              -- bit  14
+ *   reserved                -- bit  15
+ *   descriptor type         -- bit  16
+ *   ext1                    -- bit  17
+ *   MSS, checksum offset    -- bits 18-31
+ * flags[1]:
+ *   header length           -- bits 0-9
+ *   offload mode            -- bits 10-11
+ *   end of packet           -- bit  12
+ *   completion request      -- bit  13
+ *   ext2                    -- bit  14
+ *   vlan tag insertion      -- bit  15
+ *   tag to insert           -- bits 16-31
+ */
+typedef CLIB_PACKED (struct
+		     {
+		     u64 address;
+		     u32 flags[2];
+		     }) vmxnet3_tx_desc;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u32 *bufs;
+  u32 gen;
+  u16 fill;
+  u16 rid;
+  u16 produce;
+  u16 consume;
+} vmxnet3_rx_ring;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u64 next;
+  u32 gen;
+} vmxnet3_rx_comp_ring;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u16 size;
+  u8 int_mode;
+  vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE];
+  vmxnet3_rx_desc *rx_desc[VMXNET3_RX_RING_SIZE];
+  vmxnet3_rx_comp *rx_comp;
+  vmxnet3_rx_comp_ring rx_comp_ring;
+} vmxnet3_rxq_t;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u32 *bufs;
+  u32 gen;
+  u16 produce;
+  u16 consume;
+} vmxnet3_tx_ring;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u64 next;
+  u32 gen;
+} vmxnet3_tx_comp_ring;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u16 size;
+  clib_spinlock_t lock;
+
+  vmxnet3_tx_desc *tx_desc;
+  vmxnet3_tx_comp *tx_comp;
+  vmxnet3_tx_ring tx_ring;
+  vmxnet3_tx_comp_ring tx_comp_ring;
+} vmxnet3_txq_t;
+
+typedef CLIB_PACKED (struct
+		     {
+		     vmxnet3_queues queues; vmxnet3_shared shared;
+		     }) vmxnet3_dma;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u32 flags;
+  u32 per_interface_next_index;
+
+  u32 dev_instance;
+  u32 sw_if_index;
+  u32 hw_if_index;
+  vlib_pci_dev_handle_t pci_dev_handle;
+  vlib_pci_addr_t pci_addr;
+  void *bar[2];
+
+  /* queues */
+  vmxnet3_rxq_t *rxqs;
+  vmxnet3_txq_t *txqs;
+
+  u16 num_tx_queues;
+  u16 num_rx_queues;
+  u16 num_intrs;
+
+  u8 version;
+  u8 mac_addr[6];
+
+  /* error */
+  clib_error_t *error;
+
+  vmxnet3_dma *dma;
+
+} vmxnet3_device_t;
+
+typedef struct
+{
+  vmxnet3_device_t *devices;
+  vlib_physmem_region_index_t physmem_region;
+  u32 physmem_region_alloc;
+  u16 msg_id_base;
+} vmxnet3_main_t;
+
+extern vmxnet3_main_t vmxnet3_main;
+
+typedef struct
+{
+  vlib_pci_addr_t addr;
+  u32 enable_elog;
+  u16 rxq_size;
+  u16 txq_size;
+  /* return */
+  i32 rv;
+  u32 sw_if_index;
+  clib_error_t *error;
+} vmxnet3_create_if_args_t;
+
+typedef struct
+{
+  u32 next_index;
+  u32 hw_if_index;
+  vlib_buffer_t buffer;
+} vmxnet3_input_trace_t;
+
+void vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args);
+void vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * ad);
+
+extern clib_error_t *vmxnet3_plugin_api_hookup (vlib_main_t * vm);
+extern vlib_node_registration_t vmxnet3_input_node;
+extern vnet_device_class_t vmxnet3_device_class;
+
+/* format.c */
+format_function_t format_vmxnet3_device;
+format_function_t format_vmxnet3_device_name;
+format_function_t format_vmxnet3_input_trace;
+
+static_always_inline void
+vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
+{
+  *(volatile u32 *) ((u8 *) vd->bar[bar] + addr) = val;
+}
+
+static_always_inline u32
+vmxnet3_reg_read (vmxnet3_device_t * vd, u8 bar, u32 addr)
+{
+  return *(volatile u32 *) (vd->bar[bar] + addr);
+}
+
+static_always_inline uword
+vmxnet3_dma_addr (vlib_main_t * vm, vmxnet3_device_t * vd, void *p)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+
+  return (vd->flags & VMXNET3_DEVICE_F_IOVA) ? pointer_to_uword (p) :
+    vlib_physmem_virtual_to_physical (vm, vmxm->physmem_region, p);
+}
+
+static_always_inline void
+vmxnet3_rx_ring_advance_produce (vmxnet3_rxq_t * rxq, vmxnet3_rx_ring * ring)
+{
+  ring->produce++;
+  if (PREDICT_FALSE (ring->produce == rxq->size))
+    {
+      ring->produce = 0;
+      ring->gen ^= VMXNET3_RXF_GEN;
+    }
+}
+
+static_always_inline clib_error_t *
+vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd,
+			  vmxnet3_rxq_t * rxq)
+{
+  vmxnet3_rx_desc *rxd;
+  u16 n_refill, n_alloc;
+  vmxnet3_rx_ring *ring;
+
+  ring = &rxq->rx_ring[0];
+  n_refill = rxq->size - ring->fill;
+
+  if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
+    return 0;
+
+  n_alloc =
+    vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size,
+			       n_refill);
+  if (PREDICT_FALSE (n_alloc != n_refill))
+    {
+      if (n_alloc)
+	vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
+				    n_alloc);
+      return clib_error_return (0, "buffer alloc failed");
+    }
+
+  while (n_alloc)
+    {
+      rxd = &rxq->rx_desc[0][ring->produce];
+      rxd->address =
+	vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]);
+      rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE;
+
+      vmxnet3_rx_ring_advance_produce (rxq, ring);
+      ring->fill++;
+      n_alloc--;
+    }
+
+  vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD, ring->produce);
+
+  return 0;
+}
+
+static_always_inline clib_error_t *
+vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd,
+			  vmxnet3_rxq_t * rxq)
+{
+  vmxnet3_rx_desc *rxd;
+  u16 n_refill, n_alloc;
+  vmxnet3_rx_ring *ring;
+
+  ring = &rxq->rx_ring[1];
+  n_refill = rxq->size - ring->fill;
+
+  if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
+    return 0;
+
+  n_alloc =
+    vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size,
+			       n_refill);
+  if (PREDICT_FALSE (n_alloc != n_refill))
+    {
+      if (n_alloc)
+	vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
+				    n_alloc);
+      return clib_error_return (0, "buffer alloc failed");
+    }
+
+  while (n_alloc)
+    {
+      rxd = &rxq->rx_desc[1][ring->produce];
+      rxd->address =
+	vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]);
+      rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE | VMXNET3_RXF_BTYPE;
+
+      vmxnet3_rx_ring_advance_produce (rxq, ring);
+      ring->fill++;
+      n_alloc--;
+    }
+
+  vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD2, ring->produce);
+
+  return 0;
+}
+
+#endif /* __included_vmnet_vmnet_h__ */
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/vmxnet3_all_api_h.h b/src/plugins/vmxnet3/vmxnet3_all_api_h.h
new file mode 100644
index 00000000000..7dee01bf417
--- /dev/null
+++ b/src/plugins/vmxnet3/vmxnet3_all_api_h.h
@@ -0,0 +1,26 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vmxnet3/vmxnet3.api.h>
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/vmxnet3_api.c b/src/plugins/vmxnet3/vmxnet3_api.c
new file mode 100644
index 00000000000..e0c2cfa6cf0
--- /dev/null
+++ b/src/plugins/vmxnet3/vmxnet3_api.c
@@ -0,0 +1,252 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+
+#include <vmxnet3/vmxnet3.h>
+
+#include <vlibapi/api.h>
+#include <vlibmemory/api.h>
+
+/* define message IDs */
+#include <vmxnet3/vmxnet3_msg_enum.h>
+
+/* define message structures */
+#define vl_typedefs
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_printfun
+
+/* get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_api_version
+
+#include <vlibapi/api_helper_macros.h>
+
+#define foreach_vmxnet3_plugin_api_msg	\
+_(VMXNET3_CREATE, vmxnet3_create)	\
+_(VMXNET3_DELETE, vmxnet3_delete)       \
+_(VMXNET3_DUMP, vmxnet3_dump)
+
+static void
+vl_api_vmxnet3_create_t_handler (vl_api_vmxnet3_create_t * mp)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vl_api_vmxnet3_create_reply_t *rmp;
+  vmxnet3_create_if_args_t args;
+  int rv;
+
+  memset (&args, 0, sizeof (vmxnet3_create_if_args_t));
+
+  args.enable_elog = ntohl (mp->enable_elog);
+  args.addr.as_u32 = ntohl (mp->pci_addr);
+  args.rxq_size = ntohs (mp->rxq_size);
+  args.txq_size = ntohs (mp->txq_size);
+
+  vmxnet3_create_if (vm, &args);
+  rv = args.rv;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_VMXNET3_CREATE_REPLY + vmxm->msg_id_base,
+    ({
+      rmp->sw_if_index = ntohl (args.sw_if_index);
+    }));
+  /* *INDENT-ON* */
+}
+
+static void
+vl_api_vmxnet3_delete_t_handler (vl_api_vmxnet3_delete_t * mp)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vnet_main_t *vnm = vnet_get_main ();
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vl_api_vmxnet3_delete_reply_t *rmp;
+  vmxnet3_device_t *vd;
+  vnet_hw_interface_t *hw;
+  int rv = 0;
+
+  hw = vnet_get_sup_hw_interface (vnm, htonl (mp->sw_if_index));
+  if (hw == NULL || vmxnet3_device_class.index != hw->dev_class_index)
+    {
+      rv = VNET_API_ERROR_INVALID_INTERFACE;
+      goto reply;
+    }
+
+  vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
+
+  vmxnet3_delete_if (vm, vd);
+
+reply:
+  REPLY_MACRO (VL_API_VMXNET3_DELETE_REPLY + vmxm->msg_id_base);
+}
+
+static void
+send_vmxnet3_details (vl_api_registration_t * reg, vmxnet3_device_t * vd,
+		      u16 rx_qid, vmxnet3_rxq_t * rxq, u16 tx_qid,
+		      vmxnet3_txq_t * txq, vnet_sw_interface_t * swif,
+		      u8 * interface_name, u32 context)
+{
+  vl_api_vmxnet3_details_t *mp;
+  vnet_main_t *vnm = vnet_get_main ();
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vnet_hw_interface_t *hwif;
+  vmxnet3_rx_ring *ring;
+  u16 rid;
+
+  hwif = vnet_get_sup_hw_interface (vnm, swif->sw_if_index);
+
+  mp = vl_msg_api_alloc (sizeof (*mp));
+  memset (mp, 0, sizeof (*mp));
+
+  mp->_vl_msg_id = htons (VL_API_VMXNET3_DETAILS + vmxm->msg_id_base);
+  mp->context = context;
+
+  mp->sw_if_index = htonl (swif->sw_if_index);
+  strncpy ((char *) mp->if_name,
+	   (char *) interface_name, ARRAY_LEN (mp->if_name) - 1);
+
+  if (hwif->hw_address)
+    memcpy (mp->hw_addr, hwif->hw_address, ARRAY_LEN (mp->hw_addr));
+
+  mp->version = vd->version;
+  mp->pci_addr = ntohl (vd->pci_addr.as_u32);
+
+  mp->rx_qsize = htons (rxq->size);
+  mp->rx_next = htons (rxq->rx_comp_ring.next);
+  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
+    {
+      ring = &rxq->rx_ring[rid];
+      mp->rx_fill[rid] = htons (ring->fill);
+      mp->rx_produce[rid] = htons (ring->produce);
+      mp->rx_consume[rid] = htons (ring->consume);
+    }
+  mp->tx_qsize = htons (txq->size);
+  mp->tx_next = htons (txq->tx_comp_ring.next);
+  mp->tx_produce = htons (txq->tx_ring.produce);
+  mp->tx_consume = htons (txq->tx_ring.consume);
+
+  mp->admin_up_down = (swif->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? 1 : 0;
+
+  vl_api_send_msg (reg, (u8 *) mp);
+}
+
+/**
+ * @brief Message handler for vmxnet3_dump API.
+ * @param mp vl_api_vmxnet3_dump_t * mp the api message
+ */
+static void
+vl_api_vmxnet3_dump_t_handler (vl_api_vmxnet3_dump_t * mp)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_sw_interface_t *swif;
+  vmxnet3_device_t *vd;
+  u8 *if_name = 0;
+  vl_api_registration_t *reg;
+  vmxnet3_rxq_t *rxq;
+  vmxnet3_txq_t *txq;
+  u16 qid = 0;
+
+  reg = vl_api_client_index_to_registration (mp->client_index);
+  if (!reg)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (vd, vmxm->devices,
+    ({
+      swif = vnet_get_sw_interface (vnm, vd->sw_if_index);
+      if_name = format (if_name, "%U%c", format_vnet_sw_interface_name, vnm,
+			swif, 0);
+      rxq = vec_elt_at_index (vd->rxqs, qid);
+      txq = vec_elt_at_index (vd->txqs, qid);
+      send_vmxnet3_details (reg, vd, qid, rxq, qid, txq, swif, if_name,
+			    mp->context);
+      _vec_len (if_name) = 0;
+    }));
+  /* *INDENT-ON* */
+
+  vec_free (if_name);
+}
+
+#define vl_msg_name_crc_list
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_msg_name_crc_list
+
+static void
+setup_message_id_table (vmxnet3_main_t * vmxm, api_main_t * am)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + vmxm->msg_id_base);
+  foreach_vl_msg_name_crc_vmxnet3;
+#undef _
+}
+
+/* set tup the API message handling tables */
+clib_error_t *
+vmxnet3_plugin_api_hookup (vlib_main_t * vm)
+{
+  vmxnet3_main_t *vmxm = &vmxnet3_main;
+  api_main_t *am = &api_main;
+  u8 *name;
+
+  /* construct the API name */
+  name = format (0, "vmxnet3_%08x%c", api_version, 0);
+
+  /* ask for a correctly-sized block of API message decode slots */
+  vmxm->msg_id_base = vl_msg_api_get_msg_ids
+    ((char *) name, VL_MSG_FIRST_AVAILABLE);
+
+#define _(N,n)							\
+    vl_msg_api_set_handlers((VL_API_##N + vmxm->msg_id_base),	\
+			   #n,					\
+			   vl_api_##n##_t_handler,		\
+			   vl_noop_handler,			\
+			   vl_api_##n##_t_endian,		\
+			   vl_api_##n##_t_print,		\
+			   sizeof(vl_api_##n##_t), 1);
+  foreach_vmxnet3_plugin_api_msg;
+#undef _
+
+  /* set up the (msg_name, crc, message-id) table */
+  setup_message_id_table (vmxm, am);
+
+  vec_free (name);
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/vmxnet3_msg_enum.h b/src/plugins/vmxnet3/vmxnet3_msg_enum.h
new file mode 100644
index 00000000000..3086fb3bfc4
--- /dev/null
+++ b/src/plugins/vmxnet3/vmxnet3_msg_enum.h
@@ -0,0 +1,39 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#ifndef _VMXNET3_MSG_ENUM_H_
+#define _VMXNET3_MSG_ENUM_H_
+
+#include <vppinfra/byte_order.h>
+
+#define vl_msg_id(n,h) n,
+typedef enum
+{
+#include <vmxnet3/vmxnet3_all_api_h.h>
+  VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* VMXNET3_MSG_ENUM_H */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/vmxnet3/vmxnet3_test.c b/src/plugins/vmxnet3/vmxnet3_test.c
new file mode 100644
index 00000000000..b08f61b0bd9
--- /dev/null
+++ b/src/plugins/vmxnet3/vmxnet3_test.c
@@ -0,0 +1,331 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vnet/ethernet/ethernet.h>
+
+#include <vat/vat.h>
+#include <vlibapi/api.h>
+#include <vlibmemory/api.h>
+
+#include <vppinfra/error.h>
+#include <vmxnet3/vmxnet3.h>
+
+#define __plugin_msg_base vmxnet3_test_main.msg_id_base
+#include <vlibapi/vat_helper_macros.h>
+
+/* declare message IDs */
+#include <vmxnet3/vmxnet3_msg_enum.h>
+
+/* Get CRC codes of the messages defined outside of this plugin */
+#define vl_msg_name_crc_list
+#include <vpp/api/vpe_all_api_h.h>
+#undef vl_msg_name_crc_list
+
+/* define message structures */
+#define vl_typedefs
+#include <vpp/api/vpe_all_api_h.h>
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_typedefs
+
+/* declare message handlers for each api */
+#define vl_endianfun
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vl_printfun
+
+/* get API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include <vmxnet3/vmxnet3_all_api_h.h>
+#undef vp_api_version
+
+typedef struct
+{
+  /* API message ID base */
+  u16 msg_id_base;
+  u32 ping_id;
+  vat_main_t *vat_main;
+} vmxnet3_test_main_t;
+
+vmxnet3_test_main_t vmxnet3_test_main;
+
+#define foreach_standard_reply_retval_handler		\
+_(vmxnet3_delete_reply)
+
+#define _(n)                                            \
+    static void vl_api_##n##_t_handler                  \
+    (vl_api_##n##_t * mp)                               \
+    {                                                   \
+        vat_main_t * vam = vmxnet3_test_main.vat_main; 	\
+        i32 retval = ntohl(mp->retval);                 \
+        if (vam->async_mode) {                          \
+            vam->async_errors += (retval < 0);          \
+        } else {                                        \
+            vam->retval = retval;                       \
+            vam->result_ready = 1;                      \
+        }                                               \
+    }
+foreach_standard_reply_retval_handler;
+#undef _
+
+#define foreach_vpe_api_reply_msg			\
+_(VMXNET3_CREATE_REPLY, vmxnet3_create_reply)		\
+_(VMXNET3_DELETE_REPLY, vmxnet3_delete_reply)           \
+_(VMXNET3_DETAILS, vmxnet3_details)
+
+/* vmxnet3 create API */
+static int
+api_vmxnet3_create (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_vmxnet3_create_t *mp;
+  vmxnet3_create_if_args_t args;
+  int ret;
+  u32 x[4];
+
+  memset (&args, 0, sizeof (vmxnet3_create_if_args_t));
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%x:%x:%x.%x", &x[0], &x[1], &x[2], &x[3]))
+	{
+	  args.addr.domain = x[0];
+	  args.addr.bus = x[1];
+	  args.addr.slot = x[2];
+	  args.addr.function = x[3];
+	}
+      else if (unformat (i, "elog"))
+	args.enable_elog = 1;
+      else if (unformat (i, "rx-queue-size %u", &args.rxq_size))
+	;
+      else if (unformat (i, "tx-queue-size %u", &args.txq_size))
+	;
+      else
+	{
+	  clib_warning ("unknown input '%U'", format_unformat_error, i);
+	  return -99;
+	}
+    }
+
+  M (VMXNET3_CREATE, mp);
+
+  mp->pci_addr = clib_host_to_net_u32 (args.addr.as_u32);
+  mp->enable_elog = clib_host_to_net_u16 (args.enable_elog);
+  mp->rxq_size = clib_host_to_net_u16 (args.rxq_size);
+  mp->txq_size = clib_host_to_net_u16 (args.txq_size);
+
+  S (mp);
+  W (ret);
+
+  return ret;
+}
+
+/* vmxnet3-create reply handler */
+static void
+vl_api_vmxnet3_create_reply_t_handler (vl_api_vmxnet3_create_reply_t * mp)
+{
+  vat_main_t *vam = vmxnet3_test_main.vat_main;
+  i32 retval = ntohl (mp->retval);
+
+  if (retval == 0)
+    {
+      fformat (vam->ofp, "created vmxnet3 with sw_if_index %d\n",
+	       ntohl (mp->sw_if_index));
+    }
+
+  vam->retval = retval;
+  vam->result_ready = 1;
+  vam->regenerate_interface_table = 1;
+}
+
+/* vmxnet3 delete API */
+static int
+api_vmxnet3_delete (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_vmxnet3_delete_t *mp;
+  u32 sw_if_index = 0;
+  u8 index_defined = 0;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "sw_if_index %u", &sw_if_index))
+	index_defined = 1;
+      else
+	{
+	  clib_warning ("unknown input '%U'", format_unformat_error, i);
+	  return -99;
+	}
+    }
+
+  if (!index_defined)
+    {
+      errmsg ("missing sw_if_index\n");
+      return -99;
+    }
+
+  M (VMXNET3_DELETE, mp);
+
+  mp->sw_if_index = clib_host_to_net_u32 (sw_if_index);
+
+  S (mp);
+  W (ret);
+
+  return ret;
+}
+
+static int
+api_vmxnet3_dump (vat_main_t * vam)
+{
+  vmxnet3_test_main_t *vxm = &vmxnet3_test_main;
+  vl_api_vmxnet3_dump_t *mp;
+  vl_api_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for vmxnet3_dump");
+      return -99;
+    }
+
+  M (VMXNET3_DUMP, mp);
+  S (mp);
+
+  /* Use a control ping for synchronization */
+  mp_ping = vl_msg_api_alloc_as_if_client (sizeof (*mp_ping));
+  mp_ping->_vl_msg_id = htons (vxm->ping_id);
+  mp_ping->client_index = vam->my_client_index;
+
+  fformat (vam->ofp, "Sending ping id=%d\n", vxm->ping_id);
+
+  vam->result_ready = 0;
+  S (mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static void
+vl_api_vmxnet3_details_t_handler (vl_api_vmxnet3_details_t * mp)
+{
+  vat_main_t *vam = vmxnet3_test_main.vat_main;
+  u32 pci_addr = ntohl (mp->pci_addr);
+
+  fformat (vam->ofp, "%s: sw_if_index %u mac %U\n"
+	   "   version: %u\n"
+	   "   PCI Address: %U\n"
+	   "   RX completion next index %u"
+	   "   RX Queue %u\n"
+	   "    ring 0 size %u fill %u consume %u produce %u\n"
+	   "    ring 1 size %u fill %u consume %u produce %u\n"
+	   "   TX completion next index %u"
+	   "   TX Queue %u\n"
+	   "    size %u consume %u produce %u\n"
+	   "   state %s\n",
+	   mp->if_name, ntohl (mp->sw_if_index), format_ethernet_address,
+	   mp->hw_addr, mp->version,
+	   format_vlib_pci_addr, &pci_addr,
+	   ntohs (mp->rx_next),
+	   ntohs (mp->rx_qid),
+	   ntohs (mp->rx_qsize), ntohs (mp->rx_fill[0]),
+	   ntohs (mp->rx_consume[0]),
+	   ntohs (mp->rx_produce[0]),
+	   ntohs (mp->rx_qsize), ntohs (mp->rx_fill[1]),
+	   ntohs (mp->rx_consume[1]),
+	   ntohs (mp->rx_produce[1]),
+	   ntohs (mp->tx_next),
+	   ntohs (mp->tx_qid),
+	   ntohs (mp->tx_qsize), ntohs (mp->tx_consume),
+	   ntohs (mp->tx_produce), mp->admin_up_down ? "up" : "down");
+}
+
+/*
+ * List of messages that the api test plugin sends,
+ * and that the data plane plugin processes
+ */
+#define foreach_vpe_api_msg					\
+_(vmxnet3_create, "<pci-address> [rx-queue-size <size>] "	\
+              "[tx-queue-size <size>]")				\
+_(vmxnet3_delete, "<sw_if_index>")                              \
+_(vmxnet3_dump, "")
+
+static void
+vmxnet3_vat_api_hookup (vat_main_t * vam)
+{
+  vmxnet3_test_main_t *vxm __attribute__ ((unused)) = &vmxnet3_test_main;
+#define _(N,n)                                                  \
+  vl_msg_api_set_handlers((VL_API_##N + vxm->msg_id_base),      \
+                          #n,                                   \
+                          vl_api_##n##_t_handler,               \
+                          vl_noop_handler,                      \
+                          vl_api_##n##_t_endian,                \
+                          vl_api_##n##_t_print,                 \
+                          sizeof(vl_api_##n##_t), 1);
+  foreach_vpe_api_reply_msg;
+#undef _
+
+#define _(n,h)							\
+  hash_set_mem (vam->function_by_name, #n, api_##n);
+  foreach_vpe_api_msg;
+#undef _
+
+#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
+  foreach_vpe_api_msg;
+#undef _
+}
+
+clib_error_t *
+vat_plugin_register (vat_main_t * vam)
+{
+  vmxnet3_test_main_t *vxm = &vmxnet3_test_main;
+  u8 *name;
+
+  vxm->vat_main = vam;
+
+  name = format (0, "vmxnet3_%08x%c", api_version, 0);
+  vxm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  /* Get the control ping ID */
+#define _(id,n,crc) \
+  const char *id ## _CRC __attribute__ ((unused)) = #n "_" #crc;
+  foreach_vl_msg_name_crc_vpe;
+#undef _
+  vxm->ping_id = vl_msg_api_get_msg_index ((u8 *) (VL_API_CONTROL_PING_CRC));
+
+  if (vxm->msg_id_base != (u16) ~ 0)
+    vmxnet3_vat_api_hookup (vam);
+
+  vec_free (name);
+
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */