aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorStephen Hemminger <stephen@networkplumber.org>2018-10-15 12:52:30 -0700
committerDamjan Marion <damarion@cisco.com>2018-12-19 08:34:39 +0100
commit6fbef2322870bbe1768537caecc3ca06bfd70dd7 (patch)
tree9c588f17b05d5bedf57b7207694d3e92136a4cbd
parentb0b9dadc5c57b96ed43427ca78430e52fed9196e (diff)
vlib: support Hyper-v/Azure VMBus
This patch adds support for VMBus to the VPP infrastructure. Since the only device that matters is the netvsc Poll Mode Driver in DPDK, the infrastructure is much simpler than PCI. Change-Id: Ie96c897ad9c426716c2398e4528688ce2217419b Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
-rw-r--r--Makefile5
-rw-r--r--src/plugins/dpdk/device/init.c32
-rw-r--r--src/vlib/CMakeLists.txt21
-rw-r--r--src/vlib/linux/vmbus.c405
-rw-r--r--src/vlib/vmbus/vmbus.c48
-rw-r--r--src/vlib/vmbus/vmbus.h45
6 files changed, 553 insertions, 3 deletions
diff --git a/Makefile b/Makefile
index 6bc871c39a8..9868d9ae0d3 100644
--- a/Makefile
+++ b/Makefile
@@ -66,7 +66,7 @@ DEB_DEPENDS += libconfuse-dev git-review exuberant-ctags cscope pkg-config
DEB_DEPENDS += lcov chrpath autoconf indent clang-format libnuma-dev
DEB_DEPENDS += python-all python-dev python-virtualenv python-pip libffi6 check
DEB_DEPENDS += libboost-all-dev libffi-dev python-ply libmbedtls-dev
-DEB_DEPENDS += cmake ninja-build
+DEB_DEPENDS += cmake ninja-build uuid-dev
ifeq ($(OS_VERSION_ID),14.04)
DEB_DEPENDS += openjdk-8-jdk-headless
DEB_DEPENDS += libssl-dev
@@ -89,6 +89,7 @@ RPM_DEPENDS += check check-devel
RPM_DEPENDS += boost boost-devel
RPM_DEPENDS += selinux-policy selinux-policy-devel
RPM_DEPENDS += ninja-build
+RPM_DEPENDS += libuuid-devel
ifeq ($(OS_ID),fedora)
RPM_DEPENDS += dnf-utils
@@ -119,7 +120,7 @@ RPM_SUSE_BUILDTOOLS_DEPS = autoconf automake ccache check-devel chrpath
RPM_SUSE_BUILDTOOLS_DEPS += clang cmake indent libtool make ninja python-ply
RPM_SUSE_DEVEL_DEPS = glibc-devel-static java-1_8_0-openjdk-devel libnuma-devel
-RPM_SUSE_DEVEL_DEPS += libopenssl-devel openssl-devel mbedtls-devel
+RPM_SUSE_DEVEL_DEPS += libopenssl-devel openssl-devel mbedtls-devel libuuid-devel
RPM_SUSE_PYTHON_DEPS = python-devel python3-devel python-pip python3-pip
RPM_SUSE_PYTHON_DEPS += python-rpm-macros python3-rpm-macros
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index c0a927acae5..c5c98ab3359 100644
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -24,6 +24,7 @@
#include <vnet/ethernet/ethernet.h>
#include <dpdk/device/dpdk.h>
#include <vlib/pci/pci.h>
+#include <vlib/vmbus/vmbus.h>
#include <rte_ring.h>
@@ -973,6 +974,27 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf)
vlib_pci_free_device_info (d);
}
+static void
+dpdk_bind_vmbus_devices_to_uio (dpdk_config_main_t * conf)
+{
+ clib_error_t *error;
+ vlib_vmbus_addr_t *addrs, *addr = 0;
+
+ addrs = vlib_vmbus_get_all_dev_addrs ();
+
+ /* *INDENT-OFF* */
+ vec_foreach (addr, addrs)
+ {
+ error = vlib_vmbus_bind_to_uio (addr);
+
+ if (error)
+ {
+ clib_error_report (error);
+ }
+ }
+ /* *INDENT-ON* */
+}
+
static clib_error_t *
dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
unformat_input_t * input, u8 is_default)
@@ -1121,6 +1143,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
int ret, i;
int num_whitelisted = 0;
u8 no_pci = 0;
+ u8 no_vmbus = 0;
u8 no_huge = 0;
u8 huge_dir = 0;
u8 file_prefix = 0;
@@ -1214,6 +1237,12 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
vec_add1 (conf->blacklist_by_pci_vendor_and_device,
blacklist_entry);
}
+ else if (unformat (input, "no-vmbus"))
+ {
+ no_vmbus = 1;
+ tmp = format (0, "--no-vmbus%c", 0);
+ vec_add1 (conf->eal_init_args, tmp);
+ }
#define _(a) \
else if (unformat(input, #a)) \
@@ -1417,6 +1446,9 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
if (no_pci == 0 && geteuid () == 0)
dpdk_bind_devices_to_uio (conf);
+ if (no_vmbus == 0 && geteuid () == 0)
+ dpdk_bind_vmbus_devices_to_uio (conf);
+
#define _(x) \
if (devconf->x == 0 && conf->default_devconf.x > 0) \
devconf->x = conf->default_devconf.x ;
diff --git a/src/vlib/CMakeLists.txt b/src/vlib/CMakeLists.txt
index 2a6cbd54da2..2209923547a 100644
--- a/src/vlib/CMakeLists.txt
+++ b/src/vlib/CMakeLists.txt
@@ -26,6 +26,22 @@ install(
)
##############################################################################
+# Find lib and include files
+##############################################################################
+message(STATUS "Looking for libuuid")
+find_path(UUID_INCLUDE_DIR NAMES uuid/uuid.h)
+find_library(UUID_LIB NAMES uuid)
+
+if(UUID_INCLUDE_DIR AND UUID_LIB)
+ include_directories(${UUID_INCLUDE_DIR})
+ set(VMBUS_SOURCE linux/vmbus.c)
+ set(VMBUS_LIBS uuid)
+ message(STATUS "Found uuid in ${UUID_INCLUDE_DIR}")
+else()
+ message(WARNING "-- libuuid not found - vmbus support disabled")
+endif()
+
+##############################################################################
# vlib shared library
##############################################################################
add_vpp_library(vlib
@@ -56,6 +72,8 @@ add_vpp_library(vlib
unix/main.c
unix/plugin.c
unix/util.c
+ vmbus/vmbus.c
+ ${VMBUS_SOURCE}
INSTALL_HEADERS
buffer_funcs.h
@@ -89,6 +107,7 @@ add_vpp_library(vlib
unix/plugin.h
unix/unix.h
vlib.h
+ vmbus/vmbus.h
- LINK_LIBRARIES vppinfra svm ${CMAKE_DL_LIBS}
+ LINK_LIBRARIES vppinfra svm ${VMBUS_LIBS} ${CMAKE_DL_LIBS}
)
diff --git a/src/vlib/linux/vmbus.c b/src/vlib/linux/vmbus.c
new file mode 100644
index 00000000000..2af62241d4d
--- /dev/null
+++ b/src/vlib/linux/vmbus.c
@@ -0,0 +1,405 @@
+/*
+ * Copyright (c) 2018, Microsoft Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * vmbus.c: Linux user space VMBus bus management.
+ */
+
+#include <vppinfra/linux/sysfs.h>
+
+#include <vlib/vlib.h>
+#include <vlib/vmbus/vmbus.h>
+#include <vlib/unix/unix.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/ioctl.h>
+#include <net/if.h>
+#include <linux/ethtool.h>
+#include <linux/sockios.h>
+
+#include <uuid/uuid.h>
+
+static const char sysfs_vmbus_dev_path[] = "/sys/bus/vmbus/devices";
+static const char sysfs_vmbus_drv_path[] = "/sys/bus/vmbus/drivers";
+static const char sysfs_class_net_path[] = "/sys/class/net";
+static const char uio_drv_name[] = "uio_hv_generic";
+static const char netvsc_uuid[] = "f8615163-df3e-46c5-913f-f2d2f965ed0e";
+
+typedef struct
+{
+ int fd;
+ void *addr;
+ size_t size;
+} linux_vmbus_region_t;
+
+typedef struct
+{
+ int fd;
+ u32 clib_file_index;
+} linux_vmbus_irq_t;
+
+typedef struct
+{
+ vlib_vmbus_dev_handle_t handle;
+ vlib_vmbus_addr_t addr;
+
+ /* Device File descriptor */
+ int fd;
+
+ /* Minor device for uio device. */
+ u32 uio_minor;
+
+ /* private data */
+ uword private_data;
+
+} linux_vmbus_device_t;
+
+/* Pool of VMBUS devices. */
+typedef struct
+{
+ vlib_main_t *vlib_main;
+ linux_vmbus_device_t *linux_vmbus_devices;
+
+} linux_vmbus_main_t;
+
+linux_vmbus_main_t linux_vmbus_main;
+
+static linux_vmbus_device_t *
+linux_vmbus_get_device (vlib_vmbus_dev_handle_t h)
+{
+ linux_vmbus_main_t *lpm = &linux_vmbus_main;
+ return pool_elt_at_index (lpm->linux_vmbus_devices, h);
+}
+
+uword
+vlib_vmbus_get_private_data (vlib_vmbus_dev_handle_t h)
+{
+ linux_vmbus_device_t *d = linux_vmbus_get_device (h);
+ return d->private_data;
+}
+
+void
+vlib_vmbus_set_private_data (vlib_vmbus_dev_handle_t h, uword private_data)
+{
+ linux_vmbus_device_t *d = linux_vmbus_get_device (h);
+ d->private_data = private_data;
+}
+
+vlib_vmbus_addr_t *
+vlib_vmbus_get_addr (vlib_vmbus_dev_handle_t h)
+{
+ linux_vmbus_device_t *d = linux_vmbus_get_device (h);
+ return &d->addr;
+}
+
+/* Call to allocate/initialize the vmbus subsystem.
+ This is not an init function so that users can explicitly enable
+ vmbus only when it's needed. */
+clib_error_t *vmbus_bus_init (vlib_main_t * vm);
+
+linux_vmbus_main_t linux_vmbus_main;
+
+/*
+ * Take VMBus address represented in standard form like:
+ * "f2c086b2-ff2e-11e8-88de-7bad0a57de05" and convert
+ * it to u8[16]
+ */
+static uword
+unformat_vlib_vmbus_addr (unformat_input_t * input, va_list * args)
+{
+ vlib_vmbus_addr_t *addr = va_arg (*args, vlib_vmbus_addr_t *);
+ uword ret = 0;
+ u8 *s;
+
+ if (!unformat (input, "%s", &s))
+ return 0;
+
+ if (uuid_parse ((char *) s, addr->guid) == 0)
+ ret = 1;
+
+ vec_free (s);
+
+ return ret;
+}
+
+/* Convert bus address to standard UUID string */
+static u8 *
+format_vlib_vmbus_addr (u8 * s, va_list * va)
+{
+ vlib_vmbus_addr_t *addr = va_arg (*va, vlib_vmbus_addr_t *);
+ char tmp[40];
+
+ uuid_unparse (addr->guid, tmp);
+ return format (s, "%s", tmp);
+}
+
+/* workaround for mlx bug, bring lower device up before unbind */
+static clib_error_t *
+vlib_vmbus_raise_lower (int fd, const char *upper_name)
+{
+ clib_error_t *error = 0;
+ struct dirent *e;
+ struct ifreq ifr;
+ u8 *dev_net_dir;
+ DIR *dir;
+
+ memset (&ifr, 0, sizeof (ifr));
+
+ dev_net_dir = format (0, "%s/%s%c", sysfs_class_net_path, upper_name, 0);
+
+ dir = opendir ((char *) dev_net_dir);
+
+ if (!dir)
+ {
+ error = clib_error_return (0, "VMBUS failed to open %s", dev_net_dir);
+ goto done;
+ }
+
+ while ((e = readdir (dir)))
+ {
+ /* look for lower_enXXXX */
+ if (strncmp (e->d_name, "lower_", 6))
+ continue;
+
+ strncpy (ifr.ifr_name, e->d_name + 6, IFNAMSIZ);
+ break;
+ }
+ closedir (dir);
+
+ if (!e)
+ goto done; /* no lower device */
+
+ if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
+ error = clib_error_return_unix (0, "ioctl fetch intf %s flags",
+ ifr.ifr_name);
+ else if (!(ifr.ifr_flags & IFF_UP))
+ {
+ ifr.ifr_flags |= IFF_UP;
+
+ if (ioctl (fd, SIOCSIFFLAGS, &ifr) < 0)
+ error = clib_error_return_unix (0, "ioctl set intf %s flags",
+ ifr.ifr_name);
+ }
+done:
+ vec_free (dev_net_dir);
+ return error;
+}
+
+clib_error_t *
+vlib_vmbus_bind_to_uio (vlib_vmbus_addr_t * addr)
+{
+ clib_error_t *error = 0;
+ u8 *dev_dir_name;
+ char *ifname = 0;
+ static int uio_new_id_needed = 1;
+ struct dirent *e;
+ struct ifreq ifr;
+ u8 *s, *driver_name;
+ DIR *dir;
+ int fd;
+
+ dev_dir_name = format (0, "%s/%U", sysfs_vmbus_dev_path,
+ format_vlib_vmbus_addr, addr);
+ s = format (0, "%v/driver%c", dev_dir_name, 0);
+
+ driver_name = clib_sysfs_link_to_name ((char *) s);
+ vec_reset_length (s);
+
+ /* skip if not using the Linux kernel netvsc driver */
+ if (!driver_name || strcmp ("hv_netvsc", (char *) driver_name) != 0)
+ goto done;
+
+ s = format (s, "%v/net%c", dev_dir_name, 0);
+ dir = opendir ((char *) s);
+ vec_reset_length (s);
+
+ if (!dir)
+ return clib_error_return (0, "VMBUS failed to open %s", s);
+
+ while ((e = readdir (dir)))
+ {
+ if (e->d_name[0] == '.') /* skip . and .. */
+ continue;
+
+ ifname = strdup (e->d_name);
+ break;
+ }
+ closedir (dir);
+
+ if (!ifname)
+ {
+ error = clib_error_return (0,
+ "VMBUS device %U eth not found",
+ format_vlib_vmbus_addr, addr);
+ goto done;
+ }
+
+
+ memset (&ifr, 0, sizeof (ifr));
+ strncpy (ifr.ifr_name, ifname, IFNAMSIZ);
+
+ /* read up/down flags */
+ fd = socket (PF_INET, SOCK_DGRAM, 0);
+ if (fd < 0)
+ {
+ error = clib_error_return_unix (0, "socket");
+ goto done;
+ }
+
+ if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
+ {
+ error = clib_error_return_unix (0, "ioctl fetch intf %s flags",
+ ifr.ifr_name);
+ close (fd);
+ goto done;
+ }
+
+ if (ifr.ifr_flags & IFF_UP)
+ {
+ error = clib_error_return (0,
+ "Skipping VMBUS device %U as host interface %s is up",
+ format_vlib_vmbus_addr, addr, e->d_name);
+ close (fd);
+ goto done;
+ }
+
+ error = vlib_vmbus_raise_lower (fd, ifname);
+ close (fd);
+
+ if (error)
+ goto done;
+
+
+ /* tell uio_hv_generic about netvsc device type */
+ if (uio_new_id_needed)
+ {
+ uio_new_id_needed = 0;
+
+ vec_reset_length (s);
+ s = format (s, "%s/%s/new_id%c", sysfs_vmbus_drv_path, uio_drv_name, 0);
+ error = clib_sysfs_write ((char *) s, "%s", netvsc_uuid);
+
+ if (error)
+ goto done;
+
+ }
+
+ /* prefer the simplier driver_override model */
+ vec_reset_length (s);
+ s = format (s, "%/driver_override%c", dev_dir_name, 0);
+ if (access ((char *) s, F_OK) == 0)
+ {
+ clib_sysfs_write ((char *) s, "%s", uio_drv_name);
+ }
+ else
+ {
+ vec_reset_length (s);
+
+ s = format (s, "%v/driver/unbind%c", dev_dir_name, 0);
+ error =
+ clib_sysfs_write ((char *) s, "%U", format_vlib_vmbus_addr, addr);
+
+ if (error)
+ goto done;
+
+ vec_reset_length (s);
+
+ s = format (s, "%s/%s/bind%c", sysfs_vmbus_drv_path, uio_drv_name, 0);
+ error =
+ clib_sysfs_write ((char *) s, "%U", format_vlib_vmbus_addr, addr);
+ }
+ vec_reset_length (s);
+
+done:
+ free (ifname);
+ vec_free (s);
+ vec_free (dev_dir_name);
+ vec_free (driver_name);
+ return error;
+}
+
+static clib_error_t *
+scan_vmbus_addr (void *arg, u8 * dev_dir_name, u8 * ignored)
+{
+ vlib_vmbus_addr_t addr, **addrv = arg;
+ unformat_input_t input;
+ clib_error_t *err = 0;
+
+ unformat_init_string (&input, (char *) dev_dir_name,
+ vec_len (dev_dir_name));
+
+ if (!unformat (&input, "/sys/bus/vmbus/devices/%U",
+ unformat_vlib_vmbus_addr, &addr))
+ err = clib_error_return (0, "unformat error `%v`", dev_dir_name);
+
+ unformat_free (&input);
+
+ if (err)
+ return err;
+
+ vec_add1 (*addrv, addr);
+ return 0;
+}
+
+static int
+vmbus_addr_cmp (void *v1, void *v2)
+{
+ vlib_vmbus_addr_t *a1 = v1;
+ vlib_vmbus_addr_t *a2 = v2;
+
+ return uuid_compare (a1->guid, a2->guid);
+}
+
+vlib_vmbus_addr_t *
+vlib_vmbus_get_all_dev_addrs ()
+{
+ vlib_vmbus_addr_t *addrs = 0;
+ clib_error_t *err;
+
+ err =
+ foreach_directory_file ((char *) sysfs_vmbus_dev_path, scan_vmbus_addr,
+ &addrs, /* scan_dirs */ 0);
+ if (err)
+ {
+ vec_free (addrs);
+ return 0;
+ }
+
+ vec_sort_with_function (addrs, vmbus_addr_cmp);
+
+ return addrs;
+}
+
+clib_error_t *
+linux_vmbus_init (vlib_main_t * vm)
+{
+ linux_vmbus_main_t *pm = &linux_vmbus_main;
+
+ pm->vlib_main = vm;
+
+ return vlib_call_init_function (vm, unix_input_init);
+}
+
+VLIB_INIT_FUNCTION (linux_vmbus_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vlib/vmbus/vmbus.c b/src/vlib/vmbus/vmbus.c
new file mode 100644
index 00000000000..ea395ece6a8
--- /dev/null
+++ b/src/vlib/vmbus/vmbus.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018, Microsoft Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vlib/vlib.h>
+#include <vlib/vmbus/vmbus.h>
+#include <vlib/unix/unix.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/ioctl.h>
+#include <net/if.h>
+
+/* this is a stub replaced by the Linux specfic version */
+vlib_vmbus_addr_t * __attribute__ ((weak)) vlib_vmbus_get_all_dev_addrs ()
+{
+ return NULL;
+}
+
+
+clib_error_t *
+vmbus_bus_init (vlib_main_t * vm)
+{
+ return vlib_call_init_function (vm, vmbus_bus_init);
+}
+
+VLIB_INIT_FUNCTION (vmbus_bus_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vlib/vmbus/vmbus.h b/src/vlib/vmbus/vmbus.h
new file mode 100644
index 00000000000..0927b8990d6
--- /dev/null
+++ b/src/vlib/vmbus/vmbus.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2018, Microsoft Corporation.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * vmbus.h: VMBus definitions.
+ */
+
+#ifndef included_vlib_vmbus_h
+#define included_vlib_vmbus_h
+
+#include <vlib/vlib.h>
+
+typedef struct
+{
+ u8 guid[16];
+} vlib_vmbus_addr_t;
+typedef u32 vlib_vmbus_dev_handle_t;
+
+vlib_vmbus_addr_t *vlib_vmbus_get_all_dev_addrs ();
+vlib_vmbus_addr_t *vlib_vmbus_get_addr (vlib_vmbus_dev_handle_t h);
+uword vlib_vmbus_get_private_data (vlib_vmbus_dev_handle_t h);
+void vlib_vmbus_set_private_data (vlib_vmbus_dev_handle_t h,
+ uword private_data);
+
+clib_error_t *vlib_vmbus_bind_to_uio (vlib_vmbus_addr_t * addr);
+#endif /* included_vlib_vmbus_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */