From cef87f1a5eb4d69cf11ce1cd3c5506edcfba74c4 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 5 Oct 2017 15:32:41 +0200 Subject: vlib: PCI rework to support VFIO Also fixes old ixge driver, so it works with recent physmem changes and vfio. Change-Id: Id4be74b34daed47cd281a77eec43d6692340d882 Signed-off-by: Damjan Marion --- src/vlib/linux/pci.c | 918 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 646 insertions(+), 272 deletions(-) (limited to 'src/vlib/linux') diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index 790f168a355..24e2bb594a3 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -51,11 +51,16 @@ #include #include #include +#include +#include + +static const char *sysfs_pci_dev_path = "/sys/bus/pci/devices"; +static const char *sysfs_pci_drv_path = "/sys/bus/pci/drivers"; typedef struct { - /* /sys/bus/pci/devices/... directory name for this device. */ - u8 *dev_dir_name; + vlib_pci_dev_handle_t handle; + vlib_pci_addr_t addr; /* Resource file descriptors. */ int *resource_fds; @@ -69,40 +74,260 @@ typedef struct /* Minor device for uio device. */ u32 uio_minor; + /* vfio */ + int vfio_device_fd; + /* Index given by clib_file_add. */ u32 clib_file_index; + /* Interrupt handler */ + void (*interrupt_handler) (vlib_pci_dev_handle_t h); + + /* private data */ + uword private_data; + } linux_pci_device_t; +typedef struct +{ + int group; + int fd; + int refcnt; +} linux_pci_vfio_iommu_group_t; + /* Pool of PCI devices. */ typedef struct { vlib_main_t *vlib_main; linux_pci_device_t *linux_pci_devices; + + /* VFIO */ + int vfio_container_fd; + int vfio_iommu_mode; + + /* pool of IOMMU groups */ + linux_pci_vfio_iommu_group_t *iommu_groups; + + /* iommu group pool index by group id hash */ + uword *iommu_pool_index_by_group; + } linux_pci_main_t; extern linux_pci_main_t linux_pci_main; +linux_pci_device_t * +linux_pci_get_device (vlib_pci_dev_handle_t h) +{ + linux_pci_main_t *lpm = &linux_pci_main; + return pool_elt_at_index (lpm->linux_pci_devices, h); +} + +uword +vlib_pci_get_private_data (vlib_pci_dev_handle_t h) +{ + linux_pci_device_t *d = linux_pci_get_device (h); + return d->private_data; +} + +void +vlib_pci_set_private_data (vlib_pci_dev_handle_t h, uword private_data) +{ + linux_pci_device_t *d = linux_pci_get_device (h); + d->private_data = private_data; +} + +vlib_pci_addr_t * +vlib_pci_get_addr (vlib_pci_dev_handle_t h) +{ + linux_pci_device_t *lpm = linux_pci_get_device (h); + return &lpm->addr; +} + /* Call to allocate/initialize the pci subsystem. This is not an init function so that users can explicitly enable pci only when it's needed. */ clib_error_t *pci_bus_init (vlib_main_t * vm); -clib_error_t *vlib_pci_bind_to_uio (vlib_pci_device_t * d, - char *uio_driver_name); - linux_pci_main_t linux_pci_main; +vlib_pci_device_info_t * +vlib_pci_get_device_info (vlib_pci_addr_t * addr, clib_error_t ** error) +{ + linux_pci_main_t *lpm = &linux_pci_main; + clib_error_t *err; + vlib_pci_device_info_t *di; + u8 *f = 0; + u32 tmp; + int fd; + + di = clib_mem_alloc (sizeof (vlib_pci_device_info_t)); + memset (di, 0, sizeof (vlib_pci_device_info_t)); + di->addr.as_u32 = addr->as_u32; + + u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path, + format_vlib_pci_addr, addr); + + f = format (0, "%v/config%c", dev_dir_name, 0); + fd = open ((char *) f, O_RDWR); + + /* Try read-only access if write fails. */ + if (fd < 0) + fd = open ((char *) f, O_RDONLY); + + if (fd < 0) + { + err = clib_error_return_unix (0, "open `%s'", f); + goto error; + } + + /* You can only read more that 64 bytes of config space as root; so we try to + read the full space but fall back to just the first 64 bytes. */ + if (read (fd, &di->config_data, sizeof (di->config_data)) != + sizeof (di->config_data) + && read (fd, &di->config0, + sizeof (di->config0)) != sizeof (di->config0)) + { + err = clib_error_return_unix (0, "read `%s'", f); + close (fd); + goto error; + } + + { + static pci_config_header_t all_ones; + if (all_ones.vendor_id == 0) + memset (&all_ones, ~0, sizeof (all_ones)); + + if (!memcmp (&di->config0.header, &all_ones, sizeof (all_ones))) + { + err = clib_error_return (0, "invalid PCI config for `%s'", f); + close (fd); + goto error; + } + } + + if (di->config0.header.header_type == 0) + pci_config_type0_little_to_host (&di->config0); + else + pci_config_type1_little_to_host (&di->config1); + + di->numa_node = -1; + vec_reset_length (f); + f = format (f, "%v/numa_node%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "%u", &di->numa_node); + if (err) + { + di->numa_node = -1; + clib_error_free (err); + } + + vec_reset_length (f); + f = format (f, "%v/class%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "0x%x", &tmp); + if (err) + goto error; + di->device_class = tmp >> 8; + + vec_reset_length (f); + f = format (f, "%v/vendor%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "0x%x", &tmp); + if (err) + goto error; + di->vendor_id = tmp; + + vec_reset_length (f); + f = format (f, "%v/device%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "0x%x", &tmp); + if (err) + goto error; + di->device_id = tmp; + + vec_reset_length (f); + f = format (f, "%v/driver%c", dev_dir_name, 0); + di->driver_name = clib_sysfs_link_to_name ((char *) f); + + di->iommu_group = -1; + if (lpm->vfio_container_fd != -1) + { + u8 *tmpstr; + vec_reset_length (f); + f = format (f, "%v/iommu_group%c", dev_dir_name, 0); + tmpstr = clib_sysfs_link_to_name ((char *) f); + if (tmpstr) + { + di->iommu_group = atoi ((char *) tmpstr); + vec_free (tmpstr); + } + } + + vec_reset_length (f); + f = format (f, "%v/vpd%c", dev_dir_name, 0); + fd = open ((char *) f, O_RDONLY); + if (fd >= 0) + { + while (1) + { + u8 tag[3]; + u8 *data = 0; + int len; + + if (read (fd, &tag, 3) != 3) + break; + + if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91) + break; + + len = (tag[2] << 8) | tag[1]; + vec_validate (data, len); + + if (read (fd, data, len) != len) + { + vec_free (data); + break; + } + if (tag[0] == 0x82) + di->product_name = data; + else if (tag[0] == 0x90) + di->vpd_r = data; + else if (tag[0] == 0x91) + di->vpd_w = data; + + data = 0; + } + close (fd); + } + + goto done; + +error: + vlib_pci_free_device_info (di); + di = 0; + +done: + vec_free (f); + vec_free (dev_dir_name); + if (error) + *error = err; + else + clib_error_free (err); + return di; +} + clib_error_t * -vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) +vlib_pci_bind_to_uio (vlib_pci_addr_t * addr, char *uio_drv_name) { clib_error_t *error = 0; u8 *s = 0, *driver_name = 0; DIR *dir = 0; struct dirent *e; + vlib_pci_device_info_t *di; int fd, clear_driver_override = 0; - u8 *dev_dir_name = format (0, "/sys/bus/pci/devices/%U", - format_vlib_pci_addr, &d->bus_address); + u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path, + format_vlib_pci_addr, addr); + + di = vlib_pci_get_device_info (addr, &error); + + if (error) + return error; s = format (s, "%v/driver%c", dev_dir_name, 0); driver_name = clib_sysfs_link_to_name ((char *) s); @@ -117,13 +342,13 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) /* walk trough all linux interfaces and if interface belonging to this device is founf check if interface is admin up */ dir = opendir ("/sys/class/net"); - s = format (s, "%U%c", format_vlib_pci_addr, &d->bus_address, 0); + s = format (s, "%U%c", format_vlib_pci_addr, addr, 0); if (!dir) { error = clib_error_return (0, "Skipping PCI device %U: failed to " "read /sys/class/net", - format_vlib_pci_addr, &d->bus_address); + format_vlib_pci_addr, addr); goto done; } @@ -173,8 +398,7 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) { error = clib_error_return (0, "Skipping PCI device %U as host " "interface %s is up", - format_vlib_pci_addr, &d->bus_address, - e->d_name); + format_vlib_pci_addr, addr, e->d_name); close (fd); goto done; } @@ -184,26 +408,26 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) vec_reset_length (s); s = format (s, "%v/driver/unbind%c", dev_dir_name, 0); - clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr); vec_reset_length (s); s = format (s, "%v/driver_override%c", dev_dir_name, 0); if (access ((char *) s, F_OK) == 0) { - clib_sysfs_write ((char *) s, "%s", uio_driver_name); + clib_sysfs_write ((char *) s, "%s", uio_drv_name); clear_driver_override = 1; } else { vec_reset_length (s); - s = format (s, "/sys/bus/pci/drivers/%s/new_id%c", uio_driver_name, 0); - clib_sysfs_write ((char *) s, "0x%04x 0x%04x", d->vendor_id, - d->device_id); + s = format (s, "%s/%s/new_id%c", sysfs_pci_drv_path, uio_drv_name, 0); + clib_sysfs_write ((char *) s, "0x%04x 0x%04x", di->vendor_id, + di->device_id); } vec_reset_length (s); - s = format (s, "/sys/bus/pci/drivers/%s/bind%c", uio_driver_name, 0); - clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + s = format (s, "%s/%s/bind%c", sysfs_pci_drv_path, uio_drv_name, 0); + clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr); vec_reset_length (s); if (clear_driver_override) @@ -240,23 +464,41 @@ scan_uio_dir (void *arg, u8 * path_name, u8 * file_name) static clib_error_t * linux_pci_uio_read_ready (clib_file_t * uf) { - vlib_pci_main_t *pm = &pci_main; - vlib_pci_device_t *d; int __attribute__ ((unused)) rv; + vlib_pci_dev_handle_t h = uf->private_data; + linux_pci_device_t *p = linux_pci_get_device (h); u32 icount; rv = read (uf->file_descriptor, &icount, 4); - d = pool_elt_at_index (pm->pci_devs, uf->private_data); - - if (d->interrupt_handler) - d->interrupt_handler (d); + if (p->interrupt_handler) + p->interrupt_handler (h); - vlib_pci_intr_enable (d); + vlib_pci_intr_enable (h); return /* no error */ 0; } +static clib_error_t * +linux_pci_vfio_unmask_intx (linux_pci_device_t * d) +{ + clib_error_t *err = 0; + struct vfio_irq_set i = { + .argsz = sizeof (struct vfio_irq_set), + .start = 0, + .count = 1, + .index = VFIO_PCI_INTX_IRQ_INDEX, + .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, + }; + + if (ioctl (d->vfio_device_fd, VFIO_DEVICE_SET_IRQS, &i) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_SET_IRQS) '%U'", + format_vlib_pci_addr, &d->addr); + } + return err; +} + static clib_error_t * linux_pci_uio_error_ready (clib_file_t * uf) { @@ -265,74 +507,293 @@ linux_pci_uio_error_ready (clib_file_t * uf) return clib_error_return (0, "pci device %d: error", error_index); } -static void -add_device (vlib_pci_device_t * dev, linux_pci_device_t * pdev) +static clib_error_t * +linux_pci_vfio_read_ready (clib_file_t * uf) +{ + int __attribute__ ((unused)) rv; + vlib_pci_dev_handle_t h = uf->private_data; + linux_pci_device_t *p = linux_pci_get_device (h); + + u64 icount; + rv = read (uf->file_descriptor, &icount, sizeof (icount)); + + if (p->interrupt_handler) + p->interrupt_handler (h); + + linux_pci_vfio_unmask_intx (p); + + return /* no error */ 0; +} + +static clib_error_t * +linux_pci_vfio_error_ready (clib_file_t * uf) +{ + u32 error_index = (u32) uf->private_data; + + return clib_error_return (0, "pci device %d: error", error_index); +} + +static clib_error_t * +add_device_uio (vlib_main_t * vm, linux_pci_device_t * p, + vlib_pci_device_info_t * di, pci_device_registration_t * r) +{ + clib_error_t *err = 0; + clib_file_t t = { 0 }; + u8 *s = 0; + + p->addr.as_u32 = di->addr.as_u32; + p->uio_fd = -1; + + s = format (s, "%s/%U/config%c", sysfs_pci_dev_path, + format_vlib_pci_addr, &di->addr, 0); + + p->config_fd = open ((char *) s, O_RDWR); + vec_reset_length (s); + + if (p->config_fd == -1) + { + err = clib_error_return_unix (0, "open '%s'", s); + goto error; + } + + s = format (0, "%s/%U/uio", sysfs_pci_dev_path, + format_vlib_pci_addr, &di->addr); + foreach_directory_file ((char *) s, scan_uio_dir, p, /* scan_dirs */ + 1); + vec_reset_length (s); + + s = format (s, "/dev/uio%d%c", p->uio_minor, 0); + p->uio_fd = open ((char *) s, O_RDWR); + if (p->uio_fd < 0) + { + err = clib_error_return_unix (0, "open '%s'", s); + goto error; + } + + t.read_function = linux_pci_uio_read_ready; + t.file_descriptor = p->uio_fd; + t.error_function = linux_pci_uio_error_ready; + t.private_data = p->handle; + + p->clib_file_index = clib_file_add (&file_main, &t); + p->interrupt_handler = r->interrupt_handler; + err = r->init_function (vm, p->handle); + +error: + free (s); + if (err) + { + close (p->config_fd); + close (p->uio_fd); + } + return err; +} + +static linux_pci_vfio_iommu_group_t * +get_vfio_iommu_group (int group) { - vlib_pci_main_t *pm = &pci_main; linux_pci_main_t *lpm = &linux_pci_main; - linux_pci_device_t *l; + uword *p; - pool_get (lpm->linux_pci_devices, l); - l[0] = pdev[0]; + p = hash_get (lpm->iommu_pool_index_by_group, group); - l->dev_dir_name = vec_dup (l->dev_dir_name); + return p ? pool_elt_at_index (lpm->iommu_groups, p[0]) : 0; +} - dev->os_handle = l - lpm->linux_pci_devices; +static clib_error_t * +open_vfio_iommu_group (int group) +{ + linux_pci_main_t *lpm = &linux_pci_main; + linux_pci_vfio_iommu_group_t *g; + clib_error_t *err = 0; + struct vfio_group_status group_status; + u8 *s = 0; + int fd; - { - u8 *uio_dir = format (0, "%s/uio", l->dev_dir_name); - foreach_directory_file ((char *) uio_dir, scan_uio_dir, l, /* scan_dirs */ - 1); - vec_free (uio_dir); - } + g = get_vfio_iommu_group (group); + if (g) + { + g->refcnt++; + return 0; + } + s = format (s, "/dev/vfio/%u%c", group, 0); + fd = open ((char *) s, O_RDWR); + if (fd < 0) + clib_error_return_unix (0, "open '%s'", s); - { - char *uio_name = (char *) format (0, "/dev/uio%d%c", l->uio_minor, 0); - l->uio_fd = open (uio_name, O_RDWR); - if (l->uio_fd < 0) - clib_unix_error ("open `%s'", uio_name); - vec_free (uio_name); - } + group_status.argsz = sizeof (group_status); + if (ioctl (fd, VFIO_GROUP_GET_STATUS, &group_status) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_STATUS) '%s'", + s); + goto error; + } - { - clib_file_t template = { 0 }; + if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) + { + err = clib_error_return (0, "iommu group %d is not viable (not all " + "devices in this group bound to vfio-pci)", + group); + goto error; + } + + if (ioctl (fd, VFIO_GROUP_SET_CONTAINER, &lpm->vfio_container_fd) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_SET_CONTAINER) '%s'", + s); + goto error; + } - template.read_function = linux_pci_uio_read_ready; - template.file_descriptor = l->uio_fd; - template.error_function = linux_pci_uio_error_ready; - template.private_data = dev - pm->pci_devs; + if (lpm->vfio_iommu_mode == 0) + { + if (ioctl (lpm->vfio_container_fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU) < + 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_SET_IOMMU) " + "'/dev/vfio/vfio'"); + goto error; + } + lpm->vfio_iommu_mode = VFIO_TYPE1_IOMMU; + } - l->clib_file_index = clib_file_add (&file_main, &template); - } + + pool_get (lpm->iommu_groups, g); + g->fd = fd; + g->refcnt = 1; + hash_set (lpm->iommu_pool_index_by_group, group, g - lpm->iommu_groups); + vec_free (s); + return 0; +error: + close (fd); + return err; } -static void -linux_pci_device_free (linux_pci_device_t * l) +static clib_error_t * +add_device_vfio (vlib_main_t * vm, linux_pci_device_t * p, + vlib_pci_device_info_t * di, pci_device_registration_t * r) { - int i; - for (i = 0; i < vec_len (l->resource_fds); i++) - if (l->resource_fds[i] > 0) - close (l->resource_fds[i]); - if (l->config_fd > 0) - close (l->config_fd); - if (l->uio_fd > 0) - close (l->uio_fd); - vec_free (l->resource_fds); - vec_free (l->dev_dir_name); + linux_pci_vfio_iommu_group_t *g; + struct vfio_device_info device_info; + struct vfio_irq_info irq_info = { 0 }; + clib_error_t *err = 0; + u8 *s = 0; + int dfd; + + p->addr.as_u32 = di->addr.as_u32; + + if (di->driver_name == 0 || + (strcmp ("vfio-pci", (char *) di->driver_name) != 0)) + return clib_error_return (0, "Device '%U' (iommu group %d) not bound to " + "vfio-pci", format_vlib_pci_addr, &di->addr, + di->iommu_group); + + if ((err = open_vfio_iommu_group (di->iommu_group))) + return err; + + g = get_vfio_iommu_group (di->iommu_group); + + s = format (s, "%U%c", format_vlib_pci_addr, &di->addr, 0); + if ((dfd = ioctl (g->fd, VFIO_GROUP_GET_DEVICE_FD, (char *) s)) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_DEVICE_FD) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + vec_reset_length (s); + p->vfio_device_fd = dfd; + + device_info.argsz = sizeof (device_info); + if (ioctl (dfd, VFIO_DEVICE_GET_INFO, &device_info) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + + irq_info.argsz = sizeof (struct vfio_irq_info); + irq_info.index = VFIO_PCI_INTX_IRQ_INDEX; + if (ioctl (dfd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) " + "'%U'", format_vlib_pci_addr, &di->addr); + goto error; + } + + /* reset if device supports it */ + if (device_info.flags & VFIO_DEVICE_FLAGS_RESET) + if (ioctl (dfd, VFIO_DEVICE_RESET) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_RESET) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + + if ((irq_info.flags & VFIO_IRQ_INFO_EVENTFD) && irq_info.count == 1) + { + u8 buf[sizeof (struct vfio_irq_set) + sizeof (int)] = { 0 }; + struct vfio_irq_set *irq_set = (struct vfio_irq_set *) buf; + clib_file_t t = { 0 }; + int efd = eventfd (0, EFD_NONBLOCK); + + irq_set->argsz = sizeof (buf); + irq_set->count = 1; + irq_set->index = VFIO_PCI_INTX_IRQ_INDEX; + irq_set->flags = + VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + clib_memcpy (&irq_set->data, &efd, sizeof (int)); + + if (ioctl (dfd, VFIO_DEVICE_SET_IRQS, irq_set) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_SET_IRQS) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + + t.read_function = linux_pci_vfio_read_ready; + t.file_descriptor = efd; + t.error_function = linux_pci_vfio_error_ready; + t.private_data = p->handle; + p->clib_file_index = clib_file_add (&file_main, &t); + + /* unmask the interrupt */ + linux_pci_vfio_unmask_intx (p); + } + + p->interrupt_handler = r->interrupt_handler; + + s = format (s, "%s/%U/config%c", sysfs_pci_dev_path, + format_vlib_pci_addr, &di->addr, 0); + + p->config_fd = open ((char *) s, O_RDWR); + vec_reset_length (s); + + if (p->config_fd == -1) + { + err = clib_error_return_unix (0, "open '%s'", s); + goto error; + } + + err = r->init_function (vm, p->handle); + +error: + vec_free (s); + if (err) + { + close (dfd); + close (p->config_fd); + } + return err; } /* Configuration space read/write. */ clib_error_t * -vlib_pci_read_write_config (vlib_pci_device_t * dev, +vlib_pci_read_write_config (vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes) { - linux_pci_main_t *lpm = &linux_pci_main; - linux_pci_device_t *p; + linux_pci_device_t *p = linux_pci_get_device (h); int n; - p = pool_elt_at_index (lpm->linux_pci_devices, dev->os_handle); - if (read_or_write == VLIB_READ) n = pread (p->config_fd, data, n_bytes, address); else @@ -347,11 +808,10 @@ vlib_pci_read_write_config (vlib_pci_device_t * dev, } static clib_error_t * -os_map_pci_resource_internal (uword os_handle, - u32 resource, u8 * addr, void **result) +vlib_pci_map_resource_int (vlib_pci_dev_handle_t h, + u32 resource, u8 * addr, void **result) { - linux_pci_main_t *pm = &linux_pci_main; - linux_pci_device_t *p; + linux_pci_device_t *p = linux_pci_get_device (h); struct stat stat_buf; u8 *file_name; int fd; @@ -359,9 +819,10 @@ os_map_pci_resource_internal (uword os_handle, int flags = MAP_SHARED; error = 0; - p = pool_elt_at_index (pm->linux_pci_devices, os_handle); - file_name = format (0, "%v/resource%d%c", p->dev_dir_name, resource, 0); + file_name = format (0, "%s/%U/resource%d%c", sysfs_pci_dev_path, + format_vlib_pci_addr, &p->addr, resource, 0); + fd = open ((char *) file_name, O_RDWR); if (fd < 0) { @@ -402,240 +863,131 @@ done: } clib_error_t * -vlib_pci_map_resource (vlib_pci_device_t * dev, u32 resource, void **result) +vlib_pci_map_resource (vlib_pci_dev_handle_t h, u32 resource, void **result) { - return (os_map_pci_resource_internal - (dev->os_handle, resource, 0 /* addr */ , - result)); + return (vlib_pci_map_resource_int (h, resource, 0 /* addr */ , result)); } clib_error_t * -vlib_pci_map_resource_fixed (vlib_pci_device_t * dev, +vlib_pci_map_resource_fixed (vlib_pci_dev_handle_t h, u32 resource, u8 * addr, void **result) { - return (os_map_pci_resource_internal - (dev->os_handle, resource, addr, result)); + return (vlib_pci_map_resource_int (h, resource, addr, result)); } void -vlib_pci_free_device (vlib_pci_device_t * dev) -{ - linux_pci_main_t *pm = &linux_pci_main; - linux_pci_device_t *l; - - l = pool_elt_at_index (pm->linux_pci_devices, dev->os_handle); - linux_pci_device_free (l); - pool_put (pm->linux_pci_devices, l); -} - -pci_device_registration_t * __attribute__ ((unused)) -pci_device_next_registered (pci_device_registration_t * r) -{ - uword i; - - /* Null vendor id marks end of initialized list. */ - for (i = 0; r->supported_devices[i].vendor_id != 0; i++) - ; - - return clib_elf_section_data_next (r, i * sizeof (r->supported_devices[0])); -} - -static clib_error_t * -init_device_from_registered (vlib_main_t * vm, - vlib_pci_device_t * dev, - linux_pci_device_t * pdev) +init_device_from_registered (vlib_main_t * vm, vlib_pci_device_info_t * di) { vlib_pci_main_t *pm = &pci_main; + linux_pci_main_t *lpm = &linux_pci_main; pci_device_registration_t *r; pci_device_id_t *i; - clib_error_t *error; + clib_error_t *err = 0; + linux_pci_device_t *p; + + pool_get (lpm->linux_pci_devices, p); + p->handle = p - lpm->linux_pci_devices; r = pm->pci_device_registrations; while (r) { for (i = r->supported_devices; i->vendor_id != 0; i++) - if (i->vendor_id == dev->vendor_id && i->device_id == dev->device_id) + if (i->vendor_id == di->config0.header.vendor_id && + i->device_id == di->config0.header.device_id) { - error = vlib_pci_bind_to_uio (dev, "uio_pci_generic"); - if (error) - { - clib_error_report (error); - continue; - } - - add_device (dev, pdev); - dev->interrupt_handler = r->interrupt_handler; - return r->init_function (vm, dev); + if (di->iommu_group != -1) + err = add_device_vfio (vm, p, di, r); + else + err = add_device_uio (vm, p, di, r); + + if (err) + clib_error_report (err); + else + return; } r = r->next_registration; } - /* No driver, close the PCI config-space FD */ - close (pdev->config_fd); - return 0; -} -static clib_error_t * -init_device (vlib_main_t * vm, - vlib_pci_device_t * dev, linux_pci_device_t * pdev) -{ - return init_device_from_registered (vm, dev, pdev); + /* No driver, close the PCI config-space FD */ + memset (p, 0, sizeof (linux_pci_device_t)); + pool_put (lpm->linux_pci_devices, p); } static clib_error_t * -scan_device (void *arg, u8 * dev_dir_name, u8 * ignored) +scan_pci_addr (void *arg, u8 * dev_dir_name, u8 * ignored) { - vlib_main_t *vm = arg; - vlib_pci_main_t *pm = &pci_main; - int fd; - u8 *f; - clib_error_t *error = 0; - vlib_pci_device_t *dev; - linux_pci_device_t pdev = { 0 }; - u32 tmp; - - f = format (0, "%v/config%c", dev_dir_name, 0); - fd = open ((char *) f, O_RDWR); - - /* Try read-only access if write fails. */ - if (fd < 0) - fd = open ((char *) f, O_RDONLY); - - if (fd < 0) - { - error = clib_error_return_unix (0, "open `%s'", f); - goto done; - } - - pool_get (pm->pci_devs, dev); - - /* You can only read more that 64 bytes of config space as root; so we try to - read the full space but fall back to just the first 64 bytes. */ - if (read (fd, &dev->config_data, sizeof (dev->config_data)) != - sizeof (dev->config_data) - && read (fd, &dev->config0, - sizeof (dev->config0)) != sizeof (dev->config0)) - { - pool_put (pm->pci_devs, dev); - error = clib_error_return_unix (0, "read `%s'", f); - close (fd); - goto done; - } - - { - static pci_config_header_t all_ones; - if (all_ones.vendor_id == 0) - memset (&all_ones, ~0, sizeof (all_ones)); - - if (!memcmp (&dev->config0.header, &all_ones, sizeof (all_ones))) - { - pool_put (pm->pci_devs, dev); - error = clib_error_return (0, "invalid PCI config for `%s'", f); - close (fd); - goto done; - } - } - - if (dev->config0.header.header_type == 0) - pci_config_type0_little_to_host (&dev->config0); - else - pci_config_type1_little_to_host (&dev->config1); - - /* Parse bus, dev, function from directory name. */ - { - unformat_input_t input; - - unformat_init_string (&input, (char *) dev_dir_name, - vec_len (dev_dir_name)); + vlib_pci_addr_t addr, **addrv = arg; + unformat_input_t input; + clib_error_t *err = 0; - if (!unformat (&input, "/sys/bus/pci/devices/%U", - unformat_vlib_pci_addr, &dev->bus_address)) - abort (); + unformat_init_string (&input, (char *) dev_dir_name, + vec_len (dev_dir_name)); - unformat_free (&input); + if (!unformat (&input, "/sys/bus/pci/devices/%U", + unformat_vlib_pci_addr, &addr)) + err = clib_error_return (0, "unformat error `%v`", dev_dir_name); - } + unformat_free (&input); + if (err) + return err; - pdev.config_fd = fd; - pdev.dev_dir_name = dev_dir_name; + vec_add1 (*addrv, addr); + return 0; +} - hash_set (pm->pci_dev_index_by_pci_addr, dev->bus_address.as_u32, - dev - pm->pci_devs); +static int +pci_addr_cmp (void *v1, void *v2) +{ + vlib_pci_addr_t *a1 = v1; + vlib_pci_addr_t *a2 = v2; + + if (a1->domain > a2->domain) + return 1; + if (a1->domain < a2->domain) + return -1; + if (a1->bus > a2->bus) + return 1; + if (a1->bus < a2->bus) + return -1; + if (a1->slot > a2->slot) + return 1; + if (a1->slot < a2->slot) + return -1; + if (a1->function > a2->function) + return 1; + if (a1->function < a2->function) + return -1; + return 0; +} - vec_reset_length (f); - f = format (f, "%v/vpd%c", dev_dir_name, 0); - fd = open ((char *) f, O_RDONLY); - if (fd >= 0) +vlib_pci_addr_t * +vlib_pci_get_all_dev_addrs () +{ + vlib_pci_addr_t *addrs = 0; + clib_error_t *err; + err = foreach_directory_file ((char *) sysfs_pci_dev_path, scan_pci_addr, + &addrs, /* scan_dirs */ 0); + if (err) { - while (1) - { - u8 tag[3]; - u8 *data = 0; - int len; - - if (read (fd, &tag, 3) != 3) - break; - - if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91) - break; - - len = (tag[2] << 8) | tag[1]; - vec_validate (data, len); - - if (read (fd, data, len) != len) - { - vec_free (data); - break; - } - if (tag[0] == 0x82) - dev->product_name = data; - else if (tag[0] == 0x90) - dev->vpd_r = data; - else if (tag[0] == 0x91) - dev->vpd_w = data; - - data = 0; - } - close (fd); + vec_free (addrs); + return 0; } - dev->numa_node = -1; - vec_reset_length (f); - f = format (f, "%v/numa_node%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "%u", &dev->numa_node); - - vec_reset_length (f); - f = format (f, "%v/class%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->device_class = tmp >> 8; - - vec_reset_length (f); - f = format (f, "%v/vendor%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->vendor_id = tmp; - - vec_reset_length (f); - f = format (f, "%v/device%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->device_id = tmp; + vec_sort_with_function (addrs, pci_addr_cmp); - error = init_device (vm, dev, &pdev); - - vec_reset_length (f); - f = format (f, "%v/driver%c", dev_dir_name, 0); - dev->driver_name = clib_sysfs_link_to_name ((char *) f); - -done: - vec_free (f); - return error; + return addrs; } clib_error_t * linux_pci_init (vlib_main_t * vm) { vlib_pci_main_t *pm = &pci_main; + linux_pci_main_t *lpm = &linux_pci_main; + vlib_pci_addr_t *addr = 0, *addrs; clib_error_t *error; + int fd; pm->vlib_main = vm; @@ -643,14 +995,36 @@ linux_pci_init (vlib_main_t * vm) return error; ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32)); - pm->pci_dev_index_by_pci_addr = hash_create (0, sizeof (uword)); - error = foreach_directory_file ("/sys/bus/pci/devices", scan_device, vm, - /* scan_dirs */ 0); + fd = open ("/dev/vfio/vfio", O_RDWR); - /* Complain and continue. might not be root, etc. */ - if (error) - clib_error_report (error); + if ((fd != -1) && (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION)) + { + close (fd); + fd = -1; + } + + if ((fd != -1) && (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 0)) + { + close (fd); + fd = -1; + } + + lpm->vfio_container_fd = fd; + lpm->iommu_pool_index_by_group = hash_create (0, sizeof (uword)); + + addrs = vlib_pci_get_all_dev_addrs (); + /* *INDENT-OFF* */ + vec_foreach (addr, addrs) + { + vlib_pci_device_info_t *d; + if ((d = vlib_pci_get_device_info (addr, 0))) + { + init_device_from_registered (vm, d); + vlib_pci_free_device_info (d); + } + } + /* *INDENT-ON* */ return error; } -- cgit 1.2.3-korg