diff options
Diffstat (limited to 'src')
| -rwxr-xr-x | src/plugins/dpdk/device/init.c | 32 | ||||
| -rw-r--r-- | src/plugins/ixge/ixge.c | 97 | ||||
| -rw-r--r-- | src/plugins/ixge/ixge.h | 5 | ||||
| -rw-r--r-- | src/vlib/buffer_funcs.h | 12 | ||||
| -rw-r--r-- | src/vlib/linux/pci.c | 918 | ||||
| -rw-r--r-- | src/vlib/pci/pci.c | 78 | ||||
| -rw-r--r-- | src/vlib/pci/pci.h | 125 |
7 files changed, 835 insertions, 432 deletions
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index 55904f018da..20272853035 100755 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -677,24 +677,37 @@ dpdk_lib_init (dpdk_main_t * dm) static void dpdk_bind_devices_to_uio (dpdk_config_main_t * conf) { - vlib_pci_main_t *pm = &pci_main; clib_error_t *error; - vlib_pci_device_t *d; u8 *pci_addr = 0; int num_whitelisted = vec_len (conf->dev_confs); + vlib_pci_device_info_t *d = 0; + vlib_pci_addr_t *addr = 0, *addrs; + addrs = vlib_pci_get_all_dev_addrs (); /* *INDENT-OFF* */ - pool_foreach (d, pm->pci_devs, ({ + vec_foreach (addr, addrs) + { dpdk_device_config_t * devconf = 0; vec_reset_length (pci_addr); - pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, &d->bus_address, 0); + pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, addr, 0); + if (d) + { + vlib_pci_free_device_info (d); + d = 0; + } + d = vlib_pci_get_device_info (addr, &error); + if (error) + { + clib_error_report (error); + continue; + } if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && d->device_class != PCI_CLASS_PROCESSOR_CO) continue; if (num_whitelisted) { - uword * p = hash_get (conf->device_config_index_by_pci_addr, d->bus_address.as_u32); + uword * p = hash_get (conf->device_config_index_by_pci_addr, addr->as_u32); if (!p) continue; @@ -736,23 +749,24 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf) continue; } - error = vlib_pci_bind_to_uio (d, (char *) conf->uio_driver_name); + error = vlib_pci_bind_to_uio (addr, (char *) conf->uio_driver_name); if (error) { if (devconf == 0) { pool_get (conf->dev_confs, devconf); - hash_set (conf->device_config_index_by_pci_addr, d->bus_address.as_u32, + hash_set (conf->device_config_index_by_pci_addr, addr->as_u32, devconf - conf->dev_confs); - devconf->pci_addr.as_u32 = d->bus_address.as_u32; + devconf->pci_addr.as_u32 = addr->as_u32; } devconf->is_blacklisted = 1; clib_error_report (error); } - })); + } /* *INDENT-ON* */ vec_free (pci_addr); + vlib_pci_free_device_info (d); } static clib_error_t * diff --git a/src/plugins/ixge/ixge.c b/src/plugins/ixge/ixge.c index bf9424255b8..62f484e6cca 100644 --- a/src/plugins/ixge/ixge.c +++ b/src/plugins/ixge/ixge.c @@ -1395,9 +1395,8 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, vec_resize (xm->rx_buffers_to_add, n_to_alloc); _vec_len (xm->rx_buffers_to_add) = l; - n_allocated = vlib_buffer_alloc_from_free_list - (vm, xm->rx_buffers_to_add + l, n_to_alloc, - xm->vlib_buffer_free_list_index); + n_allocated = + vlib_buffer_alloc (vm, xm->rx_buffers_to_add + l, n_to_alloc); _vec_len (xm->rx_buffers_to_add) += n_allocated; /* Handle transient allocation failure */ @@ -1462,6 +1461,7 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, to_rx += 2; to_add -= 2; +#if 0 ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED == vlib_buffer_is_known (vm, bi0)); ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED == @@ -1470,6 +1470,7 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, vlib_buffer_is_known (vm, fi0)); ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED == vlib_buffer_is_known (vm, fi1)); +#endif b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); @@ -1678,10 +1679,12 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, to_rx += 1; to_add -= 1; +#if 0 ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED == vlib_buffer_is_known (vm, bi0)); ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED == vlib_buffer_is_known (vm, fi0)); +#endif b0 = vlib_get_buffer (vm, bi0); @@ -2253,8 +2256,9 @@ format_ixge_device_name (u8 * s, va_list * args) u32 i = va_arg (*args, u32); ixge_main_t *xm = &ixge_main; ixge_device_t *xd = vec_elt_at_index (xm->devices, i); - return format (s, "TenGigabitEthernet%U", - format_vlib_pci_handle, &xd->pci_device.bus_address); + vlib_pci_addr_t *addr = vlib_pci_get_addr (xd->pci_dev_handle); + return format (s, "TenGigabitEthernet%x/%x/%x/%x", + addr->domain, addr->bus, addr->slot, addr->function); } #define IXGE_COUNTER_IS_64_BIT (1 << 0) @@ -2355,8 +2359,12 @@ format_ixge_device (u8 * s, va_list * args) { - s = format (s, "\n%UPCIe %U", format_white_space, indent + 2, - format_vlib_pci_link_speed, &xd->pci_device); + vlib_pci_addr_t *addr = vlib_pci_get_addr (xd->pci_dev_handle); + vlib_pci_device_info_t *d = vlib_pci_get_device_info (addr, 0); + + if (d) + s = format (s, "\n%UPCIe %U", format_white_space, indent + 2, + format_vlib_pci_link_speed, d); } s = format (s, "\n%U", format_white_space, indent + 2); @@ -2477,13 +2485,6 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) if (!xm->n_bytes_in_rx_buffer) xm->n_bytes_in_rx_buffer = IXGE_N_BYTES_IN_RX_BUFFER; xm->n_bytes_in_rx_buffer = round_pow2 (xm->n_bytes_in_rx_buffer, 1024); - if (!xm->vlib_buffer_free_list_index) - { - xm->vlib_buffer_free_list_index = - vlib_buffer_get_or_create_free_list (vm, xm->n_bytes_in_rx_buffer, - "ixge rx"); - ASSERT (xm->vlib_buffer_free_list_index != 0); - } if (!xm->n_descriptors[rt]) xm->n_descriptors[rt] = 4 * VLIB_FRAME_SIZE; @@ -2509,28 +2510,25 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) { u32 n_alloc, i; - n_alloc = vlib_buffer_alloc_from_free_list - (vm, dq->descriptor_buffer_indices, - vec_len (dq->descriptor_buffer_indices), - xm->vlib_buffer_free_list_index); + n_alloc = vlib_buffer_alloc (vm, dq->descriptor_buffer_indices, + vec_len (dq->descriptor_buffer_indices)); ASSERT (n_alloc == vec_len (dq->descriptor_buffer_indices)); for (i = 0; i < n_alloc; i++) { - vlib_buffer_t *b = - vlib_get_buffer (vm, dq->descriptor_buffer_indices[i]); dq->descriptors[i].rx_to_hw.tail_address = - vlib_physmem_virtual_to_physical (vm, xm->physmem_region, - b->data); + vlib_get_buffer_data_physical_address (vm, + dq->descriptor_buffer_indices + [i]); } } else { u32 i; - dq->tx.head_index_write_back = - vlib_physmem_alloc (vm, - vm->buffer_main->buffer_pools[0].physmem_region, - &error, CLIB_CACHE_LINE_BYTES); + dq->tx.head_index_write_back = vlib_physmem_alloc (vm, + xm->physmem_region, + &error, + CLIB_CACHE_LINE_BYTES); for (i = 0; i < dq->n_descriptors; i++) dq->descriptors[i].tx = xm->tx_descriptor_template; @@ -2543,9 +2541,7 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) u64 a; a = - vlib_physmem_virtual_to_physical (vm, - vm->buffer_main-> - buffer_pools[0].physmem_region, + vlib_physmem_virtual_to_physical (vm, xm->physmem_region, dq->descriptors); dr->descriptor_address[0] = a & 0xFFFFFFFF; dr->descriptor_address[1] = a >> (u64) 32; @@ -2571,11 +2567,8 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) /* Make sure its initialized before hardware can get to it. */ dq->tx.head_index_write_back[0] = dq->head_index; - a = - vlib_physmem_virtual_to_physical (vm, - vm->buffer_main-> - buffer_pools[0].physmem_region, - dq->tx.head_index_write_back); + a = vlib_physmem_virtual_to_physical (vm, xm->physmem_region, + dq->tx.head_index_write_back); dr->tx.head_index_write_back_address[0] = /* enable bit */ 1 | a; dr->tx.head_index_write_back_address[1] = (u64) a >> (u64) 32; } @@ -2838,10 +2831,11 @@ VLIB_INIT_FUNCTION (ixge_init); static void -ixge_pci_intr_handler (vlib_pci_device_t * dev) +ixge_pci_intr_handler (vlib_pci_dev_handle_t h) { ixge_main_t *xm = &ixge_main; vlib_main_t *vm = xm->vlib_main; + uword private_data = vlib_pci_get_private_data (h); vlib_node_set_interrupt_pending (vm, ixge_input_node.index); @@ -2849,26 +2843,32 @@ ixge_pci_intr_handler (vlib_pci_device_t * dev) { vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ixge_input_node.index); - rt->runtime_data[0] |= 1 << dev->private_data; + rt->runtime_data[0] |= 1 << private_data; } } static clib_error_t * -ixge_pci_init (vlib_main_t * vm, vlib_pci_device_t * dev) +ixge_pci_init (vlib_main_t * vm, vlib_pci_dev_handle_t h) { ixge_main_t *xm = &ixge_main; - clib_error_t *error; + clib_error_t *error = 0; void *r; ixge_device_t *xd; + vlib_pci_addr_t *addr = vlib_pci_get_addr (h); + vlib_pci_device_info_t *d = vlib_pci_get_device_info (addr, 0); /* Allocate physmem region for DMA buffers */ - error = vlib_physmem_region_alloc (vm, "ixge decriptors", 2 << 20, 0, - VLIB_PHYSMEM_F_INIT_MHEAP, - &xm->physmem_region); + if (xm->physmem_region_allocated == 0) + { + error = vlib_physmem_region_alloc (vm, "ixge decriptors", 2 << 20, 0, + VLIB_PHYSMEM_F_INIT_MHEAP, + &xm->physmem_region); + xm->physmem_region_allocated = 1; + } if (error) return error; - error = vlib_pci_map_resource (dev, 0, &r); + error = vlib_pci_map_resource (h, 0, &r); if (error) return error; @@ -2879,13 +2879,14 @@ ixge_pci_init (vlib_main_t * vm, vlib_pci_device_t * dev) ixge_input_node.function = ixge_input_multiarch_select (); } - xd->pci_device = dev[0]; - xd->device_id = xd->pci_device.config0.header.device_id; + xd->pci_dev_handle = h; + xd->device_id = d->device_id; xd->regs = r; xd->device_index = xd - xm->devices; - xd->pci_function = dev->bus_address.function; + xd->pci_function = addr->function; xd->per_interface_next_index = ~0; + vlib_pci_set_private_data (h, xd->device_index); /* Chip found so enable node. */ { @@ -2894,7 +2895,7 @@ ixge_pci_init (vlib_main_t * vm, vlib_pci_device_t * dev) ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_INTERRUPT)); - dev->private_data = xd->device_index; + //dev->private_data = xd->device_index; } if (vec_len (xm->devices) == 1) @@ -2903,12 +2904,12 @@ ixge_pci_init (vlib_main_t * vm, vlib_pci_device_t * dev) xm->process_node_index = ixge_process_node.index; } - error = vlib_pci_bus_master_enable (dev); + error = vlib_pci_bus_master_enable (h); if (error) return error; - return vlib_pci_intr_enable (dev); + return vlib_pci_intr_enable (h); } /* *INDENT-OFF* */ diff --git a/src/plugins/ixge/ixge.h b/src/plugins/ixge/ixge.h index a87af1ab04b..c766397525b 100644 --- a/src/plugins/ixge/ixge.h +++ b/src/plugins/ixge/ixge.h @@ -1208,7 +1208,7 @@ typedef struct u32 per_interface_next_index; /* PCI bus info. */ - vlib_pci_device_t pci_device; + vlib_pci_dev_handle_t pci_dev_handle; /* From PCI config space header. */ ixge_pci_device_id_t device_id; @@ -1253,8 +1253,6 @@ typedef struct u32 n_descriptors_per_cache_line; - u32 vlib_buffer_free_list_index; - u32 process_node_index; /* Template and mask for initializing/validating TX descriptors. */ @@ -1268,6 +1266,7 @@ typedef struct f64 time_last_stats_update; vlib_physmem_region_index_t physmem_region; + int physmem_region_allocated; } ixge_main_t; extern ixge_main_t ixge_main; diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 08f5b3e9010..1ea3c0829b8 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -162,14 +162,12 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) always_inline u64 vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index) { - vlib_physmem_region_index_t pri; + vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index); - pri = vm->buffer_main->buffer_pools[b->buffer_pool_index].physmem_region; - return vlib_physmem_offset_to_physical (vm, pri, - (((uword) buffer_index) << - CLIB_LOG2_CACHE_LINE_BYTES) + - STRUCT_OFFSET_OF (vlib_buffer_t, - data)); + vlib_buffer_pool_t *pool = vec_elt_at_index (bm->buffer_pools, + b->buffer_pool_index); + + return vlib_physmem_virtual_to_physical (vm, pool->physmem_region, b->data); } /** \brief Prefetch buffer metadata by buffer index diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index 790f168a355..24e2bb594a3 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -51,11 +51,16 @@ #include <net/if.h> #include <linux/ethtool.h> #include <linux/sockios.h> +#include <linux/vfio.h> +#include <sys/eventfd.h> + +static const char *sysfs_pci_dev_path = "/sys/bus/pci/devices"; +static const char *sysfs_pci_drv_path = "/sys/bus/pci/drivers"; typedef struct { - /* /sys/bus/pci/devices/... directory name for this device. */ - u8 *dev_dir_name; + vlib_pci_dev_handle_t handle; + vlib_pci_addr_t addr; /* Resource file descriptors. */ int *resource_fds; @@ -69,40 +74,260 @@ typedef struct /* Minor device for uio device. */ u32 uio_minor; + /* vfio */ + int vfio_device_fd; + /* Index given by clib_file_add. */ u32 clib_file_index; + /* Interrupt handler */ + void (*interrupt_handler) (vlib_pci_dev_handle_t h); + + /* private data */ + uword private_data; + } linux_pci_device_t; +typedef struct +{ + int group; + int fd; + int refcnt; +} linux_pci_vfio_iommu_group_t; + /* Pool of PCI devices. */ typedef struct { vlib_main_t *vlib_main; linux_pci_device_t *linux_pci_devices; + + /* VFIO */ + int vfio_container_fd; + int vfio_iommu_mode; + + /* pool of IOMMU groups */ + linux_pci_vfio_iommu_group_t *iommu_groups; + + /* iommu group pool index by group id hash */ + uword *iommu_pool_index_by_group; + } linux_pci_main_t; extern linux_pci_main_t linux_pci_main; +linux_pci_device_t * +linux_pci_get_device (vlib_pci_dev_handle_t h) +{ + linux_pci_main_t *lpm = &linux_pci_main; + return pool_elt_at_index (lpm->linux_pci_devices, h); +} + +uword +vlib_pci_get_private_data (vlib_pci_dev_handle_t h) +{ + linux_pci_device_t *d = linux_pci_get_device (h); + return d->private_data; +} + +void +vlib_pci_set_private_data (vlib_pci_dev_handle_t h, uword private_data) +{ + linux_pci_device_t *d = linux_pci_get_device (h); + d->private_data = private_data; +} + +vlib_pci_addr_t * +vlib_pci_get_addr (vlib_pci_dev_handle_t h) +{ + linux_pci_device_t *lpm = linux_pci_get_device (h); + return &lpm->addr; +} + /* Call to allocate/initialize the pci subsystem. This is not an init function so that users can explicitly enable pci only when it's needed. */ clib_error_t *pci_bus_init (vlib_main_t * vm); -clib_error_t *vlib_pci_bind_to_uio (vlib_pci_device_t * d, - char *uio_driver_name); - linux_pci_main_t linux_pci_main; +vlib_pci_device_info_t * +vlib_pci_get_device_info (vlib_pci_addr_t * addr, clib_error_t ** error) +{ + linux_pci_main_t *lpm = &linux_pci_main; + clib_error_t *err; + vlib_pci_device_info_t *di; + u8 *f = 0; + u32 tmp; + int fd; + + di = clib_mem_alloc (sizeof (vlib_pci_device_info_t)); + memset (di, 0, sizeof (vlib_pci_device_info_t)); + di->addr.as_u32 = addr->as_u32; + + u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path, + format_vlib_pci_addr, addr); + + f = format (0, "%v/config%c", dev_dir_name, 0); + fd = open ((char *) f, O_RDWR); + + /* Try read-only access if write fails. */ + if (fd < 0) + fd = open ((char *) f, O_RDONLY); + + if (fd < 0) + { + err = clib_error_return_unix (0, "open `%s'", f); + goto error; + } + + /* You can only read more that 64 bytes of config space as root; so we try to + read the full space but fall back to just the first 64 bytes. */ + if (read (fd, &di->config_data, sizeof (di->config_data)) != + sizeof (di->config_data) + && read (fd, &di->config0, + sizeof (di->config0)) != sizeof (di->config0)) + { + err = clib_error_return_unix (0, "read `%s'", f); + close (fd); + goto error; + } + + { + static pci_config_header_t all_ones; + if (all_ones.vendor_id == 0) + memset (&all_ones, ~0, sizeof (all_ones)); + + if (!memcmp (&di->config0.header, &all_ones, sizeof (all_ones))) + { + err = clib_error_return (0, "invalid PCI config for `%s'", f); + close (fd); + goto error; + } + } + + if (di->config0.header.header_type == 0) + pci_config_type0_little_to_host (&di->config0); + else + pci_config_type1_little_to_host (&di->config1); + + di->numa_node = -1; + vec_reset_length (f); + f = format (f, "%v/numa_node%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "%u", &di->numa_node); + if (err) + { + di->numa_node = -1; + clib_error_free (err); + } + + vec_reset_length (f); + f = format (f, "%v/class%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "0x%x", &tmp); + if (err) + goto error; + di->device_class = tmp >> 8; + + vec_reset_length (f); + f = format (f, "%v/vendor%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "0x%x", &tmp); + if (err) + goto error; + di->vendor_id = tmp; + + vec_reset_length (f); + f = format (f, "%v/device%c", dev_dir_name, 0); + err = clib_sysfs_read ((char *) f, "0x%x", &tmp); + if (err) + goto error; + di->device_id = tmp; + + vec_reset_length (f); + f = format (f, "%v/driver%c", dev_dir_name, 0); + di->driver_name = clib_sysfs_link_to_name ((char *) f); + + di->iommu_group = -1; + if (lpm->vfio_container_fd != -1) + { + u8 *tmpstr; + vec_reset_length (f); + f = format (f, "%v/iommu_group%c", dev_dir_name, 0); + tmpstr = clib_sysfs_link_to_name ((char *) f); + if (tmpstr) + { + di->iommu_group = atoi ((char *) tmpstr); + vec_free (tmpstr); + } + } + + vec_reset_length (f); + f = format (f, "%v/vpd%c", dev_dir_name, 0); + fd = open ((char *) f, O_RDONLY); + if (fd >= 0) + { + while (1) + { + u8 tag[3]; + u8 *data = 0; + int len; + + if (read (fd, &tag, 3) != 3) + break; + + if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91) + break; + + len = (tag[2] << 8) | tag[1]; + vec_validate (data, len); + + if (read (fd, data, len) != len) + { + vec_free (data); + break; + } + if (tag[0] == 0x82) + di->product_name = data; + else if (tag[0] == 0x90) + di->vpd_r = data; + else if (tag[0] == 0x91) + di->vpd_w = data; + + data = 0; + } + close (fd); + } + + goto done; + +error: + vlib_pci_free_device_info (di); + di = 0; + +done: + vec_free (f); + vec_free (dev_dir_name); + if (error) + *error = err; + else + clib_error_free (err); + return di; +} + clib_error_t * -vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) +vlib_pci_bind_to_uio (vlib_pci_addr_t * addr, char *uio_drv_name) { clib_error_t *error = 0; u8 *s = 0, *driver_name = 0; DIR *dir = 0; struct dirent *e; + vlib_pci_device_info_t *di; int fd, clear_driver_override = 0; - u8 *dev_dir_name = format (0, "/sys/bus/pci/devices/%U", - format_vlib_pci_addr, &d->bus_address); + u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path, + format_vlib_pci_addr, addr); + + di = vlib_pci_get_device_info (addr, &error); + + if (error) + return error; s = format (s, "%v/driver%c", dev_dir_name, 0); driver_name = clib_sysfs_link_to_name ((char *) s); @@ -117,13 +342,13 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) /* walk trough all linux interfaces and if interface belonging to this device is founf check if interface is admin up */ dir = opendir ("/sys/class/net"); - s = format (s, "%U%c", format_vlib_pci_addr, &d->bus_address, 0); + s = format (s, "%U%c", format_vlib_pci_addr, addr, 0); if (!dir) { error = clib_error_return (0, "Skipping PCI device %U: failed to " "read /sys/class/net", - format_vlib_pci_addr, &d->bus_address); + format_vlib_pci_addr, addr); goto done; } @@ -173,8 +398,7 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) { error = clib_error_return (0, "Skipping PCI device %U as host " "interface %s is up", - format_vlib_pci_addr, &d->bus_address, - e->d_name); + format_vlib_pci_addr, addr, e->d_name); close (fd); goto done; } @@ -184,26 +408,26 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) vec_reset_length (s); s = format (s, "%v/driver/unbind%c", dev_dir_name, 0); - clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr); vec_reset_length (s); s = format (s, "%v/driver_override%c", dev_dir_name, 0); if (access ((char *) s, F_OK) == 0) { - clib_sysfs_write ((char *) s, "%s", uio_driver_name); + clib_sysfs_write ((char *) s, "%s", uio_drv_name); clear_driver_override = 1; } else { vec_reset_length (s); - s = format (s, "/sys/bus/pci/drivers/%s/new_id%c", uio_driver_name, 0); - clib_sysfs_write ((char *) s, "0x%04x 0x%04x", d->vendor_id, - d->device_id); + s = format (s, "%s/%s/new_id%c", sysfs_pci_drv_path, uio_drv_name, 0); + clib_sysfs_write ((char *) s, "0x%04x 0x%04x", di->vendor_id, + di->device_id); } vec_reset_length (s); - s = format (s, "/sys/bus/pci/drivers/%s/bind%c", uio_driver_name, 0); - clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + s = format (s, "%s/%s/bind%c", sysfs_pci_drv_path, uio_drv_name, 0); + clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr); vec_reset_length (s); if (clear_driver_override) @@ -240,24 +464,42 @@ scan_uio_dir (void *arg, u8 * path_name, u8 * file_name) static clib_error_t * linux_pci_uio_read_ready (clib_file_t * uf) { - vlib_pci_main_t *pm = &pci_main; - vlib_pci_device_t *d; int __attribute__ ((unused)) rv; + vlib_pci_dev_handle_t h = uf->private_data; + linux_pci_device_t *p = linux_pci_get_device (h); u32 icount; rv = read (uf->file_descriptor, &icount, 4); - d = pool_elt_at_index (pm->pci_devs, uf->private_data); - - if (d->interrupt_handler) - d->interrupt_handler (d); + if (p->interrupt_handler) + p->interrupt_handler (h); - vlib_pci_intr_enable (d); + vlib_pci_intr_enable (h); return /* no error */ 0; } static clib_error_t * +linux_pci_vfio_unmask_intx (linux_pci_device_t * d) +{ + clib_error_t *err = 0; + struct vfio_irq_set i = { + .argsz = sizeof (struct vfio_irq_set), + .start = 0, + .count = 1, + .index = VFIO_PCI_INTX_IRQ_INDEX, + .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, + }; + + if (ioctl (d->vfio_device_fd, VFIO_DEVICE_SET_IRQS, &i) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_SET_IRQS) '%U'", + format_vlib_pci_addr, &d->addr); + } + return err; +} + +static clib_error_t * linux_pci_uio_error_ready (clib_file_t * uf) { u32 error_index = (u32) uf->private_data; @@ -265,74 +507,293 @@ linux_pci_uio_error_ready (clib_file_t * uf) return clib_error_return (0, "pci device %d: error", error_index); } -static void -add_device (vlib_pci_device_t * dev, linux_pci_device_t * pdev) +static clib_error_t * +linux_pci_vfio_read_ready (clib_file_t * uf) +{ + int __attribute__ ((unused)) rv; + vlib_pci_dev_handle_t h = uf->private_data; + linux_pci_device_t *p = linux_pci_get_device (h); + + u64 icount; + rv = read (uf->file_descriptor, &icount, sizeof (icount)); + + if (p->interrupt_handler) + p->interrupt_handler (h); + + linux_pci_vfio_unmask_intx (p); + + return /* no error */ 0; +} + +static clib_error_t * +linux_pci_vfio_error_ready (clib_file_t * uf) +{ + u32 error_index = (u32) uf->private_data; + + return clib_error_return (0, "pci device %d: error", error_index); +} + +static clib_error_t * +add_device_uio (vlib_main_t * vm, linux_pci_device_t * p, + vlib_pci_device_info_t * di, pci_device_registration_t * r) +{ + clib_error_t *err = 0; + clib_file_t t = { 0 }; + u8 *s = 0; + + p->addr.as_u32 = di->addr.as_u32; + p->uio_fd = -1; + + s = format (s, "%s/%U/config%c", sysfs_pci_dev_path, + format_vlib_pci_addr, &di->addr, 0); + + p->config_fd = open ((char *) s, O_RDWR); + vec_reset_length (s); + + if (p->config_fd == -1) + { + err = clib_error_return_unix (0, "open '%s'", s); + goto error; + } + + s = format (0, "%s/%U/uio", sysfs_pci_dev_path, + format_vlib_pci_addr, &di->addr); + foreach_directory_file ((char *) s, scan_uio_dir, p, /* scan_dirs */ + 1); + vec_reset_length (s); + + s = format (s, "/dev/uio%d%c", p->uio_minor, 0); + p->uio_fd = open ((char *) s, O_RDWR); + if (p->uio_fd < 0) + { + err = clib_error_return_unix (0, "open '%s'", s); + goto error; + } + + t.read_function = linux_pci_uio_read_ready; + t.file_descriptor = p->uio_fd; + t.error_function = linux_pci_uio_error_ready; + t.private_data = p->handle; + + p->clib_file_index = clib_file_add (&file_main, &t); + p->interrupt_handler = r->interrupt_handler; + err = r->init_function (vm, p->handle); + +error: + free (s); + if (err) + { + close (p->config_fd); + close (p->uio_fd); + } + return err; +} + +static linux_pci_vfio_iommu_group_t * +get_vfio_iommu_group (int group) { - vlib_pci_main_t *pm = &pci_main; linux_pci_main_t *lpm = &linux_pci_main; - linux_pci_device_t *l; + uword *p; - pool_get (lpm->linux_pci_devices, l); - l[0] = pdev[0]; + p = hash_get (lpm->iommu_pool_index_by_group, group); - l->dev_dir_name = vec_dup (l->dev_dir_name); + return p ? pool_elt_at_index (lpm->iommu_groups, p[0]) : 0; +} - dev->os_handle = l - lpm->linux_pci_devices; +static clib_error_t * +open_vfio_iommu_group (int group) +{ + linux_pci_main_t *lpm = &linux_pci_main; + linux_pci_vfio_iommu_group_t *g; + clib_error_t *err = 0; + struct vfio_group_status group_status; + u8 *s = 0; + int fd; - { - u8 *uio_dir = format (0, "%s/uio", l->dev_dir_name); - foreach_directory_file ((char *) uio_dir, scan_uio_dir, l, /* scan_dirs */ - 1); - vec_free (uio_dir); - } + g = get_vfio_iommu_group (group); + if (g) + { + g->refcnt++; + return 0; + } + s = format (s, "/dev/vfio/%u%c", group, 0); + fd = open ((char *) s, O_RDWR); + if (fd < 0) + clib_error_return_unix (0, "open '%s'", s); - { - char *uio_name = (char *) format (0, "/dev/uio%d%c", l->uio_minor, 0); - l->uio_fd = open (uio_name, O_RDWR); - if (l->uio_fd < 0) - clib_unix_error ("open `%s'", uio_name); - vec_free (uio_name); - } + group_status.argsz = sizeof (group_status); + if (ioctl (fd, VFIO_GROUP_GET_STATUS, &group_status) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_STATUS) '%s'", + s); + goto error; + } - { - clib_file_t template = { 0 }; + if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) + { + err = clib_error_return (0, "iommu group %d is not viable (not all " + "devices in this group bound to vfio-pci)", + group); + goto error; + } + + if (ioctl (fd, VFIO_GROUP_SET_CONTAINER, &lpm->vfio_container_fd) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_SET_CONTAINER) '%s'", + s); + goto error; + } - template.read_function = linux_pci_uio_read_ready; - template.file_descriptor = l->uio_fd; - template.error_function = linux_pci_uio_error_ready; - template.private_data = dev - pm->pci_devs; + if (lpm->vfio_iommu_mode == 0) + { + if (ioctl (lpm->vfio_container_fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU) < + 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_SET_IOMMU) " + "'/dev/vfio/vfio'"); + goto error; + } + lpm->vfio_iommu_mode = VFIO_TYPE1_IOMMU; + } - l->clib_file_index = clib_file_add (&file_main, &template); - } + + pool_get (lpm->iommu_groups, g); + g->fd = fd; + g->refcnt = 1; + hash_set (lpm->iommu_pool_index_by_group, group, g - lpm->iommu_groups); + vec_free (s); + return 0; +error: + close (fd); + return err; } -static void -linux_pci_device_free (linux_pci_device_t * l) +static clib_error_t * +add_device_vfio (vlib_main_t * vm, linux_pci_device_t * p, + vlib_pci_device_info_t * di, pci_device_registration_t * r) { - int i; - for (i = 0; i < vec_len (l->resource_fds); i++) - if (l->resource_fds[i] > 0) - close (l->resource_fds[i]); - if (l->config_fd > 0) - close (l->config_fd); - if (l->uio_fd > 0) - close (l->uio_fd); - vec_free (l->resource_fds); - vec_free (l->dev_dir_name); + linux_pci_vfio_iommu_group_t *g; + struct vfio_device_info device_info; + struct vfio_irq_info irq_info = { 0 }; + clib_error_t *err = 0; + u8 *s = 0; + int dfd; + + p->addr.as_u32 = di->addr.as_u32; + + if (di->driver_name == 0 || + (strcmp ("vfio-pci", (char *) di->driver_name) != 0)) + return clib_error_return (0, "Device '%U' (iommu group %d) not bound to " + "vfio-pci", format_vlib_pci_addr, &di->addr, + di->iommu_group); + + if ((err = open_vfio_iommu_group (di->iommu_group))) + return err; + + g = get_vfio_iommu_group (di->iommu_group); + + s = format (s, "%U%c", format_vlib_pci_addr, &di->addr, 0); + if ((dfd = ioctl (g->fd, VFIO_GROUP_GET_DEVICE_FD, (char *) s)) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_DEVICE_FD) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + vec_reset_length (s); + p->vfio_device_fd = dfd; + + device_info.argsz = sizeof (device_info); + if (ioctl (dfd, VFIO_DEVICE_GET_INFO, &device_info) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + + irq_info.argsz = sizeof (struct vfio_irq_info); + irq_info.index = VFIO_PCI_INTX_IRQ_INDEX; + if (ioctl (dfd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) " + "'%U'", format_vlib_pci_addr, &di->addr); + goto error; + } + + /* reset if device supports it */ + if (device_info.flags & VFIO_DEVICE_FLAGS_RESET) + if (ioctl (dfd, VFIO_DEVICE_RESET) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_RESET) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + + if ((irq_info.flags & VFIO_IRQ_INFO_EVENTFD) && irq_info.count == 1) + { + u8 buf[sizeof (struct vfio_irq_set) + sizeof (int)] = { 0 }; + struct vfio_irq_set *irq_set = (struct vfio_irq_set *) buf; + clib_file_t t = { 0 }; + int efd = eventfd (0, EFD_NONBLOCK); + + irq_set->argsz = sizeof (buf); + irq_set->count = 1; + irq_set->index = VFIO_PCI_INTX_IRQ_INDEX; + irq_set->flags = + VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + clib_memcpy (&irq_set->data, &efd, sizeof (int)); + + if (ioctl (dfd, VFIO_DEVICE_SET_IRQS, irq_set) < 0) + { + err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_SET_IRQS) '%U'", + format_vlib_pci_addr, &di->addr); + goto error; + } + + t.read_function = linux_pci_vfio_read_ready; + t.file_descriptor = efd; + t.error_function = linux_pci_vfio_error_ready; + t.private_data = p->handle; + p->clib_file_index = clib_file_add (&file_main, &t); + + /* unmask the interrupt */ + linux_pci_vfio_unmask_intx (p); + } + + p->interrupt_handler = r->interrupt_handler; + + s = format (s, "%s/%U/config%c", sysfs_pci_dev_path, + format_vlib_pci_addr, &di->addr, 0); + + p->config_fd = open ((char *) s, O_RDWR); + vec_reset_length (s); + + if (p->config_fd == -1) + { + err = clib_error_return_unix (0, "open '%s'", s); + goto error; + } + + err = r->init_function (vm, p->handle); + +error: + vec_free (s); + if (err) + { + close (dfd); + close (p->config_fd); + } + return err; } /* Configuration space read/write. */ clib_error_t * -vlib_pci_read_write_config (vlib_pci_device_t * dev, +vlib_pci_read_write_config (vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes) { - linux_pci_main_t *lpm = &linux_pci_main; - linux_pci_device_t *p; + linux_pci_device_t *p = linux_pci_get_device (h); int n; - p = pool_elt_at_index (lpm->linux_pci_devices, dev->os_handle); - if (read_or_write == VLIB_READ) n = pread (p->config_fd, data, n_bytes, address); else @@ -347,11 +808,10 @@ vlib_pci_read_write_config (vlib_pci_device_t * dev, } static clib_error_t * -os_map_pci_resource_internal (uword os_handle, - u32 resource, u8 * addr, void **result) +vlib_pci_map_resource_int (vlib_pci_dev_handle_t h, + u32 resource, u8 * addr, void **result) { - linux_pci_main_t *pm = &linux_pci_main; - linux_pci_device_t *p; + linux_pci_device_t *p = linux_pci_get_device (h); struct stat stat_buf; u8 *file_name; int fd; @@ -359,9 +819,10 @@ os_map_pci_resource_internal (uword os_handle, int flags = MAP_SHARED; error = 0; - p = pool_elt_at_index (pm->linux_pci_devices, os_handle); - file_name = format (0, "%v/resource%d%c", p->dev_dir_name, resource, 0); + file_name = format (0, "%s/%U/resource%d%c", sysfs_pci_dev_path, + format_vlib_pci_addr, &p->addr, resource, 0); + fd = open ((char *) file_name, O_RDWR); if (fd < 0) { @@ -402,240 +863,131 @@ done: } clib_error_t * -vlib_pci_map_resource (vlib_pci_device_t * dev, u32 resource, void **result) +vlib_pci_map_resource (vlib_pci_dev_handle_t h, u32 resource, void **result) { - return (os_map_pci_resource_internal - (dev->os_handle, resource, 0 /* addr */ , - result)); + return (vlib_pci_map_resource_int (h, resource, 0 /* addr */ , result)); } clib_error_t * -vlib_pci_map_resource_fixed (vlib_pci_device_t * dev, +vlib_pci_map_resource_fixed (vlib_pci_dev_handle_t h, u32 resource, u8 * addr, void **result) { - return (os_map_pci_resource_internal - (dev->os_handle, resource, addr, result)); + return (vlib_pci_map_resource_int (h, resource, addr, result)); } void -vlib_pci_free_device (vlib_pci_device_t * dev) -{ - linux_pci_main_t *pm = &linux_pci_main; - linux_pci_device_t *l; - - l = pool_elt_at_index (pm->linux_pci_devices, dev->os_handle); - linux_pci_device_free (l); - pool_put (pm->linux_pci_devices, l); -} - -pci_device_registration_t * __attribute__ ((unused)) -pci_device_next_registered (pci_device_registration_t * r) -{ - uword i; - - /* Null vendor id marks end of initialized list. */ - for (i = 0; r->supported_devices[i].vendor_id != 0; i++) - ; - - return clib_elf_section_data_next (r, i * sizeof (r->supported_devices[0])); -} - -static clib_error_t * -init_device_from_registered (vlib_main_t * vm, - vlib_pci_device_t * dev, - linux_pci_device_t * pdev) +init_device_from_registered (vlib_main_t * vm, vlib_pci_device_info_t * di) { vlib_pci_main_t *pm = &pci_main; + linux_pci_main_t *lpm = &linux_pci_main; pci_device_registration_t *r; pci_device_id_t *i; - clib_error_t *error; + clib_error_t *err = 0; + linux_pci_device_t *p; + + pool_get (lpm->linux_pci_devices, p); + p->handle = p - lpm->linux_pci_devices; r = pm->pci_device_registrations; while (r) { for (i = r->supported_devices; i->vendor_id != 0; i++) - if (i->vendor_id == dev->vendor_id && i->device_id == dev->device_id) + if (i->vendor_id == di->config0.header.vendor_id && + i->device_id == di->config0.header.device_id) { - error = vlib_pci_bind_to_uio (dev, "uio_pci_generic"); - if (error) - { - clib_error_report (error); - continue; - } - - add_device (dev, pdev); - dev->interrupt_handler = r->interrupt_handler; - return r->init_function (vm, dev); + if (di->iommu_group != -1) + err = add_device_vfio (vm, p, di, r); + else + err = add_device_uio (vm, p, di, r); + + if (err) + clib_error_report (err); + else + return; } r = r->next_registration; } - /* No driver, close the PCI config-space FD */ - close (pdev->config_fd); - return 0; -} -static clib_error_t * -init_device (vlib_main_t * vm, - vlib_pci_device_t * dev, linux_pci_device_t * pdev) -{ - return init_device_from_registered (vm, dev, pdev); + /* No driver, close the PCI config-space FD */ + memset (p, 0, sizeof (linux_pci_device_t)); + pool_put (lpm->linux_pci_devices, p); } static clib_error_t * -scan_device (void *arg, u8 * dev_dir_name, u8 * ignored) +scan_pci_addr (void *arg, u8 * dev_dir_name, u8 * ignored) { - vlib_main_t *vm = arg; - vlib_pci_main_t *pm = &pci_main; - int fd; - u8 *f; - clib_error_t *error = 0; - vlib_pci_device_t *dev; - linux_pci_device_t pdev = { 0 }; - u32 tmp; - - f = format (0, "%v/config%c", dev_dir_name, 0); - fd = open ((char *) f, O_RDWR); - - /* Try read-only access if write fails. */ - if (fd < 0) - fd = open ((char *) f, O_RDONLY); - - if (fd < 0) - { - error = clib_error_return_unix (0, "open `%s'", f); - goto done; - } - - pool_get (pm->pci_devs, dev); - - /* You can only read more that 64 bytes of config space as root; so we try to - read the full space but fall back to just the first 64 bytes. */ - if (read (fd, &dev->config_data, sizeof (dev->config_data)) != - sizeof (dev->config_data) - && read (fd, &dev->config0, - sizeof (dev->config0)) != sizeof (dev->config0)) - { - pool_put (pm->pci_devs, dev); - error = clib_error_return_unix (0, "read `%s'", f); - close (fd); - goto done; - } - - { - static pci_config_header_t all_ones; - if (all_ones.vendor_id == 0) - memset (&all_ones, ~0, sizeof (all_ones)); - - if (!memcmp (&dev->config0.header, &all_ones, sizeof (all_ones))) - { - pool_put (pm->pci_devs, dev); - error = clib_error_return (0, "invalid PCI config for `%s'", f); - close (fd); - goto done; - } - } - - if (dev->config0.header.header_type == 0) - pci_config_type0_little_to_host (&dev->config0); - else - pci_config_type1_little_to_host (&dev->config1); - - /* Parse bus, dev, function from directory name. */ - { - unformat_input_t input; - - unformat_init_string (&input, (char *) dev_dir_name, - vec_len (dev_dir_name)); + vlib_pci_addr_t addr, **addrv = arg; + unformat_input_t input; + clib_error_t *err = 0; - if (!unformat (&input, "/sys/bus/pci/devices/%U", - unformat_vlib_pci_addr, &dev->bus_address)) - abort (); + unformat_init_string (&input, (char *) dev_dir_name, + vec_len (dev_dir_name)); - unformat_free (&input); + if (!unformat (&input, "/sys/bus/pci/devices/%U", + unformat_vlib_pci_addr, &addr)) + err = clib_error_return (0, "unformat error `%v`", dev_dir_name); - } + unformat_free (&input); + if (err) + return err; - pdev.config_fd = fd; - pdev.dev_dir_name = dev_dir_name; + vec_add1 (*addrv, addr); + return 0; +} - hash_set (pm->pci_dev_index_by_pci_addr, dev->bus_address.as_u32, - dev - pm->pci_devs); +static int +pci_addr_cmp (void *v1, void *v2) +{ + vlib_pci_addr_t *a1 = v1; + vlib_pci_addr_t *a2 = v2; + + if (a1->domain > a2->domain) + return 1; + if (a1->domain < a2->domain) + return -1; + if (a1->bus > a2->bus) + return 1; + if (a1->bus < a2->bus) + return -1; + if (a1->slot > a2->slot) + return 1; + if (a1->slot < a2->slot) + return -1; + if (a1->function > a2->function) + return 1; + if (a1->function < a2->function) + return -1; + return 0; +} - vec_reset_length (f); - f = format (f, "%v/vpd%c", dev_dir_name, 0); - fd = open ((char *) f, O_RDONLY); - if (fd >= 0) +vlib_pci_addr_t * +vlib_pci_get_all_dev_addrs () +{ + vlib_pci_addr_t *addrs = 0; + clib_error_t *err; + err = foreach_directory_file ((char *) sysfs_pci_dev_path, scan_pci_addr, + &addrs, /* scan_dirs */ 0); + if (err) { - while (1) - { - u8 tag[3]; - u8 *data = 0; - int len; - - if (read (fd, &tag, 3) != 3) - break; - - if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91) - break; - - len = (tag[2] << 8) | tag[1]; - vec_validate (data, len); - - if (read (fd, data, len) != len) - { - vec_free (data); - break; - } - if (tag[0] == 0x82) - dev->product_name = data; - else if (tag[0] == 0x90) - dev->vpd_r = data; - else if (tag[0] == 0x91) - dev->vpd_w = data; - - data = 0; - } - close (fd); + vec_free (addrs); + return 0; } - dev->numa_node = -1; - vec_reset_length (f); - f = format (f, "%v/numa_node%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "%u", &dev->numa_node); - - vec_reset_length (f); - f = format (f, "%v/class%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->device_class = tmp >> 8; - - vec_reset_length (f); - f = format (f, "%v/vendor%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->vendor_id = tmp; - - vec_reset_length (f); - f = format (f, "%v/device%c", dev_dir_name, 0); - clib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->device_id = tmp; + vec_sort_with_function (addrs, pci_addr_cmp); - error = init_device (vm, dev, &pdev); - - vec_reset_length (f); - f = format (f, "%v/driver%c", dev_dir_name, 0); - dev->driver_name = clib_sysfs_link_to_name ((char *) f); - -done: - vec_free (f); - return error; + return addrs; } clib_error_t * linux_pci_init (vlib_main_t * vm) { vlib_pci_main_t *pm = &pci_main; + linux_pci_main_t *lpm = &linux_pci_main; + vlib_pci_addr_t *addr = 0, *addrs; clib_error_t *error; + int fd; pm->vlib_main = vm; @@ -643,14 +995,36 @@ linux_pci_init (vlib_main_t * vm) return error; ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32)); - pm->pci_dev_index_by_pci_addr = hash_create (0, sizeof (uword)); - error = foreach_directory_file ("/sys/bus/pci/devices", scan_device, vm, - /* scan_dirs */ 0); + fd = open ("/dev/vfio/vfio", O_RDWR); - /* Complain and continue. might not be root, etc. */ - if (error) - clib_error_report (error); + if ((fd != -1) && (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION)) + { + close (fd); + fd = -1; + } + + if ((fd != -1) && (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 0)) + { + close (fd); + fd = -1; + } + + lpm->vfio_container_fd = fd; + lpm->iommu_pool_index_by_group = hash_create (0, sizeof (uword)); + + addrs = vlib_pci_get_all_dev_addrs (); + /* *INDENT-OFF* */ + vec_foreach (addr, addrs) + { + vlib_pci_device_info_t *d; + if ((d = vlib_pci_get_device_info (addr, 0))) + { + init_device_from_registered (vm, d); + vlib_pci_free_device_info (d); + } + } + /* *INDENT-ON* */ return error; } diff --git a/src/vlib/pci/pci.c b/src/vlib/pci/pci.c index 1f1edab62e6..f50ae2d9edf 100644 --- a/src/vlib/pci/pci.c +++ b/src/vlib/pci/pci.c @@ -52,25 +52,24 @@ vlib_pci_main_t pci_main; -vlib_pci_device_t * -vlib_get_pci_device (vlib_pci_addr_t * addr) +vlib_pci_device_info_t * __attribute__ ((weak)) +vlib_pci_get_device_info (vlib_pci_addr_t * addr, clib_error_t ** error) { - vlib_pci_main_t *pm = &pci_main; - uword *p; - p = hash_get (pm->pci_dev_index_by_pci_addr, addr->as_u32); - - if (p == 0) - return 0; + if (error) + *error = clib_error_return (0, "unsupported"); + return 0; +} - return vec_elt_at_index (pm->pci_devs, p[0]); +vlib_pci_addr_t * __attribute__ ((weak)) vlib_pci_get_all_dev_addrs () +{ + return 0; } static clib_error_t * show_pci_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - vlib_pci_main_t *pm = &pci_main; - vlib_pci_device_t *d; + vlib_pci_addr_t *addr = 0, *addrs; int show_all = 0; u8 *s = 0; @@ -87,28 +86,36 @@ show_pci_fn (vlib_main_t * vm, "Address", "Sock", "VID:PID", "Link Speed", "Driver", "Product Name", "Vital Product Data"); + addrs = vlib_pci_get_all_dev_addrs (); /* *INDENT-OFF* */ - pool_foreach (d, pm->pci_devs, ({ - - if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && !show_all) - continue; - - vec_reset_length (s); - - if (d->numa_node >= 0) - s = format (s, " %d", d->numa_node); - - vlib_cli_output (vm, "%-13U%-5v%04x:%04x %-13U%-16s%-32v%U", - format_vlib_pci_addr, &d->bus_address, s, - d->vendor_id, d->device_id, - format_vlib_pci_link_speed, d, - d->driver_name ? (char *) d->driver_name : "", - d->product_name, - format_vlib_pci_vpd, d->vpd_r, 0); - })); -/* *INDENT-ON* */ + vec_foreach (addr, addrs) + { + vlib_pci_device_info_t *d; + d = vlib_pci_get_device_info (addr, 0); + + if (!d) + continue; + + if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && !show_all) + continue; + + vec_reset_length (s); + if (d->numa_node >= 0) + s = format (s, " %d", d->numa_node); + + vlib_cli_output (vm, "%-13U%-5v%04x:%04x %-13U%-16s%-32v%U", + format_vlib_pci_addr, addr, s, + d->vendor_id, d->device_id, + format_vlib_pci_link_speed, d, + d->driver_name ? (char *) d->driver_name : "", + d->product_name, + format_vlib_pci_vpd, d->vpd_r, 0); + vlib_pci_free_device_info (d); + } + /* *INDENT-ON* */ vec_free (s); + vec_free (addrs); return 0; } @@ -138,16 +145,9 @@ format_vlib_pci_addr (u8 * s, va_list * va) } u8 * -format_vlib_pci_handle (u8 * s, va_list * va) -{ - vlib_pci_addr_t *addr = va_arg (*va, vlib_pci_addr_t *); - return format (s, "%x/%x/%x", addr->bus, addr->slot, addr->function); -} - -u8 * format_vlib_pci_link_speed (u8 * s, va_list * va) { - vlib_pci_device_t *d = va_arg (*va, vlib_pci_device_t *); + vlib_pci_device_info_t *d = va_arg (*va, vlib_pci_device_info_t *); pcie_config_regs_t *r = pci_config_find_capability (&d->config0, PCI_CAP_ID_PCIE); int width; @@ -250,7 +250,7 @@ VLIB_CLI_COMMAND (show_pci_command, static) = { clib_error_t * pci_bus_init (vlib_main_t * vm) { - return 0; + return vlib_call_init_function (vm, pci_bus_init); } VLIB_INIT_FUNCTION (pci_bus_init); diff --git a/src/vlib/pci/pci.h b/src/vlib/pci/pci.h index 2141080936d..e1bc4172e8b 100644 --- a/src/vlib/pci/pci.h +++ b/src/vlib/pci/pci.h @@ -43,33 +43,24 @@ #include <vlib/vlib.h> #include <vlib/pci/pci_config.h> +/* *INDENT-OFF* */ typedef CLIB_PACKED (union - { - struct - { -u16 domain; u8 bus; u8 slot: 5; u8 function:3;}; - u32 as_u32;}) vlib_pci_addr_t; - -typedef struct vlib_pci_device { - /* Operating system handle for this device. */ - uword os_handle; - - vlib_pci_addr_t bus_address; - - /* First 64 bytes of configuration space. */ - union - { - pci_config_type0_regs_t config0; - pci_config_type1_regs_t config1; - u8 config_data[256]; - }; - - /* Interrupt handler */ - void (*interrupt_handler) (struct vlib_pci_device * dev); - - /* Driver name */ - u8 *driver_name; + struct + { + u16 domain; + u8 bus; + u8 slot: 5; + u8 function:3; + }; + u32 as_u32; +}) vlib_pci_addr_t; +/* *INDENT-ON* */ + +typedef struct vlib_pci_device_info +{ + /* addr */ + vlib_pci_addr_t addr; /* Numa Node */ int numa_node; @@ -84,10 +75,42 @@ typedef struct vlib_pci_device u8 *vpd_r; u8 *vpd_w; - /* Private data */ - uword private_data; + /* Driver name */ + u8 *driver_name; + + /* First 64 bytes of configuration space. */ + union + { + pci_config_type0_regs_t config0; + pci_config_type1_regs_t config1; + u8 config_data[256]; + }; + + /* IOMMU Group */ + int iommu_group; + +} vlib_pci_device_info_t; + +typedef u32 vlib_pci_dev_handle_t; -} vlib_pci_device_t; +vlib_pci_device_info_t *vlib_pci_get_device_info (vlib_pci_addr_t * addr, + clib_error_t ** error); +vlib_pci_addr_t *vlib_pci_get_all_dev_addrs (); +vlib_pci_addr_t *vlib_pci_get_addr (vlib_pci_dev_handle_t h); +uword vlib_pci_get_private_data (vlib_pci_dev_handle_t h); +void vlib_pci_set_private_data (vlib_pci_dev_handle_t h, uword private_data); + +static inline void +vlib_pci_free_device_info (vlib_pci_device_info_t * di) +{ + if (!di) + return; + vec_free (di->product_name); + vec_free (di->vpd_r); + vec_free (di->vpd_w); + vec_free (di->driver_name); + clib_mem_free (di); +} typedef struct { @@ -97,10 +120,11 @@ typedef struct typedef struct _pci_device_registration { /* Driver init function. */ - clib_error_t *(*init_function) (vlib_main_t * vm, vlib_pci_device_t * dev); + clib_error_t *(*init_function) (vlib_main_t * vm, + vlib_pci_dev_handle_t handle); /* Interrupt handler */ - void (*interrupt_handler) (vlib_pci_device_t * dev); + void (*interrupt_handler) (vlib_pci_dev_handle_t handle); /* List of registrations */ struct _pci_device_registration *next_registration; @@ -113,9 +137,7 @@ typedef struct _pci_device_registration typedef struct { vlib_main_t *vlib_main; - vlib_pci_device_t *pci_devs; pci_device_registration_t *pci_device_registrations; - uword *pci_dev_index_by_pci_addr; } vlib_pci_main_t; extern vlib_pci_main_t pci_main; @@ -132,21 +154,21 @@ static void __vlib_add_pci_device_registration_##x (void) \ } \ __VA_ARGS__ pci_device_registration_t x -clib_error_t *vlib_pci_bind_to_uio (vlib_pci_device_t * d, +clib_error_t *vlib_pci_bind_to_uio (vlib_pci_addr_t * addr, char *uio_driver_name); /* Configuration space read/write. */ -clib_error_t *vlib_pci_read_write_config (vlib_pci_device_t * dev, +clib_error_t *vlib_pci_read_write_config (vlib_pci_dev_handle_t handle, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes); #define _(t) \ static inline clib_error_t * \ -vlib_pci_read_config_##t (vlib_pci_device_t * dev, \ +vlib_pci_read_config_##t (vlib_pci_dev_handle_t h, \ uword address, t * data) \ { \ - return vlib_pci_read_write_config (dev, VLIB_READ,address, data, \ + return vlib_pci_read_write_config (h, VLIB_READ,address, data, \ sizeof (data[0])); \ } @@ -158,10 +180,10 @@ _(u8); #define _(t) \ static inline clib_error_t * \ -vlib_pci_write_config_##t (vlib_pci_device_t * dev, uword address, \ +vlib_pci_write_config_##t (vlib_pci_dev_handle_t h, uword address, \ t * data) \ { \ - return vlib_pci_read_write_config (dev, VLIB_WRITE, \ + return vlib_pci_read_write_config (h, VLIB_WRITE, \ address, data, sizeof (data[0])); \ } @@ -172,45 +194,45 @@ _(u8); #undef _ static inline clib_error_t * -vlib_pci_intr_enable (vlib_pci_device_t * dev) +vlib_pci_intr_enable (vlib_pci_dev_handle_t h) { u16 command; clib_error_t *err; - err = vlib_pci_read_config_u16 (dev, 4, &command); + err = vlib_pci_read_config_u16 (h, 4, &command); if (err) return err; command &= ~PCI_COMMAND_INTX_DISABLE; - return vlib_pci_write_config_u16 (dev, 4, &command); + return vlib_pci_write_config_u16 (h, 4, &command); } static inline clib_error_t * -vlib_pci_intr_disable (vlib_pci_device_t * dev) +vlib_pci_intr_disable (vlib_pci_dev_handle_t h) { u16 command; clib_error_t *err; - err = vlib_pci_read_config_u16 (dev, 4, &command); + err = vlib_pci_read_config_u16 (h, 4, &command); if (err) return err; command |= PCI_COMMAND_INTX_DISABLE; - return vlib_pci_write_config_u16 (dev, 4, &command); + return vlib_pci_write_config_u16 (h, 4, &command); } static inline clib_error_t * -vlib_pci_bus_master_enable (vlib_pci_device_t * dev) +vlib_pci_bus_master_enable (vlib_pci_dev_handle_t h) { clib_error_t *err; u16 command; /* Set bus master enable (BME) */ - err = vlib_pci_read_config_u16 (dev, 4, &command); + err = vlib_pci_read_config_u16 (h, 4, &command); if (err) return err; @@ -220,23 +242,18 @@ vlib_pci_bus_master_enable (vlib_pci_device_t * dev) command |= PCI_COMMAND_BUS_MASTER; - return vlib_pci_write_config_u16 (dev, 4, &command); + return vlib_pci_write_config_u16 (h, 4, &command); } -clib_error_t *vlib_pci_map_resource (vlib_pci_device_t * dev, u32 resource, +clib_error_t *vlib_pci_map_resource (vlib_pci_dev_handle_t h, u32 resource, void **result); -clib_error_t *vlib_pci_map_resource_fixed (vlib_pci_device_t * dev, +clib_error_t *vlib_pci_map_resource_fixed (vlib_pci_dev_handle_t h, u32 resource, u8 * addr, void **result); -vlib_pci_device_t *vlib_get_pci_device (vlib_pci_addr_t * addr); -/* Free's device. */ -void vlib_pci_free_device (vlib_pci_device_t * dev); - unformat_function_t unformat_vlib_pci_addr; format_function_t format_vlib_pci_addr; -format_function_t format_vlib_pci_handle; format_function_t format_vlib_pci_link_speed; format_function_t format_vlib_pci_vpd; |