diff options
Diffstat (limited to 'src/plugins/memif/memif.c')
| -rw-r--r-- | src/plugins/memif/memif.c | 477 |
1 files changed, 293 insertions, 184 deletions
diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 9bbbe7f9d89..7e3dd44db2c 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -26,7 +26,6 @@ #include <sys/un.h> #include <sys/uio.h> #include <sys/mman.h> -#include <sys/prctl.h> #include <sys/eventfd.h> #include <inttypes.h> #include <limits.h> @@ -36,6 +35,7 @@ #include <vnet/plugin/plugin.h> #include <vnet/ethernet/ethernet.h> #include <vnet/interface/rx_queue_funcs.h> +#include <vnet/interface/tx_queue_funcs.h> #include <vpp/app/version.h> #include <memif/memif.h> #include <memif/private.h> @@ -49,6 +49,14 @@ memif_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) return 0; } +static clib_error_t * +memif_eth_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hi, + u32 flags) +{ + /* nothing for now */ + return 0; +} + static void memif_queue_intfd_close (memif_queue_t * mq) { @@ -91,6 +99,8 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) memif_region_t *mr; memif_queue_t *mq; int i; + vlib_main_t *vm = vlib_get_main (); + int with_barrier = 0; if (mif == 0) return; @@ -132,7 +142,12 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) clib_mem_free (mif->sock); } - /* *INDENT-OFF* */ + if (vlib_worker_thread_barrier_held () == 0) + { + with_barrier = 1; + vlib_worker_thread_barrier_sync (vm); + } + vec_foreach_index (i, mif->rx_queues) { mq = vec_elt_at_index (mif->rx_queues, i); @@ -146,9 +161,7 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) } } vnet_hw_if_unregister_all_rx_queues (vnm, mif->hw_if_index); - vnet_hw_if_update_runtime_data (vnm, mif->hw_if_index); - /* *INDENT-OFF* */ vec_foreach_index (i, mif->tx_queues) { mq = vec_elt_at_index (mif->tx_queues, i); @@ -158,9 +171,12 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) { memif_disconnect_free_zc_queue_buffer(mq, 0); } + clib_spinlock_free (&mq->lockp); } mq->ring = 0; } + vnet_hw_if_unregister_all_tx_queues (vnm, mif->hw_if_index); + vnet_hw_if_update_runtime_data (vnm, mif->hw_if_index); /* free tx and rx queues */ vec_foreach (mq, mif->rx_queues) @@ -182,11 +198,13 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) if (mr->fd > -1) close (mr->fd); } - /* *INDENT-ON* */ vec_free (mif->regions); vec_free (mif->remote_name); vec_free (mif->remote_if_name); clib_fifo_free (mif->msg_queue); + + if (with_barrier) + vlib_worker_thread_barrier_release (vm); } static clib_error_t * @@ -228,19 +246,22 @@ memif_int_fd_read_ready (clib_file_t * uf) clib_error_t * memif_connect (memif_if_t * mif) { + memif_main_t *mm = &memif_main; vlib_main_t *vm = vlib_get_main (); vnet_main_t *vnm = vnet_get_main (); clib_file_t template = { 0 }; memif_region_t *mr; - int i; + int i, j; + u32 n_txqs = 0, n_threads = vlib_get_n_threads (); clib_error_t *err = NULL; + u8 max_log2_ring_sz = 0; + int with_barrier = 0; memif_log_debug (mif, "connect %u", mif->dev_instance); vec_free (mif->local_disc_string); vec_free (mif->remote_disc_string); - /* *INDENT-OFF* */ vec_foreach (mr, mif->regions) { if (mr->shm) @@ -259,15 +280,21 @@ memif_connect (memif_if_t * mif) goto error; } } - /* *INDENT-ON* */ template.read_function = memif_int_fd_read_ready; template.write_function = memif_int_fd_write_ready; - /* *INDENT-OFF* */ + with_barrier = 1; + if (vlib_worker_thread_barrier_held ()) + with_barrier = 0; + + if (with_barrier) + vlib_worker_thread_barrier_sync (vm); + vec_foreach_index (i, mif->tx_queues) { memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i); + max_log2_ring_sz = clib_max (max_log2_ring_sz, mq->log2_ring_size); mq->ring = mif->regions[mq->region].shm + mq->offset; if (mq->ring->cookie != MEMIF_COOKIE) @@ -275,6 +302,50 @@ memif_connect (memif_if_t * mif) err = clib_error_return (0, "wrong cookie on tx ring %u", i); goto error; } + mq->queue_index = + vnet_hw_if_register_tx_queue (vnm, mif->hw_if_index, i); + clib_spinlock_init (&mq->lockp); + + if (mif->flags & MEMIF_IF_FLAG_USE_DMA) + { + memif_dma_info_t *dma_info; + mq->dma_head = 0; + mq->dma_tail = 0; + mq->dma_info_head = 0; + mq->dma_info_tail = 0; + mq->dma_info_size = MEMIF_DMA_INFO_SIZE; + vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE, + CLIB_CACHE_LINE_BYTES); + + vec_foreach (dma_info, mq->dma_info) + { + vec_validate_aligned (dma_info->data.desc_data, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_len, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_status, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.copy_ops, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.copy_ops); + vec_validate_aligned (dma_info->data.buffers, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.buffers); + } + } + } + + if (vec_len (mif->tx_queues) > 0) + { + n_txqs = vec_len (mif->tx_queues); + for (j = 0; j < n_threads; j++) + { + u32 qi = mif->tx_queues[j % n_txqs].queue_index; + vnet_hw_if_tx_queue_assign_thread (vnm, qi, j); + } } vec_foreach_index (i, mif->rx_queues) @@ -284,6 +355,8 @@ memif_connect (memif_if_t * mif) u32 qi; int rv; + max_log2_ring_sz = clib_max (max_log2_ring_sz, mq->log2_ring_size); + mq->ring = mif->regions[mq->region].shm + mq->offset; if (mq->ring->cookie != MEMIF_COOKIE) { @@ -293,6 +366,37 @@ memif_connect (memif_if_t * mif) qi = vnet_hw_if_register_rx_queue (vnm, mif->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY); mq->queue_index = qi; + + if (mif->flags & MEMIF_IF_FLAG_USE_DMA) + { + memif_dma_info_t *dma_info; + mq->dma_head = 0; + mq->dma_tail = 0; + mq->dma_info_head = 0; + mq->dma_info_tail = 0; + mq->dma_info_size = MEMIF_DMA_INFO_SIZE; + vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE, + CLIB_CACHE_LINE_BYTES); + vec_foreach (dma_info, mq->dma_info) + { + vec_validate_aligned (dma_info->data.desc_data, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_len, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_status, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.copy_ops, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.copy_ops); + vec_validate_aligned (dma_info->data.buffers, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.buffers); + } + } + if (mq->int_fd > -1) { template.file_descriptor = mq->int_fd; @@ -324,7 +428,23 @@ memif_connect (memif_if_t * mif) vnet_hw_if_rx_queue_set_int_pending (vnm, qi); } } - /* *INDENT-ON* */ + + if (1 << max_log2_ring_sz > vec_len (mm->per_thread_data[0].desc_data)) + { + memif_per_thread_data_t *ptd; + + vec_foreach (ptd, mm->per_thread_data) + { + vec_validate_aligned (ptd->desc_data, pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (ptd->desc_len, pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (ptd->desc_status, pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + } + } + if (with_barrier) + vlib_worker_thread_barrier_release (vm); mif->flags &= ~MEMIF_IF_FLAG_CONNECTING; mif->flags |= MEMIF_IF_FLAG_CONNECTED; @@ -334,6 +454,8 @@ memif_connect (memif_if_t * mif) return 0; error: + if (with_barrier) + vlib_worker_thread_barrier_release (vm); memif_log_err (mif, "%U", format_clib_error, err); return err; } @@ -405,7 +527,6 @@ memif_init_regions_and_queues (memif_if_t * mif) if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) { vlib_buffer_pool_t *bp; - /* *INDENT-OFF* */ vec_foreach (bp, vm->buffer_main->buffer_pools) { vlib_physmem_map_t *pm; @@ -416,7 +537,6 @@ memif_init_regions_and_queues (memif_if_t * mif) r->shm = pm->base; r->is_external = 1; } - /* *INDENT-ON* */ } for (i = 0; i < mif->run.num_s2m_rings; i++) @@ -461,7 +581,6 @@ memif_init_regions_and_queues (memif_if_t * mif) vec_validate_aligned (mif->tx_queues, mif->run.num_s2m_rings - 1, CLIB_CACHE_LINE_BYTES); - /* *INDENT-OFF* */ vec_foreach_index (i, mif->tx_queues) { memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i); @@ -470,6 +589,7 @@ memif_init_regions_and_queues (memif_if_t * mif) err = clib_error_return_unix (0, "eventfd[tx queue %u]", i); goto error; } + mq->int_clib_file_index = ~0; mq->ring = memif_get_ring (mif, MEMIF_RING_S2M, i); mq->log2_ring_size = mif->cfg.log2_ring_size; @@ -481,13 +601,11 @@ memif_init_regions_and_queues (memif_if_t * mif) vec_validate_aligned (mq->buffers, 1 << mq->log2_ring_size, CLIB_CACHE_LINE_BYTES); } - /* *INDENT-ON* */ ASSERT (mif->rx_queues == 0); vec_validate_aligned (mif->rx_queues, mif->run.num_m2s_rings - 1, CLIB_CACHE_LINE_BYTES); - /* *INDENT-OFF* */ vec_foreach_index (i, mif->rx_queues) { memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i); @@ -507,7 +625,6 @@ memif_init_regions_and_queues (memif_if_t * mif) vec_validate_aligned (mq->buffers, 1 << mq->log2_ring_size, CLIB_CACHE_LINE_BYTES); } - /* *INDENT-ON* */ return 0; @@ -558,7 +675,6 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) } last_run_duration = start_time = vlib_time_now (vm); - /* *INDENT-OFF* */ pool_foreach (mif, mm->interfaces) { memif_socket_file_t * msf = vec_elt_at_index (mm->socket_files, mif->socket_file_index); @@ -583,8 +699,8 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { clib_memset (sock, 0, sizeof(clib_socket_t)); sock->config = (char *) msf->filename; - sock->flags = CLIB_SOCKET_F_IS_CLIENT | CLIB_SOCKET_F_SEQPACKET | - CLIB_SOCKET_F_BLOCKING; + sock->is_seqpacket = 1; + sock->is_blocking = 1; if ((err = clib_socket_init (sock))) { @@ -611,162 +727,160 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) } } } - /* *INDENT-ON* */ last_run_duration = vlib_time_now (vm) - last_run_duration; } return 0; } -/* *INDENT-OFF* */ VLIB_REGISTER_NODE (memif_process_node,static) = { .function = memif_process, .type = VLIB_NODE_TYPE_PROCESS, .name = "memif-process", }; -/* *INDENT-ON* */ -static int -memif_add_socket_file (u32 sock_id, u8 * socket_filename) +/* + * Returns an unused socket id, and ~0 if it can't find one. + */ +u32 +memif_get_unused_socket_id () { memif_main_t *mm = &memif_main; uword *p; - memif_socket_file_t *msf; + int i, j; - p = hash_get (mm->socket_file_index_by_sock_id, sock_id); - if (p) + static u32 seed = 0; + /* limit to 1M tries */ + for (j = 0; j < 1 << 10; j++) { - msf = pool_elt_at_index (mm->socket_files, *p); - if (strcmp ((char *) msf->filename, (char *) socket_filename) == 0) + seed = random_u32 (&seed); + for (i = 0; i < 1 << 10; i++) { - /* Silently accept identical "add". */ - return 0; + /* look around randomly generated id */ + seed += (2 * (i % 2) - 1) * i; + if (seed == (u32) ~0) + continue; + p = hash_get (mm->socket_file_index_by_sock_id, seed); + if (!p) + return seed; } - - /* But don't allow a direct add of a different filename. */ - return VNET_API_ERROR_ENTRY_ALREADY_EXISTS; } - pool_get (mm->socket_files, msf); - clib_memset (msf, 0, sizeof (memif_socket_file_t)); - - msf->filename = socket_filename; - msf->socket_id = sock_id; - - hash_set (mm->socket_file_index_by_sock_id, sock_id, - msf - mm->socket_files); - - return 0; + return ~0; } -static int -memif_delete_socket_file (u32 sock_id) +clib_error_t * +memif_socket_filename_add_del (u8 is_add, u32 sock_id, char *sock_filename) { memif_main_t *mm = &memif_main; uword *p; memif_socket_file_t *msf; + clib_error_t *err = 0; + char *dir = 0, *tmp; + u32 idx = 0; + u8 *name = 0; - p = hash_get (mm->socket_file_index_by_sock_id, sock_id); - if (!p) - { - /* Don't delete non-existent entries. */ - return VNET_API_ERROR_INVALID_ARGUMENT; - } + /* allow adding socket id 0 */ + if (sock_id == 0 && is_add == 0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, "cannot delete socket id 0"); - msf = pool_elt_at_index (mm->socket_files, *p); - if (msf->ref_cnt > 0) + if (sock_id == ~0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socked id is not specified"); + + if (is_add == 0) { - return VNET_API_ERROR_UNEXPECTED_INTF_STATE; - } + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (!p) + /* Don't delete non-existent entries. */ + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socket file with id %u does not exist", sock_id); - vec_free (msf->filename); - pool_put (mm->socket_files, msf); + msf = pool_elt_at_index (mm->socket_files, *p); + if (msf->ref_cnt > 0) + return vnet_error (VNET_ERR_UNEXPECTED_INTF_STATE, + "socket file '%s' is in use", msf->filename); - hash_unset (mm->socket_file_index_by_sock_id, sock_id); + vec_free (msf->filename); + pool_put (mm->socket_files, msf); - return 0; -} - -int -memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 * sock_filename) -{ - char *dir = 0, *tmp; - u32 idx = 0; + hash_unset (mm->socket_file_index_by_sock_id, sock_id); - /* allow adding socket id 0 */ - if ((sock_id == 0 && is_add == 0) || sock_id == ~0) - { - return VNET_API_ERROR_INVALID_ARGUMENT; + return 0; } - if (is_add == 0) + if (sock_filename == 0 || sock_filename[0] == 0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socket filename not specified"); + + if (clib_socket_prefix_is_valid (sock_filename)) { - return memif_delete_socket_file (sock_id); + name = format (0, "%s%c", sock_filename, 0); } - - if (sock_filename == 0 || sock_filename[0] == 0) + else if (sock_filename[0] == '/') { - return VNET_API_ERROR_INVALID_ARGUMENT; + name = format (0, "%s%c", sock_filename, 0); } - - if (sock_filename[0] != '/') + else { - clib_error_t *error; - /* copy runtime dir path */ vec_add (dir, vlib_unix_get_runtime_dir (), strlen (vlib_unix_get_runtime_dir ())); vec_add1 (dir, '/'); /* if sock_filename contains dirs, add them to path */ - tmp = strrchr ((char *) sock_filename, '/'); + tmp = strrchr (sock_filename, '/'); if (tmp) { - idx = tmp - (char *) sock_filename; + idx = tmp - sock_filename; vec_add (dir, sock_filename, idx); } vec_add1 (dir, '\0'); /* create socket dir */ - error = vlib_unix_recursive_mkdir (dir); - if (error) + if ((err = vlib_unix_recursive_mkdir (dir))) { - clib_error_free (error); - return VNET_API_ERROR_SYSCALL_ERROR_1; + clib_error_free (err); + err = vnet_error (VNET_ERR_SYSCALL_ERROR_1, + "unable to create socket dir"); + goto done; } - sock_filename = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), - sock_filename, 0); + name = + format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), sock_filename, 0); } - else - { - sock_filename = vec_dup (sock_filename); - /* check if directory exists */ - tmp = strrchr ((char *) sock_filename, '/'); - if (tmp) + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (p) + { + msf = pool_elt_at_index (mm->socket_files, *p); + if (strcmp ((char *) msf->filename, (char *) name) == 0) { - idx = tmp - (char *) sock_filename; - vec_add (dir, sock_filename, idx); - vec_add1 (dir, '\0'); + /* Silently accept identical "add". */ + goto done; } - /* check dir existance and access rights for effective user/group IDs */ - if ((dir == NULL) - || - (faccessat ( /* ignored */ -1, dir, F_OK | R_OK | W_OK, AT_EACCESS) - < 0)) - { - vec_free (dir); - return VNET_API_ERROR_INVALID_ARGUMENT; - } + /* But don't allow a direct add of a different filename. */ + err = vnet_error (VNET_ERR_ENTRY_ALREADY_EXISTS, "entry already exists"); + goto done; } - vec_free (dir); - return memif_add_socket_file (sock_id, sock_filename); + pool_get (mm->socket_files, msf); + clib_memset (msf, 0, sizeof (memif_socket_file_t)); + + msf->filename = name; + msf->socket_id = sock_id; + name = 0; + + hash_set (mm->socket_file_index_by_sock_id, sock_id, msf - mm->socket_files); + +done: + vec_free (name); + vec_free (dir); + return err; } -int -memif_delete_if (vlib_main_t * vm, memif_if_t * mif) +clib_error_t * +memif_delete_if (vlib_main_t *vm, memif_if_t *mif) { vnet_main_t *vnm = vnet_get_main (); memif_main_t *mm = &memif_main; @@ -797,7 +911,6 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif) } /* free interface data structures */ - clib_spinlock_free (&mif->lockp); mhash_unset (&msf->dev_instance_by_id, &mif->id, 0); /* remove socket file */ @@ -806,10 +919,8 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif) if (msf->is_listener) { int i; - /* *INDENT-OFF* */ vec_foreach_index (i, msf->pending_clients) memif_socket_close (msf->pending_clients + i); - /* *INDENT-ON* */ memif_socket_close (&msf->sock); vec_free (msf->pending_clients); } @@ -827,6 +938,7 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif) } } + vec_free (mif->local_disc_string); clib_memset (mif, 0, sizeof (*mif)); pool_put (mm->interfaces, mif); @@ -837,33 +949,39 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif) return 0; } -/* *INDENT-OFF* */ -VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = -{ +VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = { .name = "memif-ip", .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P, + .tx_hash_fn_type = VNET_HASH_FN_TYPE_IP, }; -/* *INDENT-ON* */ -int -memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) +static void +memif_prepare_dma_args (vlib_dma_config_t *args) +{ + args->max_batches = 256; + args->max_transfer_size = VLIB_BUFFER_DEFAULT_DATA_SIZE; + args->barrier_before_last = 1; + args->sw_fallback = 1; + args->callback_fn = NULL; +} + +clib_error_t * +memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args) { memif_main_t *mm = &memif_main; vlib_thread_main_t *tm = vlib_get_thread_main (); vnet_main_t *vnm = vnet_get_main (); + vnet_eth_interface_registration_t eir = {}; memif_if_t *mif = 0; vnet_sw_interface_t *sw; - clib_error_t *error = 0; - int ret = 0; uword *p; - vnet_hw_interface_t *hw; memif_socket_file_t *msf = 0; - int rv = 0; + clib_error_t *err = 0; p = hash_get (mm->socket_file_index_by_sock_id, args->socket_id); if (p == 0) { - rv = VNET_API_ERROR_INVALID_ARGUMENT; + err = vnet_error (VNET_ERR_INVALID_ARGUMENT, "unknown socket id"); goto done; } @@ -874,14 +992,17 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) { if ((!msf->is_listener != !args->is_master)) { - rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + err = + vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS, + "socket file cannot be used by both master and slave"); goto done; } p = mhash_get (&msf->dev_instance_by_id, &args->id); if (p) { - rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + err = vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS, + "interface already exists"); goto done; } } @@ -889,25 +1010,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) /* Create new socket file */ if (msf->ref_cnt == 0) { - struct stat file_stat; - - /* If we are creating listener make sure file doesn't exist or if it - * exists thn delete it if it is old socket file */ - if (args->is_master && (stat ((char *) msf->filename, &file_stat) == 0)) - { - if (S_ISSOCK (file_stat.st_mode)) - { - unlink ((char *) msf->filename); - } - else - { - error = clib_error_return (0, "File exists for %s", - msf->filename); - rv = VNET_API_ERROR_VALUE_EXIST; - goto done; - } - } - mhash_init (&msf->dev_instance_by_id, sizeof (uword), sizeof (memif_interface_id_t)); msf->dev_instance_by_fd = hash_create (0, sizeof (uword)); @@ -933,8 +1035,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) bt->total_length_not_including_first_buffer = 0; vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0; - /* initially prealloc copy_ops so we can use - _vec_len instead of vec_elen */ vec_validate_aligned (ptd->copy_ops, 0, CLIB_CACHE_LINE_BYTES); vec_reset_length (ptd->copy_ops); vec_validate_aligned (ptd->buffers, 0, CLIB_CACHE_LINE_BYTES); @@ -952,8 +1052,19 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) if (args->secret) mif->secret = vec_dup (args->secret); - if (tm->n_vlib_mains > 1) - clib_spinlock_init (&mif->lockp); + /* register dma config if enabled */ + if (args->use_dma) + { + vlib_dma_config_t dma_args; + bzero (&dma_args, sizeof (dma_args)); + memif_prepare_dma_args (&dma_args); + + dma_args.max_transfers = 1 << args->log2_ring_size; + dma_args.callback_fn = memif_dma_completion_cb; + mif->dma_input_config = vlib_dma_config_add (vm, &dma_args); + dma_args.callback_fn = memif_tx_dma_completion_cb; + mif->dma_tx_config = vlib_dma_config_add (vm, &dma_args); + } if (mif->mode == MEMIF_INTERFACE_MODE_ETHERNET) { @@ -969,10 +1080,13 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) args->hw_addr[0] = 2; args->hw_addr[1] = 0xfe; } - error = ethernet_register_interface (vnm, memif_device_class.index, - mif->dev_instance, args->hw_addr, - &mif->hw_if_index, - memif_eth_flag_change); + + eir.dev_class_index = memif_device_class.index; + eir.dev_instance = mif->dev_instance; + eir.address = args->hw_addr; + eir.cb.flag_change = memif_eth_flag_change; + eir.cb.set_max_frame_size = memif_eth_set_max_frame_size; + mif->hw_if_index = vnet_eth_register_interface (vnm, &eir); } else if (mif->mode == MEMIF_INTERFACE_MODE_IP) { @@ -983,11 +1097,9 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) mif->dev_instance); } else - error = clib_error_return (0, "unsupported interface mode"); - - if (error) { - ret = VNET_API_ERROR_SYSCALL_ERROR_2; + err = + vnet_error (VNET_ERR_SYSCALL_ERROR_2, "unsupported interface mode"); goto error; } @@ -1006,7 +1118,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) /* If this is new one, start listening */ if (msf->is_listener && msf->ref_cnt == 0) { - struct stat file_stat; clib_socket_t *s = clib_mem_alloc (sizeof (clib_socket_t)); ASSERT (msf->sock == 0); @@ -1014,19 +1125,15 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) clib_memset (s, 0, sizeof (clib_socket_t)); s->config = (char *) msf->filename; - s->flags = CLIB_SOCKET_F_IS_SERVER | - CLIB_SOCKET_F_ALLOW_GROUP_WRITE | - CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_PASSCRED; + s->local_only = 1; + s->is_server = 1; + s->allow_group_write = 1; + s->is_seqpacket = 1; + s->passcred = 1; - if ((error = clib_socket_init (s))) + if ((err = clib_socket_init (s))) { - ret = VNET_API_ERROR_SYSCALL_ERROR_4; - goto error; - } - - if (stat ((char *) msf->filename, &file_stat) == -1) - { - ret = VNET_API_ERROR_SYSCALL_ERROR_8; + err->code = VNET_ERR_SYSCALL_ERROR_4; goto error; } @@ -1047,8 +1154,10 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) mif->flags |= MEMIF_IF_FLAG_ZERO_COPY; } - hw = vnet_get_hw_interface (vnm, mif->hw_if_index); - hw->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_INT_MODE; + if (args->use_dma) + mif->flags |= MEMIF_IF_FLAG_USE_DMA; + + vnet_hw_if_set_caps (vnm, mif->hw_if_index, VNET_HW_IF_CAP_INT_MODE); vnet_hw_if_set_input_node (vnm, mif->hw_if_index, memif_input_node.index); mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0); @@ -1061,15 +1170,12 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) error: memif_delete_if (vm, mif); - if (error) - { - memif_log_err (mif, "%U", format_clib_error, error); - clib_error_free (error); - } - return ret; + if (err) + memif_log_err (mif, "%U", format_clib_error, err); + return err; done: - return rv; + return err; } clib_error_t * @@ -1081,7 +1187,14 @@ memif_interface_admin_up_down (vnet_main_t *vnm, u32 hw_if_index, u32 flags) static clib_error_t *error = 0; if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) - mif->flags |= MEMIF_IF_FLAG_ADMIN_UP; + { + if (mif->flags & MEMIF_IF_FLAG_CONNECTED) + { + vnet_hw_interface_set_flags (vnm, mif->hw_if_index, + VNET_HW_INTERFACE_FLAG_LINK_UP); + } + mif->flags |= MEMIF_IF_FLAG_ADMIN_UP; + } else mif->flags &= ~MEMIF_IF_FLAG_ADMIN_UP; @@ -1108,19 +1221,15 @@ memif_init (vlib_main_t * vm) * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the * default run-time directory. */ - memif_socket_filename_add_del (1, 0, (u8 *) MEMIF_DEFAULT_SOCKET_FILENAME); - - return 0; + return memif_socket_filename_add_del (1, 0, MEMIF_DEFAULT_SOCKET_FILENAME); } VLIB_INIT_FUNCTION (memif_init); -/* *INDENT-OFF* */ VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, .description = "Packet Memory Interface (memif) -- Experimental", }; -/* *INDENT-ON* */ /* * fd.io coding-style-patch-verification: ON |