diff options
| -rw-r--r-- | vlib/vlib/threads.c | 4 | ||||
| -rw-r--r-- | vlib/vlib/threads.h | 3 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/device.c | 10 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/dpdk.h | 13 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/init.c | 20 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/node.c | 895 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/threads.c | 59 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/threads.h | 1 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/vhost_user.c | 5 | ||||
| -rw-r--r-- | vnet/vnet/devices/virtio/vhost-user.c | 10 |
10 files changed, 19 insertions, 1001 deletions
diff --git a/vlib/vlib/threads.c b/vlib/vlib/threads.c index 3b815be4..1808f36f 100644 --- a/vlib/vlib/threads.c +++ b/vlib/vlib/threads.c @@ -983,9 +983,7 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input) while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "main-thread-io")) - tm->main_thread_is_io_node = 1; - else if (unformat (input, "use-pthreads")) + if (unformat (input, "use-pthreads")) tm->use_pthreads = 1; else if (unformat (input, "thread-prefix %v", &tm->thread_prefix)) ; diff --git a/vlib/vlib/threads.h b/vlib/vlib/threads.h index f13d3768..f81be798 100644 --- a/vlib/vlib/threads.h +++ b/vlib/vlib/threads.h @@ -264,9 +264,6 @@ typedef struct { vlib_worker_thread_t * worker_threads; - /* thread / cpu / io thread parameters */ - u32 main_thread_is_io_node; - /* * Launch all threads as pthreads, * not eal_rte_launch (strict affinity) threads diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c index bad149f0..f4d4fe77 100644 --- a/vnet/vnet/devices/dpdk/device.c +++ b/vnet/vnet/devices/dpdk/device.c @@ -1309,16 +1309,6 @@ u32 dpdk_num_mbufs (void) } /* - * Return the io_thread_release - */ -int dpdk_io_thread_release (void) -{ - dpdk_main_t * dm = &dpdk_main; - - return dm->io_thread_release; -} - -/* * Return the pmd type for a given hardware interface */ dpdk_pmd_t dpdk_get_pmd_type (vnet_hw_interface_t *hi) diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h index 1e7d3684..ded9d2db 100644 --- a/vnet/vnet/devices/dpdk/dpdk.h +++ b/vnet/vnet/devices/dpdk/dpdk.h @@ -67,7 +67,6 @@ extern vnet_device_class_t dpdk_device_class; extern vlib_node_registration_t dpdk_input_node; -extern vlib_node_registration_t dpdk_io_input_node; extern vlib_node_registration_t handoff_dispatch_node; typedef enum { @@ -391,7 +390,7 @@ typedef struct { u32 ethernet_input_node_index; /* dpdk i/o thread initialization barrier */ - volatile u32 io_thread_release; + volatile u32 worker_thread_release; /* pcap tracing [only works if (CLIB_DEBUG > 0)] */ int tx_pcap_enable; @@ -417,7 +416,6 @@ typedef struct { */ u8 admin_up_down_in_progress; - u8 have_io_threads; u8 use_rss; /* which cpus are running dpdk-input */ @@ -479,13 +477,6 @@ clib_error_t * dpdk_set_mac_address (vnet_hw_interface_t * hi, char * address); clib_error_t * dpdk_set_mc_filter (vnet_hw_interface_t * hi, struct ether_addr mc_addr_vec[], int naddr); -typedef void (*dpdk_io_thread_callback_t) (vlib_main_t *vm); - -void dpdk_io_thread (vlib_worker_thread_t * w, - u32 instances, - u32 instance_id, - char *worker_name, - dpdk_io_thread_callback_t callback); void dpdk_thread_input (dpdk_main_t * dm, dpdk_device_t * xd); clib_error_t * dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd); @@ -607,8 +598,6 @@ u32 dpdk_get_admin_up_down_in_progress (void); u32 dpdk_num_mbufs (void); -int dpdk_io_thread_release (void); - dpdk_pmd_t dpdk_get_pmd_type (vnet_hw_interface_t *hi); i8 dpdk_get_cpu_socket (vnet_hw_interface_t *hi); diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c index 6b6f63a7..398bd19e 100644 --- a/vnet/vnet/devices/dpdk/init.c +++ b/vnet/vnet/devices/dpdk/init.c @@ -252,20 +252,9 @@ dpdk_lib_init (dpdk_main_t * dm) rt->function = dpdk_input_multiarch_select(); /* find out which cpus will be used for input */ - p = hash_get_mem (tm->thread_registrations_by_name, "io"); + p = hash_get_mem (tm->thread_registrations_by_name, "workers"); tr = p ? (vlib_thread_registration_t *) p[0] : 0; - if (!tr || tr->count == 0) - { - /* no io threads, workers doing input */ - p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - tr = p ? (vlib_thread_registration_t *) p[0] : 0; - } - else - { - dm->have_io_threads = 1; - } - if (tr && tr->count > 0) { dm->input_cpu_first_index = tr->first_index; @@ -1521,10 +1510,7 @@ dpdk_process (vlib_main_t * vm, if (tm->n_vlib_mains == 1) vlib_node_set_state (vm, dpdk_input_node.index, VLIB_NODE_STATE_POLLING); - else if (tm->main_thread_is_io_node) - vlib_node_set_state (vm, dpdk_io_input_node.index, - VLIB_NODE_STATE_POLLING); - else if (!dm->have_io_threads) + else for (i=0; i < tm->n_vlib_mains; i++) if (vec_len(dm->devices_by_cpu[i]) > 0) vlib_node_set_state (vlib_mains[i], dpdk_input_node.index, @@ -1536,7 +1522,7 @@ dpdk_process (vlib_main_t * vm, dpdk_vhost_user_process_init(&vu_state); - dm->io_thread_release = 1; + dm->worker_thread_release = 1; f64 now = vlib_time_now (vm); vec_foreach (xd, dm->devices) diff --git a/vnet/vnet/devices/dpdk/node.c b/vnet/vnet/devices/dpdk/node.c index 27670f37..0d453085 100644 --- a/vnet/vnet/devices/dpdk/node.c +++ b/vnet/vnet/devices/dpdk/node.c @@ -687,7 +687,6 @@ CLIB_MULTIARCH_SELECT_FN(dpdk_input_efd); void dpdk_set_next_node (dpdk_rx_next_t next, char *name) { vlib_node_registration_t *r = &dpdk_input_node; - vlib_node_registration_t *r_io = &dpdk_io_input_node; vlib_node_registration_t *r_handoff = &handoff_dispatch_node; switch (next) @@ -697,7 +696,6 @@ void dpdk_set_next_node (dpdk_rx_next_t next, char *name) case DPDK_RX_NEXT_MPLS_INPUT: case DPDK_RX_NEXT_ETHERNET_INPUT: r->next_nodes[next] = name; - r_io->next_nodes[next] = name; r_handoff->next_nodes[next] = name; break; @@ -708,899 +706,6 @@ void dpdk_set_next_node (dpdk_rx_next_t next, char *name) } /* - * This function is used when dedicated IO threads feed the worker threads. - * - * Devices are allocated to this thread based on instances and instance_id. - * If instances==0 then the function automatically determines the number - * of instances of this thread, and allocates devices between them. - * If instances != 0, then instance_id must be in the range 0..instances-1. - * The function allocates devices among the specified number of instances, - * with this thread having the given instance id. This option is used for - * splitting devices among differently named "io"-type threads. - */ -void dpdk_io_thread (vlib_worker_thread_t * w, - u32 instances, - u32 instance_id, - char *worker_name, - dpdk_io_thread_callback_t callback) -{ - vlib_main_t * vm = vlib_get_main(); - vlib_thread_main_t * tm = vlib_get_thread_main(); - vlib_thread_registration_t * tr; - dpdk_main_t * dm = &dpdk_main; - char *io_name = w->registration->name; - dpdk_device_t * xd; - dpdk_device_t ** my_devices = 0; - vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index = 0; - vlib_frame_queue_t ** congested_handoff_queue_by_worker_index = 0; - vlib_frame_queue_elt_t * hf = 0; - int i; - u32 n_left_to_next_worker = 0, * to_next_worker = 0; - u32 next_worker_index = 0; - u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number(); - u32 num_workers = 0; - u32 num_devices = 0; - uword * p; - u16 queue_id = 0; - vlib_node_runtime_t * node_trace = 0; - u32 first_worker_index = 0; - u32 buffer_flags_template; - - /* Wait until the dpdk init sequence is complete */ - while (dm->io_thread_release == 0) - vlib_worker_thread_barrier_check(); - - clib_time_init (&vm->clib_time); - - p = hash_get_mem (tm->thread_registrations_by_name, worker_name); - ASSERT (p); - tr = (vlib_thread_registration_t *) p[0]; - if (tr) - { - num_workers = tr->count; - first_worker_index = tr->first_index; - } - - /* Allocate devices to this thread */ - if (instances == 0) - { - /* auto-assign */ - instance_id = w->instance_id; - - p = hash_get_mem (tm->thread_registrations_by_name, io_name); - tr = (vlib_thread_registration_t *) p[0]; - /* Otherwise, how did we get here */ - ASSERT (tr && tr->count); - instances = tr->count; - } - else - { - /* manually assign */ - ASSERT (instance_id < instances); - } - - vec_validate (handoff_queue_elt_by_worker_index, - first_worker_index + num_workers - 1); - - vec_validate_init_empty (congested_handoff_queue_by_worker_index, - first_worker_index + num_workers - 1, - (vlib_frame_queue_t *)(~0)); - - buffer_flags_template = dm->buffer_flags_template; - - /* And handle them... */ - while (1) - { - u32 n_buffers; - u32 mb_index; - uword n_rx_bytes = 0; - u32 n_trace, trace_cnt __attribute__((unused)); - vlib_buffer_free_list_t * fl; - u32 hash; - u64 hash_key; - u8 efd_discard_burst; - - vlib_worker_thread_barrier_check (); - - /* Invoke callback if supplied */ - if (PREDICT_FALSE(callback != NULL)) - callback(vm); - - if (PREDICT_FALSE(vec_len(dm->devices) != num_devices)) - { - vec_reset_length(my_devices); - vec_foreach (xd, dm->devices) - { - if (((xd - dm->devices) % tr->count) == instance_id) - { - fprintf(stderr, "i/o thread %d (cpu %d) takes port %d\n", - instance_id, (int) os_get_cpu_number(), (int) (xd - dm->devices)); - vec_add1 (my_devices, xd); - } - } - num_devices = vec_len(dm->devices); - } - - for (i = 0; i < vec_len (my_devices); i++) - { - xd = my_devices[i]; - - if (!xd->admin_up) - continue; - - n_buffers = dpdk_rx_burst(dm, xd, 0 /* queue_id */); - - if (n_buffers == 0) - { - /* check if EFD (dpdk) is enabled */ - if (PREDICT_FALSE(dm->efd.enabled)) - { - /* reset a few stats */ - xd->efd_agent.last_poll_time = 0; - xd->efd_agent.last_burst_sz = 0; - } - continue; - } - - trace_cnt = n_trace = 0; - if (PREDICT_FALSE(vm->trace_main.trace_active_hint)) - { - /* - * packet tracing is triggered on the dpdk-input node for - * ease-of-use. Re-fetch the node_runtime for dpdk-input - * in case it has changed. - */ - node_trace = vlib_node_get_runtime (vm, dpdk_input_node.index); - - vec_reset_length (xd->d_trace_buffers); - trace_cnt = n_trace = vlib_get_trace_count (vm, node_trace); - } - - /* - * DAW-FIXME: VMXNET3 device stop/start doesn't work, - * therefore fake the stop in the dpdk driver by - * silently dropping all of the incoming pkts instead of - * stopping the driver / hardware. - */ - if (PREDICT_FALSE(xd->admin_up != 1)) - { - for (mb_index = 0; mb_index < n_buffers; mb_index++) - rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]); - continue; - } - - /* reset EFD action for the burst */ - efd_discard_burst = 0; - - /* Check for congestion if EFD (Early-Fast-Discard) is enabled - * in any mode (e.g. dpdk, monitor, or drop_all) - */ - if (PREDICT_FALSE(dm->efd.enabled)) - { - /* update EFD counters */ - dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled); - - if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED)) - { - /* drop all received packets */ - for (mb_index = 0; mb_index < n_buffers; mb_index++) - rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]); - - xd->efd_agent.discard_cnt += n_buffers; - increment_efd_drop_counter(vm, - DPDK_ERROR_VLAN_EFD_DROP_PKTS, - n_buffers); - - continue; - } - - if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >= - dm->efd.consec_full_frames_hi_thresh)) - { - u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index, - queue_id); - if (device_queue_sz >= dm->efd.queue_hi_thresh) - { - /* dpdk device queue has reached the critical threshold */ - xd->efd_agent.congestion_cnt++; - - /* apply EFD to packets from the burst */ - efd_discard_burst = 1; - } - } - } - - fl = vlib_buffer_get_free_list - (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - - mb_index = 0; - - while (n_buffers > 0) - { - u32 bi0; - u8 next0, error0; - u32 l3_offset0; - vlib_buffer_t * b0, * b_seg, * b_chain = 0; - ethernet_header_t * h0; - u8 nb_seg = 1; - struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index]; - struct rte_mbuf *mb_seg = mb->next; - - if (PREDICT_TRUE(n_buffers > 1)) - { - struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2]; - vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); - CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD); - } - - b0 = vlib_buffer_from_rte_mbuf(mb); - - /* check whether EFD is looking for packets to discard */ - if (PREDICT_FALSE(efd_discard_burst)) - { - u32 cntr_type; - if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb))) - { - rte_pktmbuf_free(mb); - xd->efd_agent.discard_cnt++; - increment_efd_drop_counter(vm, - cntr_type, - 1); - - n_buffers--; - mb_index++; - continue; - } - } - - /* Prefetch one next segment if it exists */ - if (PREDICT_FALSE(mb->nb_segs > 1)) - { - struct rte_mbuf *pfmb = mb->next; - vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); - CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); - b_chain = b0; - } - - bi0 = vlib_get_buffer_index (vm, b0); - vlib_buffer_init_for_free_list (b0, fl); - b0->clone_count = 0; - - dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0, - &next0, &error0); -#ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS - /* - * Clear overloaded TX offload flags when a DPDK driver - * is using them for RX flags (e.g. Cisco VIC Ethernet driver) - */ - if (PREDICT_TRUE(trace_cnt == 0)) - mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK; - else - trace_cnt--; -#endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */ - - if (error0) - clib_warning ("bi %d error %d", bi0, error0); - - b0->error = 0; - - l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT || - next0 == DPDK_RX_NEXT_IP6_INPUT || - next0 == DPDK_RX_NEXT_MPLS_INPUT) ? - sizeof (ethernet_header_t) : 0); - - b0->current_data = l3_offset0; - /* Some drivers like fm10k receive frames with - mb->data_off > RTE_PKTMBUF_HEADROOM */ - b0->current_data += mb->data_off - RTE_PKTMBUF_HEADROOM; - b0->current_length = mb->data_len - l3_offset0; - - b0->flags = buffer_flags_template; - - if (VMWARE_LENGTH_BUG_WORKAROUND) - b0->current_length -= 4; - - vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0; - vnet_buffer(b0)->handoff.next_index = next0; - n_rx_bytes += mb->pkt_len; - - /* Process subsequent segments of multi-segment packets */ - while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs)) - { - ASSERT(mb_seg != 0); - - b_seg = vlib_buffer_from_rte_mbuf(mb_seg); - vlib_buffer_init_for_free_list (b_seg, fl); - b_seg->clone_count = 0; - - ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - ASSERT(b_seg->current_data == 0); - - /* - * The driver (e.g. virtio) may not put the packet data at the start - * of the segment, so don't assume b_seg->current_data == 0 is correct. - */ - b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data; - - b_seg->current_length = mb_seg->data_len; - b0->total_length_not_including_first_buffer += - mb_seg->data_len; - - b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT; - b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg); - - b_chain = b_seg; - mb_seg = mb_seg->next; - nb_seg++; - } - - /* - * Turn this on if you run into - * "bad monkey" contexts, and you want to know exactly - * which nodes they've visited... See main.c... - */ - VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0); - - if (PREDICT_FALSE (n_trace > mb_index)) - vec_add1 (xd->d_trace_buffers, bi0); - - next_worker_index = first_worker_index; - - /* - * Force unknown traffic onto worker 0, - * and into ethernet-input. $$$$ add more hashes. - */ - h0 = (ethernet_header_t *) b0->data; - - /* Compute ingress LB hash */ - hash_key = eth_get_key(h0); - hash = (u32)clib_xxhash(hash_key); - - if (PREDICT_TRUE (is_pow2(num_workers))) - next_worker_index += hash & (num_workers - 1); - else - next_worker_index += hash % num_workers; - - /* if EFD is enabled and not already discarding from dpdk, - * check the worker ring/queue for congestion - */ - if (PREDICT_FALSE(tm->efd.enabled && !efd_discard_burst)) - { - vlib_frame_queue_t *fq; - - /* fq will be valid if the ring is congested */ - fq = is_vlib_handoff_queue_congested( - next_worker_index, tm->efd.queue_hi_thresh, - congested_handoff_queue_by_worker_index); - - if (PREDICT_FALSE(fq != NULL)) - { - u32 cntr_type; - if (PREDICT_TRUE(cntr_type = - is_efd_discardable(tm, b0, mb))) - { - /* discard the packet */ - fq->enqueue_efd_discards++; - increment_efd_drop_counter(vm, cntr_type, 1); - rte_pktmbuf_free(mb); - n_buffers--; - mb_index++; - continue; - } - } - } - - if (next_worker_index != current_worker_index) - { - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - hf = dpdk_get_handoff_queue_elt( - next_worker_index, - handoff_queue_elt_by_worker_index); - - n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors; - to_next_worker = &hf->buffer_index[hf->n_vectors]; - current_worker_index = next_worker_index; - } - - /* enqueue to correct worker thread */ - to_next_worker[0] = bi0; - to_next_worker++; - n_left_to_next_worker--; - - if (n_left_to_next_worker == 0) - { - hf->n_vectors = VLIB_FRAME_SIZE; - vlib_put_handoff_queue_elt(hf); - current_worker_index = ~0; - handoff_queue_elt_by_worker_index[next_worker_index] = 0; - hf = 0; - } - - n_buffers--; - mb_index++; - } - - if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0)) - { - /* credit the trace to the trace node */ - dpdk_rx_trace (dm, node_trace, xd, queue_id, xd->d_trace_buffers, - vec_len (xd->d_trace_buffers)); - vlib_set_trace_count (vm, node_trace, n_trace - vec_len (xd->d_trace_buffers)); - } - - vlib_increment_combined_counter - (vnet_get_main()->interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, - cpu_index, - xd->vlib_sw_if_index, - mb_index, n_rx_bytes); - - dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index); - dw->aggregate_rx_packets += mb_index; - } - - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - /* Ship frames to the worker nodes */ - for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++) - { - if (handoff_queue_elt_by_worker_index[i]) - { - hf = handoff_queue_elt_by_worker_index[i]; - /* - * It works better to let the handoff node - * rate-adapt, always ship the handoff queue element. - */ - if (1 || hf->n_vectors == hf->last_n_vectors) - { - vlib_put_handoff_queue_elt(hf); - handoff_queue_elt_by_worker_index[i] = 0; - } - else - hf->last_n_vectors = hf->n_vectors; - } - congested_handoff_queue_by_worker_index[i] = (vlib_frame_queue_t *)(~0); - } - hf = 0; - current_worker_index = ~0; - - vlib_increment_main_loop_counter (vm); - } -} - -/* - * This function is used when the main thread performs IO and feeds the - * worker threads. - */ -static uword -dpdk_io_input (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * f) -{ - dpdk_main_t * dm = &dpdk_main; - dpdk_device_t * xd; - vlib_thread_main_t * tm = vlib_get_thread_main(); - uword n_rx_packets = 0; - static vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index; - static vlib_frame_queue_t ** congested_handoff_queue_by_worker_index = 0; - vlib_frame_queue_elt_t * hf = 0; - int i; - u32 n_left_to_next_worker = 0, * to_next_worker = 0; - u32 next_worker_index = 0; - u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number(); - static int num_workers_set; - static u32 num_workers; - u16 queue_id = 0; - vlib_node_runtime_t * node_trace; - static u32 first_worker_index; - u32 buffer_flags_template; - - if (PREDICT_FALSE(num_workers_set == 0)) - { - uword * p; - vlib_thread_registration_t * tr; - /* Only the standard vnet worker threads are supported */ - p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - tr = (vlib_thread_registration_t *) p[0]; - if (tr) - { - num_workers = tr->count; - first_worker_index = tr->first_index; - } - num_workers_set = 1; - } - - if (PREDICT_FALSE(handoff_queue_elt_by_worker_index == 0)) - { - vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1); - - vec_validate_init_empty (congested_handoff_queue_by_worker_index, - first_worker_index + num_workers - 1, - (vlib_frame_queue_t *)(~0)); - } - - /* packet tracing is triggered on the dpdk-input node for ease-of-use */ - node_trace = vlib_node_get_runtime (vm, dpdk_input_node.index); - - buffer_flags_template = dm->buffer_flags_template; - - vec_foreach (xd, dm->devices) - { - u32 n_buffers; - u32 mb_index; - uword n_rx_bytes = 0; - u32 n_trace, trace_cnt __attribute__((unused)); - vlib_buffer_free_list_t * fl; - u32 hash; - u64 hash_key; - u8 efd_discard_burst = 0; - - if (!xd->admin_up) - continue; - - n_buffers = dpdk_rx_burst(dm, xd, queue_id ); - - if (n_buffers == 0) - { - /* check if EFD (dpdk) is enabled */ - if (PREDICT_FALSE(dm->efd.enabled)) - { - /* reset a few stats */ - xd->efd_agent.last_poll_time = 0; - xd->efd_agent.last_burst_sz = 0; - } - continue; - } - - vec_reset_length (xd->d_trace_buffers); - trace_cnt = n_trace = vlib_get_trace_count (vm, node_trace); - - /* - * DAW-FIXME: VMXNET3 device stop/start doesn't work, - * therefore fake the stop in the dpdk driver by - * silently dropping all of the incoming pkts instead of - * stopping the driver / hardware. - */ - if (PREDICT_FALSE(xd->admin_up != 1)) - { - for (mb_index = 0; mb_index < n_buffers; mb_index++) - rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]); - continue; - } - - /* Check for congestion if EFD (Early-Fast-Discard) is enabled - * in any mode (e.g. dpdk, monitor, or drop_all) - */ - if (PREDICT_FALSE(dm->efd.enabled)) - { - /* update EFD counters */ - dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled); - - if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED)) - { - /* discard all received packets */ - for (mb_index = 0; mb_index < n_buffers; mb_index++) - rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]); - - xd->efd_agent.discard_cnt += n_buffers; - increment_efd_drop_counter(vm, - DPDK_ERROR_VLAN_EFD_DROP_PKTS, - n_buffers); - - continue; - } - - if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >= - dm->efd.consec_full_frames_hi_thresh)) - { - u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index, - queue_id); - if (device_queue_sz >= dm->efd.queue_hi_thresh) - { - /* dpdk device queue has reached the critical threshold */ - xd->efd_agent.congestion_cnt++; - - /* apply EFD to packets from the burst */ - efd_discard_burst = 1; - } - } - } - - fl = vlib_buffer_get_free_list - (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - - mb_index = 0; - - while (n_buffers > 0) - { - u32 bi0; - u8 next0, error0; - u32 l3_offset0; - vlib_buffer_t * b0, * b_seg, * b_chain = 0; - ethernet_header_t * h0; - u8 nb_seg = 1; - struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index]; - struct rte_mbuf *mb_seg = mb->next; - - if (PREDICT_TRUE(n_buffers > 1)) - { - struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2]; - vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); - CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD); - } - - b0 = vlib_buffer_from_rte_mbuf(mb); - - /* check whether EFD is looking for packets to discard */ - if (PREDICT_FALSE(efd_discard_burst)) - { - u32 cntr_type; - if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb))) - { - rte_pktmbuf_free(mb); - xd->efd_agent.discard_cnt++; - increment_efd_drop_counter(vm, - cntr_type, - 1); - - n_buffers--; - mb_index++; - continue; - } - } - - /* Prefetch one next segment if it exists */ - if (PREDICT_FALSE(mb->nb_segs > 1)) - { - struct rte_mbuf *pfmb = mb->next; - vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); - CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); - b_chain = b0; - } - - bi0 = vlib_get_buffer_index (vm, b0); - vlib_buffer_init_for_free_list (b0, fl); - b0->clone_count = 0; - - dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0, - &next0, &error0); -#ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS - /* - * Clear overloaded TX offload flags when a DPDK driver - * is using them for RX flags (e.g. Cisco VIC Ethernet driver) - */ - if (PREDICT_TRUE(trace_cnt == 0)) - mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK; - else - trace_cnt--; -#endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */ - - if (error0) - clib_warning ("bi %d error %d", bi0, error0); - - b0->error = 0; - - l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT || - next0 == DPDK_RX_NEXT_IP6_INPUT || - next0 == DPDK_RX_NEXT_MPLS_INPUT) ? - sizeof (ethernet_header_t) : 0); - - b0->current_data = l3_offset0; - b0->current_length = mb->data_len - l3_offset0; - - b0->flags = buffer_flags_template; - - if (VMWARE_LENGTH_BUG_WORKAROUND) - b0->current_length -= 4; - - vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0; - vnet_buffer(b0)->handoff.next_index = next0; - n_rx_bytes += mb->pkt_len; - - /* Process subsequent segments of multi-segment packets */ - while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs)) - { - ASSERT(mb_seg != 0); - - b_seg = vlib_buffer_from_rte_mbuf(mb_seg); - vlib_buffer_init_for_free_list (b_seg, fl); - b_seg->clone_count = 0; - - ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - ASSERT(b_seg->current_data == 0); - - /* - * The driver (e.g. virtio) may not put the packet data at the start - * of the segment, so don't assume b_seg->current_data == 0 is correct. - */ - b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data; - - b_seg->current_length = mb_seg->data_len; - b0->total_length_not_including_first_buffer += - mb_seg->data_len; - - b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT; - b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg); - - b_chain = b_seg; - mb_seg = mb_seg->next; - nb_seg++; - } - - /* - * Turn this on if you run into - * "bad monkey" contexts, and you want to know exactly - * which nodes they've visited... See main.c... - */ - VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0); - - if (PREDICT_FALSE (n_trace > mb_index)) - vec_add1 (xd->d_trace_buffers, bi0); - - next_worker_index = first_worker_index; - - /* - * Force unknown traffic onto worker 0, - * and into ethernet-input. $$$$ add more hashes. - */ - h0 = (ethernet_header_t *) b0->data; - - /* Compute ingress LB hash */ - hash_key = eth_get_key(h0); - hash = (u32)clib_xxhash(hash_key); - - if (PREDICT_TRUE (is_pow2(num_workers))) - next_worker_index += hash & (num_workers - 1); - else - next_worker_index += hash % num_workers; - - /* if EFD is enabled and not already discarding from dpdk, - * check the worker ring/queue for congestion - */ - if (PREDICT_FALSE(tm->efd.enabled && !efd_discard_burst)) - { - vlib_frame_queue_t *fq; - - /* fq will be valid if the ring is congested */ - fq = is_vlib_handoff_queue_congested( - next_worker_index, tm->efd.queue_hi_thresh, - congested_handoff_queue_by_worker_index); - - if (PREDICT_FALSE(fq != NULL)) - { - u32 cntr_type; - if (PREDICT_TRUE(cntr_type = - is_efd_discardable(tm, b0, mb))) - { - /* discard the packet */ - fq->enqueue_efd_discards++; - increment_efd_drop_counter(vm, cntr_type, 1); - rte_pktmbuf_free(mb); - n_buffers--; - mb_index++; - continue; - } - } - } - - if (next_worker_index != current_worker_index) - { - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - hf = dpdk_get_handoff_queue_elt( - next_worker_index, - handoff_queue_elt_by_worker_index); - - n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors; - to_next_worker = &hf->buffer_index[hf->n_vectors]; - current_worker_index = next_worker_index; - } - - /* enqueue to correct worker thread */ - to_next_worker[0] = bi0; - to_next_worker++; - n_left_to_next_worker--; - - if (n_left_to_next_worker == 0) - { - hf->n_vectors = VLIB_FRAME_SIZE; - vlib_put_handoff_queue_elt(hf); - current_worker_index = ~0; - handoff_queue_elt_by_worker_index[next_worker_index] = 0; - hf = 0; - } - - n_buffers--; - mb_index++; - } - - if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0)) - { - /* credit the trace to the trace node */ - dpdk_rx_trace (dm, node_trace, xd, queue_id, xd->d_trace_buffers, - vec_len (xd->d_trace_buffers)); - vlib_set_trace_count (vm, node_trace, n_trace - vec_len (xd->d_trace_buffers)); - } - - vlib_increment_combined_counter - (vnet_get_main()->interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, - cpu_index, - xd->vlib_sw_if_index, - mb_index, n_rx_bytes); - - dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index); - dw->aggregate_rx_packets += mb_index; - n_rx_packets += mb_index; - } - - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - /* Ship frames to the worker nodes */ - for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++) - { - if (handoff_queue_elt_by_worker_index[i]) - { - hf = handoff_queue_elt_by_worker_index[i]; - /* - * It works better to let the handoff node - * rate-adapt, always ship the handoff queue element. - */ - if (1 || hf->n_vectors == hf->last_n_vectors) - { - vlib_put_handoff_queue_elt(hf); - handoff_queue_elt_by_worker_index[i] = 0; - } - else - hf->last_n_vectors = hf->n_vectors; - } - congested_handoff_queue_by_worker_index[i] = (vlib_frame_queue_t *)(~0); - } - hf = 0; - current_worker_index = ~0; - return n_rx_packets; -} - -VLIB_REGISTER_NODE (dpdk_io_input_node) = { - .function = dpdk_io_input, - .type = VLIB_NODE_TYPE_INPUT, - .name = "dpdk-io-input", - - /* Will be enabled if/when hardware is detected. */ - .state = VLIB_NODE_STATE_DISABLED, - - .format_buffer = format_ethernet_header_with_length, - .format_trace = format_dpdk_rx_dma_trace, - - .n_errors = DPDK_N_ERROR, - .error_strings = dpdk_error_strings, - - .n_next_nodes = DPDK_RX_N_NEXT, - .next_nodes = { - [DPDK_RX_NEXT_DROP] = "error-drop", - [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input", - [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input-no-checksum", - [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input", - [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input", - }, -}; - -/* * set_efd_bitmap() * Based on the operation type, set lower/upper bits for the given index value */ diff --git a/vnet/vnet/devices/dpdk/threads.c b/vnet/vnet/devices/dpdk/threads.c index c66db722..48ee13d1 100644 --- a/vnet/vnet/devices/dpdk/threads.c +++ b/vnet/vnet/devices/dpdk/threads.c @@ -166,7 +166,7 @@ static inline int vlib_frame_queue_dequeue_internal (vlib_main_t *vm) return processed; } -int dpdk_frame_queue_dequeue (vlib_main_t *vm) +int dpdk_frame_queue_dequeue (vlib_main_t *vm) { return vlib_frame_queue_dequeue_internal (vm); } @@ -176,17 +176,12 @@ int dpdk_frame_queue_dequeue (vlib_main_t *vm) * * w * Information for the current thread - * io_name - * The name of thread performing dpdk device IO (if any). If there are no - * instances of that thread, then the current thread will do dpdk device - * polling. Ports will be divided among instances of the current thread. * callback * If not null, this function will be called once during each main loop. */ static_always_inline void dpdk_worker_thread_internal (vlib_main_t *vm, - dpdk_worker_thread_callback_t callback, - int have_io_threads) + dpdk_worker_thread_callback_t callback) { vlib_node_main_t * nm = &vm->node_main; u64 cpu_time_now = clib_cpu_time_now (); @@ -201,17 +196,13 @@ dpdk_worker_thread_internal (vlib_main_t *vm, if (PREDICT_FALSE(callback != NULL)) callback(vm); - if (!have_io_threads) - { - vlib_node_runtime_t * n; - vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) - { - cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, - VLIB_NODE_STATE_POLLING, /* frame */ 0, - cpu_time_now); - } - - } + vlib_node_runtime_t * n; + vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) + { + cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, + VLIB_NODE_STATE_POLLING, /* frame */ 0, + cpu_time_now); + } if (_vec_len (nm->pending_frames)) { @@ -235,13 +226,9 @@ dpdk_worker_thread_internal (vlib_main_t *vm, } void dpdk_worker_thread (vlib_worker_thread_t * w, - char *io_name, dpdk_worker_thread_callback_t callback) { vlib_main_t *vm; - uword * p; - vlib_thread_main_t * tm = vlib_get_thread_main(); - vlib_thread_registration_t * tr; dpdk_main_t * dm = &dpdk_main; vm = vlib_get_main(); @@ -252,24 +239,17 @@ void dpdk_worker_thread (vlib_worker_thread_t * w, clib_mem_set_heap (w->thread_mheap); /* Wait until the dpdk init sequence is complete */ - while (dm->io_thread_release == 0) + while (dm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); - /* any I/O threads? */ - p = hash_get_mem (tm->thread_registrations_by_name, io_name); - tr = (vlib_thread_registration_t *)p[0]; - - if (tr && tr->count > 0) - dpdk_worker_thread_internal(vm, callback, /* have_io_threads */ 1); - else - dpdk_worker_thread_internal(vm, callback, /* have_io_threads */ 0); + dpdk_worker_thread_internal(vm, callback); } void dpdk_worker_thread_fn (void * arg) { vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; vlib_worker_thread_init (w); - dpdk_worker_thread (w, "io", 0); + dpdk_worker_thread (w, 0); } #if VIRL == 0 @@ -280,21 +260,6 @@ VLIB_REGISTER_THREAD (worker_thread_reg, static) = { }; #endif -void dpdk_io_thread_fn (void * arg) -{ - vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; - vlib_worker_thread_init (w); - dpdk_io_thread (w, 0, 0, "workers", 0); -} - -#if VIRL == 0 -VLIB_REGISTER_THREAD (io_thread_reg, static) = { - .name = "io", - .short_name = "io", - .function = dpdk_io_thread_fn, -}; -#endif - static clib_error_t * dpdk_thread_init (vlib_main_t *vm) { diff --git a/vnet/vnet/devices/dpdk/threads.h b/vnet/vnet/devices/dpdk/threads.h index f2b0532b..c7d301be 100644 --- a/vnet/vnet/devices/dpdk/threads.h +++ b/vnet/vnet/devices/dpdk/threads.h @@ -20,7 +20,6 @@ typedef void (*dpdk_worker_thread_callback_t) (vlib_main_t *vm); void dpdk_worker_thread (vlib_worker_thread_t * w, - char *io_name, dpdk_worker_thread_callback_t callback); int dpdk_frame_queue_dequeue (vlib_main_t *vm); diff --git a/vnet/vnet/devices/dpdk/vhost_user.c b/vnet/vnet/devices/dpdk/vhost_user.c index b1615b5a..92b5d0e5 100644 --- a/vnet/vnet/devices/dpdk/vhost_user.c +++ b/vnet/vnet/devices/dpdk/vhost_user.c @@ -392,10 +392,7 @@ dpdk_create_vhost_user_if_internal (u32 * hw_if_index, u32 if_id, u8 *hwaddr) if (tm->n_vlib_mains == 1 && dpdk_input_node.state != VLIB_NODE_STATE_POLLING) vlib_node_set_state (vm, dpdk_input_node.index, VLIB_NODE_STATE_POLLING); - if (tm->n_vlib_mains > 1 && tm->main_thread_is_io_node) - vlib_node_set_state (vm, dpdk_io_input_node.index, VLIB_NODE_STATE_POLLING); - - if (tm->n_vlib_mains > 1 && !tm->main_thread_is_io_node) + if (tm->n_vlib_mains > 1) vlib_node_set_state (vlib_mains[cpu], dpdk_input_node.index, VLIB_NODE_STATE_POLLING); next_cpu++; diff --git a/vnet/vnet/devices/virtio/vhost-user.c b/vnet/vnet/devices/virtio/vhost-user.c index 4ed45e38..afb2e009 100644 --- a/vnet/vnet/devices/virtio/vhost-user.c +++ b/vnet/vnet/devices/virtio/vhost-user.c @@ -1648,7 +1648,7 @@ static void vhost_user_vui_register(vlib_main_t * vm, vhost_user_intf_t *vui) if (tm->n_vlib_mains == 1) vlib_node_set_state (vm, vhost_user_input_node.index, VLIB_NODE_STATE_POLLING); - else if (!dm->have_io_threads) + else vlib_node_set_state (vlib_mains[cpu_index], vhost_user_input_node.index, VLIB_NODE_STATE_POLLING); @@ -1665,18 +1665,10 @@ int vhost_user_create_if(vnet_main_t * vnm, vlib_main_t * vm, u8 *hwaddr) { vhost_user_intf_t * vui = NULL; - dpdk_main_t * dm = &dpdk_main; - vlib_thread_main_t * tm = vlib_get_thread_main(); u32 sw_if_idx = ~0; int sockfd = -1; int rv = 0; - if (tm->n_vlib_mains > 1 && dm->have_io_threads) - { - clib_warning("vhost-user interfaces are not supported with multiple io threads"); - return -1; - } - if (is_server) { if ((rv = vhost_user_init_server_sock (sock_filename, &sockfd)) != 0) { return rv; |