summaryrefslogtreecommitdiffstats
path: root/src/vnet/devices
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2017-04-05 19:18:20 +0200
committerDave Barach <openvpp@barachs.net>2017-04-06 11:31:39 +0000
commit586afd762bfa149f5ca167bd5fd5a0cd59ce94fe (patch)
tree808b57c61e0fe1a181871bb1ad94398c5ba42671 /src/vnet/devices
parentbc799c92d761a2d45105aa6a1685b3663687d2a4 (diff)
Use thread local storage for thread index
This patch deprecates stack-based thread identification, Also removes requirement that thread stacks are adjacent. Finally, possibly annoying for some folks, it renames all occurences of cpu_index and cpu_number with thread index. Using word "cpu" is misleading here as thread can be migrated ti different CPU, and also it is not related to linux cpu index. Change-Id: I68cdaf661e701d2336fc953dcb9978d10a70f7c1 Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'src/vnet/devices')
-rw-r--r--src/vnet/devices/af_packet/node.c20
-rw-r--r--src/vnet/devices/devices.c61
-rw-r--r--src/vnet/devices/devices.h18
-rw-r--r--src/vnet/devices/netmap/node.c24
-rw-r--r--src/vnet/devices/ssvm/node.c6
-rw-r--r--src/vnet/devices/virtio/vhost-user.c127
6 files changed, 130 insertions, 126 deletions
diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c
index ba337f3f70b..76980102f71 100644
--- a/src/vnet/devices/af_packet/node.c
+++ b/src/vnet/devices/af_packet/node.c
@@ -124,7 +124,7 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
u32 frame_num = apif->rx_req->tp_frame_nr;
u8 *block_start = apif->rx_ring + block * block_size;
uword n_trace = vlib_get_trace_count (vm, node);
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm,
VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
u32 min_bufs = apif->rx_req->tp_frame_size / n_buffer_bytes;
@@ -132,15 +132,15 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
if (apif->per_interface_next_index != ~0)
next_index = apif->per_interface_next_index;
- n_free_bufs = vec_len (apm->rx_buffers[cpu_index]);
+ n_free_bufs = vec_len (apm->rx_buffers[thread_index]);
if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE))
{
- vec_validate (apm->rx_buffers[cpu_index],
+ vec_validate (apm->rx_buffers[thread_index],
VLIB_FRAME_SIZE + n_free_bufs - 1);
n_free_bufs +=
- vlib_buffer_alloc (vm, &apm->rx_buffers[cpu_index][n_free_bufs],
+ vlib_buffer_alloc (vm, &apm->rx_buffers[thread_index][n_free_bufs],
VLIB_FRAME_SIZE);
- _vec_len (apm->rx_buffers[cpu_index]) = n_free_bufs;
+ _vec_len (apm->rx_buffers[thread_index]) = n_free_bufs;
}
rx_frame = apif->next_rx_frame;
@@ -163,11 +163,11 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
{
/* grab free buffer */
u32 last_empty_buffer =
- vec_len (apm->rx_buffers[cpu_index]) - 1;
+ vec_len (apm->rx_buffers[thread_index]) - 1;
prev_bi0 = bi0;
- bi0 = apm->rx_buffers[cpu_index][last_empty_buffer];
+ bi0 = apm->rx_buffers[thread_index][last_empty_buffer];
b0 = vlib_get_buffer (vm, bi0);
- _vec_len (apm->rx_buffers[cpu_index]) = last_empty_buffer;
+ _vec_len (apm->rx_buffers[thread_index]) = last_empty_buffer;
n_free_bufs--;
/* copy data */
@@ -236,9 +236,9 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_increment_combined_counter
(vnet_get_main ()->interface_main.combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_RX,
- os_get_cpu_number (), apif->hw_if_index, n_rx_packets, n_rx_bytes);
+ vlib_get_thread_index (), apif->hw_if_index, n_rx_packets, n_rx_bytes);
- vnet_device_increment_rx_packets (cpu_index, n_rx_packets);
+ vnet_device_increment_rx_packets (thread_index, n_rx_packets);
return n_rx_packets;
}
diff --git a/src/vnet/devices/devices.c b/src/vnet/devices/devices.c
index 4164522099e..5e5e812c48b 100644
--- a/src/vnet/devices/devices.c
+++ b/src/vnet/devices/devices.c
@@ -104,7 +104,7 @@ vnet_device_queue_sort (void *a1, void *a2)
void
vnet_device_input_assign_thread (u32 hw_if_index,
- u16 queue_id, uword cpu_index)
+ u16 queue_id, uword thread_index)
{
vnet_main_t *vnm = vnet_get_main ();
vnet_device_main_t *vdm = &vnet_device_main;
@@ -115,19 +115,19 @@ vnet_device_input_assign_thread (u32 hw_if_index,
ASSERT (hw->input_node_index > 0);
- if (vdm->first_worker_cpu_index == 0)
- cpu_index = 0;
+ if (vdm->first_worker_thread_index == 0)
+ thread_index = 0;
- if (cpu_index != 0 &&
- (cpu_index < vdm->first_worker_cpu_index ||
- cpu_index > vdm->last_worker_cpu_index))
+ if (thread_index != 0 &&
+ (thread_index < vdm->first_worker_thread_index ||
+ thread_index > vdm->last_worker_thread_index))
{
- cpu_index = vdm->next_worker_cpu_index++;
- if (vdm->next_worker_cpu_index > vdm->last_worker_cpu_index)
- vdm->next_worker_cpu_index = vdm->first_worker_cpu_index;
+ thread_index = vdm->next_worker_thread_index++;
+ if (vdm->next_worker_thread_index > vdm->last_worker_thread_index)
+ vdm->next_worker_thread_index = vdm->first_worker_thread_index;
}
- vm = vlib_mains[cpu_index];
+ vm = vlib_mains[thread_index];
rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
vec_add2 (rt->devices_and_queues, dq, 1);
@@ -136,33 +136,33 @@ vnet_device_input_assign_thread (u32 hw_if_index,
dq->queue_id = queue_id;
vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort);
- vec_validate (hw->input_node_cpu_index_by_queue, queue_id);
- hw->input_node_cpu_index_by_queue[queue_id] = cpu_index;
+ vec_validate (hw->input_node_thread_index_by_queue, queue_id);
+ hw->input_node_thread_index_by_queue[queue_id] = thread_index;
}
static int
vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id,
- uword cpu_index)
+ uword thread_index)
{
vnet_main_t *vnm = vnet_get_main ();
vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
vnet_device_input_runtime_t *rt;
vnet_device_and_queue_t *dq;
- uword old_cpu_index;
+ uword old_thread_index;
- if (hw->input_node_cpu_index_by_queue == 0)
+ if (hw->input_node_thread_index_by_queue == 0)
return VNET_API_ERROR_INVALID_INTERFACE;
- if (vec_len (hw->input_node_cpu_index_by_queue) < queue_id + 1)
+ if (vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1)
return VNET_API_ERROR_INVALID_INTERFACE;
- old_cpu_index = hw->input_node_cpu_index_by_queue[queue_id];
+ old_thread_index = hw->input_node_thread_index_by_queue[queue_id];
- if (old_cpu_index == cpu_index)
+ if (old_thread_index == thread_index)
return 0;
rt =
- vlib_node_get_runtime_data (vlib_mains[old_cpu_index],
+ vlib_node_get_runtime_data (vlib_mains[old_thread_index],
hw->input_node_index);
vec_foreach (dq, rt->devices_and_queues)
@@ -240,7 +240,7 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input,
vnet_device_main_t *vdm = &vnet_device_main;
u32 hw_if_index = (u32) ~ 0;
u32 queue_id = (u32) 0;
- u32 cpu_index = (u32) ~ 0;
+ u32 thread_index = (u32) ~ 0;
int rv;
if (!unformat_user (input, unformat_line_input, line_input))
@@ -253,10 +253,10 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input,
;
else if (unformat (line_input, "queue %d", &queue_id))
;
- else if (unformat (line_input, "main", &cpu_index))
- cpu_index = 0;
- else if (unformat (line_input, "worker %d", &cpu_index))
- cpu_index += vdm->first_worker_cpu_index;
+ else if (unformat (line_input, "main", &thread_index))
+ thread_index = 0;
+ else if (unformat (line_input, "worker %d", &thread_index))
+ thread_index += vdm->first_worker_thread_index;
else
{
error = clib_error_return (0, "parse error: '%U'",
@@ -271,16 +271,17 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input,
if (hw_if_index == (u32) ~ 0)
return clib_error_return (0, "please specify valid interface name");
- if (cpu_index > vdm->last_worker_cpu_index)
+ if (thread_index > vdm->last_worker_thread_index)
return clib_error_return (0,
"please specify valid worker thread or main");
- rv = vnet_device_input_unassign_thread (hw_if_index, queue_id, cpu_index);
+ rv =
+ vnet_device_input_unassign_thread (hw_if_index, queue_id, thread_index);
if (rv)
return clib_error_return (0, "not found");
- vnet_device_input_assign_thread (hw_if_index, queue_id, cpu_index);
+ vnet_device_input_assign_thread (hw_if_index, queue_id, thread_index);
return 0;
}
@@ -326,9 +327,9 @@ vnet_device_init (vlib_main_t * vm)
tr = p ? (vlib_thread_registration_t *) p[0] : 0;
if (tr && tr->count > 0)
{
- vdm->first_worker_cpu_index = tr->first_index;
- vdm->next_worker_cpu_index = tr->first_index;
- vdm->last_worker_cpu_index = tr->first_index + tr->count - 1;
+ vdm->first_worker_thread_index = tr->first_index;
+ vdm->next_worker_thread_index = tr->first_index;
+ vdm->last_worker_thread_index = tr->first_index + tr->count - 1;
}
return 0;
}
diff --git a/src/vnet/devices/devices.h b/src/vnet/devices/devices.h
index bbb29fe301c..966f8302230 100644
--- a/src/vnet/devices/devices.h
+++ b/src/vnet/devices/devices.h
@@ -50,9 +50,9 @@ typedef struct
typedef struct
{
vnet_device_per_worker_data_t *workers;
- uword first_worker_cpu_index;
- uword last_worker_cpu_index;
- uword next_worker_cpu_index;
+ uword first_worker_thread_index;
+ uword last_worker_thread_index;
+ uword next_worker_thread_index;
} vnet_device_main_t;
typedef struct
@@ -80,7 +80,7 @@ vnet_set_device_input_node (u32 hw_if_index, u32 node_index)
}
void vnet_device_input_assign_thread (u32 hw_if_index, u16 queue_id,
- uword cpu_index);
+ uword thread_index);
static inline u64
vnet_get_aggregate_rx_packets (void)
@@ -95,12 +95,12 @@ vnet_get_aggregate_rx_packets (void)
}
static inline void
-vnet_device_increment_rx_packets (u32 cpu_index, u64 count)
+vnet_device_increment_rx_packets (u32 thread_index, u64 count)
{
vnet_device_main_t *vdm = &vnet_device_main;
vnet_device_per_worker_data_t *pwd;
- pwd = vec_elt_at_index (vdm->workers, cpu_index);
+ pwd = vec_elt_at_index (vdm->workers, thread_index);
pwd->aggregate_rx_packets += count;
}
@@ -117,9 +117,9 @@ vnet_device_input_set_interrupt_pending (vnet_main_t * vnm, u32 hw_if_index,
{
vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
- ASSERT (queue_id < vec_len (hw->input_node_cpu_index_by_queue));
- u32 cpu_index = hw->input_node_cpu_index_by_queue[queue_id];
- vlib_node_set_interrupt_pending (vlib_mains[cpu_index],
+ ASSERT (queue_id < vec_len (hw->input_node_thread_index_by_queue));
+ u32 thread_index = hw->input_node_thread_index_by_queue[queue_id];
+ vlib_node_set_interrupt_pending (vlib_mains[thread_index],
hw->input_node_index);
}
diff --git a/src/vnet/devices/netmap/node.c b/src/vnet/devices/netmap/node.c
index 68ea7832614..e120eeaeabd 100644
--- a/src/vnet/devices/netmap/node.c
+++ b/src/vnet/devices/netmap/node.c
@@ -98,22 +98,22 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
u32 n_free_bufs;
struct netmap_ring *ring;
int cur_ring;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm,
VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
if (nif->per_interface_next_index != ~0)
next_index = nif->per_interface_next_index;
- n_free_bufs = vec_len (nm->rx_buffers[cpu_index]);
+ n_free_bufs = vec_len (nm->rx_buffers[thread_index]);
if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE))
{
- vec_validate (nm->rx_buffers[cpu_index],
+ vec_validate (nm->rx_buffers[thread_index],
VLIB_FRAME_SIZE + n_free_bufs - 1);
n_free_bufs +=
- vlib_buffer_alloc (vm, &nm->rx_buffers[cpu_index][n_free_bufs],
+ vlib_buffer_alloc (vm, &nm->rx_buffers[thread_index][n_free_bufs],
VLIB_FRAME_SIZE);
- _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs;
+ _vec_len (nm->rx_buffers[thread_index]) = n_free_bufs;
}
cur_ring = nif->first_rx_ring;
@@ -163,11 +163,11 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_buffer_t *b0;
/* grab free buffer */
u32 last_empty_buffer =
- vec_len (nm->rx_buffers[cpu_index]) - 1;
+ vec_len (nm->rx_buffers[thread_index]) - 1;
prev_bi0 = bi0;
- bi0 = nm->rx_buffers[cpu_index][last_empty_buffer];
+ bi0 = nm->rx_buffers[thread_index][last_empty_buffer];
b0 = vlib_get_buffer (vm, bi0);
- _vec_len (nm->rx_buffers[cpu_index]) = last_empty_buffer;
+ _vec_len (nm->rx_buffers[thread_index]) = last_empty_buffer;
n_free_bufs--;
/* copy data */
@@ -247,9 +247,9 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_increment_combined_counter
(vnet_get_main ()->interface_main.combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_RX,
- os_get_cpu_number (), nif->hw_if_index, n_rx_packets, n_rx_bytes);
+ vlib_get_thread_index (), nif->hw_if_index, n_rx_packets, n_rx_bytes);
- vnet_device_increment_rx_packets (cpu_index, n_rx_packets);
+ vnet_device_increment_rx_packets (thread_index, n_rx_packets);
return n_rx_packets;
}
@@ -260,7 +260,7 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
{
int i;
u32 n_rx_packets = 0;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
netmap_main_t *nm = &netmap_main;
netmap_if_t *nmi;
@@ -269,7 +269,7 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
nmi = vec_elt_at_index (nm->interfaces, i);
if (nmi->is_admin_up &&
(i % nm->input_cpu_count) ==
- (cpu_index - nm->input_cpu_first_index))
+ (thread_index - nm->input_cpu_first_index))
n_rx_packets += netmap_device_input_fn (vm, node, frame, nmi);
}
diff --git a/src/vnet/devices/ssvm/node.c b/src/vnet/devices/ssvm/node.c
index a6c9dfd771d..539b4161d7a 100644
--- a/src/vnet/devices/ssvm/node.c
+++ b/src/vnet/devices/ssvm/node.c
@@ -89,7 +89,7 @@ ssvm_eth_device_input (ssvm_eth_main_t * em,
ethernet_header_t *eh0;
u16 type0;
u32 n_rx_bytes = 0, l3_offset0;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
u32 trace_cnt __attribute__ ((unused)) = vlib_get_trace_count (vm, node);
volatile u32 *lock;
u32 *elt_indices;
@@ -284,10 +284,10 @@ out:
vlib_increment_combined_counter
(vnet_get_main ()->interface_main.combined_sw_if_counters
- + VNET_INTERFACE_COUNTER_RX, cpu_index,
+ + VNET_INTERFACE_COUNTER_RX, thread_index,
intfc->vlib_hw_if_index, rx_queue_index, n_rx_bytes);
- vnet_device_increment_rx_packets (cpu_index, rx_queue_index);
+ vnet_device_increment_rx_packets (thread_index, rx_queue_index);
return rx_queue_index;
}
diff --git a/src/vnet/devices/virtio/vhost-user.c b/src/vnet/devices/virtio/vhost-user.c
index 00807dc0398..5e720f6510a 100644
--- a/src/vnet/devices/virtio/vhost-user.c
+++ b/src/vnet/devices/virtio/vhost-user.c
@@ -331,7 +331,7 @@ vhost_user_tx_thread_placement (vhost_user_intf_t * vui)
{
//Let's try to assign one queue to each thread
u32 qid = 0;
- u32 cpu_index = 0;
+ u32 thread_index = 0;
vui->use_tx_spinlock = 0;
while (1)
{
@@ -341,20 +341,21 @@ vhost_user_tx_thread_placement (vhost_user_intf_t * vui)
if (!rxvq->started || !rxvq->enabled)
continue;
- vui->per_cpu_tx_qid[cpu_index] = qid;
- cpu_index++;
- if (cpu_index == vlib_get_thread_main ()->n_vlib_mains)
+ vui->per_cpu_tx_qid[thread_index] = qid;
+ thread_index++;
+ if (thread_index == vlib_get_thread_main ()->n_vlib_mains)
return;
}
//We need to loop, meaning the spinlock has to be used
vui->use_tx_spinlock = 1;
- if (cpu_index == 0)
+ if (thread_index == 0)
{
//Could not find a single valid one
- for (cpu_index = 0;
- cpu_index < vlib_get_thread_main ()->n_vlib_mains; cpu_index++)
+ for (thread_index = 0;
+ thread_index < vlib_get_thread_main ()->n_vlib_mains;
+ thread_index++)
{
- vui->per_cpu_tx_qid[cpu_index] = 0;
+ vui->per_cpu_tx_qid[thread_index] = 0;
}
return;
}
@@ -368,7 +369,7 @@ vhost_user_rx_thread_placement ()
vhost_user_intf_t *vui;
vhost_cpu_t *vhc;
u32 *workers = 0;
- u32 cpu_index;
+ u32 thread_index;
vlib_main_t *vm;
//Let's list all workers cpu indexes
@@ -400,9 +401,9 @@ vhost_user_rx_thread_placement ()
continue;
i %= vec_len (vui_workers);
- cpu_index = vui_workers[i];
+ thread_index = vui_workers[i];
i++;
- vhc = &vum->cpus[cpu_index];
+ vhc = &vum->cpus[thread_index];
iaq.qid = qid;
iaq.vhost_iface_index = vui - vum->vhost_user_interfaces;
@@ -429,14 +430,14 @@ vhost_user_rx_thread_placement ()
vhc->operation_mode = mode;
}
- for (cpu_index = vum->input_cpu_first_index;
- cpu_index < vum->input_cpu_first_index + vum->input_cpu_count;
- cpu_index++)
+ for (thread_index = vum->input_cpu_first_index;
+ thread_index < vum->input_cpu_first_index + vum->input_cpu_count;
+ thread_index++)
{
vlib_node_state_t state = VLIB_NODE_STATE_POLLING;
- vhc = &vum->cpus[cpu_index];
- vm = vlib_mains ? vlib_mains[cpu_index] : &vlib_global_main;
+ vhc = &vum->cpus[thread_index];
+ vm = vlib_mains ? vlib_mains[thread_index] : &vlib_global_main;
switch (vhc->operation_mode)
{
case VHOST_USER_INTERRUPT_MODE:
@@ -532,7 +533,7 @@ vhost_user_set_interrupt_pending (vhost_user_intf_t * vui, u32 ifq)
{
vhost_user_main_t *vum = &vhost_user_main;
vhost_cpu_t *vhc;
- u32 cpu_index;
+ u32 thread_index;
vhost_iface_and_queue_t *vhiq;
vlib_main_t *vm;
u32 ifq2;
@@ -553,8 +554,8 @@ vhost_user_set_interrupt_pending (vhost_user_intf_t * vui, u32 ifq)
if ((vhiq->vhost_iface_index == (ifq >> 8)) &&
(VHOST_VRING_IDX_TX (vhiq->qid) == (ifq & 0xff)))
{
- cpu_index = vhc - vum->cpus;
- vm = vlib_mains ? vlib_mains[cpu_index] : &vlib_global_main;
+ thread_index = vhc - vum->cpus;
+ vm = vlib_mains ? vlib_mains[thread_index] : &vlib_global_main;
/*
* Convert RX virtqueue number in the lower byte to vring
* queue index for the input node process. Top bytes contain
@@ -1592,7 +1593,7 @@ vhost_user_if_input (vlib_main_t * vm,
u32 n_trace = vlib_get_trace_count (vm, node);
u16 qsz_mask;
u32 map_hint = 0;
- u16 cpu_index = os_get_cpu_number ();
+ u16 thread_index = vlib_get_thread_index ();
u16 copy_len = 0;
{
@@ -1651,32 +1652,32 @@ vhost_user_if_input (vlib_main_t * vm,
* in the loop and come back later. This is not an issue as for big packet,
* processing cost really comes from the memory copy.
*/
- if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len < n_left + 1))
+ if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len < n_left + 1))
{
- u32 curr_len = vum->cpus[cpu_index].rx_buffers_len;
- vum->cpus[cpu_index].rx_buffers_len +=
+ u32 curr_len = vum->cpus[thread_index].rx_buffers_len;
+ vum->cpus[thread_index].rx_buffers_len +=
vlib_buffer_alloc_from_free_list (vm,
- vum->cpus[cpu_index].rx_buffers +
+ vum->cpus[thread_index].rx_buffers +
curr_len,
VHOST_USER_RX_BUFFERS_N - curr_len,
VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
if (PREDICT_FALSE
- (vum->cpus[cpu_index].rx_buffers_len <
+ (vum->cpus[thread_index].rx_buffers_len <
VHOST_USER_RX_BUFFER_STARVATION))
{
/* In case of buffer starvation, discard some packets from the queue
* and log the event.
* We keep doing best effort for the remaining packets. */
- u32 flush = (n_left + 1 > vum->cpus[cpu_index].rx_buffers_len) ?
- n_left + 1 - vum->cpus[cpu_index].rx_buffers_len : 1;
+ u32 flush = (n_left + 1 > vum->cpus[thread_index].rx_buffers_len) ?
+ n_left + 1 - vum->cpus[thread_index].rx_buffers_len : 1;
flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush);
n_left -= flush;
vlib_increment_simple_counter (vnet_main.
interface_main.sw_if_counters +
VNET_INTERFACE_COUNTER_DROP,
- os_get_cpu_number (),
+ vlib_get_thread_index (),
vui->sw_if_index, flush);
vlib_error_count (vm, vhost_user_input_node.index,
@@ -1696,7 +1697,7 @@ vhost_user_if_input (vlib_main_t * vm,
u32 desc_data_offset;
vring_desc_t *desc_table = txvq->desc;
- if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len <= 1))
+ if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len <= 1))
{
/* Not enough rx_buffers
* Note: We yeld on 1 so we don't need to do an additional
@@ -1707,17 +1708,18 @@ vhost_user_if_input (vlib_main_t * vm,
}
desc_current = txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
- vum->cpus[cpu_index].rx_buffers_len--;
- bi_current = (vum->cpus[cpu_index].rx_buffers)
- [vum->cpus[cpu_index].rx_buffers_len];
+ vum->cpus[thread_index].rx_buffers_len--;
+ bi_current = (vum->cpus[thread_index].rx_buffers)
+ [vum->cpus[thread_index].rx_buffers_len];
b_head = b_current = vlib_get_buffer (vm, bi_current);
to_next[0] = bi_current; //We do that now so we can forget about bi_current
to_next++;
n_left_to_next--;
vlib_prefetch_buffer_with_index (vm,
- (vum->cpus[cpu_index].rx_buffers)
- [vum->cpus[cpu_index].
+ (vum->
+ cpus[thread_index].rx_buffers)
+ [vum->cpus[thread_index].
rx_buffers_len - 1], LOAD);
/* Just preset the used descriptor id and length for later */
@@ -1791,7 +1793,7 @@ vhost_user_if_input (vlib_main_t * vm,
(b_current->current_length == VLIB_BUFFER_DATA_SIZE))
{
if (PREDICT_FALSE
- (vum->cpus[cpu_index].rx_buffers_len == 0))
+ (vum->cpus[thread_index].rx_buffers_len == 0))
{
/* Cancel speculation */
to_next--;
@@ -1805,17 +1807,18 @@ vhost_user_if_input (vlib_main_t * vm,
* but valid.
*/
vhost_user_input_rewind_buffers (vm,
- &vum->cpus[cpu_index],
+ &vum->cpus
+ [thread_index],
b_head);
n_left = 0;
goto stop;
}
/* Get next output */
- vum->cpus[cpu_index].rx_buffers_len--;
+ vum->cpus[thread_index].rx_buffers_len--;
u32 bi_next =
- (vum->cpus[cpu_index].rx_buffers)[vum->cpus
- [cpu_index].rx_buffers_len];
+ (vum->cpus[thread_index].rx_buffers)[vum->cpus
+ [thread_index].rx_buffers_len];
b_current->next_buffer = bi_next;
b_current->flags |= VLIB_BUFFER_NEXT_PRESENT;
bi_current = bi_next;
@@ -1823,7 +1826,7 @@ vhost_user_if_input (vlib_main_t * vm,
}
/* Prepare a copy order executed later for the data */
- vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len];
+ vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len];
copy_len++;
u32 desc_data_l =
desc_table[desc_current].len - desc_data_offset;
@@ -1880,7 +1883,7 @@ vhost_user_if_input (vlib_main_t * vm,
if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD))
{
if (PREDICT_FALSE
- (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy,
+ (vhost_user_input_copy (vui, vum->cpus[thread_index].copy,
copy_len, &map_hint)))
{
clib_warning
@@ -1905,7 +1908,7 @@ vhost_user_if_input (vlib_main_t * vm,
/* Do the memory copies */
if (PREDICT_FALSE
- (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy,
+ (vhost_user_input_copy (vui, vum->cpus[thread_index].copy,
copy_len, &map_hint)))
{
clib_warning ("Memory mapping error on interface hw_if_index=%d "
@@ -1933,9 +1936,9 @@ vhost_user_if_input (vlib_main_t * vm,
vlib_increment_combined_counter
(vnet_main.interface_main.combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_RX,
- os_get_cpu_number (), vui->sw_if_index, n_rx_packets, n_rx_bytes);
+ vlib_get_thread_index (), vui->sw_if_index, n_rx_packets, n_rx_bytes);
- vnet_device_increment_rx_packets (cpu_index, n_rx_packets);
+ vnet_device_increment_rx_packets (thread_index, n_rx_packets);
return n_rx_packets;
}
@@ -1946,15 +1949,15 @@ vhost_user_input (vlib_main_t * vm,
{
vhost_user_main_t *vum = &vhost_user_main;
uword n_rx_packets = 0;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
vhost_iface_and_queue_t *vhiq;
vhost_user_intf_t *vui;
vhost_cpu_t *vhc;
- vhc = &vum->cpus[cpu_index];
+ vhc = &vum->cpus[thread_index];
if (PREDICT_TRUE (vhc->operation_mode == VHOST_USER_POLLING_MODE))
{
- vec_foreach (vhiq, vum->cpus[cpu_index].rx_queues)
+ vec_foreach (vhiq, vum->cpus[thread_index].rx_queues)
{
vui = &vum->vhost_user_interfaces[vhiq->vhost_iface_index];
n_rx_packets += vhost_user_if_input (vm, vum, vui, vhiq->qid, node);
@@ -2096,7 +2099,7 @@ vhost_user_tx (vlib_main_t * vm,
vhost_user_vring_t *rxvq;
u16 qsz_mask;
u8 error;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
u32 map_hint = 0;
u8 retry = 8;
u16 copy_len;
@@ -2116,7 +2119,7 @@ vhost_user_tx (vlib_main_t * vm,
qid =
VHOST_VRING_IDX_RX (*vec_elt_at_index
- (vui->per_cpu_tx_qid, os_get_cpu_number ()));
+ (vui->per_cpu_tx_qid, vlib_get_thread_index ()));
rxvq = &vui->vrings[qid];
if (PREDICT_FALSE (vui->use_tx_spinlock))
vhost_user_vring_lock (vui, qid);
@@ -2143,10 +2146,10 @@ retry:
if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
{
- vum->cpus[cpu_index].current_trace =
+ vum->cpus[thread_index].current_trace =
vlib_add_trace (vm, node, b0,
- sizeof (*vum->cpus[cpu_index].current_trace));
- vhost_user_tx_trace (vum->cpus[cpu_index].current_trace,
+ sizeof (*vum->cpus[thread_index].current_trace));
+ vhost_user_tx_trace (vum->cpus[thread_index].current_trace,
vui, qid / 2, b0, rxvq);
}
@@ -2188,14 +2191,14 @@ retry:
{
// Get a header from the header array
virtio_net_hdr_mrg_rxbuf_t *hdr =
- &vum->cpus[cpu_index].tx_headers[tx_headers_len];
+ &vum->cpus[thread_index].tx_headers[tx_headers_len];
tx_headers_len++;
hdr->hdr.flags = 0;
hdr->hdr.gso_type = 0;
hdr->num_buffers = 1; //This is local, no need to check
// Prepare a copy order executed later for the header
- vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len];
+ vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len];
copy_len++;
cpy->len = vui->virtio_net_hdr_sz;
cpy->dst = buffer_map_addr;
@@ -2220,7 +2223,7 @@ retry:
else if (vui->virtio_net_hdr_sz == 12) //MRG is available
{
virtio_net_hdr_mrg_rxbuf_t *hdr =
- &vum->cpus[cpu_index].tx_headers[tx_headers_len - 1];
+ &vum->cpus[thread_index].tx_headers[tx_headers_len - 1];
//Move from available to used buffer
rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id =
@@ -2282,7 +2285,7 @@ retry:
}
{
- vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len];
+ vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len];
copy_len++;
cpy->len = bytes_left;
cpy->len = (cpy->len > buffer_len) ? buffer_len : cpy->len;
@@ -2325,8 +2328,8 @@ retry:
if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
{
- vum->cpus[cpu_index].current_trace->hdr =
- vum->cpus[cpu_index].tx_headers[tx_headers_len - 1];
+ vum->cpus[thread_index].current_trace->hdr =
+ vum->cpus[thread_index].tx_headers[tx_headers_len - 1];
}
n_left--; //At the end for error counting when 'goto done' is invoked
@@ -2336,7 +2339,7 @@ retry:
done:
//Do the memory copies
if (PREDICT_FALSE
- (vhost_user_tx_copy (vui, vum->cpus[cpu_index].copy,
+ (vhost_user_tx_copy (vui, vum->cpus[thread_index].copy,
copy_len, &map_hint)))
{
clib_warning ("Memory mapping error on interface hw_if_index=%d "
@@ -2386,7 +2389,7 @@ done3:
vlib_increment_simple_counter
(vnet_main.interface_main.sw_if_counters
+ VNET_INTERFACE_COUNTER_DROP,
- os_get_cpu_number (), vui->sw_if_index, n_left);
+ vlib_get_thread_index (), vui->sw_if_index, n_left);
}
vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
@@ -2773,11 +2776,11 @@ vhost_user_send_interrupt_process (vlib_main_t * vm,
case ~0:
vec_foreach (vhc, vum->cpus)
{
- u32 cpu_index = vhc - vum->cpus;
+ u32 thread_index = vhc - vum->cpus;
f64 next_timeout;
next_timeout = timeout;
- vec_foreach (vhiq, vum->cpus[cpu_index].rx_queues)
+ vec_foreach (vhiq, vum->cpus[thread_index].rx_queues)
{
vui = &vum->vhost_user_interfaces[vhiq->vhost_iface_index];
vhost_user_vring_t *rxvq =