diff options
-rw-r--r-- | src/plugins/flowperpkt/l2_node.c | 20 | ||||
-rw-r--r-- | src/plugins/flowperpkt/node.c | 20 | ||||
-rw-r--r-- | src/plugins/snat/in2out.c | 2 | ||||
-rw-r--r-- | src/plugins/snat/out2in.c | 2 | ||||
-rw-r--r-- | src/vlib/main.h | 2 | ||||
-rw-r--r-- | src/vlib/threads.c | 12 | ||||
-rw-r--r-- | src/vlib/threads.h | 3 | ||||
-rw-r--r-- | src/vlib/unix/main.c | 2 | ||||
-rw-r--r-- | src/vlibmemory/memory_vlib.c | 2 | ||||
-rw-r--r-- | src/vnet/dpo/interface_dpo.c | 8 | ||||
-rw-r--r-- | src/vnet/lisp-gpe/lisp_gpe_adjacency.c | 2 | ||||
-rw-r--r-- | src/vppinfra/bihash_template.c | 16 | ||||
-rw-r--r-- | src/vppinfra/lock.h | 6 | ||||
-rw-r--r-- | src/vppinfra/mem.h | 6 | ||||
-rw-r--r-- | src/vppinfra/mhash.c | 2 | ||||
-rw-r--r-- | src/vppinfra/mhash.h | 2 | ||||
-rw-r--r-- | src/vppinfra/mheap.c | 4 | ||||
-rw-r--r-- | src/vppinfra/os.h | 20 | ||||
-rw-r--r-- | src/vppinfra/smp.c | 2 | ||||
-rw-r--r-- | src/vppinfra/unix-misc.c | 19 |
20 files changed, 77 insertions, 75 deletions
diff --git a/src/plugins/flowperpkt/l2_node.c b/src/plugins/flowperpkt/l2_node.c index fdaf81d1a2e..db80e990352 100644 --- a/src/plugins/flowperpkt/l2_node.c +++ b/src/plugins/flowperpkt/l2_node.c @@ -102,7 +102,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, u8 * src_mac, u8 * dst_mac, u16 ethertype, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->thread_index; + u32 my_thread_index = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; @@ -116,7 +116,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, vlib_buffer_free_list_t *fl; /* Find or allocate a buffer */ - b0 = fm->l2_buffers_per_worker[my_cpu_number]; + b0 = fm->l2_buffers_per_worker[my_thread_index]; /* Need to allocate a buffer? */ if (PREDICT_FALSE (b0 == 0)) @@ -130,7 +130,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, return; /* Initialize the buffer */ - b0 = fm->l2_buffers_per_worker[my_cpu_number] = + b0 = fm->l2_buffers_per_worker[my_thread_index] = vlib_get_buffer (vm, bi0); fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); @@ -142,16 +142,16 @@ add_to_flow_record_l2 (vlib_main_t * vm, { /* use the current buffer */ bi0 = vlib_get_buffer_index (vm, b0); - offset = fm->l2_next_record_offset_per_worker[my_cpu_number]; + offset = fm->l2_next_record_offset_per_worker[my_thread_index]; } /* Find or allocate a frame */ - f = fm->l2_frames_per_worker[my_cpu_number]; + f = fm->l2_frames_per_worker[my_thread_index]; if (PREDICT_FALSE (f == 0)) { u32 *to_next; f = vlib_get_frame_to_node (vm, ip4_lookup_node.index); - fm->l2_frames_per_worker[my_cpu_number] = f; + fm->l2_frames_per_worker[my_thread_index] = f; /* Enqueue the buffer */ to_next = vlib_frame_vector_args (f); @@ -299,13 +299,13 @@ add_to_flow_record_l2 (vlib_main_t * vm, } vlib_put_frame_to_node (vm, ip4_lookup_node.index, - fm->l2_frames_per_worker[my_cpu_number]); - fm->l2_frames_per_worker[my_cpu_number] = 0; - fm->l2_buffers_per_worker[my_cpu_number] = 0; + fm->l2_frames_per_worker[my_thread_index]); + fm->l2_frames_per_worker[my_thread_index] = 0; + fm->l2_buffers_per_worker[my_thread_index] = 0; offset = 0; } - fm->l2_next_record_offset_per_worker[my_cpu_number] = offset; + fm->l2_next_record_offset_per_worker[my_thread_index] = offset; } void diff --git a/src/plugins/flowperpkt/node.c b/src/plugins/flowperpkt/node.c index 0277682dc87..9bac4166a97 100644 --- a/src/plugins/flowperpkt/node.c +++ b/src/plugins/flowperpkt/node.c @@ -101,7 +101,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, u32 src_address, u32 dst_address, u8 tos, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->thread_index; + u32 my_thread_index = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; @@ -115,7 +115,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, vlib_buffer_free_list_t *fl; /* Find or allocate a buffer */ - b0 = fm->ipv4_buffers_per_worker[my_cpu_number]; + b0 = fm->ipv4_buffers_per_worker[my_thread_index]; /* Need to allocate a buffer? */ if (PREDICT_FALSE (b0 == 0)) @@ -129,7 +129,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, return; /* Initialize the buffer */ - b0 = fm->ipv4_buffers_per_worker[my_cpu_number] = + b0 = fm->ipv4_buffers_per_worker[my_thread_index] = vlib_get_buffer (vm, bi0); fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); @@ -141,16 +141,16 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, { /* use the current buffer */ bi0 = vlib_get_buffer_index (vm, b0); - offset = fm->ipv4_next_record_offset_per_worker[my_cpu_number]; + offset = fm->ipv4_next_record_offset_per_worker[my_thread_index]; } /* Find or allocate a frame */ - f = fm->ipv4_frames_per_worker[my_cpu_number]; + f = fm->ipv4_frames_per_worker[my_thread_index]; if (PREDICT_FALSE (f == 0)) { u32 *to_next; f = vlib_get_frame_to_node (vm, ip4_lookup_node.index); - fm->ipv4_frames_per_worker[my_cpu_number] = f; + fm->ipv4_frames_per_worker[my_thread_index] = f; /* Enqueue the buffer */ to_next = vlib_frame_vector_args (f); @@ -300,13 +300,13 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, } vlib_put_frame_to_node (vm, ip4_lookup_node.index, - fm->ipv4_frames_per_worker[my_cpu_number]); - fm->ipv4_frames_per_worker[my_cpu_number] = 0; - fm->ipv4_buffers_per_worker[my_cpu_number] = 0; + fm->ipv4_frames_per_worker[my_thread_index]); + fm->ipv4_frames_per_worker[my_thread_index] = 0; + fm->ipv4_buffers_per_worker[my_thread_index] = 0; offset = 0; } - fm->ipv4_next_record_offset_per_worker[my_cpu_number] = offset; + fm->ipv4_next_record_offset_per_worker[my_thread_index] = offset; } void diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c index f7d29c69fe1..bc86a7a4d23 100644 --- a/src/plugins/snat/in2out.c +++ b/src/plugins/snat/in2out.c @@ -1514,7 +1514,7 @@ snat_det_in2out_node_fn (vlib_main_t * vm, u32 pkts_processed = 0; snat_main_t * sm = &snat_main; u32 now = (u32) vlib_time_now (vm); - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; diff --git a/src/plugins/snat/out2in.c b/src/plugins/snat/out2in.c index 3d7b106a210..824406ab425 100644 --- a/src/plugins/snat/out2in.c +++ b/src/plugins/snat/out2in.c @@ -1168,7 +1168,7 @@ snat_det_out2in_node_fn (vlib_main_t * vm, snat_out2in_next_t next_index; u32 pkts_processed = 0; snat_main_t * sm = &snat_main; - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; diff --git a/src/vlib/main.h b/src/vlib/main.h index 329bf0733e4..0e8026d17b0 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -320,7 +320,7 @@ always_inline void vlib_set_queue_signal_callback /* Main routine. */ int vlib_main (vlib_main_t * vm, unformat_input_t * input); -/* Thread stacks, for os_get_cpu_number */ +/* Thread stacks, for os_get_thread_index */ extern u8 **vlib_thread_stacks; /* Number of thread stacks that the application needs */ diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 9ccfd3a2901..b7bc9e26c0c 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -35,16 +35,8 @@ vl (void *p) vlib_worker_thread_t *vlib_worker_threads; vlib_thread_main_t vlib_thread_main; -__thread uword vlib_thread_index = 0; - -uword -os_get_cpu_number (void) -{ - return vlib_thread_index; -} - uword -os_get_ncpus (void) +os_get_nthreads (void) { u32 len; @@ -467,7 +459,7 @@ vlib_worker_thread_bootstrap_fn (void *arg) w->lwp = syscall (SYS_gettid); w->thread_id = pthread_self (); - vlib_thread_index = w - vlib_worker_threads; + __os_thread_index = w - vlib_worker_threads; rv = (void *) clib_calljmp ((uword (*)(uword)) w->thread_function, diff --git a/src/vlib/threads.h b/src/vlib/threads.h index 101d3d4ac1f..17d35a24b34 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -181,11 +181,10 @@ u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); void vlib_worker_thread_barrier_sync (vlib_main_t * vm); void vlib_worker_thread_barrier_release (vlib_main_t * vm); -extern __thread uword vlib_thread_index; static_always_inline uword vlib_get_thread_index (void) { - return vlib_thread_index; + return __os_thread_index; } always_inline void diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index db5ddd64d31..103576db01c 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -565,7 +565,7 @@ vlib_unix_main (int argc, char *argv[]) vlib_thread_stack_init (0); - vlib_thread_index = 0; + __os_thread_index = 0; i = clib_calljmp (thread0, (uword) vm, (void *) (vlib_thread_stacks[0] + diff --git a/src/vlibmemory/memory_vlib.c b/src/vlibmemory/memory_vlib.c index acba8b3f89e..e5d88732c2e 100644 --- a/src/vlibmemory/memory_vlib.c +++ b/src/vlibmemory/memory_vlib.c @@ -1333,7 +1333,7 @@ vl_api_rpc_call_main_thread (void *fp, u8 * data, u32 data_length) unix_shared_memory_queue_t *q; /* Main thread: call the function directly */ - if (os_get_cpu_number () == 0) + if (vlib_get_thread_index () == 0) { vlib_main_t *vm = vlib_get_main (); void (*call_fp) (void *); diff --git a/src/vnet/dpo/interface_dpo.c b/src/vnet/dpo/interface_dpo.c index 50ca756f9da..8d700c23e46 100644 --- a/src/vnet/dpo/interface_dpo.c +++ b/src/vnet/dpo/interface_dpo.c @@ -231,7 +231,7 @@ interface_dpo_inline (vlib_main_t * vm, vlib_frame_t * from_frame) { u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index (); vnet_interface_main_t *im; im = &vnet_get_main ()->interface_main; @@ -274,13 +274,13 @@ interface_dpo_inline (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, ido0->ido_sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, ido1->ido_sw_if_index, 1, vlib_buffer_length_in_chain (vm, b1)); @@ -331,7 +331,7 @@ interface_dpo_inline (vlib_main_t * vm, /* Bump the interface's RX coutners */ vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, ido0->ido_sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); diff --git a/src/vnet/lisp-gpe/lisp_gpe_adjacency.c b/src/vnet/lisp-gpe/lisp_gpe_adjacency.c index d5f3a28ac9d..7db1c9bbb72 100644 --- a/src/vnet/lisp-gpe/lisp_gpe_adjacency.c +++ b/src/vnet/lisp-gpe/lisp_gpe_adjacency.c @@ -302,7 +302,7 @@ lisp_gpe_increment_stats_counters (lisp_cp_main_t * lcm, ip_adjacency_t * adj, /* compute payload length starting after GPE */ u32 bytes = b->current_length - (lisp_data - b->data - b->current_data); - vlib_increment_combined_counter (&lgm->counters, os_get_cpu_number (), + vlib_increment_combined_counter (&lgm->counters, vlib_get_thread_index (), p[0], 1, bytes); } diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c index d8b97b5f3d0..51fadeb8adf 100644 --- a/src/vppinfra/bihash_template.c +++ b/src/vppinfra/bihash_template.c @@ -96,12 +96,12 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) clib_bihash_bucket_t working_bucket __attribute__ ((aligned (8))); void *oldheap; BVT (clib_bihash_value) * working_copy; - u32 cpu_number = os_get_cpu_number (); + u32 thread_index = os_get_thread_index (); - if (cpu_number >= vec_len (h->working_copies)) + if (thread_index >= vec_len (h->working_copies)) { oldheap = clib_mem_set_heap (h->mheap); - vec_validate (h->working_copies, cpu_number); + vec_validate (h->working_copies, thread_index); clib_mem_set_heap (oldheap); } @@ -110,7 +110,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) * updates from multiple threads will not result in sporadic, spurious * lookup failures. */ - working_copy = h->working_copies[cpu_number]; + working_copy = h->working_copies[thread_index]; h->saved_bucket.as_u64 = b->as_u64; oldheap = clib_mem_set_heap (h->mheap); @@ -119,7 +119,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) { vec_validate_aligned (working_copy, (1 << b->log2_pages) - 1, sizeof (u64)); - h->working_copies[cpu_number] = working_copy; + h->working_copies[thread_index] = working_copy; } _vec_len (working_copy) = 1 << b->log2_pages; @@ -132,7 +132,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) working_bucket.offset = BV (clib_bihash_get_offset) (h, working_copy); CLIB_MEMORY_BARRIER (); b->as_u64 = working_bucket.as_u64; - h->working_copies[cpu_number] = working_copy; + h->working_copies[thread_index] = working_copy; } static @@ -233,7 +233,7 @@ int BV (clib_bihash_add_del) int i, limit; u64 hash, new_hash; u32 new_log2_pages; - u32 cpu_number = os_get_cpu_number (); + u32 thread_index = os_get_thread_index (); int mark_bucket_linear; int resplit_once; @@ -323,7 +323,7 @@ int BV (clib_bihash_add_del) new_log2_pages = h->saved_bucket.log2_pages + 1; mark_bucket_linear = 0; - working_copy = h->working_copies[cpu_number]; + working_copy = h->working_copies[thread_index]; resplit_once = 0; new_v = BV (split_and_rehash) (h, working_copy, new_log2_pages); diff --git a/src/vppinfra/lock.h b/src/vppinfra/lock.h index c60ff414612..0cd2b4fea53 100644 --- a/src/vppinfra/lock.h +++ b/src/vppinfra/lock.h @@ -24,7 +24,7 @@ typedef struct u32 lock; #if CLIB_DEBUG > 0 pid_t pid; - uword cpu_index; + uword thread_index; void *frame_address; #endif } *clib_spinlock_t; @@ -57,7 +57,7 @@ clib_spinlock_lock (clib_spinlock_t * p) #if CLIB_DEBUG > 0 (*p)->frame_address = __builtin_frame_address (0); (*p)->pid = getpid (); - (*p)->cpu_index = os_get_cpu_number (); + (*p)->thread_index = os_get_thread_index (); #endif } @@ -75,7 +75,7 @@ clib_spinlock_unlock (clib_spinlock_t * p) #if CLIB_DEBUG > 0 (*p)->frame_address = 0; (*p)->pid = 0; - (*p)->cpu_index = 0; + (*p)->thread_index = 0; #endif } diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 1260eab28c0..63c5ac16a02 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -54,14 +54,14 @@ extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; always_inline void * clib_mem_get_per_cpu_heap (void) { - int cpu = os_get_cpu_number (); + int cpu = os_get_thread_index (); return clib_per_cpu_mheaps[cpu]; } always_inline void * clib_mem_set_per_cpu_heap (u8 * new_heap) { - int cpu = os_get_cpu_number (); + int cpu = os_get_thread_index (); void *old = clib_per_cpu_mheaps[cpu]; clib_per_cpu_mheaps[cpu] = new_heap; return old; @@ -83,7 +83,7 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, align_offset = align; } - cpu = os_get_cpu_number (); + cpu = os_get_thread_index (); heap = clib_per_cpu_mheaps[cpu]; heap = mheap_get_aligned (heap, size, align, align_offset, &offset); clib_per_cpu_mheaps[cpu] = heap; diff --git a/src/vppinfra/mhash.c b/src/vppinfra/mhash.c index c917e164cd9..00b67c4959f 100644 --- a/src/vppinfra/mhash.c +++ b/src/vppinfra/mhash.c @@ -226,7 +226,7 @@ static uword mhash_set_tmp_key (mhash_t * h, const void *key) { u8 *key_tmp; - int my_cpu = os_get_cpu_number (); + int my_cpu = os_get_thread_index (); vec_validate (h->key_tmps, my_cpu); key_tmp = h->key_tmps[my_cpu]; diff --git a/src/vppinfra/mhash.h b/src/vppinfra/mhash.h index 102adf4eb83..7eb1918384e 100644 --- a/src/vppinfra/mhash.h +++ b/src/vppinfra/mhash.h @@ -93,7 +93,7 @@ mhash_key_to_mem (mhash_t * h, uword key) { u8 *key_tmp; - int my_cpu = os_get_cpu_number (); + int my_cpu = os_get_thread_index (); vec_validate (h->key_tmps, my_cpu); key_tmp = h->key_tmps[my_cpu]; return key_tmp; diff --git a/src/vppinfra/mheap.c b/src/vppinfra/mheap.c index 192732db5f9..d4010ceb297 100644 --- a/src/vppinfra/mheap.c +++ b/src/vppinfra/mheap.c @@ -56,7 +56,7 @@ mheap_maybe_lock (void *v) mheap_t *h = mheap_header (v); if (v && (h->flags & MHEAP_FLAG_THREAD_SAFE)) { - u32 my_cpu = os_get_cpu_number (); + u32 my_cpu = os_get_thread_index (); if (h->owner_cpu == my_cpu) { h->recursion_count++; @@ -77,7 +77,7 @@ mheap_maybe_unlock (void *v) mheap_t *h = mheap_header (v); if (v && h->flags & MHEAP_FLAG_THREAD_SAFE) { - ASSERT (os_get_cpu_number () == h->owner_cpu); + ASSERT (os_get_thread_index () == h->owner_cpu); if (--h->recursion_count == 0) { h->owner_cpu = ~0; diff --git a/src/vppinfra/os.h b/src/vppinfra/os.h index a5c74f8ce72..33300716103 100644 --- a/src/vppinfra/os.h +++ b/src/vppinfra/os.h @@ -56,8 +56,24 @@ void os_out_of_memory (void); /* Estimate, measure or divine CPU timestamp clock frequency. */ f64 os_cpu_clock_frequency (void); -uword os_get_cpu_number (void); -uword os_get_ncpus (void); +extern __thread uword __os_thread_index; + +static_always_inline uword +os_get_thread_index (void) +{ + return __os_thread_index; +} + +static_always_inline uword +os_get_cpu_number (void) __attribute__ ((deprecated)); + +static_always_inline uword +os_get_cpu_number (void) +{ + return __os_thread_index; +} + +uword os_get_nthreads (void); #include <vppinfra/smp.h> diff --git a/src/vppinfra/smp.c b/src/vppinfra/smp.c index 8ac19960982..f603283ef61 100644 --- a/src/vppinfra/smp.c +++ b/src/vppinfra/smp.c @@ -53,7 +53,7 @@ allocate_per_cpu_mheap (uword cpu) void *heap; uword vm_size, stack_size, mheap_flags; - ASSERT (os_get_cpu_number () == cpu); + ASSERT (os_get_thread_index () == cpu); vm_size = (uword) 1 << m->log2_n_per_cpu_vm_bytes; stack_size = (uword) 1 << m->log2_n_per_cpu_stack_bytes; diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index 2928369d52e..361015b4f67 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -45,6 +45,8 @@ #include <fcntl.h> #include <stdio.h> /* for sprintf */ +__thread uword __os_thread_index = 0; + clib_error_t * unix_file_n_bytes (char *file, uword * result) { @@ -188,14 +190,14 @@ void os_puts (u8 * string, uword string_length, uword is_error) void os_puts (u8 * string, uword string_length, uword is_error) { - int cpu = os_get_cpu_number (); - int ncpus = os_get_ncpus (); + int cpu = os_get_thread_index (); + int nthreads = os_get_nthreads (); char buf[64]; int fd = is_error ? 2 : 1; struct iovec iovs[2]; int n_iovs = 0; - if (ncpus > 1) + if (nthreads > 1) { snprintf (buf, sizeof (buf), "%d: ", cpu); @@ -219,16 +221,9 @@ os_out_of_memory (void) os_panic (); } -uword os_get_cpu_number (void) __attribute__ ((weak)); -uword -os_get_cpu_number (void) -{ - return 0; -} - -uword os_get_ncpus (void) __attribute__ ((weak)); +uword os_get_nthreads (void) __attribute__ ((weak)); uword -os_get_ncpus (void) +os_get_nthreads (void) { return 1; } |