From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 19 Dec 2016 23:05:39 +0100 Subject: Reorganize source tree to use single autotools instance Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion --- src/vppinfra/mhash.c | 408 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 src/vppinfra/mhash.c (limited to 'src/vppinfra/mhash.c') diff --git a/src/vppinfra/mhash.c b/src/vppinfra/mhash.c new file mode 100644 index 00000000..c917e164 --- /dev/null +++ b/src/vppinfra/mhash.c @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + Copyright (c) 2010 Eliot Dresselhaus + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include + +always_inline u32 +load_partial_u32 (void *d, uword n) +{ + if (n == 4) + return ((u32 *) d)[0]; + if (n == 3) + return ((u16 *) d)[0] | (((u8 *) d)[2] << 16); + if (n == 2) + return ((u16 *) d)[0]; + if (n == 1) + return ((u8 *) d)[0]; + ASSERT (0); + return 0; +} + +always_inline u32 +mhash_key_sum_inline (void *data, uword n_data_bytes, u32 seed) +{ + u32 *d32 = data; + u32 a, b, c, n_left; + + a = b = c = seed; + n_left = n_data_bytes; + a ^= n_data_bytes; + + while (n_left > 12) + { + a += d32[0]; + b += d32[1]; + c += d32[2]; + hash_v3_mix32 (a, b, c); + n_left -= 12; + d32 += 3; + } + + if (n_left > 8) + { + c += load_partial_u32 (d32 + 2, n_left - 8); + n_left = 8; + } + if (n_left > 4) + { + b += load_partial_u32 (d32 + 1, n_left - 4); + n_left = 4; + } + if (n_left > 0) + a += load_partial_u32 (d32 + 0, n_left - 0); + + hash_v3_finalize32 (a, b, c); + + return c; +} + +#define foreach_mhash_key_size \ + _ (2) _ (3) _ (4) _ (5) _ (6) _ (7) \ + _ (8) _ (12) _ (16) _ (20) \ + _ (24) _ (28) _ (32) _ (36) \ + _ (40) _ (44) _ (48) _ (52) \ + _ (56) _ (60) _ (64) + +#define _(N_KEY_BYTES) \ + static uword \ + mhash_key_sum_##N_KEY_BYTES (hash_t * h, uword key) \ + { \ + mhash_t * hv = uword_to_pointer (h->user, mhash_t *); \ + return mhash_key_sum_inline (mhash_key_to_mem (hv, key), \ + (N_KEY_BYTES), \ + hv->hash_seed); \ + } \ + \ + static uword \ + mhash_key_equal_##N_KEY_BYTES (hash_t * h, uword key1, uword key2) \ + { \ + mhash_t * hv = uword_to_pointer (h->user, mhash_t *); \ + void * k1 = mhash_key_to_mem (hv, key1); \ + void * k2 = mhash_key_to_mem (hv, key2); \ + return ! memcmp (k1, k2, (N_KEY_BYTES)); \ + } + +foreach_mhash_key_size +#undef _ +static uword +mhash_key_sum_c_string (hash_t * h, uword key) +{ + mhash_t *hv = uword_to_pointer (h->user, mhash_t *); + void *k = mhash_key_to_mem (hv, key); + return mhash_key_sum_inline (k, strlen (k), hv->hash_seed); +} + +static uword +mhash_key_equal_c_string (hash_t * h, uword key1, uword key2) +{ + mhash_t *hv = uword_to_pointer (h->user, mhash_t *); + void *k1 = mhash_key_to_mem (hv, key1); + void *k2 = mhash_key_to_mem (hv, key2); + return strcmp (k1, k2) == 0; +} + +static uword +mhash_key_sum_vec_string (hash_t * h, uword key) +{ + mhash_t *hv = uword_to_pointer (h->user, mhash_t *); + void *k = mhash_key_to_mem (hv, key); + return mhash_key_sum_inline (k, vec_len (k), hv->hash_seed); +} + +static uword +mhash_key_equal_vec_string (hash_t * h, uword key1, uword key2) +{ + mhash_t *hv = uword_to_pointer (h->user, mhash_t *); + void *k1 = mhash_key_to_mem (hv, key1); + void *k2 = mhash_key_to_mem (hv, key2); + return vec_len (k1) == vec_len (k2) && memcmp (k1, k2, vec_len (k1)) == 0; +} + +/* The CLIB hash user pointer must always point to a valid mhash_t. + Now, the address of mhash_t can change (think vec_resize). + So we must always be careful that it points to the correct + address. */ +always_inline void +mhash_sanitize_hash_user (mhash_t * mh) +{ + uword *hash = mh->hash; + hash_t *h = hash_header (hash); + h->user = pointer_to_uword (mh); +} + +void +mhash_init (mhash_t * h, uword n_value_bytes, uword n_key_bytes) +{ + static struct + { + hash_key_sum_function_t *key_sum; + hash_key_equal_function_t *key_equal; + } t[] = + { +#define _(N_KEY_BYTES) \ + [N_KEY_BYTES] = { \ + .key_sum = mhash_key_sum_##N_KEY_BYTES, \ + .key_equal = mhash_key_equal_##N_KEY_BYTES, \ + }, + + foreach_mhash_key_size +#undef _ + [MHASH_C_STRING_KEY] = + { + .key_sum = mhash_key_sum_c_string,.key_equal = mhash_key_equal_c_string,}, + [MHASH_VEC_STRING_KEY] = + { + .key_sum = mhash_key_sum_vec_string,.key_equal = + mhash_key_equal_vec_string,},}; + + if (mhash_key_vector_is_heap (h)) + heap_free (h->key_vector_or_heap); + else + vec_free (h->key_vector_or_heap); + vec_free (h->key_vector_free_indices); + { + int i; + for (i = 0; i < vec_len (h->key_tmps); i++) + vec_free (h->key_tmps[i]); + } + vec_free (h->key_tmps); + hash_free (h->hash); + + memset (h, 0, sizeof (h[0])); + h->n_key_bytes = n_key_bytes; + +#if 0 + if (h->n_key_bytes > 0) + { + vec_validate (h->key_tmp, h->n_key_bytes - 1); + _vec_len (h->key_tmp) = 0; + } +#endif + + ASSERT (n_key_bytes < ARRAY_LEN (t)); + h->hash = hash_create2 ( /* elts */ 0, + /* user */ pointer_to_uword (h), + /* value_bytes */ n_value_bytes, + t[n_key_bytes].key_sum, t[n_key_bytes].key_equal, + /* format pair/arg */ + 0, 0); +} + +static uword +mhash_set_tmp_key (mhash_t * h, const void *key) +{ + u8 *key_tmp; + int my_cpu = os_get_cpu_number (); + + vec_validate (h->key_tmps, my_cpu); + key_tmp = h->key_tmps[my_cpu]; + + vec_reset_length (key_tmp); + + if (mhash_key_vector_is_heap (h)) + { + uword is_c_string = h->n_key_bytes == MHASH_C_STRING_KEY; + + if (is_c_string) + vec_add (key_tmp, key, strlen (key) + 1); + else + vec_add (key_tmp, key, vec_len (key)); + } + else + vec_add (key_tmp, key, h->n_key_bytes); + + h->key_tmps[my_cpu] = key_tmp; + + return ~0; +} + +hash_pair_t * +mhash_get_pair (mhash_t * h, const void *key) +{ + uword ikey; + mhash_sanitize_hash_user (h); + ikey = mhash_set_tmp_key (h, key); + return hash_get_pair (h->hash, ikey); +} + +typedef struct +{ + u32 heap_handle; + + /* Must conincide with vec_header. */ + vec_header_t vec; +} mhash_string_key_t; + +uword +mhash_set_mem (mhash_t * h, void *key, uword * new_value, uword * old_value) +{ + u8 *k; + uword ikey, i, l = 0, n_key_bytes, old_n_elts, key_alloc_from_free_list = 0; + + mhash_sanitize_hash_user (h); + + if (mhash_key_vector_is_heap (h)) + { + mhash_string_key_t *sk; + uword is_c_string = h->n_key_bytes == MHASH_C_STRING_KEY; + uword handle; + + n_key_bytes = is_c_string ? (strlen (key) + 1) : vec_len (key); + i = + heap_alloc (h->key_vector_or_heap, n_key_bytes + sizeof (sk[0]), + handle); + + sk = (void *) (h->key_vector_or_heap + i); + sk->heap_handle = handle; + sk->vec.len = n_key_bytes; + clib_memcpy (sk->vec.vector_data, key, n_key_bytes); + + /* Advance key past vector header. */ + i += sizeof (sk[0]); + } + else + { + key_alloc_from_free_list = (l = + vec_len (h->key_vector_free_indices)) > 0; + if (key_alloc_from_free_list) + { + i = h->key_vector_free_indices[l - 1]; + k = vec_elt_at_index (h->key_vector_or_heap, i); + _vec_len (h->key_vector_free_indices) = l - 1; + } + else + { + vec_add2 (h->key_vector_or_heap, k, h->n_key_bytes); + i = k - h->key_vector_or_heap; + } + + n_key_bytes = h->n_key_bytes; + clib_memcpy (k, key, n_key_bytes); + } + ikey = i; + + old_n_elts = hash_elts (h->hash); + h->hash = _hash_set3 (h->hash, ikey, new_value, old_value); + + /* If element already existed remove duplicate key. */ + if (hash_elts (h->hash) == old_n_elts) + { + hash_pair_t *p; + + /* Fetch old key for return value. */ + p = hash_get_pair (h->hash, ikey); + ikey = p->key; + + /* Remove duplicate key. */ + if (mhash_key_vector_is_heap (h)) + { + mhash_string_key_t *sk; + sk = (void *) (h->key_vector_or_heap + i - sizeof (sk[0])); + heap_dealloc (h->key_vector_or_heap, sk->heap_handle); + } + else + { + if (key_alloc_from_free_list) + { + h->key_vector_free_indices[l] = i; + _vec_len (h->key_vector_free_indices) = l + 1; + } + else + _vec_len (h->key_vector_or_heap) -= h->n_key_bytes; + } + } + + return ikey; +} + +uword +mhash_unset (mhash_t * h, void *key, uword * old_value) +{ + hash_pair_t *p; + uword i; + + mhash_sanitize_hash_user (h); + i = mhash_set_tmp_key (h, key); + + p = hash_get_pair (h->hash, i); + if (!p) + return 0; + + ASSERT (p->key != ~0); + i = p->key; + + if (mhash_key_vector_is_heap (h)) + { + mhash_string_key_t *sk; + sk = (void *) (h->key_vector_or_heap + i) - sizeof (sk[0]); + heap_dealloc (h->key_vector_or_heap, sk->heap_handle); + } + else + vec_add1 (h->key_vector_free_indices, i); + + hash_unset3 (h->hash, i, old_value); + return 1; +} + +u8 * +format_mhash_key (u8 * s, va_list * va) +{ + mhash_t *h = va_arg (*va, mhash_t *); + u32 ki = va_arg (*va, u32); + void *k = mhash_key_to_mem (h, ki); + + if (mhash_key_vector_is_heap (h)) + { + uword is_c_string = h->n_key_bytes == MHASH_C_STRING_KEY; + u32 l = is_c_string ? strlen (k) : vec_len (k); + vec_add (s, k, l); + } + else if (h->format_key) + s = format (s, "%U", h->format_key, k); + else + s = format (s, "%U", format_hex_bytes, k, h->n_key_bytes); + + return s; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg From f55f9b851f59264d737d92c6277a87588c565d24 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 10 May 2017 21:06:28 +0200 Subject: completelly deprecate os_get_cpu_number, replace new occurences Change-Id: I82c663bc0866c6c68ba354104b0bb059387f4b9d Signed-off-by: Damjan Marion --- src/plugins/flowperpkt/l2_node.c | 20 ++++++++++---------- src/plugins/flowperpkt/node.c | 20 ++++++++++---------- src/plugins/snat/in2out.c | 2 +- src/plugins/snat/out2in.c | 2 +- src/vlib/main.h | 2 +- src/vlib/threads.c | 12 ++---------- src/vlib/threads.h | 3 +-- src/vlib/unix/main.c | 2 +- src/vlibmemory/memory_vlib.c | 2 +- src/vnet/dpo/interface_dpo.c | 8 ++++---- src/vnet/lisp-gpe/lisp_gpe_adjacency.c | 2 +- src/vppinfra/bihash_template.c | 16 ++++++++-------- src/vppinfra/lock.h | 6 +++--- src/vppinfra/mem.h | 6 +++--- src/vppinfra/mhash.c | 2 +- src/vppinfra/mhash.h | 2 +- src/vppinfra/mheap.c | 4 ++-- src/vppinfra/os.h | 20 ++++++++++++++++++-- src/vppinfra/smp.c | 2 +- src/vppinfra/unix-misc.c | 19 +++++++------------ 20 files changed, 77 insertions(+), 75 deletions(-) (limited to 'src/vppinfra/mhash.c') diff --git a/src/plugins/flowperpkt/l2_node.c b/src/plugins/flowperpkt/l2_node.c index fdaf81d1..db80e990 100644 --- a/src/plugins/flowperpkt/l2_node.c +++ b/src/plugins/flowperpkt/l2_node.c @@ -102,7 +102,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, u8 * src_mac, u8 * dst_mac, u16 ethertype, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->thread_index; + u32 my_thread_index = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; @@ -116,7 +116,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, vlib_buffer_free_list_t *fl; /* Find or allocate a buffer */ - b0 = fm->l2_buffers_per_worker[my_cpu_number]; + b0 = fm->l2_buffers_per_worker[my_thread_index]; /* Need to allocate a buffer? */ if (PREDICT_FALSE (b0 == 0)) @@ -130,7 +130,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, return; /* Initialize the buffer */ - b0 = fm->l2_buffers_per_worker[my_cpu_number] = + b0 = fm->l2_buffers_per_worker[my_thread_index] = vlib_get_buffer (vm, bi0); fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); @@ -142,16 +142,16 @@ add_to_flow_record_l2 (vlib_main_t * vm, { /* use the current buffer */ bi0 = vlib_get_buffer_index (vm, b0); - offset = fm->l2_next_record_offset_per_worker[my_cpu_number]; + offset = fm->l2_next_record_offset_per_worker[my_thread_index]; } /* Find or allocate a frame */ - f = fm->l2_frames_per_worker[my_cpu_number]; + f = fm->l2_frames_per_worker[my_thread_index]; if (PREDICT_FALSE (f == 0)) { u32 *to_next; f = vlib_get_frame_to_node (vm, ip4_lookup_node.index); - fm->l2_frames_per_worker[my_cpu_number] = f; + fm->l2_frames_per_worker[my_thread_index] = f; /* Enqueue the buffer */ to_next = vlib_frame_vector_args (f); @@ -299,13 +299,13 @@ add_to_flow_record_l2 (vlib_main_t * vm, } vlib_put_frame_to_node (vm, ip4_lookup_node.index, - fm->l2_frames_per_worker[my_cpu_number]); - fm->l2_frames_per_worker[my_cpu_number] = 0; - fm->l2_buffers_per_worker[my_cpu_number] = 0; + fm->l2_frames_per_worker[my_thread_index]); + fm->l2_frames_per_worker[my_thread_index] = 0; + fm->l2_buffers_per_worker[my_thread_index] = 0; offset = 0; } - fm->l2_next_record_offset_per_worker[my_cpu_number] = offset; + fm->l2_next_record_offset_per_worker[my_thread_index] = offset; } void diff --git a/src/plugins/flowperpkt/node.c b/src/plugins/flowperpkt/node.c index 0277682d..9bac4166 100644 --- a/src/plugins/flowperpkt/node.c +++ b/src/plugins/flowperpkt/node.c @@ -101,7 +101,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, u32 src_address, u32 dst_address, u8 tos, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->thread_index; + u32 my_thread_index = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; @@ -115,7 +115,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, vlib_buffer_free_list_t *fl; /* Find or allocate a buffer */ - b0 = fm->ipv4_buffers_per_worker[my_cpu_number]; + b0 = fm->ipv4_buffers_per_worker[my_thread_index]; /* Need to allocate a buffer? */ if (PREDICT_FALSE (b0 == 0)) @@ -129,7 +129,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, return; /* Initialize the buffer */ - b0 = fm->ipv4_buffers_per_worker[my_cpu_number] = + b0 = fm->ipv4_buffers_per_worker[my_thread_index] = vlib_get_buffer (vm, bi0); fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); @@ -141,16 +141,16 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, { /* use the current buffer */ bi0 = vlib_get_buffer_index (vm, b0); - offset = fm->ipv4_next_record_offset_per_worker[my_cpu_number]; + offset = fm->ipv4_next_record_offset_per_worker[my_thread_index]; } /* Find or allocate a frame */ - f = fm->ipv4_frames_per_worker[my_cpu_number]; + f = fm->ipv4_frames_per_worker[my_thread_index]; if (PREDICT_FALSE (f == 0)) { u32 *to_next; f = vlib_get_frame_to_node (vm, ip4_lookup_node.index); - fm->ipv4_frames_per_worker[my_cpu_number] = f; + fm->ipv4_frames_per_worker[my_thread_index] = f; /* Enqueue the buffer */ to_next = vlib_frame_vector_args (f); @@ -300,13 +300,13 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, } vlib_put_frame_to_node (vm, ip4_lookup_node.index, - fm->ipv4_frames_per_worker[my_cpu_number]); - fm->ipv4_frames_per_worker[my_cpu_number] = 0; - fm->ipv4_buffers_per_worker[my_cpu_number] = 0; + fm->ipv4_frames_per_worker[my_thread_index]); + fm->ipv4_frames_per_worker[my_thread_index] = 0; + fm->ipv4_buffers_per_worker[my_thread_index] = 0; offset = 0; } - fm->ipv4_next_record_offset_per_worker[my_cpu_number] = offset; + fm->ipv4_next_record_offset_per_worker[my_thread_index] = offset; } void diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c index f7d29c69..bc86a7a4 100644 --- a/src/plugins/snat/in2out.c +++ b/src/plugins/snat/in2out.c @@ -1514,7 +1514,7 @@ snat_det_in2out_node_fn (vlib_main_t * vm, u32 pkts_processed = 0; snat_main_t * sm = &snat_main; u32 now = (u32) vlib_time_now (vm); - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; diff --git a/src/plugins/snat/out2in.c b/src/plugins/snat/out2in.c index 3d7b106a..824406ab 100644 --- a/src/plugins/snat/out2in.c +++ b/src/plugins/snat/out2in.c @@ -1168,7 +1168,7 @@ snat_det_out2in_node_fn (vlib_main_t * vm, snat_out2in_next_t next_index; u32 pkts_processed = 0; snat_main_t * sm = &snat_main; - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; diff --git a/src/vlib/main.h b/src/vlib/main.h index 329bf073..0e8026d1 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -320,7 +320,7 @@ always_inline void vlib_set_queue_signal_callback /* Main routine. */ int vlib_main (vlib_main_t * vm, unformat_input_t * input); -/* Thread stacks, for os_get_cpu_number */ +/* Thread stacks, for os_get_thread_index */ extern u8 **vlib_thread_stacks; /* Number of thread stacks that the application needs */ diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 9ccfd3a2..b7bc9e26 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -35,16 +35,8 @@ vl (void *p) vlib_worker_thread_t *vlib_worker_threads; vlib_thread_main_t vlib_thread_main; -__thread uword vlib_thread_index = 0; - -uword -os_get_cpu_number (void) -{ - return vlib_thread_index; -} - uword -os_get_ncpus (void) +os_get_nthreads (void) { u32 len; @@ -467,7 +459,7 @@ vlib_worker_thread_bootstrap_fn (void *arg) w->lwp = syscall (SYS_gettid); w->thread_id = pthread_self (); - vlib_thread_index = w - vlib_worker_threads; + __os_thread_index = w - vlib_worker_threads; rv = (void *) clib_calljmp ((uword (*)(uword)) w->thread_function, diff --git a/src/vlib/threads.h b/src/vlib/threads.h index 101d3d4a..17d35a24 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -181,11 +181,10 @@ u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); void vlib_worker_thread_barrier_sync (vlib_main_t * vm); void vlib_worker_thread_barrier_release (vlib_main_t * vm); -extern __thread uword vlib_thread_index; static_always_inline uword vlib_get_thread_index (void) { - return vlib_thread_index; + return __os_thread_index; } always_inline void diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index db5ddd64..103576db 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -565,7 +565,7 @@ vlib_unix_main (int argc, char *argv[]) vlib_thread_stack_init (0); - vlib_thread_index = 0; + __os_thread_index = 0; i = clib_calljmp (thread0, (uword) vm, (void *) (vlib_thread_stacks[0] + diff --git a/src/vlibmemory/memory_vlib.c b/src/vlibmemory/memory_vlib.c index acba8b3f..e5d88732 100644 --- a/src/vlibmemory/memory_vlib.c +++ b/src/vlibmemory/memory_vlib.c @@ -1333,7 +1333,7 @@ vl_api_rpc_call_main_thread (void *fp, u8 * data, u32 data_length) unix_shared_memory_queue_t *q; /* Main thread: call the function directly */ - if (os_get_cpu_number () == 0) + if (vlib_get_thread_index () == 0) { vlib_main_t *vm = vlib_get_main (); void (*call_fp) (void *); diff --git a/src/vnet/dpo/interface_dpo.c b/src/vnet/dpo/interface_dpo.c index 50ca756f..8d700c23 100644 --- a/src/vnet/dpo/interface_dpo.c +++ b/src/vnet/dpo/interface_dpo.c @@ -231,7 +231,7 @@ interface_dpo_inline (vlib_main_t * vm, vlib_frame_t * from_frame) { u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index (); vnet_interface_main_t *im; im = &vnet_get_main ()->interface_main; @@ -274,13 +274,13 @@ interface_dpo_inline (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, ido0->ido_sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, ido1->ido_sw_if_index, 1, vlib_buffer_length_in_chain (vm, b1)); @@ -331,7 +331,7 @@ interface_dpo_inline (vlib_main_t * vm, /* Bump the interface's RX coutners */ vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, ido0->ido_sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); diff --git a/src/vnet/lisp-gpe/lisp_gpe_adjacency.c b/src/vnet/lisp-gpe/lisp_gpe_adjacency.c index d5f3a28a..7db1c9bb 100644 --- a/src/vnet/lisp-gpe/lisp_gpe_adjacency.c +++ b/src/vnet/lisp-gpe/lisp_gpe_adjacency.c @@ -302,7 +302,7 @@ lisp_gpe_increment_stats_counters (lisp_cp_main_t * lcm, ip_adjacency_t * adj, /* compute payload length starting after GPE */ u32 bytes = b->current_length - (lisp_data - b->data - b->current_data); - vlib_increment_combined_counter (&lgm->counters, os_get_cpu_number (), + vlib_increment_combined_counter (&lgm->counters, vlib_get_thread_index (), p[0], 1, bytes); } diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c index d8b97b5f..51fadeb8 100644 --- a/src/vppinfra/bihash_template.c +++ b/src/vppinfra/bihash_template.c @@ -96,12 +96,12 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) clib_bihash_bucket_t working_bucket __attribute__ ((aligned (8))); void *oldheap; BVT (clib_bihash_value) * working_copy; - u32 cpu_number = os_get_cpu_number (); + u32 thread_index = os_get_thread_index (); - if (cpu_number >= vec_len (h->working_copies)) + if (thread_index >= vec_len (h->working_copies)) { oldheap = clib_mem_set_heap (h->mheap); - vec_validate (h->working_copies, cpu_number); + vec_validate (h->working_copies, thread_index); clib_mem_set_heap (oldheap); } @@ -110,7 +110,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) * updates from multiple threads will not result in sporadic, spurious * lookup failures. */ - working_copy = h->working_copies[cpu_number]; + working_copy = h->working_copies[thread_index]; h->saved_bucket.as_u64 = b->as_u64; oldheap = clib_mem_set_heap (h->mheap); @@ -119,7 +119,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) { vec_validate_aligned (working_copy, (1 << b->log2_pages) - 1, sizeof (u64)); - h->working_copies[cpu_number] = working_copy; + h->working_copies[thread_index] = working_copy; } _vec_len (working_copy) = 1 << b->log2_pages; @@ -132,7 +132,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, clib_bihash_bucket_t * b) working_bucket.offset = BV (clib_bihash_get_offset) (h, working_copy); CLIB_MEMORY_BARRIER (); b->as_u64 = working_bucket.as_u64; - h->working_copies[cpu_number] = working_copy; + h->working_copies[thread_index] = working_copy; } static @@ -233,7 +233,7 @@ int BV (clib_bihash_add_del) int i, limit; u64 hash, new_hash; u32 new_log2_pages; - u32 cpu_number = os_get_cpu_number (); + u32 thread_index = os_get_thread_index (); int mark_bucket_linear; int resplit_once; @@ -323,7 +323,7 @@ int BV (clib_bihash_add_del) new_log2_pages = h->saved_bucket.log2_pages + 1; mark_bucket_linear = 0; - working_copy = h->working_copies[cpu_number]; + working_copy = h->working_copies[thread_index]; resplit_once = 0; new_v = BV (split_and_rehash) (h, working_copy, new_log2_pages); diff --git a/src/vppinfra/lock.h b/src/vppinfra/lock.h index c60ff414..0cd2b4fe 100644 --- a/src/vppinfra/lock.h +++ b/src/vppinfra/lock.h @@ -24,7 +24,7 @@ typedef struct u32 lock; #if CLIB_DEBUG > 0 pid_t pid; - uword cpu_index; + uword thread_index; void *frame_address; #endif } *clib_spinlock_t; @@ -57,7 +57,7 @@ clib_spinlock_lock (clib_spinlock_t * p) #if CLIB_DEBUG > 0 (*p)->frame_address = __builtin_frame_address (0); (*p)->pid = getpid (); - (*p)->cpu_index = os_get_cpu_number (); + (*p)->thread_index = os_get_thread_index (); #endif } @@ -75,7 +75,7 @@ clib_spinlock_unlock (clib_spinlock_t * p) #if CLIB_DEBUG > 0 (*p)->frame_address = 0; (*p)->pid = 0; - (*p)->cpu_index = 0; + (*p)->thread_index = 0; #endif } diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 1260eab2..63c5ac16 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -54,14 +54,14 @@ extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; always_inline void * clib_mem_get_per_cpu_heap (void) { - int cpu = os_get_cpu_number (); + int cpu = os_get_thread_index (); return clib_per_cpu_mheaps[cpu]; } always_inline void * clib_mem_set_per_cpu_heap (u8 * new_heap) { - int cpu = os_get_cpu_number (); + int cpu = os_get_thread_index (); void *old = clib_per_cpu_mheaps[cpu]; clib_per_cpu_mheaps[cpu] = new_heap; return old; @@ -83,7 +83,7 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, align_offset = align; } - cpu = os_get_cpu_number (); + cpu = os_get_thread_index (); heap = clib_per_cpu_mheaps[cpu]; heap = mheap_get_aligned (heap, size, align, align_offset, &offset); clib_per_cpu_mheaps[cpu] = heap; diff --git a/src/vppinfra/mhash.c b/src/vppinfra/mhash.c index c917e164..00b67c49 100644 --- a/src/vppinfra/mhash.c +++ b/src/vppinfra/mhash.c @@ -226,7 +226,7 @@ static uword mhash_set_tmp_key (mhash_t * h, const void *key) { u8 *key_tmp; - int my_cpu = os_get_cpu_number (); + int my_cpu = os_get_thread_index (); vec_validate (h->key_tmps, my_cpu); key_tmp = h->key_tmps[my_cpu]; diff --git a/src/vppinfra/mhash.h b/src/vppinfra/mhash.h index 102adf4e..7eb19183 100644 --- a/src/vppinfra/mhash.h +++ b/src/vppinfra/mhash.h @@ -93,7 +93,7 @@ mhash_key_to_mem (mhash_t * h, uword key) { u8 *key_tmp; - int my_cpu = os_get_cpu_number (); + int my_cpu = os_get_thread_index (); vec_validate (h->key_tmps, my_cpu); key_tmp = h->key_tmps[my_cpu]; return key_tmp; diff --git a/src/vppinfra/mheap.c b/src/vppinfra/mheap.c index 192732db..d4010ceb 100644 --- a/src/vppinfra/mheap.c +++ b/src/vppinfra/mheap.c @@ -56,7 +56,7 @@ mheap_maybe_lock (void *v) mheap_t *h = mheap_header (v); if (v && (h->flags & MHEAP_FLAG_THREAD_SAFE)) { - u32 my_cpu = os_get_cpu_number (); + u32 my_cpu = os_get_thread_index (); if (h->owner_cpu == my_cpu) { h->recursion_count++; @@ -77,7 +77,7 @@ mheap_maybe_unlock (void *v) mheap_t *h = mheap_header (v); if (v && h->flags & MHEAP_FLAG_THREAD_SAFE) { - ASSERT (os_get_cpu_number () == h->owner_cpu); + ASSERT (os_get_thread_index () == h->owner_cpu); if (--h->recursion_count == 0) { h->owner_cpu = ~0; diff --git a/src/vppinfra/os.h b/src/vppinfra/os.h index a5c74f8c..33300716 100644 --- a/src/vppinfra/os.h +++ b/src/vppinfra/os.h @@ -56,8 +56,24 @@ void os_out_of_memory (void); /* Estimate, measure or divine CPU timestamp clock frequency. */ f64 os_cpu_clock_frequency (void); -uword os_get_cpu_number (void); -uword os_get_ncpus (void); +extern __thread uword __os_thread_index; + +static_always_inline uword +os_get_thread_index (void) +{ + return __os_thread_index; +} + +static_always_inline uword +os_get_cpu_number (void) __attribute__ ((deprecated)); + +static_always_inline uword +os_get_cpu_number (void) +{ + return __os_thread_index; +} + +uword os_get_nthreads (void); #include diff --git a/src/vppinfra/smp.c b/src/vppinfra/smp.c index 8ac19960..f603283e 100644 --- a/src/vppinfra/smp.c +++ b/src/vppinfra/smp.c @@ -53,7 +53,7 @@ allocate_per_cpu_mheap (uword cpu) void *heap; uword vm_size, stack_size, mheap_flags; - ASSERT (os_get_cpu_number () == cpu); + ASSERT (os_get_thread_index () == cpu); vm_size = (uword) 1 << m->log2_n_per_cpu_vm_bytes; stack_size = (uword) 1 << m->log2_n_per_cpu_stack_bytes; diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index 2928369d..361015b4 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -45,6 +45,8 @@ #include #include /* for sprintf */ +__thread uword __os_thread_index = 0; + clib_error_t * unix_file_n_bytes (char *file, uword * result) { @@ -188,14 +190,14 @@ void os_puts (u8 * string, uword string_length, uword is_error) void os_puts (u8 * string, uword string_length, uword is_error) { - int cpu = os_get_cpu_number (); - int ncpus = os_get_ncpus (); + int cpu = os_get_thread_index (); + int nthreads = os_get_nthreads (); char buf[64]; int fd = is_error ? 2 : 1; struct iovec iovs[2]; int n_iovs = 0; - if (ncpus > 1) + if (nthreads > 1) { snprintf (buf, sizeof (buf), "%d: ", cpu); @@ -219,16 +221,9 @@ os_out_of_memory (void) os_panic (); } -uword os_get_cpu_number (void) __attribute__ ((weak)); -uword -os_get_cpu_number (void) -{ - return 0; -} - -uword os_get_ncpus (void) __attribute__ ((weak)); +uword os_get_nthreads (void) __attribute__ ((weak)); uword -os_get_ncpus (void) +os_get_nthreads (void) { return 1; } -- cgit 1.2.3-korg