From 79934e855fc58aefde498cd0564796cdd6ccbc59 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Tue, 5 Apr 2022 12:40:31 +0200 Subject: vppinfra: refactor address sanitizer Type: refactor Change-Id: I5ca142ec1557d5b5c3806b43553ad9d3b5ea1112 Signed-off-by: Damjan Marion --- src/CMakeLists.txt | 1 - src/plugins/unittest/string_test.c | 2 +- src/svm/fifo_segment.c | 4 +- src/svm/ssvm.c | 2 +- src/svm/svm.c | 14 ++-- src/svm/svm_fifo.c | 6 +- src/vlib/unix/plugin.h | 4 +- src/vlibapi/api_common.h | 12 +-- src/vlibmemory/memory_client.c | 6 +- src/vlibmemory/memory_shared.c | 4 +- src/vlibmemory/socket_api.c | 2 +- src/vlibmemory/vlib_api_cli.c | 2 +- src/vppinfra/CMakeLists.txt | 2 - src/vppinfra/clib.h | 15 ++++ src/vppinfra/dlmalloc.c | 61 ++++++++------- src/vppinfra/elf.c | 2 +- src/vppinfra/linux/mem.c | 4 +- src/vppinfra/mem.h | 22 +++++- src/vppinfra/mem_bulk.c | 10 +-- src/vppinfra/mem_dlmalloc.c | 15 ++-- src/vppinfra/pool.h | 12 +-- src/vppinfra/sanitizer.c | 7 -- src/vppinfra/sanitizer.h | 147 ------------------------------------- src/vppinfra/vec.c | 6 +- src/vppinfra/vec_bootstrap.h | 6 +- 25 files changed, 121 insertions(+), 247 deletions(-) delete mode 100644 src/vppinfra/sanitizer.c delete mode 100644 src/vppinfra/sanitizer.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2b5b7dbceed..a69942c6fd2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -133,7 +133,6 @@ set(VPP_SANITIZE_ADDR_OPTIONS if (VPP_ENABLE_SANITIZE_ADDR) add_compile_options(-fsanitize=address) - add_compile_definitions(CLIB_SANITIZE_ADDR) add_link_options(-fsanitize=address) endif (VPP_ENABLE_SANITIZE_ADDR) diff --git a/src/plugins/unittest/string_test.c b/src/plugins/unittest/string_test.c index c59a05ced08..f0e40c5e142 100644 --- a/src/plugins/unittest/string_test.c +++ b/src/plugins/unittest/string_test.c @@ -508,7 +508,7 @@ test_clib_strncmp (vlib_main_t * vm, unformat_input_t * input) /* unterminated s1 */ s1[s1len] = 0x1; - CLIB_MEM_UNPOISON (s1, CLIB_STRING_MACRO_MAX); + clib_mem_unpoison (s1, CLIB_STRING_MACRO_MAX); indicator = clib_strncmp (s1, "Every moment is a fresh beginning", sizeof ("every moment is a fresh beginning") - 1); if (indicator != 0) diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index 77177a14a69..489ca2a4929 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -312,7 +312,7 @@ fifo_segment_init (fifo_segment_t * fs) seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align); fsh = uword_to_pointer (seg_start, void *); - CLIB_MEM_UNPOISON (fsh, seg_sz); + clib_mem_unpoison (fsh, seg_sz); memset (fsh, 0, sizeof (*fsh) + slices_sz); fsh->byte_index = sizeof (*fsh) + slices_sz; @@ -781,7 +781,7 @@ fsh_slice_collect_chunks (fifo_segment_header_t * fsh, while (c) { - CLIB_MEM_UNPOISON (c, sizeof (*c)); + clib_mem_unpoison (c, sizeof (*c)); next = fs_chunk_ptr (fsh, c->next); fl_index = fs_freelist_for_size (c->length); fss_chunk_free_list_push (fsh, fss, fl_index, c); diff --git a/src/svm/ssvm.c b/src/svm/ssvm.c index e404906cb29..9cdde128561 100644 --- a/src/svm/ssvm.c +++ b/src/svm/ssvm.c @@ -95,7 +95,7 @@ ssvm_server_init_shm (ssvm_private_t * ssvm) close (ssvm_fd); - CLIB_MEM_UNPOISON (sh, sizeof (*sh)); + clib_mem_unpoison (sh, sizeof (*sh)); sh->server_pid = ssvm->my_pid; sh->ssvm_size = ssvm->ssvm_size; sh->ssvm_va = pointer_to_uword (sh); diff --git a/src/svm/svm.c b/src/svm/svm.c index b844e20b4cc..0a910558f6e 100644 --- a/src/svm/svm.c +++ b/src/svm/svm.c @@ -327,7 +327,7 @@ svm_data_region_create (svm_map_region_args_t * a, svm_region_t * rp) return -3; } close (fd); - CLIB_MEM_UNPOISON (rp->data_base, map_size); + clib_mem_unpoison (rp->data_base, map_size); rp->backing_file = (char *) format (0, "%s%c", a->backing_file, 0); rp->flags |= SVM_FLAGS_FILE; } @@ -412,7 +412,7 @@ svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp) return -3; } close (fd); - CLIB_MEM_UNPOISON (rp->data_base, map_size); + clib_mem_unpoison (rp->data_base, map_size); } return 0; } @@ -605,7 +605,7 @@ svm_map_region (svm_map_region_args_t * a) return (0); } close (svm_fd); - CLIB_MEM_UNPOISON (rp, a->size); + clib_mem_unpoison (rp, a->size); svm_region_init_mapped_region (a, rp); @@ -663,7 +663,7 @@ svm_map_region (svm_map_region_args_t * a) return (0); } - CLIB_MEM_UNPOISON (rp, MMAP_PAGESIZE); + clib_mem_unpoison (rp, MMAP_PAGESIZE); /* * We lost the footrace to create this region; make sure @@ -701,7 +701,7 @@ svm_map_region (svm_map_region_args_t * a) close (svm_fd); - CLIB_MEM_UNPOISON (rp, a->size); + clib_mem_unpoison (rp, a->size); if ((uword) rp != rp->virtual_base) { @@ -1051,7 +1051,7 @@ svm_region_unmap_internal (void *rp_arg, u8 is_client) oldheap = svm_push_pvt_heap (rp); /* nb vec_delete() in the loop */ /* Remove the caller from the list of mappers */ - CLIB_MEM_UNPOISON (rp->client_pids, vec_bytes (rp->client_pids)); + clib_mem_unpoison (rp->client_pids, vec_bytes (rp->client_pids)); for (i = 0; i < vec_len (rp->client_pids); i++) { if (rp->client_pids[i] == mypid) @@ -1184,7 +1184,7 @@ svm_region_exit_internal (u8 is_client) virtual_base = root_rp->virtual_base; virtual_size = root_rp->virtual_size; - CLIB_MEM_UNPOISON (root_rp->client_pids, vec_bytes (root_rp->client_pids)); + clib_mem_unpoison (root_rp->client_pids, vec_bytes (root_rp->client_pids)); for (i = 0; i < vec_len (root_rp->client_pids); i++) { if (root_rp->client_pids[i] == mypid) diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 83cad42ee9b..49b3d1728f3 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -72,8 +72,8 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f, c = f_cptr (f, c->next); while ((to_copy -= n_chunk)) { - CLIB_MEM_UNPOISON (c, sizeof (*c)); - CLIB_MEM_UNPOISON (c->data, c->length); + clib_mem_unpoison (c, sizeof (*c)); + clib_mem_unpoison (c->data, c->length); n_chunk = clib_min (c->length, to_copy); clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk); c = c->length <= to_copy ? f_cptr (f, c->next) : c; @@ -1155,7 +1155,7 @@ svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst) len = clib_min (cursize - offset, len); head_idx = head + offset; - CLIB_MEM_UNPOISON (f->ooo_deq, sizeof (*f->ooo_deq)); + clib_mem_unpoison (f->ooo_deq, sizeof (*f->ooo_deq)); if (!f->ooo_deq || !f_chunk_includes_pos (f->ooo_deq, head_idx)) f_update_ooo_deq (f, head_idx, head_idx + len); diff --git a/src/vlib/unix/plugin.h b/src/vlib/unix/plugin.h index 1c46e6b196c..2c8a2f2083c 100644 --- a/src/vlib/unix/plugin.h +++ b/src/vlib/unix/plugin.h @@ -58,6 +58,7 @@ typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); u8 default_disabled : 1; u8 deep_bind : 1; const char version[32]; @@ -65,7 +66,7 @@ typedef struct const char overrides[256]; const char *early_init; const char *description; -} __clib_packed vlib_plugin_registration_t; +} vlib_plugin_registration_t; /* * Plugins may also use this registration format, which is @@ -150,7 +151,6 @@ u8 *vlib_get_vat_plugin_path (void); #define VLIB_PLUGIN_REGISTER() \ vlib_plugin_registration_t vlib_plugin_registration \ - CLIB_NOSANITIZE_PLUGIN_REG_SECTION \ __clib_export __clib_section(".vlib_plugin_registration") /* Call a plugin init function: used for init function dependencies. */ diff --git a/src/vlibapi/api_common.h b/src/vlibapi/api_common.h index 122f402d5f7..4c48b003e1e 100644 --- a/src/vlibapi/api_common.h +++ b/src/vlibapi/api_common.h @@ -150,24 +150,24 @@ typedef struct msgbuf_ u8 data[0]; /**< actual message begins here */ } msgbuf_t; -CLIB_NOSANITIZE_ADDR static inline void +__clib_nosanitize_addr static inline void VL_MSG_API_UNPOISON (const void *a) { const msgbuf_t *m = &((const msgbuf_t *) a)[-1]; - CLIB_MEM_UNPOISON (m, sizeof (*m) + ntohl (m->data_len)); + clib_mem_unpoison (m, sizeof (*m) + ntohl (m->data_len)); } -CLIB_NOSANITIZE_ADDR static inline void -VL_MSG_API_SVM_QUEUE_UNPOISON (const svm_queue_t * q) +__clib_nosanitize_addr static inline void +VL_MSG_API_SVM_QUEUE_UNPOISON (const svm_queue_t *q) { - CLIB_MEM_UNPOISON (q, sizeof (*q) + q->elsize * q->maxsize); + clib_mem_unpoison (q, sizeof (*q) + q->elsize * q->maxsize); } static inline void VL_MSG_API_POISON (const void *a) { const msgbuf_t *m = &((const msgbuf_t *) a)[-1]; - CLIB_MEM_POISON (m, sizeof (*m) + ntohl (m->data_len)); + clib_mem_poison (m, sizeof (*m) + ntohl (m->data_len)); } /* api_shared.c prototypes */ diff --git a/src/vlibmemory/memory_client.c b/src/vlibmemory/memory_client.c index 54bc8d8f859..585fca67998 100644 --- a/src/vlibmemory/memory_client.c +++ b/src/vlibmemory/memory_client.c @@ -114,11 +114,11 @@ vl_api_name_and_crc_free (void) hash_free (am->msg_index_by_name_and_crc); } -CLIB_NOSANITIZE_ADDR static void +__clib_nosanitize_addr static void VL_API_VEC_UNPOISON (const void *v) { const vec_header_t *vh = &((vec_header_t *) v)[-1]; - CLIB_MEM_UNPOISON (vh, sizeof (*vh) + vec_len (v)); + clib_mem_unpoison (vh, sizeof (*vh) + vec_len (v)); } static void @@ -192,7 +192,7 @@ vl_client_connect (const char *name, int ctx_quota, int input_queue_size) return -1; } - CLIB_MEM_UNPOISON (shmem_hdr, sizeof (*shmem_hdr)); + clib_mem_unpoison (shmem_hdr, sizeof (*shmem_hdr)); VL_MSG_API_SVM_QUEUE_UNPOISON (shmem_hdr->vl_input_queue); oldheap = vl_msg_push_heap (); diff --git a/src/vlibmemory/memory_shared.c b/src/vlibmemory/memory_shared.c index 5244b27ad37..b70f45fae14 100644 --- a/src/vlibmemory/memory_shared.c +++ b/src/vlibmemory/memory_shared.c @@ -43,8 +43,8 @@ #define DEBUG_MESSAGE_BUFFER_OVERRUN 0 -CLIB_NOSANITIZE_ADDR static inline void * -vl_msg_api_alloc_internal (svm_region_t * vlib_rp, int nbytes, int pool, +__clib_nosanitize_addr static inline void * +vl_msg_api_alloc_internal (svm_region_t *vlib_rp, int nbytes, int pool, int may_return_null) { int i; diff --git a/src/vlibmemory/socket_api.c b/src/vlibmemory/socket_api.c index 9f9773ead1d..3df182b68b6 100644 --- a/src/vlibmemory/socket_api.c +++ b/src/vlibmemory/socket_api.c @@ -697,7 +697,7 @@ vl_api_sock_init_shm_t_handler (vl_api_sock_init_shm_t * mp) /* delete the unused heap created in ssvm_server_init_memfd and mark it * accessible again for ASAN */ clib_mem_destroy_heap (memfd->sh->heap); - CLIB_MEM_UNPOISON ((void *) memfd->sh->ssvm_va, memfd->ssvm_size); + clib_mem_unpoison ((void *) memfd->sh->ssvm_va, memfd->ssvm_size); /* Remember to close this fd when the socket connection goes away */ vec_add1 (regp->additional_fds_to_close, memfd->fd); diff --git a/src/vlibmemory/vlib_api_cli.c b/src/vlibmemory/vlib_api_cli.c index 732ced7634c..bdbedec009b 100644 --- a/src/vlibmemory/vlib_api_cli.c +++ b/src/vlibmemory/vlib_api_cli.c @@ -435,7 +435,7 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename, } close (fd); - CLIB_MEM_UNPOISON (hp, file_size); + clib_mem_unpoison (hp, file_size); nitems = ntohl (hp->nitems); diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt index 2061e4fdc4a..8f7129aa488 100644 --- a/src/vppinfra/CMakeLists.txt +++ b/src/vppinfra/CMakeLists.txt @@ -78,7 +78,6 @@ set(VPPINFRA_SRCS random.c random_isaac.c rbtree.c - sanitizer.c serialize.c socket.c std-formats.c @@ -103,7 +102,6 @@ set(VPPINFRA_SRCS ) set(VPPINFRA_HEADERS - sanitizer.h bihash_12_4.h bihash_16_8.h bihash_24_8.h diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h index 8d1674945dd..2db611d969b 100644 --- a/src/vppinfra/clib.h +++ b/src/vppinfra/clib.h @@ -95,6 +95,15 @@ /* Make a string from the macro's argument */ #define CLIB_STRING_MACRO(x) #x +/* sanitizers */ +#ifdef __has_feature +#if __has_feature(address_sanitizer) +#define CLIB_SANITIZE_ADDR 1 +#endif +#elif defined(__SANITIZE_ADDRESS__) +#define CLIB_SANITIZE_ADDR 1 +#endif + #define __clib_unused __attribute__ ((unused)) #define __clib_weak __attribute__ ((weak)) #define __clib_packed __attribute__ ((packed)) @@ -117,6 +126,12 @@ __attribute__ ((optimize ("no-optimize-sibling-calls"))) #endif +#ifdef CLIB_SANITIZE_ADDR +#define __clib_nosanitize_addr __attribute__ ((no_sanitize_address)) +#else +#define __clib_nosanitize_addr +#endif + #define never_inline __attribute__ ((__noinline__)) #if CLIB_DEBUG > 0 diff --git a/src/vppinfra/dlmalloc.c b/src/vppinfra/dlmalloc.c index 03e7d874032..143db33c4b4 100644 --- a/src/vppinfra/dlmalloc.c +++ b/src/vppinfra/dlmalloc.c @@ -6,7 +6,6 @@ */ #include -#include /*------------------------------ internal #includes ---------------------- */ @@ -460,7 +459,7 @@ static FORCEINLINE void x86_clear_lock(int* sl) { #if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0 /* Plain spin locks use single word (embedded in malloc_states) */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static int spin_acquire_lock(int *sl) { int spins = 0; while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) { @@ -1286,7 +1285,7 @@ static struct malloc_state _gm_; ((char*)(A) >= S->base && (char*)(A) < S->base + S->size) /* Return segment holding given address */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static msegmentptr segment_holding(mstate m, char* addr) { msegmentptr sp = &m->seg; for (;;) { @@ -1298,7 +1297,7 @@ static msegmentptr segment_holding(mstate m, char* addr) { } /* Return true if segment contains a segment link */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static int has_segment_link(mstate m, msegmentptr ss) { msegmentptr sp = &m->seg; for (;;) { @@ -1616,7 +1615,7 @@ static size_t traverse_and_check(mstate m); #if (FOOTERS && !INSECURE) /* Check if (alleged) mstate m has expected magic field */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static inline int ok_magic (const mstate m) { @@ -2083,7 +2082,7 @@ static void do_check_malloc_state(mstate m) { /* ----------------------------- statistics ------------------------------ */ #if !NO_MALLINFO -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static struct dlmallinfo internal_mallinfo(mstate m) { struct dlmallinfo nm = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; ensure_initialization(); @@ -2493,7 +2492,7 @@ static mchunkptr mmap_resize(mstate m, mchunkptr oldp, size_t nb, int flags) { /* -------------------------- mspace management -------------------------- */ /* Initialize top chunk and its size */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void init_top(mstate m, mchunkptr p, size_t psize) { /* Ensure alignment */ size_t offset = align_offset(chunk2mem(p)); @@ -2538,7 +2537,7 @@ static void reset_on_error(mstate m) { #endif /* PROCEED_ON_ERROR */ /* Allocate chunk and prepend remainder with chunk in successor base. */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void* prepend_alloc(mstate m, char* newbase, char* oldbase, size_t nb) { mchunkptr p = align_as_chunk(newbase); @@ -2581,7 +2580,7 @@ static void* prepend_alloc(mstate m, char* newbase, char* oldbase, } /* Add a segment to hold a new noncontiguous region */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) { /* Determine locations and sizes of segment, fenceposts, old top */ char* old_top = (char*)m->top; @@ -2637,7 +2636,7 @@ static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) { /* -------------------------- System allocation -------------------------- */ /* Get memory from system using MORECORE or MMAP */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void* sys_alloc(mstate m, size_t nb) { char* tbase = CMFAIL; size_t tsize = 0; @@ -2852,7 +2851,7 @@ static void* sys_alloc(mstate m, size_t nb) { /* ----------------------- system deallocation -------------------------- */ /* Unmap and unlink any mmapped segments that don't contain used chunks */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static size_t release_unused_segments(mstate m) { size_t released = 0; int nsegs = 0; @@ -2900,7 +2899,7 @@ static size_t release_unused_segments(mstate m) { return released; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static int sys_trim(mstate m, size_t pad) { size_t released = 0; ensure_initialization(); @@ -2969,7 +2968,7 @@ static int sys_trim(mstate m, size_t pad) { /* Consolidate and bin a chunk. Differs from exported versions of free mainly in that the chunk need not be marked as inuse. */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void dispose_chunk(mstate m, mchunkptr p, size_t psize) { mchunkptr next = chunk_plus_offset(p, psize); if (!pinuse(p)) { @@ -3041,7 +3040,7 @@ static void dispose_chunk(mstate m, mchunkptr p, size_t psize) { /* ---------------------------- malloc --------------------------- */ /* allocate a large request from the best fitting chunk in a treebin */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void* tmalloc_large(mstate m, size_t nb) { tchunkptr v = 0; size_t rsize = -nb; /* Unsigned negation */ @@ -3113,7 +3112,7 @@ static void* tmalloc_large(mstate m, size_t nb) { } /* allocate a small request from the best fitting chunk in a treebin */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void* tmalloc_small(mstate m, size_t nb) { tchunkptr t, v; size_t rsize; @@ -3420,7 +3419,7 @@ void* dlcalloc(size_t n_elements, size_t elem_size) { /* ------------ Internal support for realloc, memalign, etc -------------- */ /* Try to realloc; only in-place unless can_move true */ -static CLIB_NOSANITIZE_ADDR mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb, +static __clib_nosanitize_addr mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb, int can_move) { mchunkptr newp = 0; size_t oldsize = chunksize(p); @@ -3499,7 +3498,7 @@ static CLIB_NOSANITIZE_ADDR mchunkptr try_realloc_chunk(mstate m, mchunkptr p, s return newp; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr static void* internal_memalign(mstate m, size_t alignment, size_t bytes) { void* mem = 0; if (alignment < MIN_CHUNK_SIZE) /* must be at least a minimum chunk size */ @@ -4082,7 +4081,7 @@ int mspace_track_large_chunks(mspace msp, int enable) { return ret; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr size_t destroy_mspace(mspace msp) { size_t freed = 0; mstate ms = (mstate)msp; @@ -4118,7 +4117,7 @@ void mspace_get_address_and_size (mspace msp, char **addrp, size_t *sizep) *sizep = this_seg->size; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr int mspace_is_heap_object (mspace msp, void *p) { msegment *this_seg; @@ -4144,7 +4143,7 @@ int mspace_is_heap_object (mspace msp, void *p) return 0; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr void *mspace_least_addr (mspace msp) { mstate ms = (mstate) msp; @@ -4158,7 +4157,7 @@ void mspace_disable_expand (mspace msp) disable_expand (ms); } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr int mspace_enable_disable_trace (mspace msp, int enable) { mstate ms = (mstate)msp; @@ -4175,7 +4174,7 @@ int mspace_enable_disable_trace (mspace msp, int enable) return (was_enabled); } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr int mspace_is_traced (mspace msp) { mstate ms = (mstate)msp; @@ -4185,7 +4184,7 @@ int mspace_is_traced (mspace msp) return 0; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr void* mspace_get_aligned (mspace msp, unsigned long n_user_data_bytes, unsigned long align, @@ -4265,7 +4264,7 @@ void* mspace_get_aligned (mspace msp, return (void *) searchp; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr void mspace_put (mspace msp, void *p_arg) { char *object_header; @@ -4315,7 +4314,7 @@ void mspace_put_no_offset (mspace msp, void *p_arg) mspace_free (msp, p_arg); } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr size_t mspace_usable_size_with_delta (const void *p) { size_t usable_size; @@ -4341,7 +4340,7 @@ size_t mspace_usable_size_with_delta (const void *p) versions. This is not so nice but better than the alternatives. */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr void* mspace_malloc(mspace msp, size_t bytes) { mstate ms = (mstate)msp; if (!ok_magic(ms)) { @@ -4456,7 +4455,7 @@ void* mspace_malloc(mspace msp, size_t bytes) { return 0; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr void mspace_free(mspace msp, void* mem) { if (mem != 0) { mchunkptr p = mem2chunk(mem); @@ -4623,7 +4622,7 @@ void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) { return mem; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) { void* mem = 0; if (oldmem != 0) { @@ -4656,7 +4655,7 @@ void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) { return mem; } -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) { mstate ms = (mstate)msp; if (!ok_magic(ms)) { @@ -4796,7 +4795,7 @@ size_t mspace_set_footprint_limit(mspace msp, size_t bytes) { } #if !NO_MALLINFO -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr struct dlmallinfo mspace_mallinfo(mspace msp) { mstate ms = (mstate)msp; if (!ok_magic(ms)) { @@ -4806,7 +4805,7 @@ struct dlmallinfo mspace_mallinfo(mspace msp) { } #endif /* NO_MALLINFO */ -CLIB_NOSANITIZE_ADDR +__clib_nosanitize_addr size_t mspace_usable_size(const void* mem) { if (mem != 0) { mchunkptr p = mem2chunk(mem); diff --git a/src/vppinfra/elf.c b/src/vppinfra/elf.c index d11fd7e99a9..af0b2b8cb6f 100644 --- a/src/vppinfra/elf.c +++ b/src/vppinfra/elf.c @@ -1357,7 +1357,7 @@ elf_read_file (elf_main_t * em, char *file_name) goto done; } - CLIB_MEM_UNPOISON (data, mmap_length); + clib_mem_unpoison (data, mmap_length); em->file_name = file_name; diff --git a/src/vppinfra/linux/mem.c b/src/vppinfra/linux/mem.c index 9cf598d3c6f..4365281ce8c 100644 --- a/src/vppinfra/linux/mem.c +++ b/src/vppinfra/linux/mem.c @@ -455,7 +455,7 @@ clib_mem_vm_map_internal (void *base, clib_mem_page_sz_t log2_page_sz, else mm->first_map = hdr; - CLIB_MEM_UNPOISON (hdr, sys_page_sz); + clib_mem_unpoison (hdr, sys_page_sz); hdr->next = 0; hdr->prev = mm->last_map; snprintf (hdr->name, CLIB_VM_MAP_HDR_NAME_MAX_LEN - 1, "%s", (char *) name); @@ -470,7 +470,7 @@ clib_mem_vm_map_internal (void *base, clib_mem_page_sz_t log2_page_sz, map_unlock (); - CLIB_MEM_UNPOISON (base, size); + clib_mem_unpoison (base, size); return base; } diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index e33ab37e877..75015d59a4a 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -47,7 +47,9 @@ #include #include /* memcpy, clib_memset */ -#include +#ifdef CLIB_SANITIZE_ADDR +#include +#endif #define CLIB_MAX_MHEAPS 256 #define CLIB_MAX_NUMAS 16 @@ -163,6 +165,22 @@ extern clib_mem_main_t clib_mem_main; /* Unspecified NUMA socket */ #define VEC_NUMA_UNSPECIFIED (0xFF) +static_always_inline void +clib_mem_poison (const void volatile *p, uword s) +{ +#ifdef CLIB_SANITIZE_ADDR + ASAN_POISON_MEMORY_REGION (p, s); +#endif +} + +static_always_inline void +clib_mem_unpoison (const void volatile *p, uword s) +{ +#ifdef CLIB_SANITIZE_ADDR + ASAN_UNPOISON_MEMORY_REGION (p, s); +#endif +} + always_inline clib_mem_heap_t * clib_mem_get_per_cpu_heap (void) { @@ -336,7 +354,7 @@ clib_mem_vm_alloc (uword size) if (mmap_addr == (void *) -1) mmap_addr = 0; else - CLIB_MEM_UNPOISON (mmap_addr, size); + clib_mem_unpoison (mmap_addr, size); return mmap_addr; } diff --git a/src/vppinfra/mem_bulk.c b/src/vppinfra/mem_bulk.c index ba8b2e94909..4dd6a168217 100644 --- a/src/vppinfra/mem_bulk.c +++ b/src/vppinfra/mem_bulk.c @@ -66,7 +66,7 @@ clib_mem_bulk_init (u32 elt_sz, u32 align, u32 min_elts_per_chunk) if (min_elts_per_chunk == 0) min_elts_per_chunk = CLIB_MEM_BULK_DEFAULT_MIN_ELTS_PER_CHUNK; - CLIB_MEM_UNPOISON (b, sizeof (clib_mem_bulk_t)); + clib_mem_unpoison (b, sizeof (clib_mem_bulk_t)); clib_memset (b, 0, sizeof (clib_mem_bulk_t)); b->mspace = heap->mspace; b->align = align; @@ -92,7 +92,7 @@ again: while (c) { next = c->next; - CLIB_MEM_POISON (c, bulk_chunk_size (b)); + clib_mem_poison (c, bulk_chunk_size (b)); mspace_free (ms, c); c = next; } @@ -104,7 +104,7 @@ again: goto again; } - CLIB_MEM_POISON (b, sizeof (clib_mem_bulk_t)); + clib_mem_poison (b, sizeof (clib_mem_bulk_t)); mspace_free (ms, b); } @@ -148,7 +148,7 @@ clib_mem_bulk_alloc (clib_mem_bulk_handle_t h) { u32 i, sz = bulk_chunk_size (b); c = mspace_memalign (b->mspace, b->chunk_align, sz); - CLIB_MEM_UNPOISON (c, sz); + clib_mem_unpoison (c, sz); clib_memset (c, 0, sizeof (clib_mem_bulk_chunk_hdr_t)); b->avail_chunks = c; c->n_free = b->elts_per_chunk; @@ -192,7 +192,7 @@ clib_mem_bulk_free (clib_mem_bulk_handle_t h, void *p) { /* chunk is empty - give it back */ remove_from_chunk_list (&b->avail_chunks, c); - CLIB_MEM_POISON (c, bulk_chunk_size (b)); + clib_mem_poison (c, bulk_chunk_size (b)); mspace_free (b->mspace, c); return; } diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c index fdde720bc73..4150fbe8b4b 100644 --- a/src/vppinfra/mem_dlmalloc.c +++ b/src/vppinfra/mem_dlmalloc.c @@ -19,7 +19,6 @@ #include #include #include -#include typedef struct { @@ -235,7 +234,7 @@ clib_mem_create_heap_internal (void *base, uword size, mspace_disable_expand (h->mspace); - CLIB_MEM_POISON (mspace_least_addr (h->mspace), + clib_mem_poison (mspace_least_addr (h->mspace), mspace_footprint (h->mspace)); return h; @@ -619,7 +618,7 @@ clib_mem_heap_alloc_inline (void *heap, uword size, uword align, if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED)) mheap_get_trace (pointer_to_uword (p), clib_mem_size (p)); - CLIB_MEM_UNPOISON (p, size); + clib_mem_unpoison (p, size); return p; } @@ -699,16 +698,16 @@ clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size, if (p && pointer_is_aligned (p, align) && mspace_realloc_in_place (h->mspace, p, new_size)) { - CLIB_MEM_UNPOISON (p, new_size); + clib_mem_unpoison (p, new_size); } else { new = clib_mem_heap_alloc_inline (h, new_size, align, 1); - CLIB_MEM_UNPOISON (new, new_size); + clib_mem_unpoison (new, new_size); if (old_alloc_size) { - CLIB_MEM_UNPOISON (p, old_alloc_size); + clib_mem_unpoison (p, old_alloc_size); clib_memcpy_fast (new, p, clib_min (new_size, old_alloc_size)); clib_mem_heap_free (h, p); } @@ -760,7 +759,7 @@ clib_mem_heap_free (void *heap, void *p) if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED)) mheap_put_trace (pointer_to_uword (p), size); - CLIB_MEM_POISON (p, clib_mem_size (p)); + clib_mem_poison (p, clib_mem_size (p)); mspace_free (h->mspace, p); } @@ -781,7 +780,7 @@ __clib_export void clib_mem_free_s (void *p) { uword size = clib_mem_size (p); - CLIB_MEM_UNPOISON (p, size); + clib_mem_unpoison (p, size); memset_s_inline (p, size, 0, size); clib_mem_free (p); } diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index 9f9194f59e2..8330d64b0bd 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -185,7 +185,7 @@ _pool_get (void **pp, void **ep, uword align, int zero, uword elt_sz) ph->free_bitmap = clib_bitmap_andnoti_notrim (ph->free_bitmap, index); vec_set_len (ph->free_indices, n_free - 1); - CLIB_MEM_UNPOISON (e, elt_sz); + clib_mem_unpoison (e, elt_sz); goto done; } @@ -299,7 +299,7 @@ _pool_put_index (void *p, uword index, uword elt_sz) else vec_add1 (ph->free_indices, index); - CLIB_MEM_POISON (p + index * elt_sz, elt_sz); + clib_mem_poison (p + index * elt_sz, elt_sz); } #define pool_put_index(P, I) _pool_put_index ((void *) (P), I, _vec_elt_sz (P)) @@ -322,7 +322,7 @@ _pool_alloc (void **pp, uword n_elts, uword align, void *heap, uword elt_sz) pp[0] = _vec_realloc_inline (pp[0], len + n_elts, elt_sz, sizeof (pool_header_t), align, heap); _vec_set_len (pp[0], len, elt_sz); - CLIB_MEM_POISON (pp[0] + len * elt_sz, n_elts * elt_sz); + clib_mem_poison (pp[0] + len * elt_sz, n_elts * elt_sz); ph = pool_header (pp[0]); vec_resize (ph->free_indices, n_elts); @@ -358,7 +358,7 @@ _pool_dup (void *p, uword align, uword elt_sz) { u32 *fi; vec_foreach (fi, ph->free_indices) - CLIB_MEM_UNPOISON (p + elt_sz * fi[0], elt_sz); + clib_mem_unpoison (p + elt_sz * fi[0], elt_sz); clib_memcpy_fast (n, p, len * elt_sz); @@ -368,8 +368,8 @@ _pool_dup (void *p, uword align, uword elt_sz) vec_foreach (fi, ph->free_indices) { uword offset = elt_sz * fi[0]; - CLIB_MEM_POISON (p + offset, elt_sz); - CLIB_MEM_POISON (n + offset, elt_sz); + clib_mem_poison (p + offset, elt_sz); + clib_mem_poison (n + offset, elt_sz); } } diff --git a/src/vppinfra/sanitizer.c b/src/vppinfra/sanitizer.c deleted file mode 100644 index fab1cdca136..00000000000 --- a/src/vppinfra/sanitizer.c +++ /dev/null @@ -1,7 +0,0 @@ -#ifdef CLIB_SANITIZE_ADDR - -#include - -__clib_export clib_sanitizer_main_t sanitizer_main = { .shadow_scale = ~0 }; - -#endif /* CLIB_SANITIZE_ADDR */ diff --git a/src/vppinfra/sanitizer.h b/src/vppinfra/sanitizer.h deleted file mode 100644 index cf2da071225..00000000000 --- a/src/vppinfra/sanitizer.h +++ /dev/null @@ -1,147 +0,0 @@ -#ifndef _included_clib_sanitizer_h -#define _included_clib_sanitizer_h - -#ifdef CLIB_SANITIZE_ADDR - -#include -#include -#include - -typedef struct -{ - size_t shadow_scale; - size_t shadow_offset; -} clib_sanitizer_main_t; - -extern clib_sanitizer_main_t sanitizer_main; - -#define CLIB_NOSANITIZE_ADDR __attribute__((no_sanitize_address)) -#define CLIB_MEM_POISON(a, s) ASAN_POISON_MEMORY_REGION((a), (s)) -#define CLIB_MEM_UNPOISON(a, s) ASAN_UNPOISON_MEMORY_REGION((a), (s)) - -#define CLIB_MEM_OVERFLOW_MAX 64 - -static_always_inline void -sanitizer_unpoison__ (u64 *restrict *shadow_ptr, size_t *shadow_len, - const void *ptr, size_t len) -{ - size_t scale, off; - - if (PREDICT_FALSE (~0 == sanitizer_main.shadow_scale)) - __asan_get_shadow_mapping (&sanitizer_main.shadow_scale, - &sanitizer_main.shadow_offset); - - scale = sanitizer_main.shadow_scale; - off = sanitizer_main.shadow_offset; - - /* compute the shadow address and length */ - *shadow_len = len >> scale; - ASSERT (*shadow_len <= CLIB_MEM_OVERFLOW_MAX); - *shadow_ptr = (void *) (((clib_address_t) ptr >> scale) + off); -} - -static_always_inline CLIB_NOSANITIZE_ADDR void -sanitizer_unpoison_push__ (u64 *restrict shadow, const void *ptr, size_t len) -{ - u64 *restrict shadow_ptr; - size_t shadow_len; - int i; - - sanitizer_unpoison__ (&shadow_ptr, &shadow_len, ptr, len); - - /* save the shadow area */ - for (i = 0; i < shadow_len; i++) - shadow[i] = shadow_ptr[i]; - - /* unpoison */ - for (i = 0; i < shadow_len; i++) - shadow_ptr[i] = 0; -} - -static_always_inline CLIB_NOSANITIZE_ADDR void -sanitizer_unpoison_pop__ (const u64 *restrict shadow, const void *ptr, - size_t len) -{ - u64 *restrict shadow_ptr; - size_t shadow_len; - int i; - - sanitizer_unpoison__ (&shadow_ptr, &shadow_len, ptr, len); - - /* restore the shadow area */ - for (i = 0; i < shadow_len; i++) - { - ASSERT (0 == shadow_ptr[i]); - shadow_ptr[i] = shadow[i]; - } -} - -#define CLIB_MEM_OVERFLOW_PUSH(src, n) \ - do \ - { \ - const void *clib_mem_overflow_src__ = (src); \ - size_t clib_mem_overflow_n__ = (n); \ - u64 clib_mem_overflow_shadow__; \ - sanitizer_unpoison_push__ (&clib_mem_overflow_shadow__, \ - clib_mem_overflow_src__, \ - clib_mem_overflow_n__) - -#define CLIB_MEM_OVERFLOW_POP() \ - sanitizer_unpoison_pop__ (&clib_mem_overflow_shadow__, \ - clib_mem_overflow_src__, clib_mem_overflow_n__); \ - } \ - while (0) - -#define CLIB_MEM_OVERFLOW_LOAD(src) \ - ({ \ - typeof (*(src)) *clib_mem_overflow_load_src__ = (src), \ - clib_mem_overflow_load_ret__; \ - CLIB_MEM_OVERFLOW_PUSH (clib_mem_overflow_load_src__, \ - sizeof (*clib_mem_overflow_load_src__)); \ - clib_mem_overflow_load_ret__ = *clib_mem_overflow_load_src__; \ - CLIB_MEM_OVERFLOW_POP (); \ - clib_mem_overflow_load_ret__; \ - }) - -static_always_inline void -CLIB_MEM_POISON_LEN (void *src, size_t oldlen, size_t newlen) -{ - if (oldlen > newlen) - CLIB_MEM_POISON (src + newlen, oldlen - newlen); - else if (newlen > oldlen) - CLIB_MEM_UNPOISON (src + oldlen, newlen - oldlen); -} - -#else /* CLIB_SANITIZE_ADDR */ - -#define CLIB_NOSANITIZE_ADDR -#define CLIB_MEM_POISON(a, s) (void)(a) -#define CLIB_MEM_UNPOISON(a, s) (void)(a) -#define CLIB_MEM_OVERFLOW_PUSH(a, b) (void) (a) -#define CLIB_MEM_OVERFLOW_POP() -#define CLIB_MEM_OVERFLOW_LOAD(src) (*(src)) -#define CLIB_MEM_POISON_LEN(a, b, c) - -#endif /* CLIB_SANITIZE_ADDR */ - -/* - * clang tends to force alignment of all sections when compiling for address - * sanitizer. This confuse VPP plugin infra, prevent clang to do that - * On the contrary, GCC does not support this kind of attribute on sections - * sigh. - */ -#ifdef __clang__ -#define CLIB_NOSANITIZE_PLUGIN_REG_SECTION CLIB_NOSANITIZE_ADDR -#else -#define CLIB_NOSANITIZE_PLUGIN_REG_SECTION -#endif - -#endif /* _included_clib_sanitizer_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/vec.c b/src/vppinfra/vec.c index 8bd0ec01f42..4dc8f18ce24 100644 --- a/src/vppinfra/vec.c +++ b/src/vppinfra/vec.c @@ -55,7 +55,7 @@ _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align, v = p + data_offset; } - CLIB_MEM_UNPOISON (p, alloc_size); + clib_mem_unpoison (p, alloc_size); clib_memset_u8 (p + old_data_size, 0, alloc_size - old_data_size); } else @@ -68,7 +68,7 @@ _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align, new_data_size = data_offset + n_data_bytes; p = clib_mem_heap_alloc_aligned (heap, new_data_size, align); alloc_size = clib_mem_size (p); - CLIB_MEM_UNPOISON (p, alloc_size); + clib_mem_unpoison (p, alloc_size); clib_memset_u8 (p, 0, alloc_size); v = p + data_offset; _vec_find (v)->hdr_size = data_offset / VEC_MIN_ALIGN; @@ -82,7 +82,7 @@ _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align, _vec_find (v)->default_heap = 1; } - CLIB_MEM_POISON (p + new_data_size, alloc_size - new_data_size); + clib_mem_poison (p + new_data_size, alloc_size - new_data_size); _vec_find (v)->len = n_elts; return v; } diff --git a/src/vppinfra/vec_bootstrap.h b/src/vppinfra/vec_bootstrap.h index 0054802e8c1..a94c1a19ba7 100644 --- a/src/vppinfra/vec_bootstrap.h +++ b/src/vppinfra/vec_bootstrap.h @@ -85,7 +85,7 @@ always_inline uword __vec_elt_sz (uword elt_sz, int is_void); #define _vec_elt_sz(V) __vec_elt_sz (sizeof ((V)[0]), _vec_is_void (V)) #define _vec_align(V, A) __vec_align (__alignof__((V)[0]), A) -always_inline CLIB_NOSANITIZE_ADDR uword +always_inline __clib_nosanitize_addr uword vec_get_header_size (void *v) { uword header_size = _vec_find (v)->hdr_size * VEC_MIN_ALIGN; @@ -176,9 +176,9 @@ _vec_set_len (void *v, uword len, uword elt_sz) uword old_len = _vec_len (v); if (len > old_len) - CLIB_MEM_UNPOISON (v + old_len * elt_sz, (len - old_len) * elt_sz); + clib_mem_unpoison (v + old_len * elt_sz, (len - old_len) * elt_sz); else if (len > old_len) - CLIB_MEM_POISON (v + len * elt_sz, (old_len - len) * elt_sz); + clib_mem_poison (v + len * elt_sz, (old_len - len) * elt_sz); _vec_find (v)->len = len; } -- cgit 1.2.3-korg