diff options
| -rw-r--r-- | src/svm/ssvm.c | 52 | ||||
| -rw-r--r-- | src/svm/ssvm.h | 3 | ||||
| -rw-r--r-- | src/vppinfra/linux/mem.c | 40 | ||||
| -rw-r--r-- | src/vppinfra/mem.h | 1 |
4 files changed, 70 insertions, 26 deletions
diff --git a/src/svm/ssvm.c b/src/svm/ssvm.c index 69463b1f192..66e38ebf864 100644 --- a/src/svm/ssvm.c +++ b/src/svm/ssvm.c @@ -90,6 +90,7 @@ ssvm_master_init_shm (ssvm_private_t * ssvm) mapa.requested_va = requested_va; mapa.size = ssvm->ssvm_size; mapa.fd = ssvm_fd; + mapa.numa_node = ssvm->numa; if (clib_mem_vm_ext_map (&mapa)) { clib_unix_warning ("mmap"); @@ -237,6 +238,11 @@ ssvm_master_init_memfd (ssvm_private_t * memfd) alloc.size = memfd->ssvm_size; alloc.flags = CLIB_MEM_VM_F_SHARED; alloc.requested_va = memfd->requested_va; + if (memfd->numa) + { + alloc.numa_node = memfd->numa; + alloc.flags |= CLIB_MEM_VM_F_NUMA_PREFER; + } if ((err = clib_mem_vm_ext_alloc (&alloc))) { clib_error_report (err); @@ -348,27 +354,31 @@ int ssvm_master_init_private (ssvm_private_t * ssvm) { uword pagesize = clib_mem_get_page_size (), rnd_size = 0; + clib_mem_vm_alloc_t alloc = { 0 }; + struct dlmallinfo dlminfo; ssvm_shared_header_t *sh; + clib_error_t *err; u8 *heap; rnd_size = clib_max (ssvm->ssvm_size + (pagesize - 1), ssvm->ssvm_size); rnd_size &= ~(pagesize - 1); -#if USE_DLMALLOC == 0 - { - mheap_t *heap_header; - - heap = mheap_alloc (0, rnd_size); - if (heap == 0) - { - clib_unix_warning ("mheap alloc"); - return -1; - } - heap_header = mheap_header (heap); - heap_header->flags |= MHEAP_FLAG_THREAD_SAFE; - } -#else - heap = create_mspace (rnd_size, 1 /* locked */ ); + alloc.name = (char *) ssvm->name; + alloc.size = rnd_size + pagesize; + if (ssvm->numa) + { + alloc.numa_node = ssvm->numa; + alloc.flags |= CLIB_MEM_VM_F_NUMA_PREFER; + } + + if ((err = clib_mem_vm_ext_alloc (&alloc))) + { + clib_error_report (err); + return SSVM_API_ERROR_CREATE_FAILURE; + } + + heap = create_mspace_with_base ((u8 *) alloc.addr + pagesize, rnd_size, + 1 /* locked */ ); if (heap == 0) { clib_unix_warning ("mheap alloc"); @@ -378,18 +388,16 @@ ssvm_master_init_private (ssvm_private_t * ssvm) mspace_disable_expand (heap); /* Find actual size because mspace size is rounded up by dlmalloc */ - struct dlmallinfo dlminfo; dlminfo = mspace_mallinfo (heap); rnd_size = dlminfo.fordblks; -#endif ssvm->ssvm_size = rnd_size; ssvm->i_am_master = 1; ssvm->my_pid = getpid (); ssvm->requested_va = ~0; - /* Allocate a [sic] shared memory header, in process memory... */ - sh = clib_mem_alloc_aligned (sizeof (*sh), CLIB_CACHE_LINE_BYTES); + /* First page in allocated memory is set aside for the shared header */ + sh = alloc.addr; ssvm->sh = sh; clib_memset (sh, 0, sizeof (*sh)); @@ -413,12 +421,8 @@ void ssvm_delete_private (ssvm_private_t * ssvm) { vec_free (ssvm->name); -#if USE_DLMALLOC == 0 - mheap_free (ssvm->sh->heap); -#else destroy_mspace (ssvm->sh->heap); -#endif - clib_mem_free (ssvm->sh); + clib_mem_vm_free (ssvm->sh, ssvm->ssvm_size + clib_mem_get_page_size ()); } int diff --git a/src/svm/ssvm.h b/src/svm/ssvm.h index 82951d614b0..6225a63cae4 100644 --- a/src/svm/ssvm.h +++ b/src/svm/ssvm.h @@ -83,9 +83,10 @@ typedef struct { ssvm_shared_header_t *sh; uword ssvm_size; + uword requested_va; u32 my_pid; u8 *name; - uword requested_va; + u8 numa; /**< Numa requested at alloc time */ int i_am_master; union diff --git a/src/vppinfra/linux/mem.c b/src/vppinfra/linux/mem.c index 69ac871567c..3b7294f038a 100644 --- a/src/vppinfra/linux/mem.c +++ b/src/vppinfra/linux/mem.c @@ -386,21 +386,59 @@ done: clib_error_t * clib_mem_vm_ext_map (clib_mem_vm_map_t * a) { + long unsigned int old_mask[16] = { 0 }; int mmap_flags = MAP_SHARED; + clib_error_t *err = 0; + int old_mpol = -1; void *addr; + int rv; + + if (a->numa_node) + { + rv = get_mempolicy (&old_mpol, old_mask, sizeof (old_mask) * 8 + 1, 0, + 0); + + if (rv == -1) + { + err = clib_error_return_unix (0, "get_mempolicy"); + goto done; + } + } if (a->requested_va) mmap_flags |= MAP_FIXED; + if (old_mpol != -1) + { + long unsigned int mask[16] = { 0 }; + mask[0] = 1 << a->numa_node; + rv = set_mempolicy (MPOL_BIND, mask, sizeof (mask) * 8 + 1); + if (rv == -1) + { + err = clib_error_return_unix (0, "set_mempolicy"); + goto done; + } + } + addr = (void *) mmap (uword_to_pointer (a->requested_va, void *), a->size, PROT_READ | PROT_WRITE, mmap_flags, a->fd, 0); if (addr == MAP_FAILED) return clib_error_return_unix (0, "mmap"); + /* re-apply old numa memory policy */ + if (old_mpol != -1 && + set_mempolicy (old_mpol, old_mask, sizeof (old_mask) * 8 + 1) == -1) + { + err = clib_error_return_unix (0, "set_mempolicy"); + goto done; + } + a->addr = addr; CLIB_MEM_UNPOISON (addr, a->size); - return 0; + +done: + return err; } /* diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 0367c4a1213..f35c495eb25 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -455,6 +455,7 @@ typedef struct int fd; /**< File descriptor to be mapped */ uword requested_va; /**< Request fixed position mapping */ void *addr; /**< Pointer to mapped memory, if successful */ + u8 numa_node; } clib_mem_vm_map_t; clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a); |