aboutsummaryrefslogtreecommitdiffstats
path: root/src/vppinfra
diff options
context:
space:
mode:
Diffstat (limited to 'src/vppinfra')
-rw-r--r--src/vppinfra/CMakeLists.txt12
-rw-r--r--src/vppinfra/cache.h24
-rw-r--r--src/vppinfra/clib.h7
-rw-r--r--src/vppinfra/cpu.h1
-rw-r--r--src/vppinfra/crypto/aes_cbc.h203
-rw-r--r--src/vppinfra/devicetree.c355
-rw-r--r--src/vppinfra/devicetree.h118
-rw-r--r--src/vppinfra/format.c10
-rw-r--r--src/vppinfra/format.h2
-rw-r--r--src/vppinfra/jsonformat.c13
-rw-r--r--src/vppinfra/linux/mem.c42
-rw-r--r--src/vppinfra/mem.h17
-rw-r--r--src/vppinfra/mem_dlmalloc.c51
-rw-r--r--src/vppinfra/stack.c25
-rw-r--r--src/vppinfra/stack.h1
-rw-r--r--src/vppinfra/time.c7
-rw-r--r--src/vppinfra/time_range.c9
-rw-r--r--src/vppinfra/unix-misc.c22
18 files changed, 832 insertions, 87 deletions
diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt
index 233e75d6e2a..83a8b2a7e57 100644
--- a/src/vppinfra/CMakeLists.txt
+++ b/src/vppinfra/CMakeLists.txt
@@ -16,15 +16,7 @@ enable_language(ASM)
##############################################################################
# find libdl
##############################################################################
-vpp_find_path(LIBDL_INCLUDE_DIR dlfcn.h)
-vpp_find_library(LIBDL_LIB NAMES dl)
-
-if (LIBDL_INCLUDE_DIR AND LIBDL_LIB)
- message(STATUS "libdl found at ${LIBDL_LIB}")
- list(APPEND VPPINFRA_LIBS ${LIBDL_LIB})
-else()
- message(FATAL_ERROR "libdl not found")
-endif()
+list(APPEND VPPINFRA_LIBS ${CMAKE_DL_LIBS})
##############################################################################
# find libunwind
@@ -77,6 +69,7 @@ set(VPPINFRA_SRCS
bitmap.c
bihash_all_vector.c
cpu.c
+ devicetree.c
dlmalloc.c
elf.c
elog.c
@@ -159,6 +152,7 @@ set(VPPINFRA_HEADERS
crypto/aes_ctr.h
crypto/aes_gcm.h
crypto/poly1305.h
+ devicetree.h
dlist.h
dlmalloc.h
elf_clib.h
diff --git a/src/vppinfra/cache.h b/src/vppinfra/cache.h
index 4229a068486..13778a423fd 100644
--- a/src/vppinfra/cache.h
+++ b/src/vppinfra/cache.h
@@ -68,6 +68,17 @@
#define CLIB_PREFETCH_WRITE 1
#define CLIB_PREFETCH_STORE 1 /* alias for write */
+/* locality arguments to __builtin_prefetch. */
+#define CLIB_PREFETCH_TO_STREAM 0 // NTA
+#define CLIB_PREFETCH_TO_L3 1 // T2
+#define CLIB_PREFETCH_TO_L2 2 // T1
+#define CLIB_PREFETCH_TO_L1 3 // T0
+
+#define _CLIB_TARGETED_PREFETCH(n, size, type, loc) \
+ if ((size) > (n) *CLIB_CACHE_PREFETCH_BYTES) \
+ __builtin_prefetch (_addr + (n) *CLIB_CACHE_PREFETCH_BYTES, \
+ CLIB_PREFETCH_##type, CLIB_PREFETCH_TO_##loc);
+
#define _CLIB_PREFETCH(n, size, type) \
if ((size) > (n) *CLIB_CACHE_PREFETCH_BYTES) \
__builtin_prefetch (_addr + (n) *CLIB_CACHE_PREFETCH_BYTES, \
@@ -86,6 +97,19 @@
} \
while (0)
+#define CLIB_TARGETED_PREFETCH(addr, size, type, locality) \
+ do \
+ { \
+ void *_addr = (addr); \
+ \
+ ASSERT ((size) <= 4 * CLIB_CACHE_PREFETCH_BYTES); \
+ _CLIB_TARGETED_PREFETCH (0, size, type, locality); \
+ _CLIB_TARGETED_PREFETCH (1, size, type, locality); \
+ _CLIB_TARGETED_PREFETCH (2, size, type, locality); \
+ _CLIB_TARGETED_PREFETCH (3, size, type, locality); \
+ } \
+ while (0)
+
#undef _
static_always_inline void
diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h
index 75cebc65672..5348738ec6a 100644
--- a/src/vppinfra/clib.h
+++ b/src/vppinfra/clib.h
@@ -106,6 +106,13 @@
#define CLIB_STRING_ARRAY(...) \
(char *[]) { __VA_ARGS__, 0 }
+#define CLIB_SWAP(a, b) \
+ { \
+ typeof (a) __tmp = a; \
+ a = b; \
+ b = __tmp; \
+ }
+
/* sanitizers */
#ifdef __has_feature
#if __has_feature(address_sanitizer)
diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h
index 7a1b75fcf7d..b3743d4c26d 100644
--- a/src/vppinfra/cpu.h
+++ b/src/vppinfra/cpu.h
@@ -150,6 +150,7 @@ _CLIB_MARCH_FN_REGISTRATION(fn)
_ (movdir64b, 7, ecx, 28) \
_ (enqcmd, 7, ecx, 29) \
_ (avx512_fp16, 7, edx, 23) \
+ _ (aperfmperf, 0x00000006, ecx, 0) \
_ (invariant_tsc, 0x80000007, edx, 8) \
_ (monitorx, 0x80000001, ecx, 29)
diff --git a/src/vppinfra/crypto/aes_cbc.h b/src/vppinfra/crypto/aes_cbc.h
index cb3d0784051..ee9263df260 100644
--- a/src/vppinfra/crypto/aes_cbc.h
+++ b/src/vppinfra/crypto/aes_cbc.h
@@ -539,4 +539,207 @@ clib_aes256_cbc_decrypt (const aes_cbc_key_data_t *kd, const u8 *ciphertext,
clib_aes_cbc_decrypt (kd, ciphertext, len, iv, AES_KEY_256, plaintext);
}
+#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
+#pragma GCC optimize("O3")
+#endif
+
+#if defined(__VAES__) && defined(__AVX512F__)
+#define u8xN u8x64
+#define u32xN u32x16
+#define u32xN_min_scalar u32x16_min_scalar
+#define u32xN_is_all_zero u32x16_is_all_zero
+#define u32xN_splat u32x16_splat
+#elif defined(__VAES__)
+#define u8xN u8x32
+#define u32xN u32x8
+#define u32xN_min_scalar u32x8_min_scalar
+#define u32xN_is_all_zero u32x8_is_all_zero
+#define u32xN_splat u32x8_splat
+#else
+#define u8xN u8x16
+#define u32xN u32x4
+#define u32xN_min_scalar u32x4_min_scalar
+#define u32xN_is_all_zero u32x4_is_all_zero
+#define u32xN_splat u32x4_splat
+#endif
+
+static_always_inline u32
+clib_aes_cbc_encrypt_multi (aes_cbc_key_data_t **key_data,
+ const uword *key_indices, u8 **plaintext,
+ const uword *oplen, u8 **iv, aes_key_size_t ks,
+ u8 **ciphertext, uword n_ops)
+{
+ int rounds = AES_KEY_ROUNDS (ks);
+ u8 placeholder[8192];
+ u32 i, j, count, n_left = n_ops;
+ u32xN placeholder_mask = {};
+ u32xN len = {};
+ u32 key_index[4 * N_AES_LANES];
+ u8 *src[4 * N_AES_LANES] = {};
+ u8 *dst[4 * N_AES_LANES] = {};
+ u8xN r[4] = {};
+ u8xN k[15][4] = {};
+
+ for (i = 0; i < 4 * N_AES_LANES; i++)
+ key_index[i] = ~0;
+
+more:
+ for (i = 0; i < 4 * N_AES_LANES; i++)
+ if (len[i] == 0)
+ {
+ if (n_left == 0)
+ {
+ /* no more work to enqueue, so we are enqueueing placeholder buffer
+ */
+ src[i] = dst[i] = placeholder;
+ len[i] = sizeof (placeholder);
+ placeholder_mask[i] = 0;
+ }
+ else
+ {
+ u8x16 t = aes_block_load (iv[0]);
+ ((u8x16 *) r)[i] = t;
+
+ src[i] = plaintext[0];
+ dst[i] = ciphertext[0];
+ len[i] = oplen[0];
+ placeholder_mask[i] = ~0;
+ if (key_index[i] != key_indices[0])
+ {
+ aes_cbc_key_data_t *kd;
+ key_index[i] = key_indices[0];
+ kd = key_data[key_index[i]];
+ for (j = 0; j < rounds + 1; j++)
+ ((u8x16 *) k[j])[i] = kd->encrypt_key[j];
+ }
+ n_left--;
+ iv++;
+ ciphertext++;
+ plaintext++;
+ key_indices++;
+ oplen++;
+ }
+ }
+
+ count = u32xN_min_scalar (len);
+
+ ASSERT (count % 16 == 0);
+
+ for (i = 0; i < count; i += 16)
+ {
+#if defined(__VAES__) && defined(__AVX512F__)
+ r[0] = u8x64_xor3 (r[0], aes_block_load_x4 (src, i), k[0][0]);
+ r[1] = u8x64_xor3 (r[1], aes_block_load_x4 (src + 4, i), k[0][1]);
+ r[2] = u8x64_xor3 (r[2], aes_block_load_x4 (src + 8, i), k[0][2]);
+ r[3] = u8x64_xor3 (r[3], aes_block_load_x4 (src + 12, i), k[0][3]);
+
+ for (j = 1; j < rounds; j++)
+ {
+ r[0] = aes_enc_round_x4 (r[0], k[j][0]);
+ r[1] = aes_enc_round_x4 (r[1], k[j][1]);
+ r[2] = aes_enc_round_x4 (r[2], k[j][2]);
+ r[3] = aes_enc_round_x4 (r[3], k[j][3]);
+ }
+ r[0] = aes_enc_last_round_x4 (r[0], k[j][0]);
+ r[1] = aes_enc_last_round_x4 (r[1], k[j][1]);
+ r[2] = aes_enc_last_round_x4 (r[2], k[j][2]);
+ r[3] = aes_enc_last_round_x4 (r[3], k[j][3]);
+
+ aes_block_store_x4 (dst, i, r[0]);
+ aes_block_store_x4 (dst + 4, i, r[1]);
+ aes_block_store_x4 (dst + 8, i, r[2]);
+ aes_block_store_x4 (dst + 12, i, r[3]);
+#elif defined(__VAES__)
+ r[0] = u8x32_xor3 (r[0], aes_block_load_x2 (src, i), k[0][0]);
+ r[1] = u8x32_xor3 (r[1], aes_block_load_x2 (src + 2, i), k[0][1]);
+ r[2] = u8x32_xor3 (r[2], aes_block_load_x2 (src + 4, i), k[0][2]);
+ r[3] = u8x32_xor3 (r[3], aes_block_load_x2 (src + 6, i), k[0][3]);
+
+ for (j = 1; j < rounds; j++)
+ {
+ r[0] = aes_enc_round_x2 (r[0], k[j][0]);
+ r[1] = aes_enc_round_x2 (r[1], k[j][1]);
+ r[2] = aes_enc_round_x2 (r[2], k[j][2]);
+ r[3] = aes_enc_round_x2 (r[3], k[j][3]);
+ }
+ r[0] = aes_enc_last_round_x2 (r[0], k[j][0]);
+ r[1] = aes_enc_last_round_x2 (r[1], k[j][1]);
+ r[2] = aes_enc_last_round_x2 (r[2], k[j][2]);
+ r[3] = aes_enc_last_round_x2 (r[3], k[j][3]);
+
+ aes_block_store_x2 (dst, i, r[0]);
+ aes_block_store_x2 (dst + 2, i, r[1]);
+ aes_block_store_x2 (dst + 4, i, r[2]);
+ aes_block_store_x2 (dst + 6, i, r[3]);
+#else
+#if __x86_64__
+ r[0] = u8x16_xor3 (r[0], aes_block_load (src[0] + i), k[0][0]);
+ r[1] = u8x16_xor3 (r[1], aes_block_load (src[1] + i), k[0][1]);
+ r[2] = u8x16_xor3 (r[2], aes_block_load (src[2] + i), k[0][2]);
+ r[3] = u8x16_xor3 (r[3], aes_block_load (src[3] + i), k[0][3]);
+
+ for (j = 1; j < rounds; j++)
+ {
+ r[0] = aes_enc_round_x1 (r[0], k[j][0]);
+ r[1] = aes_enc_round_x1 (r[1], k[j][1]);
+ r[2] = aes_enc_round_x1 (r[2], k[j][2]);
+ r[3] = aes_enc_round_x1 (r[3], k[j][3]);
+ }
+
+ r[0] = aes_enc_last_round_x1 (r[0], k[j][0]);
+ r[1] = aes_enc_last_round_x1 (r[1], k[j][1]);
+ r[2] = aes_enc_last_round_x1 (r[2], k[j][2]);
+ r[3] = aes_enc_last_round_x1 (r[3], k[j][3]);
+
+ aes_block_store (dst[0] + i, r[0]);
+ aes_block_store (dst[1] + i, r[1]);
+ aes_block_store (dst[2] + i, r[2]);
+ aes_block_store (dst[3] + i, r[3]);
+#else
+ r[0] ^= aes_block_load (src[0] + i);
+ r[1] ^= aes_block_load (src[1] + i);
+ r[2] ^= aes_block_load (src[2] + i);
+ r[3] ^= aes_block_load (src[3] + i);
+ for (j = 0; j < rounds - 1; j++)
+ {
+ r[0] = vaesmcq_u8 (vaeseq_u8 (r[0], k[j][0]));
+ r[1] = vaesmcq_u8 (vaeseq_u8 (r[1], k[j][1]));
+ r[2] = vaesmcq_u8 (vaeseq_u8 (r[2], k[j][2]));
+ r[3] = vaesmcq_u8 (vaeseq_u8 (r[3], k[j][3]));
+ }
+ r[0] = vaeseq_u8 (r[0], k[j][0]) ^ k[rounds][0];
+ r[1] = vaeseq_u8 (r[1], k[j][1]) ^ k[rounds][1];
+ r[2] = vaeseq_u8 (r[2], k[j][2]) ^ k[rounds][2];
+ r[3] = vaeseq_u8 (r[3], k[j][3]) ^ k[rounds][3];
+ aes_block_store (dst[0] + i, r[0]);
+ aes_block_store (dst[1] + i, r[1]);
+ aes_block_store (dst[2] + i, r[2]);
+ aes_block_store (dst[3] + i, r[3]);
+#endif
+#endif
+ }
+
+ len -= u32xN_splat (count);
+
+ for (i = 0; i < 4 * N_AES_LANES; i++)
+ {
+ src[i] += count;
+ dst[i] += count;
+ }
+
+ if (n_left > 0)
+ goto more;
+
+ if (!u32xN_is_all_zero (len & placeholder_mask))
+ goto more;
+
+ return n_ops;
+}
+
+#undef u8xN
+#undef u32xN
+#undef u32xN_min_scalar
+#undef u32xN_is_all_zero
+#undef u32xN_splat
+
#endif /* __crypto_aes_cbc_h__ */
diff --git a/src/vppinfra/devicetree.c b/src/vppinfra/devicetree.c
new file mode 100644
index 00000000000..df5a24f198e
--- /dev/null
+++ b/src/vppinfra/devicetree.c
@@ -0,0 +1,355 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2024 Cisco Systems, Inc.
+ */
+
+#include <vppinfra/clib.h>
+#include <vppinfra/devicetree.h>
+
+#ifdef __linux
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <dirent.h>
+
+static_always_inline clib_dt_node_t *
+clib_dt_node_add_child (clib_dt_main_t *dm, clib_dt_node_t *n, char *name)
+{
+ clib_dt_node_t *cn;
+
+ cn = clib_mem_alloc (sizeof (clib_dt_node_t));
+ *cn = (clib_dt_node_t){ .parent = n, .depth = n ? n->depth + 1 : 0 };
+ vec_add1 (dm->nodes, cn);
+
+ if (n == 0)
+ {
+ ASSERT (dm->root == 0);
+ dm->root = cn;
+ return cn;
+ }
+
+ vec_add1 (n->child_nodes, cn);
+ cn->path = format (0, "%v/%s", n->path, name);
+ cn->dt_main = dm;
+ hash_set_mem (dm->node_by_path, cn->path, cn);
+ if (vec_len (n->child_nodes) > 1)
+ {
+ clib_dt_node_t *prev = n->child_nodes[vec_len (n->child_nodes) - 2];
+ prev->next = cn;
+ cn->prev = prev;
+ }
+
+ return cn;
+}
+#endif
+
+void
+clib_dt_main_free (clib_dt_main_t *dm)
+{
+ vec_foreach_pointer (n, dm->nodes)
+ {
+ vec_foreach_pointer (p, n->properties)
+ clib_mem_free (p);
+ vec_free (n->child_nodes);
+ vec_free (n->path);
+ vec_free (n->properties);
+ }
+
+ vec_free (dm->nodes);
+ hash_free (dm->node_by_path);
+ hash_free (dm->node_by_phandle);
+}
+
+#ifdef __linux
+__clib_export clib_error_t *
+clib_dt_read_from_sysfs (clib_dt_main_t *dm)
+{
+ DIR *dir, **dir_stack = 0;
+ struct dirent *e;
+ clib_dt_node_t *n;
+ u8 *path = 0;
+ u32 path_prefix_len;
+ clib_error_t *err = 0;
+
+ path = format (0, CLIB_DT_LINUX_PREFIX);
+ path_prefix_len = vec_len (path);
+ vec_add1 (path, 0);
+
+ dir = opendir ((char *) path);
+ if (!dir)
+ {
+ err = clib_error_return (0, "'%s' opendir failed", path);
+ goto done;
+ }
+
+ dm->node_by_path = hash_create_vec (0, sizeof (u8), sizeof (uword));
+ dm->node_by_phandle = hash_create (0, sizeof (uword));
+ vec_set_len (path, path_prefix_len);
+ n = clib_dt_node_add_child (dm, 0, 0);
+
+ while (1)
+ {
+ e = readdir (dir);
+
+ if (!e)
+ {
+ closedir (dir);
+ if (vec_len (dir_stack) == 0)
+ break;
+
+ dir = dir_stack[vec_len (dir_stack) - 1];
+ vec_pop (dir_stack);
+ n = n->parent;
+ continue;
+ }
+
+ if (e->d_type == DT_REG)
+ {
+ path = format (path, "%v/%s%c", n->path, e->d_name, 0);
+ int fd = open ((char *) path, 0);
+ if (fd >= 0)
+ {
+ struct stat st;
+ if (fstat (fd, &st) == 0)
+ {
+ u32 sz = sizeof (clib_dt_property_t) + st.st_size;
+ clib_dt_property_t *p = clib_mem_alloc (sz);
+ clib_memset (p, 0, sz);
+
+ if (read (fd, p->data, st.st_size) == st.st_size)
+ {
+ strncpy (p->name, e->d_name, sizeof (p->name));
+ p->size = st.st_size;
+ vec_add1 (n->properties, p);
+ if (strncmp ("name", p->name, 5) == 0)
+ n->name = p;
+ if ((strncmp ("phandle", p->name, 8) == 0) &&
+ (p->size == 4))
+ {
+ u32 phandle =
+ clib_net_to_host_u32 (*(u32u *) p->data);
+ hash_set (dm->node_by_phandle, phandle, n);
+ }
+ }
+ else
+ {
+ clib_mem_free (p);
+ err = clib_error_return (0, "'%s' read failed", path);
+ close (fd);
+ goto done;
+ }
+ }
+ else
+ {
+ err = clib_error_return (0, "'%s' fstat failed", path);
+ close (fd);
+ goto done;
+ }
+ close (fd);
+ }
+ else
+ {
+ err = clib_error_return (0, "'%s' open failed", path);
+ goto done;
+ }
+
+ vec_set_len (path, path_prefix_len);
+ }
+ else if (e->d_type == DT_DIR)
+ {
+ DIR *subdir;
+ if (strncmp (".", e->d_name, 2) == 0 ||
+ strncmp ("..", e->d_name, 3) == 0)
+ continue;
+
+ path = format (path, "%v/%s%c", n->path, e->d_name, 0);
+ subdir = opendir ((char *) path);
+ vec_set_len (path, path_prefix_len);
+ if (subdir)
+ {
+ vec_add1 (dir_stack, dir);
+ dir = subdir;
+ n = clib_dt_node_add_child (dm, n, e->d_name);
+ }
+ else
+ {
+ err = clib_error_return (0, "'%s' opendir failed", path);
+ goto done;
+ }
+ }
+ else
+ err =
+ clib_error_return (0, "unknown entry %s [%u]", e->d_name, e->d_type);
+ }
+
+done:
+ if (err)
+ clib_dt_main_free (dm);
+ while (vec_len (dir_stack))
+ closedir (vec_pop (dir_stack));
+ vec_free (dir_stack);
+ vec_free (path);
+ return err;
+}
+#endif
+
+__clib_export clib_dt_node_t *
+clib_dt_get_child_node (clib_dt_node_t *n, char *fmt, ...)
+{
+ u8 *s;
+ va_list va;
+ va_start (va, fmt);
+ s = va_format (0, fmt, &va);
+ va_end (va);
+ vec_add1 (s, 0);
+
+ vec_foreach_pointer (cn, n->child_nodes)
+ {
+ u8 *p = cn->path + vec_len (cn->path) - 1;
+ u32 i = 0;
+
+ while (p > cn->path && p[-1] != '/')
+ p--;
+
+ if (p[-1] != '/')
+ continue;
+
+ while (p[i] == s[i] && s[i] != 0)
+ i++;
+
+ if (s[i] != 0)
+ continue;
+
+ vec_free (s);
+ return cn;
+ }
+
+ vec_free (s);
+ return 0;
+}
+
+__clib_export clib_dt_node_t *
+clib_dt_get_node_with_path (clib_dt_main_t *dm, char *fmt, ...)
+{
+ u8 *s;
+ uword *p;
+
+ va_list va;
+ va_start (va, fmt);
+ s = va_format (0, fmt, &va);
+ va_end (va);
+
+ if (s[0] != '/')
+ return 0;
+
+ p = hash_get_mem (dm->node_by_path, s);
+ if (p)
+ return (clib_dt_node_t *) p[0];
+
+ return 0;
+}
+
+__clib_export clib_dt_property_t *
+clib_dt_get_node_property_by_name (clib_dt_node_t *n, char *name)
+{
+ vec_foreach_pointer (p, n->properties)
+ if (strncmp (name, p->name, sizeof (p->name)) == 0)
+ return p;
+ return 0;
+}
+
+__clib_export int
+clib_dt_node_is_compatible (clib_dt_node_t *n, char *comp)
+{
+ clib_dt_property_t *p;
+ char *s;
+
+ p = clib_dt_get_node_property_by_name (n, "compatible");
+
+ if (!p)
+ return 0;
+
+ s = (char *) p->data;
+ for (u32 i = 1, len = 1; i <= p->size; i++)
+ {
+ if (p->data[i - 1] == 0)
+ {
+ if (strncmp (comp, s, len) == 0)
+ return 1;
+ s = (char *) p->data + i;
+ len = 1;
+ }
+ else
+ len++;
+ }
+
+ return 0;
+}
+
+__clib_export u8 *
+format_clib_dt_property_data (u8 *s, va_list *args)
+{
+ clib_dt_property_t *p = va_arg (*args, clib_dt_property_t *);
+ u32 sz = p->size, is_printable = 0;
+ u32 n_nulls = 0;
+
+ if (sz > 2 && p->data[sz - 1] == 0 && p->data[0] != 0)
+ {
+ is_printable = 1;
+ for (u32 i = 1; i < sz - 1; i++)
+ {
+ u8 c = p->data[i];
+ if (c == 0)
+ {
+ if (p->data[i - 1] == 0)
+ {
+ is_printable = 0;
+ break;
+ }
+ n_nulls++;
+ }
+ else if ((c < 0x20) || (c > 0x7f))
+ {
+ is_printable = 0;
+ break;
+ }
+ }
+ }
+
+ if (is_printable)
+ {
+ s = format (s, "'%s'", p->data);
+ if (n_nulls)
+ {
+ for (u32 i = 2; i < p->size; i++)
+ if (((u8 *) p->data)[i - 1] == 0)
+ s = format (s, ", '%s'", ((u8 *) p->data) + i);
+ }
+ }
+ else
+ {
+ s = format (s, "< %02x", p->data[0]);
+ for (u32 i = 0; i < p->size; i++)
+ s = format (s, " %02x", p->data[i]);
+ s = format (s, " >");
+ }
+ return s;
+}
+
+__clib_export clib_dt_node_t *
+clib_dt_dereference_node (clib_dt_node_t *n, char *name)
+{
+ clib_dt_property_t *p;
+ uword *h;
+
+ p = clib_dt_get_node_property_by_name (n, name);
+ if (!p || (p->size != sizeof (u32)))
+ return 0;
+
+ h = hash_get (n->dt_main->node_by_phandle,
+ clib_net_to_host_u32 (*(u32u *) p->data));
+
+ if (h)
+ return (clib_dt_node_t *) h[0];
+
+ return 0;
+}
diff --git a/src/vppinfra/devicetree.h b/src/vppinfra/devicetree.h
new file mode 100644
index 00000000000..db7d8411a11
--- /dev/null
+++ b/src/vppinfra/devicetree.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2024 Cisco Systems, Inc.
+ */
+
+#ifndef CLIB_DEVICETREE_H_
+#define CLIB_DEVICETREE_H_
+
+#include <vppinfra/clib.h>
+#include <vlib/vlib.h>
+
+#ifdef __linux
+#define CLIB_DT_LINUX_PREFIX "/sys/firmware/devicetree/base"
+#endif
+
+typedef struct
+{
+ char name[32];
+ u32 size;
+ u8 data[];
+} clib_dt_property_t;
+
+typedef struct clib_dt_main clib_dt_main_t;
+
+typedef struct clib_dt_node
+{
+ u8 *path;
+ struct clib_dt_node *parent;
+ struct clib_dt_node *prev;
+ struct clib_dt_node *next;
+ struct clib_dt_node **child_nodes;
+ u8 depth;
+ clib_dt_property_t *name;
+ clib_dt_property_t **properties;
+ clib_dt_main_t *dt_main;
+} clib_dt_node_t;
+
+typedef struct clib_dt_main
+{
+ clib_dt_node_t **nodes;
+ clib_dt_node_t *root;
+ uword *node_by_path;
+ uword *node_by_phandle;
+} clib_dt_main_t;
+
+__clib_export clib_dt_node_t *clib_dt_get_child_node (clib_dt_node_t *n,
+ char *fmt, ...);
+clib_dt_node_t *clib_dt_get_node_with_path (clib_dt_main_t *dm, char *fmt,
+ ...);
+clib_dt_property_t *clib_dt_get_node_property_by_name (clib_dt_node_t *,
+ char *);
+int clib_dt_node_is_compatible (clib_dt_node_t *, char *);
+clib_dt_node_t *clib_dt_dereference_node (clib_dt_node_t *, char *);
+#ifdef __linux
+clib_error_t *clib_dt_read_from_sysfs (clib_dt_main_t *dm);
+#endif
+
+format_function_t format_clib_dt_desc;
+format_function_t format_clib_dt_property_data;
+
+static_always_inline int
+clib_dt_property_is_u32 (clib_dt_property_t *p)
+{
+ if (p == 0 || p->size != 4)
+ return 0;
+ return 1;
+}
+
+static_always_inline u32
+clib_dt_property_get_u32 (clib_dt_property_t *p)
+{
+ return clib_net_to_host_u32 (*(u32u *) p->data);
+}
+
+static_always_inline char *
+clib_dt_property_get_string (clib_dt_property_t *p)
+{
+ return (char *) p->data;
+}
+
+static_always_inline clib_dt_node_t *
+clib_dt_get_root_node (clib_dt_node_t *n)
+{
+ return n->dt_main->root;
+}
+
+static_always_inline clib_dt_node_t *
+foreach_clib_dt_tree_node_helper (clib_dt_node_t *first, clib_dt_node_t **prev,
+ clib_dt_node_t *n)
+{
+ clib_dt_node_t *next;
+
+again:
+ if ((!*prev || (*prev)->parent != n) && vec_len (n->child_nodes) > 0)
+ next = n->child_nodes[0];
+ else if (n->next)
+ next = n->next;
+ else
+ {
+ next = n->parent;
+ *prev = n;
+ n = next;
+ if (n == first)
+ return 0;
+ goto again;
+ }
+
+ *prev = n;
+ return next == first ? 0 : next;
+}
+
+#define foreach_clib_dt_child_node(_cn, _n) \
+ vec_foreach_pointer (_cn, (_n)->child_nodes)
+
+#define foreach_clib_dt_tree_node(_n, _first) \
+ for (clib_dt_node_t *__last = 0, *(_n) = _first; _n; \
+ _n = foreach_clib_dt_tree_node_helper (_first, &__last, _n))
+
+#endif /* CLIB_DEVICETREE_H_ */
diff --git a/src/vppinfra/format.c b/src/vppinfra/format.c
index cf17b8a1acb..642d3e20654 100644
--- a/src/vppinfra/format.c
+++ b/src/vppinfra/format.c
@@ -833,6 +833,16 @@ done:
return s;
}
+__clib_export char *
+format_c_string (u8 *s, const char *fmt, ...)
+{
+ va_list args;
+ va_start (args, fmt);
+ s = va_format (s, fmt, &args);
+ va_end (args);
+ vec_add1 (s, '\0');
+ return (char *) s;
+}
/*
* fd.io coding-style-patch-verification: ON
diff --git a/src/vppinfra/format.h b/src/vppinfra/format.h
index a1a70a2d64f..14bac869f89 100644
--- a/src/vppinfra/format.h
+++ b/src/vppinfra/format.h
@@ -372,6 +372,8 @@ int test_unformat_main (unformat_input_t * input);
created circular dependency problems. */
int test_vec_main (unformat_input_t * input);
+char *format_c_string (u8 *s, const char *fmt, ...);
+
#endif /* included_format_h */
/*
diff --git a/src/vppinfra/jsonformat.c b/src/vppinfra/jsonformat.c
index 1aa3864be04..73cb94769d8 100644
--- a/src/vppinfra/jsonformat.c
+++ b/src/vppinfra/jsonformat.c
@@ -500,12 +500,13 @@ format_vl_api_mac_address_t (u8 * s, va_list * args)
mac->bytes[0], mac->bytes[1], mac->bytes[2],
mac->bytes[3], mac->bytes[4], mac->bytes[5]);
}
-#define _(T) \
- cJSON *vl_api_ ##T## _t_tojson (vl_api_ ##T## _t *a) { \
- u8 *s = format(0, "%U", format_vl_api_ ##T## _t, a); \
- cJSON *o = cJSON_CreateString((char *)s); \
- vec_free(s); \
- return o; \
+#define _(T) \
+ cJSON *vl_api_##T##_t_tojson (vl_api_##T##_t *a) \
+ { \
+ char *s = format_c_string (0, "%U", format_vl_api_##T##_t, a, 0); \
+ cJSON *o = cJSON_CreateString (s); \
+ vec_free (s); \
+ return o; \
}
foreach_type_tojson
#undef _
diff --git a/src/vppinfra/linux/mem.c b/src/vppinfra/linux/mem.c
index 734f5c4788c..651ea107b4d 100644
--- a/src/vppinfra/linux/mem.c
+++ b/src/vppinfra/linux/mem.c
@@ -101,11 +101,13 @@ legacy_get_log2_default_hugepage_size (void)
void
clib_mem_main_init (void)
{
+ unsigned long nodemask = 0, maxnode = CLIB_MAX_NUMAS;
+ unsigned long flags = MPOL_F_MEMS_ALLOWED;
clib_mem_main_t *mm = &clib_mem_main;
long sysconf_page_size;
uword page_size;
- void *va;
- int fd;
+ void *va = 0;
+ int fd, mode;
if (mm->log2_page_sz != CLIB_MEM_PAGE_SZ_UNKNOWN)
return;
@@ -131,23 +133,8 @@ clib_mem_main_init (void)
mm->log2_sys_default_hugepage_sz = mm->log2_default_hugepage_sz;
/* numa nodes */
- va = mmap (0, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
- MAP_ANONYMOUS, -1, 0);
- if (va == MAP_FAILED)
- return;
-
- if (mlock (va, page_size))
- goto done;
-
- for (int i = 0; i < CLIB_MAX_NUMAS; i++)
- {
- int status;
- if (syscall (__NR_move_pages, 0, 1, &va, &i, &status, 0) == 0)
- mm->numa_node_bitmap |= 1ULL << i;
- }
-
-done:
- munmap (va, page_size);
+ if (syscall (__NR_get_mempolicy, &mode, &nodemask, maxnode, va, flags) == 0)
+ mm->numa_node_bitmap = nodemask;
}
__clib_export u64
@@ -528,8 +515,10 @@ __clib_export void
clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
uword n_pages, clib_mem_page_stats_t * stats)
{
- int i, *status = 0;
+ int *status = 0;
+ uword i;
void **ptr = 0;
+ unsigned char incore;
log2_page_size = clib_mem_log2_page_size_validate (log2_page_size);
@@ -551,6 +540,19 @@ clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
for (i = 0; i < n_pages; i++)
{
+ /* move_pages() returns -ENONET in status for huge pages on 5.19+ kernel.
+ * Retry with get_mempolicy() to obtain NUMA node info only if the pages
+ * are allocated and in memory, which is checked by mincore(). */
+ if (status[i] == -ENOENT &&
+ syscall (__NR_mincore, ptr[i], 1, &incore) == 0 && (incore & 1) != 0)
+ {
+ if (syscall (__NR_get_mempolicy, &status[i], 0, 0, ptr[i],
+ MPOL_F_NODE | MPOL_F_ADDR) != 0)
+ {
+ /* if get_mempolicy fails, keep the original value in status */
+ status[i] = -ENONET;
+ }
+ }
if (status[i] >= 0 && status[i] < CLIB_MAX_NUMAS)
{
stats->mapped++;
diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index 75015d59a4a..6211bb51f0a 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -299,10 +299,27 @@ void *clib_mem_init_thread_safe (void *memory, uword memory_size);
void clib_mem_exit (void);
+typedef struct
+{
+ /* Address of callers: outer first, inner last. */
+ void *callers[12];
+
+ /* Count of allocations with this traceback. */
+ u32 n_allocations;
+
+ /* Count of bytes allocated with this traceback. */
+ u32 n_bytes;
+
+ /* Offset of this item */
+ uword offset;
+} mheap_trace_t;
+
void clib_mem_trace (int enable);
int clib_mem_is_traced (void);
+mheap_trace_t *clib_mem_trace_dup (clib_mem_heap_t *heap);
+
typedef struct
{
/* Total number of objects allocated. */
diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c
index e98687fff2a..7944240390b 100644
--- a/src/vppinfra/mem_dlmalloc.c
+++ b/src/vppinfra/mem_dlmalloc.c
@@ -23,21 +23,6 @@
typedef struct
{
- /* Address of callers: outer first, inner last. */
- uword callers[12];
-
- /* Count of allocations with this traceback. */
- u32 n_allocations;
-
- /* Count of bytes allocated with this traceback. */
- u32 n_bytes;
-
- /* Offset of this item */
- uword offset;
-} mheap_trace_t;
-
-typedef struct
-{
clib_spinlock_t lock;
mheap_trace_t *traces;
@@ -68,7 +53,7 @@ mheap_get_trace_internal (const clib_mem_heap_t *heap, uword offset,
mheap_trace_t *t;
uword i, trace_index, *p;
mheap_trace_t trace = {};
- int index;
+ int n_callers;
if (heap != tm->current_traced_mheap || mheap_trace_thread_disable)
return;
@@ -82,19 +67,10 @@ mheap_get_trace_internal (const clib_mem_heap_t *heap, uword offset,
/* Turn off tracing for this thread to avoid embarrassment... */
mheap_trace_thread_disable = 1;
- index = -2; /* skip first 2 stack frames */
- foreach_clib_stack_frame (sf)
- {
- if (index >= 0)
- {
- if (index == ARRAY_LEN (trace.callers))
- break;
- trace.callers[index] = sf->ip;
- }
- index++;
- }
-
- if (index < 1)
+ /* Skip our frame and mspace_get_aligned's frame */
+ n_callers =
+ clib_stack_frame_get_raw (trace.callers, ARRAY_LEN (trace.callers), 2);
+ if (n_callers == 0)
goto out;
if (!tm->trace_by_callers)
@@ -574,6 +550,23 @@ clib_mem_trace_enable_disable (uword enable)
return rv;
}
+__clib_export mheap_trace_t *
+clib_mem_trace_dup (clib_mem_heap_t *heap)
+{
+ mheap_trace_main_t *tm = &mheap_trace_main;
+ mheap_trace_t *traces_copy = 0;
+
+ clib_spinlock_lock (&tm->lock);
+ if (vec_len (tm->traces) > 0 && heap == tm->current_traced_mheap)
+ {
+ traces_copy = vec_dup (tm->traces);
+ qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]),
+ mheap_trace_sort);
+ }
+ clib_spinlock_unlock (&tm->lock);
+ return traces_copy;
+}
+
__clib_export clib_mem_heap_t *
clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt, ...)
{
diff --git a/src/vppinfra/stack.c b/src/vppinfra/stack.c
index 190e880c228..12b24e3189f 100644
--- a/src/vppinfra/stack.c
+++ b/src/vppinfra/stack.c
@@ -17,7 +17,30 @@
static __thread unw_cursor_t cursor;
static __thread unw_context_t context;
-#endif
+#endif /* HAVE_LIBUNWIND */
+
+__clib_export int
+clib_stack_frame_get_raw (void **sf, int n, int skip)
+{
+#if HAVE_LIBUNWIND == 1
+ void *sf__[20];
+ int n__;
+
+ /* Also skip current frame. */
+ skip++;
+ n__ = unw_backtrace (sf__, clib_min (ARRAY_LEN (sf__), n + skip));
+
+ if (n__ <= skip)
+ return 0;
+ else if (n__ - skip < n)
+ n = n__ - skip;
+
+ clib_memcpy_fast (&sf[0], &sf__[skip], n * sizeof (sf[0]));
+ return n;
+#else /* HAVE_LIBUNWIND */
+ return 0;
+#endif /* HAVE_LIBUNWIND */
+}
__clib_export clib_stack_frame_t *
clib_stack_frame_get (clib_stack_frame_t *sf)
diff --git a/src/vppinfra/stack.h b/src/vppinfra/stack.h
index 98a621d4176..5b833a3811e 100644
--- a/src/vppinfra/stack.h
+++ b/src/vppinfra/stack.h
@@ -17,6 +17,7 @@ typedef struct
u8 is_signal_frame;
} clib_stack_frame_t;
+int clib_stack_frame_get_raw (void **sf, int n, int skip);
clib_stack_frame_t *clib_stack_frame_get (clib_stack_frame_t *);
#define foreach_clib_stack_frame(sf) \
diff --git a/src/vppinfra/time.c b/src/vppinfra/time.c
index 5a6aaf182e4..f1736499a0a 100644
--- a/src/vppinfra/time.c
+++ b/src/vppinfra/time.c
@@ -76,8 +76,11 @@ clock_frequency_from_proc_filesystem (void)
f64 ppc_timebase = 0; /* warnings be gone */
unformat_input_t input;
-/* $$$$ aarch64 kernel doesn't report "cpu MHz" */
-#if defined(__aarch64__)
+#if defined(__x86_64__)
+ if (clib_cpu_supports_aperfmperf ())
+ return 0.0;
+#elif defined(__aarch64__)
+ /* $$$$ aarch64 kernel doesn't report "cpu MHz" */
return 0.0;
#endif
diff --git a/src/vppinfra/time_range.c b/src/vppinfra/time_range.c
index 4b5e1303763..54f5629641a 100644
--- a/src/vppinfra/time_range.c
+++ b/src/vppinfra/time_range.c
@@ -264,11 +264,10 @@ format_clib_timebase_time (u8 * s, va_list * args)
clib_timebase_time_to_components (now, cp);
- s = format (s, "%s, %u %s %u %u:%02u:%02u",
- day_names_epoch_order[cp->day_name_index],
- cp->day,
- month_short_names[cp->month],
- cp->year, cp->hour, cp->minute, cp->second);
+ s = format (s, "%s, %02u %s %u %02u:%02u:%02u",
+ day_names_epoch_order[cp->day_name_index], cp->day,
+ month_short_names[cp->month], cp->year, cp->hour, cp->minute,
+ cp->second);
return (s);
}
diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c
index 88a56d88afc..05ca2f901c6 100644
--- a/src/vppinfra/unix-misc.c
+++ b/src/vppinfra/unix-misc.c
@@ -67,6 +67,8 @@
__clib_export __thread uword __os_thread_index = 0;
__clib_export __thread uword __os_numa_index = 0;
+__clib_export clib_bitmap_t *os_get_cpu_affinity_bitmap (int pid);
+
clib_error_t *
clib_file_n_bytes (char *file, uword * result)
{
@@ -275,6 +277,8 @@ os_get_online_cpu_core_bitmap ()
{
#if __linux__
return clib_sysfs_read_bitmap ("/sys/devices/system/cpu/online");
+#elif defined(__FreeBSD__)
+ return os_get_cpu_affinity_bitmap (0);
#else
return 0;
#endif
@@ -309,6 +313,9 @@ os_get_cpu_affinity_bitmap (int pid)
cpuset_t mask;
uword *r = NULL;
+ clib_bitmap_alloc (r, sizeof (CPU_SETSIZE));
+ clib_bitmap_zero (r);
+
if (cpuset_getaffinity (CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, -1,
sizeof (mask), &mask) != 0)
{
@@ -330,21 +337,6 @@ os_get_online_cpu_node_bitmap ()
{
#if __linux__
return clib_sysfs_read_bitmap ("/sys/devices/system/node/online");
-#elif defined(__FreeBSD__)
- domainset_t domain;
- uword *r = NULL;
- int policy;
-
- if (cpuset_getdomain (CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, -1,
- sizeof (domain), &domain, &policy) != 0)
- {
- clib_bitmap_free (r);
- return NULL;
- }
-
- for (int bit = 0; bit < CPU_SETSIZE; bit++)
- clib_bitmap_set (r, bit, CPU_ISSET (bit, &domain));
- return r;
#else
return 0;
#endif