diff options
Diffstat (limited to 'src/vppinfra')
31 files changed, 1609 insertions, 2864 deletions
diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt index f34ceed9d15..83a8b2a7e57 100644 --- a/src/vppinfra/CMakeLists.txt +++ b/src/vppinfra/CMakeLists.txt @@ -14,6 +14,26 @@ enable_language(ASM) ############################################################################## +# find libdl +############################################################################## +list(APPEND VPPINFRA_LIBS ${CMAKE_DL_LIBS}) + +############################################################################## +# find libunwind +############################################################################## +vpp_find_path(LIBUNWIND_INCLUDE_DIR unwind.h) +vpp_find_library(LIBUNWIND_LIB NAMES unwind libunwind) + +if (LIBUNWIND_INCLUDE_DIR AND LIBUNWIND_LIB) + message(STATUS "libunwind found at ${LIBUNWIND_LIB}") + list(APPEND VPPINFRA_LIBS ${LIBUNWIND_LIB}) + add_definitions(-DHAVE_LIBUNWIND=1) +else() + message(WARNING "libunwind not found - stack traces disabled") + add_definitions(-DHAVE_LIBUNWIND=0) +endif() + +############################################################################## # Generate vppinfra/config.h ############################################################################## set(LOG2_CACHE_LINE_BYTES ${VPP_LOG2_CACHE_LINE_SIZE}) @@ -42,15 +62,14 @@ add_definitions(-fvisibility=hidden) set_source_files_properties( cJSON.c jsonformat.c PROPERTIES COMPILE_DEFINITIONS " CJSON_API_VISIBILITY " ) - ############################################################################## # vppinfra sources ############################################################################## set(VPPINFRA_SRCS - backtrace.c bitmap.c bihash_all_vector.c cpu.c + devicetree.c dlmalloc.c elf.c elog.c @@ -80,6 +99,7 @@ set(VPPINFRA_SRCS rbtree.c serialize.c socket.c + stack.c std-formats.c string.c time.c @@ -132,6 +152,7 @@ set(VPPINFRA_HEADERS crypto/aes_ctr.h crypto/aes_gcm.h crypto/poly1305.h + devicetree.h dlist.h dlmalloc.h elf_clib.h @@ -142,6 +163,7 @@ set(VPPINFRA_HEADERS fifo.h file.h format.h + format_ansi.h format_table.h hash.h heap.h @@ -175,6 +197,7 @@ set(VPPINFRA_HEADERS smp.h socket.h sparse_vec.h + stack.h string.h time.h time_range.h @@ -229,13 +252,9 @@ elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") ) endif() -option(VPP_USE_EXTERNAL_LIBEXECINFO "Use external libexecinfo (useful for non-glibc targets)." OFF) -if(VPP_USE_EXTERNAL_LIBEXECINFO) - set(EXECINFO_LIB execinfo) -endif() add_vpp_library(vppinfra SOURCES ${VPPINFRA_SRCS} - LINK_LIBRARIES m ${EXECINFO_LIB} + LINK_LIBRARIES m ${VPPINFRA_LIBS} INSTALL_HEADERS ${VPPINFRA_HEADERS} COMPONENT libvppinfra LTO @@ -260,6 +279,7 @@ if(VPP_BUILD_VPPINFRA_TESTS) longjmp macros maplog + mhash pmalloc pool_alloc pool_iterate diff --git a/src/vppinfra/asm_mips.h b/src/vppinfra/asm_mips.h deleted file mode 100644 index 7c9e69586f4..00000000000 --- a/src/vppinfra/asm_mips.h +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - Copyright (c) 2004 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#ifndef included_asm_mips_h -#define included_asm_mips_h - -/* Encoding of MIPS instructions. */ -/* Encoding of opcode field (op). */ -#define mips_foreach_opcode \ - _(SPECIAL) _(REGIMM) _(j) _(jal) _(beq) _(bne) _(blez) _(bgtz) \ - _(addi) _(addiu) _(slti) _(sltiu) _(andi) _(ori) _(xori) _(lui) \ - _(COP0) _(COP1) _(COP2) _(COP1X) _(beql) _(bnel) _(blezl) _(bgtzl) \ - _(daddi) _(daddiu) _(ldl) _(ldr) _(SPECIAL2) _(jalx) _(MDMX) _(O37) \ - _(lb) _(lh) _(lwl) _(lw) _(lbu) _(lhu) _(lwr) _(lwu) \ - _(sb) _(sh) _(swl) _(sw) _(sdl) _(sdr) _(swr) _(cache) \ - _(ll) _(lwc1) _(lwc2) _(pref) _(lld) _(ldc1) _(ldc2) _(ld) \ - _(sc) _(swc1) _(swc2) _(o73) _(scd) _(sdc1) _(sdc2) _(sd) - -/* Encoding of funct field. */ -#define mips_foreach_special_funct \ - _(sll) _(MOVCI) _(srl) _(sra) _(sllv) _(o05) _(srlv) _(srav) \ - _(jr) _(jalr) _(movz) _(movn) _(syscall) _(break) _(o16) _(sync) \ - _(mfhi) _(mthi) _(mflo) _(mtlo) _(dsllv) _(o25) _(dsrlv) _(dsrav) \ - _(mult) _(multu) _(div) _(divu) _(dmult) _(dmultu) _(ddiv) _(ddivu) \ - _(add) _(addu) _(sub) _(subu) _(and) _(or) _(xor) _(nor) \ - _(o50) _(o51) _(slt) _(sltu) _(dadd) _(daddu) _(dsub) _(dsubu) \ - _(tge) _(tgeu) _(tlt) _(tltu) _(teq) _(o65) _(tne) _(o67) \ - _(dsll) _(o71) _(dsrl) _(dsra) _(dsll32) _(o75) _(dsrl32) _(dsra32) - -/* SPECIAL2 encoding of funct field. */ -#define mips_foreach_special2_funct \ - _(madd) _(maddu) _(mul) _(o03) _(msub) _(msubu) _(o06) _(o07) \ - _(o10) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(o20) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) \ - _(clz) _(clo) _(o42) _(o43) _(dclz) _(dclo) _(o46) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(o60) _(o61) _(o62) _(o63) _(o64) _(o65) _(o66) _(o67) \ - _(o70) _(o71) _(o72) _(o73) _(o74) _(o75) _(o76) _(sdbbp) - -/* REGIMM encoding of rt field. */ -#define mips_foreach_regimm_rt \ - _(bltz) _(bgez) _(bltzl) _(bgezl) _(o04) _(o05) _(o06) _(o07) \ - _(tgei) _(tgeiu) _(tltiu) _(teqi) _(o14) _(tnei) _(o16) _(o17) \ - _(bltzal) _(bgezal) _(bltzall) _(bgezall) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) - -/* COP0 encoding of rs field. */ -#define mips_foreach_cop0_rs \ - _(mfc0) _(dmfc0) _(o02) _(o03) _(mtc0) _(dmtc0) _(o06) _(o07) \ - _(o10) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(C0) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) - -/* COP0 encoding of funct when rs == RS_CO */ -#define mips_foreach_cop0_funct \ - _(o00) _(tlbr) _(tlbwi) _(o03) _(o04) _(o05) _(tlbwr) _(o07) \ - _(tlbp) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(o20) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(eret) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(deret) \ - _(wait) _(o41) _(o42) _(o43) _(o44) _(o45) _(o46) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(o60) _(o61) _(o62) _(o63) _(o64) _(o65) _(o66) _(o67) \ - _(o70) _(o71) _(o72) _(o73) _(o74) _(o75) _(o76) _(o77) - -/* COP1 encoding of rs field. */ -#define mips_foreach_cop1_rs \ - _(mfc1) _(dmfc1) _(cfc1) _(o03) _(mtc1) _(dmtc1) _(ctc1) _(o07) \ - _(BC1) _(o11) _(o12) _(o13) _(o14) _(o15) _(o16) _(o17) \ - _(S) _(D) _(o22) _(o23) _(W) _(L) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) - -/* COP1 encoding of funct for S and D */ -#define mips_foreach_cop1_funct \ - _(add) _(sub) _(mul) _(div) _(sqrt) _(abs) _(mov) _(neg) \ - _(roundl) _(truncl) _(ceill) _(floorl) _(roundw) _(truncw) _(ceilw) _(floorw) \ - _(o20) _(MOVCF) _(movz) _(movn) _(o24) _(recip) _(rsqrt) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) \ - _(cvts) _(cvtd) _(o42) _(o43) _(cvtw) _(cvtl) _(o46) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(cf) _(cun) _(ceq) _(cueq) _(colt) _(cult) _(cole) _(cule) \ - _(csf) _(cngle) _(cseq) _(cngl) _(clt) _(cnge) _(cle) _(cngt) - -/* COP1X encoding of funct */ -#define mips_foreach_cop1x_funct \ - _(lwxc1) _(ldxc1) _(o02) _(o03) _(o04) _(luxc1) _(o06) _(o07) \ - _(swxc1) _(sdxc1) _(o12) _(o13) _(o14) _(suxc1) _(o16) _(prefx) \ - _(o20) _(o21) _(o22) _(o23) _(o24) _(o25) _(o26) _(o27) \ - _(o30) _(o31) _(o32) _(o33) _(o34) _(o35) _(o36) _(o37) \ - _(madds) _(maddd) _(o42) _(o43) _(o44) _(o45) _(o46) _(o47) \ - _(msubs) _(msubd) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(nmadds) _(nmaddd) _(o62) _(o63) _(o64) _(o65) _(o66) _(o67) \ - _(nmsubs) _(nmsubd) _(o72) _(o73) _(o74) _(o75) _(o76) _(o77) - -#define mips_foreach_mdmx_funct \ - _(msgn) _(ceq) _(pickf) _(pickt) _(clt) _(cle) _(min) _(max) \ - _(o10) _(o11) _(sub) _(add) _(and) _(xor) _(or) _(nor) \ - _(sll) _(o21) _(srl) _(sra) _(o24) _(o25) _(o26) _(o27) \ - _(alniob) _(alnvob) _(alniqh) _(alnvqh) _(o34) _(o35) _(o36) _(shfl) \ - _(rzu) _(rnau) _(rneu) _(o43) _(rzs) _(rnas) _(rnes) _(o47) \ - _(o50) _(o51) _(o52) _(o53) _(o54) _(o55) _(o56) _(o57) \ - _(mul) _(o61) _(muls) _(mula) _(o64) _(o65) _(suba) _(adda) \ - _(o70) _(o71) _(o72) _(o73) _(o74) _(o75) _(wac) _(rac) - -#define _(f) MIPS_OPCODE_##f, -typedef enum -{ - mips_foreach_opcode -} mips_insn_opcode_t; -#undef _ - -#define _(f) MIPS_SPECIAL_FUNCT_##f, -typedef enum -{ - mips_foreach_special_funct -} mips_insn_special_funct_t; -#undef _ - -#define _(f) MIPS_SPECIAL2_FUNCT_##f, -typedef enum -{ - mips_foreach_special2_funct -} mips_insn_special2_funct_t; -#undef _ - -#define _(f) MIPS_REGIMM_RT_##f, -typedef enum -{ - mips_foreach_regimm_rt -} mips_insn_regimm_rt_t; -#undef _ - -#define _(f) MIPS_COP0_RS_##f, -typedef enum -{ - mips_foreach_cop0_rs -} mips_insn_cop0_rs_t; -#undef _ - -#define _(f) MIPS_COP0_FUNCT_##f, -typedef enum -{ - mips_foreach_cop0_funct -} mips_insn_cop0_funct_t; -#undef _ - -#define _(f) MIPS_COP1_RS_##f, -typedef enum -{ - mips_foreach_cop1_rs -} mips_insn_cop1_rs_t; -#undef _ - -#define _(f) MIPS_COP1_FUNCT_##f, -typedef enum -{ - mips_foreach_cop1_funct -} mips_insn_cop1_funct_t; -#undef _ - -#define _(f) MIPS_COP1X_FUNCT_##f, -typedef enum -{ - mips_foreach_cop1x_funct -} mips_insn_cop1x_funct_t; -#undef _ - -#define _(f) MIPS_MDMX_FUNCT_##f, -typedef enum -{ - mips_foreach_mdmx_funct -} mips_insn_mdmx_funct_t; -#undef _ - -always_inline mips_insn_opcode_t -mips_insn_get_op (u32 insn) -{ - return (insn >> 26) & 0x3f; -} - -always_inline u32 -mips_insn_get_rs (u32 insn) -{ - return (insn >> 21) & 0x1f; -} - -always_inline u32 -mips_insn_get_rt (u32 insn) -{ - return (insn >> 16) & 0x1f; -} - -always_inline u32 -mips_insn_get_rd (u32 insn) -{ - return (insn >> 11) & 0x1f; -} - -always_inline u32 -mips_insn_get_sa (u32 insn) -{ - return (insn >> 6) & 0x1f; -} - -always_inline u32 -mips_insn_get_funct (u32 insn) -{ - return (insn >> 0) & 0x3f; -} - -always_inline i32 -mips_insn_get_immediate (u32 insn) -{ - return (((i32) insn) << 16) >> 16; -} - -always_inline u32 -mips_insn_encode_i_type (int op, int rs, int rt, int immediate) -{ - u32 insn; - insn = immediate; - insn |= rt << 16; - insn |= rs << 21; - insn |= op << 26; - - ASSERT (mips_insn_get_immediate (insn) == immediate); - ASSERT (mips_insn_get_rt (insn) == rt); - ASSERT (mips_insn_get_rs (insn) == rt); - ASSERT (mips_insn_get_op (insn) == op); - - return insn; -} - -always_inline u32 -mips_insn_encode_j_type (int op, u32 addr) -{ - u32 insn; - - insn = (addr & ((1 << 28) - 1)) / 4; - insn |= op << 26; - - return insn; -} - -always_inline u32 -mips_insn_encode_r_type (int op, int rs, int rt, int rd, int sa, int funct) -{ - u32 insn; - insn = funct; - insn |= sa << 6; - insn |= rd << 11; - insn |= rt << 16; - insn |= rs << 21; - insn |= op << 26; - - ASSERT (mips_insn_get_funct (insn) == funct); - ASSERT (mips_insn_get_sa (insn) == sa); - ASSERT (mips_insn_get_rd (insn) == rd); - ASSERT (mips_insn_get_rt (insn) == rt); - ASSERT (mips_insn_get_rs (insn) == rt); - ASSERT (mips_insn_get_op (insn) == op); - - return insn; -} - -#define mips_insn_r(op,funct,rd,rs,rt,sa) \ - mips_insn_encode_r_type (MIPS_OPCODE_##op, \ - (rs), (rt), (rd), (sa), \ - MIPS_##op##_FUNCT_##funct) - -#define mips_insn_i(op,rs,rt,imm) \ - mips_insn_encode_i_type (MIPS_OPCODE_##op, (rs), (rt), (imm)) - -#define mips_insn_j(op,target) \ - mips_insn_encode_i_type (MIPS_OPCODE_##op, (rs), (rt), (imm)) - -/* Generate unsigned load instructions of data of various sizes. */ -always_inline u32 -mips_insn_load (u32 rd, i32 offset, u32 base, u32 log2_bytes) -{ - int op; - - ASSERT (log2_bytes < 4); - switch (log2_bytes) - { - case 0: - op = MIPS_OPCODE_lbu; - break; - case 1: - op = MIPS_OPCODE_lhu; - break; - case 2: - op = MIPS_OPCODE_lwu; - break; - case 3: - op = MIPS_OPCODE_ld; - break; - } - - return mips_insn_encode_i_type (op, base, rd, offset); -} - -typedef enum -{ - MIPS_REG_SP = 29, - MIPS_REG_RA = 31, -} mips_reg_t; - -#endif /* included_asm_mips_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/asm_x86.c b/src/vppinfra/asm_x86.c deleted file mode 100644 index e6e00ce5543..00000000000 --- a/src/vppinfra/asm_x86.c +++ /dev/null @@ -1,1947 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* FIXME - opcode name remove to save table space; enum - x87 - 3dnow - cbw naming -*/ - -#include <vppinfra/error.h> -#include <vppinfra/byte_order.h> -#include <vppinfra/asm_x86.h> - -#define foreach_x86_gp_register \ - _ (AX) _ (CX) _ (DX) _ (BX) \ - _ (SP) _ (BP) _ (SI) _ (DI) - -typedef enum { -#define _(r) X86_INSN_GP_REG_##r, - foreach_x86_gp_register -#undef _ -} x86_insn_gp_register_t; - -typedef union { - struct { - u8 rm : 3; - u8 reg : 3; - u8 mode : 2; - }; - u8 byte; -} x86_insn_modrm_byte_t; - -typedef union { - struct { - u8 base : 3; - u8 index : 3; - u8 log2_scale : 2; - }; - u8 byte; -} x86_insn_sib_byte_t; - -always_inline uword -x86_insn_has_modrm_byte (x86_insn_t * insn) -{ - int i; - for (i = 0; i < ARRAY_LEN (insn->operands); i++) - switch (insn->operands[i].code) - { - case 'G': case 'E': case 'M': case 'R': - return 1; - } - return 0; -} - -always_inline uword -x86_insn_immediate_type (x86_insn_t * insn) -{ - int i; - for (i = 0; i < ARRAY_LEN (insn->operands); i++) - switch (insn->operands[i].code) - { - case 'J': - case 'I': - case 'O': - return insn->operands[i].type; - } - return 0; -} - -/* Opcode extension in modrm byte reg field. */ -#define foreach_x86_insn_modrm_reg_group \ - _ (1) _ (1a) _ (2) _ (3) _ (4) _ (5) _ (6) _ (7) \ - _ (8) _ (9) _ (10) _ (11) _ (12) _ (13) _ (14) \ - _ (15) _ (16) _ (p) - -#define foreach_x86_insn_sse_group \ - _ (10) _ (28) _ (50) _ (58) _ (60) _ (68) _ (70) _ (78) \ - _ (c0) _ (d0) _ (d8) _ (e0) _ (e8) _ (f0) _ (f8) - -enum { -#define _(x) X86_INSN_MODRM_REG_GROUP_##x, - foreach_x86_insn_modrm_reg_group -#undef _ -#define _(x) X86_INSN_SSE_GROUP_##x, - foreach_x86_insn_sse_group -#undef _ -}; - -enum { -#define _(x) \ - X86_INSN_FLAG_MODRM_REG_GROUP_##x \ - = X86_INSN_FLAG_SET_MODRM_REG_GROUP (1 + X86_INSN_MODRM_REG_GROUP_##x), - foreach_x86_insn_modrm_reg_group -#undef _ - -#define _(x) \ - X86_INSN_FLAG_SSE_GROUP_##x \ - = X86_INSN_FLAG_SET_SSE_GROUP (1 + X86_INSN_SSE_GROUP_##x), - foreach_x86_insn_sse_group -#undef _ -}; - -#define foreach_x86_gp_reg \ - _ (AX) _ (CX) _ (DX) _ (BX) \ - _ (SP) _ (BP) _ (SI) _ (DI) - -#define foreach_x86_condition \ - _ (o) _ (no) _ (b) _ (nb) \ - _ (z) _ (nz) _ (be) _ (nbe) \ - _ (s) _ (ns) _ (p) _ (np) \ - _ (l) _ (nl) _ (le) _ (nle) - -#define _3f(x,f,o0,o1,o2) \ -{ \ - .name = #x, \ - .flags = (f), \ - .operands[0] = { .data = #o0 }, \ - .operands[1] = { .data = #o1 }, \ - .operands[2] = { .data = #o2 }, \ -} - -#define _2f(x,f,o0,o1) _3f(x,f,o0,o1,__) -#define _1f(x,f,o0) _2f(x,f,o0,__) -#define _0f(x,f) _1f(x,f,__) - -#define _3(x,o0,o1,o2) _3f(x,0,o0,o1,o2) -#define _2(x,o0,o1) _2f(x,0,o0,o1) -#define _1(x,o0) _1f(x,0,o0) -#define _0(x) _0f(x,0) - -static x86_insn_t x86_insns_one_byte[256] = { - -#define _(x) \ - _2 (x, Eb, Gb), \ - _2 (x, Ev, Gv), \ - _2 (x, Gb, Eb), \ - _2 (x, Gv, Ev), \ - _2 (x, AL, Ib), \ - _2 (x, AX, Iz) - - /* 0x00 */ - _ (add), - _0 (push_es), - _0 (pop_es), - _ (or), - _0 (push_cs), - _0 (escape_two_byte), - - /* 0x10 */ - _ (adc), - _0 (push_ss), - _0 (pop_ss), - _ (sbb), - _0 (push_ds), - _0 (pop_ds), - - /* 0x20 */ - _ (and), - _0 (segment_es), - _0 (daa), - _ (sub), - _0 (segment_cs), - _0 (das), - - /* 0x30 */ - _ (xor), - _0 (segment_ss), - _0 (aaa), - _ (cmp), - _0 (segment_ds), - _0 (aas), - -#undef _ - - /* 0x40 */ -#define _(r) _1 (inc, r), - foreach_x86_gp_reg -#undef _ -#define _(r) _1 (dec, r), - foreach_x86_gp_reg -#undef _ - - /* 0x50 */ -#define _(r) _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, r), - foreach_x86_gp_reg -#undef _ -#define _(r) _1f (pop, X86_INSN_FLAG_DEFAULT_64_BIT, r), - foreach_x86_gp_reg -#undef _ - - /* 0x60 */ - _0 (pusha), - _0 (popa), - _2 (bound, Gv, Ma), - _2 (movsxd, Gv, Ed), - _0 (segment_fs), - _0 (segment_gs), - _0 (operand_type), - _0 (address_size), - _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, Iz), - _3 (imul, Gv, Ev, Iz), - _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, Ib), - _3 (imul, Gv, Ev, Ib), - _1 (insb, DX), - _1 (insw, DX), - _1 (outsb, DX), - _1 (outsw, DX), - - /* 0x70 */ -#define _(x) _1 (j##x, Jb), - foreach_x86_condition -#undef _ - - /* 0x80 */ - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Eb, Ib), - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Ev, Iz), - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Eb, Ib), - _2f (modrm_group_1, X86_INSN_FLAG_MODRM_REG_GROUP_1, Ev, Ib), - _2 (test, Eb, Gb), - _2 (test, Ev, Gv), - _2 (xchg, Eb, Gb), - _2 (xchg, Ev, Gv), - _2 (mov, Eb, Gb), - _2 (mov, Ev, Gv), - _2 (mov, Gb, Eb), - _2 (mov, Gv, Ev), - _2 (mov, Ev, Sw), - _2 (lea, Gv, Ev), - _2 (mov, Sw, Ew), - _1f (modrm_group_1a, X86_INSN_FLAG_MODRM_REG_GROUP_1a, Ev), - - /* 0x90 */ - _0 (nop), - _1 (xchg, CX), - _1 (xchg, DX), - _1 (xchg, BX), - _1 (xchg, SP), - _1 (xchg, BP), - _1 (xchg, SI), - _1 (xchg, DI), - _0 (cbw), - _0 (cwd), - _1 (call, Ap), - _0 (wait), - _0 (pushf), - _0 (popf), - _0 (sahf), - _0 (lahf), - - /* 0xa0 */ - _2 (mov, AL, Ob), - _2 (mov, AX, Ov), - _2 (mov, Ob, AL), - _2 (mov, Ov, AX), - _0 (movsb), - _0 (movsw), - _0 (cmpsb), - _0 (cmpsw), - _2 (test, AL, Ib), - _2 (test, AX, Iz), - _1 (stosb, AL), - _1 (stosw, AX), - _1 (lodsb, AL), - _1 (lodsw, AX), - _1 (scasb, AL), - _1 (scasw, AX), - - /* 0xb0 */ - _2 (mov, AL, Ib), - _2 (mov, CL, Ib), - _2 (mov, DL, Ib), - _2 (mov, BL, Ib), - _2 (mov, AH, Ib), - _2 (mov, CH, Ib), - _2 (mov, DH, Ib), - _2 (mov, BH, Ib), -#define _(r) _2 (mov, r, Iv), - foreach_x86_gp_reg -#undef _ - - /* 0xc0 */ - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Eb, Ib), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Ev, Ib), - _1 (ret, Iw), - _0 (ret), - _2 (les, Gz, Mp), - _2 (lds, Gz, Mp), - _2f (modrm_group_11, X86_INSN_FLAG_MODRM_REG_GROUP_11, Eb, Ib), - _2f (modrm_group_11, X86_INSN_FLAG_MODRM_REG_GROUP_11, Ev, Iz), - _2 (enter, Iw, Ib), - _0 (leave), - _1 (ret, Iw), - _0 (ret), - _0 (int3), - _1 (int, Ib), - _0 (into), - _0 (iret), - - /* 0xd0 */ - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Eb, 1b), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Ev, 1b), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Eb, CL), - _2f (modrm_group_2, X86_INSN_FLAG_MODRM_REG_GROUP_2, Ev, CL), - _0 (aam), - _0 (aad), - _0 (salc), - _0 (xlat), - /* FIXME x87 */ - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - - /* 0xe0 */ - _1 (loopnz, Jb), - _1 (loopz, Jb), - _1 (loop, Jb), - _1 (jcxz, Jb), - _2 (in, AL, Ib), - _2 (in, AX, Ib), - _2 (out, Ib, AL), - _2 (out, Ib, AX), - _1f (call, X86_INSN_FLAG_DEFAULT_64_BIT, Jz), - _1f ( jmp, X86_INSN_FLAG_DEFAULT_64_BIT, Jz), - _1 (jmp, Ap), - _1 (jmp, Jb), - _2 (in, AL, DX), - _2 (in, AX, DX), - _2 (out, DX, AL), - _2 (out, DX, AX), - - /* 0xf0 */ - _0 (lock), - _0 (int1), - _0 (repne), - _0 (rep), - _0 (hlt), - _0 (cmc), - _0f (modrm_group_3, X86_INSN_FLAG_MODRM_REG_GROUP_3), - _0f (modrm_group_3, X86_INSN_FLAG_MODRM_REG_GROUP_3), - _0 (clc), - _0 (stc), - _0 (cli), - _0 (sti), - _0 (cld), - _0 (std), - _1f (modrm_group_4, X86_INSN_FLAG_MODRM_REG_GROUP_4, Eb), - _0f (modrm_group_5, X86_INSN_FLAG_MODRM_REG_GROUP_5), -}; - -static x86_insn_t x86_insns_two_byte[256] = { - /* 0x00 */ - _0f (modrm_group_6, X86_INSN_FLAG_MODRM_REG_GROUP_6), - _0f (modrm_group_7, X86_INSN_FLAG_MODRM_REG_GROUP_7), - _2 (lar, Gv, Ew), - _2 (lsl, Gv, Ew), - _0 (bad), - _0 (syscall), - _0 (clts), - _0 (sysret), - _0 (invd), - _0 (wbinvd), - _0 (bad), - _0 (ud2), - _0 (bad), - _0f (modrm_group_p, X86_INSN_FLAG_MODRM_REG_GROUP_p), - _0 (femms), - _0 (escape_3dnow), - - /* 0x10 */ - _2f (movups, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (movups, X86_INSN_FLAG_SSE_GROUP_10, Ex, Gx), - _2f (movlps, X86_INSN_FLAG_SSE_GROUP_10, Ex, Gx), - _2f (movlps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (unpcklps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (unpckhps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _2f (movhps, X86_INSN_FLAG_SSE_GROUP_10, Ex, Gx), - _2f (movhps, X86_INSN_FLAG_SSE_GROUP_10, Gx, Ex), - _0f (modrm_group_16, X86_INSN_FLAG_MODRM_REG_GROUP_16), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - _0 (nop), - - /* 0x20 */ - _2 (mov, Rv, Cv), - _2 (mov, Rv, Dv), - _2 (mov, Cv, Rv), - _2 (mov, Dv, Rv), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2f (movaps, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (movaps, X86_INSN_FLAG_SSE_GROUP_28, Ex, Gx), - _2f (cvtpi2ps, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (movntps, X86_INSN_FLAG_SSE_GROUP_28, Mx, Gx), - _2f (cvttps2pi, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (cvtps2pi, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (ucomiss, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - _2f (comiss, X86_INSN_FLAG_SSE_GROUP_28, Gx, Ex), - - /* 0x30 */ - _0 (wrmsr), - _0 (rdtsc), - _0 (rdmsr), - _0 (rdpmc), - _0 (sysenter), - _0 (sysexit), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - - /* 0x40 */ -#define _(x) _2 (cmov##x, Gv, Ev), - foreach_x86_condition -#undef _ - - /* 0x50 */ - _2f (movmskps, X86_INSN_FLAG_SSE_GROUP_50, Gd, Rx), - _2f (sqrtps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (rsqrtps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (rcpps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (andps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (andnps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (orps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (xorps, X86_INSN_FLAG_SSE_GROUP_50, Gx, Ex), - _2f (addps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (mulps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (cvtps2pd, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (cvtdq2ps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (subps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (minps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (divps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - _2f (maxps, X86_INSN_FLAG_SSE_GROUP_58, Gx, Ex), - - /* 0x60 */ - _2f (punpcklbw, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (punpcklwd, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (punpckldq, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (packsswb, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (pcmpgtb, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (pcmpgtw, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (pcmpgtd, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (packuswb, X86_INSN_FLAG_SSE_GROUP_60, Gm, Em), - _2f (punpckhbw, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (punpckhwd, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (punpckhdq, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (packssdw, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_68), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_68), - _2f (movd, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - _2f (movq, X86_INSN_FLAG_SSE_GROUP_68, Gm, Em), - - /* 0x70 */ - _3f (pshufw, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em, Ib), - _0f (modrm_group_12, X86_INSN_FLAG_MODRM_REG_GROUP_12), - _0f (modrm_group_13, X86_INSN_FLAG_MODRM_REG_GROUP_13), - _0f (modrm_group_14, X86_INSN_FLAG_MODRM_REG_GROUP_14), - _2f (pcmpeqb, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em), - _2f (pcmpeqw, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em), - _2f (pcmpeqd, X86_INSN_FLAG_SSE_GROUP_70, Gm, Em), - _0f (emms, X86_INSN_FLAG_SSE_GROUP_70), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_78), - _2f (movd, X86_INSN_FLAG_SSE_GROUP_78, Em, Gm), - _2f (movq, X86_INSN_FLAG_SSE_GROUP_78, Em, Gm), - - /* 0x80 */ -#define _(x) _1 (jmp##x, Jz), - foreach_x86_condition -#undef _ - - /* 0x90 */ -#define _(x) _1 (set##x, Eb), - foreach_x86_condition -#undef _ - - /* 0xa0 */ - _0 (push_fs), - _0 (pop_fs), - _0 (cpuid), - _2 (bt, Ev, Gv), - _3 (shld, Ev, Gv, Ib), - _3 (shld, Ev, Gv, CL), - _0 (bad), - _0 (bad), - _0 (push_gs), - _0 (pop_gs), - _0 (rsm), - _2 (bts, Ev, Gv), - _3 (shrd, Ev, Gv, Ib), - _3 (shrd, Ev, Gv, CL), - _0f (modrm_group_15, X86_INSN_FLAG_MODRM_REG_GROUP_15), - _2 (imul, Gv, Ev), - - /* 0xb0 */ - _2 (cmpxchg, Eb, Gb), - _2 (cmpxchg, Ev, Gv), - _2 (lss, Gz, Mp), - _2 (btr, Ev, Gv), - _2 (lfs, Gz, Mp), - _2 (lgs, Gz, Mp), - _2 (movzbl, Gv, Eb), - _2 (movzwl, Gv, Ew), - _0 (bad), - _0f (modrm_group_10, X86_INSN_FLAG_MODRM_REG_GROUP_10), - _2f (modrm_group_8, X86_INSN_FLAG_MODRM_REG_GROUP_8, Ev, Ib), - _2 (btc, Ev, Gv), - _2 (bsf, Gv, Ev), - _2 (bsr, Gv, Ev), - _2 (movsx, Gv, Eb), - _2 (movsx, Gv, Ew), - - /* 0xc0 */ - _2 (xadd, Eb, Gb), - _2 (xadd, Ev, Gv), - _3f (cmpps, X86_INSN_FLAG_SSE_GROUP_c0, Gx, Ex, Ib), - _2 (movnti, Mv, Gv), - _3f (pinsrw, X86_INSN_FLAG_SSE_GROUP_c0, Gm, Ew, Ib), - _3f (pextrw, X86_INSN_FLAG_SSE_GROUP_c0, Gd, Rm, Ib), - _3f (shufps, X86_INSN_FLAG_SSE_GROUP_c0, Gx, Ex, Ib), - _1f (modrm_group_9, X86_INSN_FLAG_MODRM_REG_GROUP_9, Mx), -#define _(r) _1 (bswap, r), - foreach_x86_gp_reg -#undef _ - - /* 0xd0 */ - _0f (bad, X86_INSN_FLAG_SSE_GROUP_d0), - _2f (psrlw, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (psrld, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (psrlq, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (paddq, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _2f (pmullw, X86_INSN_FLAG_SSE_GROUP_d0, Gm, Em), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_d0), - _2f (pmovmskb, X86_INSN_FLAG_SSE_GROUP_d0, Gd, Rm), - _2f (psubusb, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (psubusw, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pminub, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pand, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (paddusb, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (paddusw, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pmaxub, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - _2f (pandn, X86_INSN_FLAG_SSE_GROUP_d8, Gm, Em), - - /* 0xe0 */ - _2f (pavgb, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (psraw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (psrad, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (pavgw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (pmulhuw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (pmulhw, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (bad, X86_INSN_FLAG_SSE_GROUP_e0, Gm, Em), - _2f (movntq, X86_INSN_FLAG_SSE_GROUP_e0, Mm, Gm), - _2f (psubsb, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (psubsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (pminsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (por, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (paddsb, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (paddsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (pmaxsw, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - _2f (pxor, X86_INSN_FLAG_SSE_GROUP_e8, Gm, Em), - - /* 0xf0 */ - _0f (bad, X86_INSN_FLAG_SSE_GROUP_f0), - _2f (psllw, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (pslld, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (psllq, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (pmuludq, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (pmaddwd, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (psadbw, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (maskmovq, X86_INSN_FLAG_SSE_GROUP_f0, Gm, Em), - _2f (psubb, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (psubw, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (psubd, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (psubq, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (paddb, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (paddw, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _2f (paddd, X86_INSN_FLAG_SSE_GROUP_f8, Gm, Em), - _0f (bad, X86_INSN_FLAG_SSE_GROUP_f8), -}; - -typedef struct { - x86_insn_t insns[8]; -} x86_insn_group8_t; - -/* Escape groups are indexed by modrm reg field. */ -static x86_insn_group8_t x86_insn_modrm_reg_groups[] = { - [X86_INSN_MODRM_REG_GROUP_1].insns = { - _0 (add), _0 ( or), _0 (adc), _0 (sbb), - _0 (and), _0 (sub), _0 (xor), _0 (cmp), - }, - - [X86_INSN_MODRM_REG_GROUP_1a].insns = { - _0f (pop, X86_INSN_FLAG_DEFAULT_64_BIT), - _0 (bad), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_2].insns = { - _0 (rol), _0 (ror), _0 (rcl), _0 (rcr), - _0 (shl), _0 (shr), _0 (sal), _0 (sar), - }, - - [X86_INSN_MODRM_REG_GROUP_3].insns = { - _0 (test), _0 (test), _0 (not), _0 (neg), - _0 (mul), _0 (imul), _0 (div), _0 (idiv), - }, - - [X86_INSN_MODRM_REG_GROUP_4].insns = { - _0 (inc), _0 (dec), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_5].insns = { - _1 (inc, Ev), - _1 (dec, Ev), - _1f (call, X86_INSN_FLAG_DEFAULT_64_BIT, Ev), - _1 (call, Mp), - _1f (jmp, X86_INSN_FLAG_DEFAULT_64_BIT, Ev), - _1 (jmp, Mp), - _1f (push, X86_INSN_FLAG_DEFAULT_64_BIT, Ev), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_6].insns = { - _1 (sldt, Ev), - _1 (str, Ev), - _1 (lldt, Ev), - _1 (ltr, Ev), - _1 (verr, Ev), - _1 (verw, Ev), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_7].insns = { - _1 (sgdt, Mv), - _1 (sidt, Mv), - _1 (lgdt, Mv), - _1 (lidt, Mv), - _1 (smsw, Ev), - _0 (bad), - _1 (lmsw, Ew), - _1 (invlpg, Mv), - }, - - [X86_INSN_MODRM_REG_GROUP_8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (bt, Ev, Ib), - _2 (bts, Ev, Ib), - _2 (btr, Ev, Ib), - _2 (btc, Ev, Ib), - }, - - [X86_INSN_MODRM_REG_GROUP_9].insns = { - _0 (bad), - _1 (cmpxchg, Mx), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_10].insns = { - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_11].insns = { - _0 (mov), _0 (bad), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_12].insns = { - _0 (bad), - _0 (bad), - _2 (psrlw, Rm, Ib), - _0 (bad), - _2 (psraw, Rm, Ib), - _0 (bad), - _2 (psllw, Rm, Ib), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_13].insns = { - _0 (bad), - _0 (bad), - _2 (psrld, Rm, Ib), - _0 (bad), - _2 (psrad, Rm, Ib), - _0 (bad), - _2 (pslld, Rm, Ib), - _0 (bad), - }, - - [X86_INSN_MODRM_REG_GROUP_14].insns = { - _0 (bad), - _0 (bad), - _2 (psrlq, Rm, Ib), - _0f (bad, 0), - _0 (bad), - _0 (bad), - _2 (psllq, Rm, Ib), - _0f (bad, 0), - }, - - [X86_INSN_MODRM_REG_GROUP_15].insns = { - _1 (fxsave, Mv), - _1 (fxrstor, Mv), - _1 (ldmxcsr, Mv), - _1 (stmxcsr, Mv), - _0 (bad), - _1 (lfence, Mv), - _1 (mfence, Mv), - _1 (sfence, Mv), - }, - - [X86_INSN_MODRM_REG_GROUP_16].insns = { - _1 (prefetch_nta, Mv), - _1 (prefetch_t0, Mv), - _1 (prefetch_t1, Mv), - _1 (prefetch_t2, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - }, - - [X86_INSN_MODRM_REG_GROUP_p].insns = { - _1 (prefetch_exclusive, Mv), - _1 (prefetch_modified, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_modified, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - _1 (prefetch_nop, Mv), - }, -}; - -static x86_insn_group8_t x86_insn_sse_groups_repz[] = { - [X86_INSN_SSE_GROUP_10].insns = { - _2 (movss, Gx, Ex), - _2 (movss, Ex, Gx), - _2 (movsldup, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movshdup, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_28].insns = { - _0 (bad), - _0 (bad), - _2 (cvtsi2ss, Gx, Ev), - _0 (bad), - _2 (cvttss2si, Gv, Ex), - _2 (cvtss2si, Gv, Ex), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_50].insns = { - _0 (bad), - _2 (sqrtss, Gx, Ex), - _2 (rsqrtps, Gx, Ex), - _2 (rcpss, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_58].insns = { - _2 (addss, Gx, Ex), - _2 (mulss, Gx, Ex), - _2 (cvtss2sd, Gx, Ex), - _2 (cvttps2dq, Gx, Ex), - _2 (subss, Gx, Ex), - _2 (minss, Gx, Ex), - _2 (divss, Gx, Ex), - _2 (maxss, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_60].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_68].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movdqu, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_70].insns = { - _3 (pshufhw, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_78].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movq, Gx, Ex), - _2 (movdqu, Ex, Gx), - }, - - [X86_INSN_SSE_GROUP_c0].insns = { - _0 (bad), - _0 (bad), - _3 (cmpss, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movq2dq, Gx, Em), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (cvtdq2pd, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, -}; - -static x86_insn_group8_t x86_insn_sse_groups_operand_size[] = { - [X86_INSN_SSE_GROUP_10].insns = { - _2 (movupd, Gx, Ex), - _2 (movupd, Ex, Gx), - _2 (movlpd, Gx, Ex), - _2 (movlpd, Ex, Gx), - _2 (unpcklpd, Gx, Ex), - _2 (unpckhpd, Gx, Ex), - _2 (movhpd, Gx, Mx), - _2 (movhpd, Mx, Gx), - }, - - [X86_INSN_SSE_GROUP_28].insns = { - _2 (movapd, Gx, Ex), - _2 (movapd, Ex, Gx), - _2 (cvtpi2pd, Gx, Ex), - _2 (movntpd, Mx, Gx), - _2 (cvttpd2pi, Gx, Mx), - _2 (cvtpd2pi, Gx, Mx), - _2 (ucomisd, Gx, Ex), - _2 (comisd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_50].insns = { - _2 (movmskpd, Gd, Rx), - _2 (sqrtpd, Gx, Ex), - _0 (bad), - _0 (bad), - _2 (andpd, Gx, Ex), - _2 (andnpd, Gx, Ex), - _2 (orpd, Gx, Ex), - _2 (xorpd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_58].insns = { - _2 (addpd, Gx, Ex), - _2 (mulpd, Gx, Ex), - _2 (cvtpd2ps, Gx, Ex), - _2 (cvtps2dq, Gx, Ex), - _2 (subpd, Gx, Ex), - _2 (minpd, Gx, Ex), - _2 (divpd, Gx, Ex), - _2 (maxpd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_60].insns = { - _2 (punpcklbw, Gx, Ex), - _2 (punpcklwd, Gx, Ex), - _2 (punpckldq, Gx, Ex), - _2 (packsswb, Gx, Ex), - _2 (pcmpgtb, Gx, Ex), - _2 (pcmpgtw, Gx, Ex), - _2 (pcmpgtd, Gx, Ex), - _2 (packuswb, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_68].insns = { - _2 (punpckhbw, Gx, Ex), - _2 (punpckhwd, Gx, Ex), - _2 (punpckhdq, Gx, Ex), - _2 (packssdw, Gx, Ex), - _2 (punpcklqdq, Gx, Ex), - _2 (punpckhqdq, Gx, Ex), - _2 (movd, Gx, Ev), - _2 (movdqa, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_70].insns = { - _3 (pshufd, Gx, Ex, Ib), - _0f (modrm_group_12, X86_INSN_FLAG_MODRM_REG_GROUP_12), - _0f (modrm_group_13, X86_INSN_FLAG_MODRM_REG_GROUP_13), - _0f (modrm_group_14, X86_INSN_FLAG_MODRM_REG_GROUP_14), - _2 (pcmpeqb, Gx, Ex), - _2 (pcmpeqw, Gx, Ex), - _2 (pcmpeqd, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_78].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (haddpd, Gx, Ex), - _2 (hsubpd, Gx, Ex), - _2 (movd, Ev, Gx), - _2 (movdqa, Ex, Gx), - }, - - [X86_INSN_SSE_GROUP_c0].insns = { - _0 (bad), - _0 (bad), - _3 (cmppd, Gx, Ex, Ib), - _0 (bad), - _3 (pinsrw, Gx, Ew, Ib), - _3 (pextrw, Gd, Gx, Ib), - _3 (shufpd, Gx, Ex, Ib), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d0].insns = { - _2 (addsubpd, Gx, Ex), - _2 (psrlw, Gx, Ex), - _2 (psrld, Gx, Ex), - _2 (psrlq, Gx, Ex), - _2 (paddq, Gx, Ex), - _2 (pmullw, Gx, Ex), - _2 (movq, Ex, Gx), - _2 (pmovmskb, Gd, Rx), - }, - - [X86_INSN_SSE_GROUP_d8].insns = { - _2 (psubusb, Gx, Ex), - _2 (psubusw, Gx, Ex), - _2 (pminub, Gx, Ex), - _2 (pand, Gx, Ex), - _2 (paddusb, Gx, Ex), - _2 (paddusw, Gx, Ex), - _2 (pmaxub, Gx, Ex), - _2 (pandn, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_e0].insns = { - _2 (pavgb, Gx, Ex), - _2 (psraw, Gx, Ex), - _2 (psrad, Gx, Ex), - _2 (pavgw, Gx, Ex), - _2 (pmulhuw, Gx, Ex), - _2 (pmulhw, Gx, Ex), - _2 (cvttpd2dq, Gx, Ex), - _2 (movntdq, Mx, Gx), - }, - - [X86_INSN_SSE_GROUP_e8].insns = { - _2 (psubsb, Gx, Ex), - _2 (psubsw, Gx, Ex), - _2 (pminsw, Gx, Ex), - _2 (por, Gx, Ex), - _2 (paddsb, Gx, Ex), - _2 (paddsw, Gx, Ex), - _2 (pmaxsw, Gx, Ex), - _2 (pxor, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_f0].insns = { - _0 (bad), - _2 (psllw, Gx, Ex), - _2 (pslld, Gx, Ex), - _2 (psllq, Gx, Ex), - _2 (pmuludq, Gx, Ex), - _2 (pmaddwd, Gx, Ex), - _2 (psadbw, Gx, Ex), - _2 (maskmovdqu, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_f8].insns = { - _2 (psubb, Gx, Ex), - _2 (psubw, Gx, Ex), - _2 (psubd, Gx, Ex), - _2 (psubq, Gx, Ex), - _2 (paddb, Gx, Ex), - _2 (paddw, Gx, Ex), - _2 (paddd, Gx, Ex), - _0 (bad), - }, -}; - -static x86_insn_group8_t x86_insn_sse_groups_repnz[] = { - [X86_INSN_SSE_GROUP_10].insns = { - _2 (movsd, Gx, Ex), - _2 (movsd, Ex, Gx), - _2 (movddup, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_28].insns = { - _0 (bad), - _0 (bad), - _2 (cvtsi2sd, Gx, Ev), - _0 (bad), - _2 (cvttsd2si, Gv, Ex), - _2 (cvtsd2si, Gv, Ex), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_50].insns = { - _0 (bad), - _2 (sqrtsd, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_58].insns = { - _2 (addsd, Gx, Ex), - _2 (mulsd, Gx, Ex), - _2 (cvtsd2ss, Gx, Ex), - _0 (bad), - _2 (subsd, Gx, Ex), - _2 (minsd, Gx, Ex), - _2 (divsd, Gx, Ex), - _2 (maxsd, Gx, Ex), - }, - - [X86_INSN_SSE_GROUP_60].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_68].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_70].insns = { - _3 (pshuflw, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_78].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (haddps, Gx, Ex), - _2 (hsubps, Gx, Ex), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_c0].insns = { - _0 (bad), - _0 (bad), - _3 (cmpsd, Gx, Ex, Ib), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d0].insns = { - _2 (addsubps, Gx, Ex), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (movdq2q, Gm, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_d8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e0].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _2 (cvtpd2dq, Gx, Ex), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_e8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f0].insns = { - _2 (lddqu, Gx, Mx), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, - - [X86_INSN_SSE_GROUP_f8].insns = { - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - _0 (bad), - }, -}; - -#undef _ - -/* Parses memory displacements and immediates. */ -static u8 * x86_insn_parse_number (u32 log2_n_bytes, - u8 * code, u8 * code_end, - i64 * result) -{ - i64 x = 0; - - if (code + (1 << log2_n_bytes) > code_end) - return 0; - - switch (log2_n_bytes) - { - case 3: - x = clib_little_to_host_unaligned_mem_u64 ((u64 *) code); - break; - - case 2: - x = (i32) clib_little_to_host_unaligned_mem_u32 ((u32 *) code); - break; - - case 1: - x = (i16) clib_little_to_host_unaligned_mem_u16 ((u16 *) code); - break; - - case 0: - x = (i8) code[0]; - break; - - default: - ASSERT (0); - } - - *result = x; - return code + (1 << log2_n_bytes); -} - -static u32 -x86_insn_log2_immediate_bytes (x86_insn_parse_t * p, x86_insn_t * insn) -{ - u32 i = ~0; - switch (x86_insn_immediate_type (insn)) - { - case 'b': i = 0; break; - case 'w': i = 1; break; - case 'd': i = 2; break; - case 'q': i = 3; break; - - case 'z': - i = p->log2_effective_operand_bytes; - if (i > 2) i = 2; - break; - - case 'v': - i = p->log2_effective_operand_bytes; - break; - - default: - i = ~0; - break; - } - - return i; -} - -static u8 * -x86_insn_parse_modrm_byte (x86_insn_parse_t * x, - x86_insn_modrm_byte_t modrm, - u32 parse_flags, - u8 * code, - u8 * code_end) -{ - u8 effective_address_bits; - - if (parse_flags & X86_INSN_PARSE_64_BIT) - effective_address_bits = (x->flags & X86_INSN_ADDRESS_SIZE) ? 32 : 64; - else if (parse_flags & X86_INSN_PARSE_32_BIT) - effective_address_bits = (x->flags & X86_INSN_ADDRESS_SIZE) ? 16 : 32; - else - effective_address_bits = (x->flags & X86_INSN_ADDRESS_SIZE) ? 32 : 16; - - x->log2_effective_address_bytes = 1; - x->log2_effective_address_bytes += effective_address_bits > 16; - x->log2_effective_address_bytes += effective_address_bits > 32; - - x->regs[0] |= modrm.reg; - if (modrm.mode == 3) - x->regs[1] |= modrm.rm; - else - { - u32 log2_disp_bytes = ~0; - - x->flags |= X86_INSN_IS_ADDRESS; - - if (effective_address_bits != 16) - { - u8 has_sib_byte = 0; - - switch (modrm.mode) - { - case 0: - /* When base is bp displacement is present for mode 0. */ - if (modrm.rm == X86_INSN_GP_REG_BP) - { - log2_disp_bytes = x->log2_effective_address_bytes; - break; - } - else if (modrm.rm == X86_INSN_GP_REG_SP - && effective_address_bits != 16) - { - has_sib_byte = 1; - break; - } - /* fall through */ - case 1: - case 2: - x->regs[1] |= modrm.rm; - x->flags |= X86_INSN_HAS_BASE; - if (modrm.mode != 0) - { - log2_disp_bytes = (modrm.mode == 1 - ? 0 - : x->log2_effective_address_bytes); - if (log2_disp_bytes > 2) - log2_disp_bytes = 2; - } - break; - } - - if (has_sib_byte) - { - x86_insn_sib_byte_t sib; - - if (code >= code_end) - return 0; - sib.byte = *code++; - - x->log2_index_scale = 1 << sib.log2_scale; - x->regs[1] |= sib.base; - x->flags |= X86_INSN_HAS_BASE; - - if (sib.index != X86_INSN_GP_REG_SP) - { - x->regs[2] |= sib.index; - x->flags |= X86_INSN_HAS_INDEX; - } - } - } - else - { - /* effective_address_bits == 16 */ - switch (modrm.mode) - { - case 0: - if (modrm.rm == 6) - { - /* [disp16] */ - log2_disp_bytes = 1; - break; - } - /* fall through */ - case 1: - case 2: - switch (modrm.rm) - { - case 0: /* [bx + si/di] */ - case 1: - x->regs[1] = X86_INSN_GP_REG_BX; - x->regs[2] = X86_INSN_GP_REG_SI + (modrm.rm & 1); - x->flags |= X86_INSN_HAS_BASE | X86_INSN_HAS_INDEX; - break; - - case 2: /* [bp + si/di] */ - case 3: - x->regs[1] = X86_INSN_GP_REG_BP; - x->regs[2] = X86_INSN_GP_REG_SI + (modrm.rm & 1); - x->flags |= X86_INSN_HAS_BASE | X86_INSN_HAS_INDEX; - break; - - case 4: /* [si/di] */ - case 5: - x->regs[1] = X86_INSN_GP_REG_SI + (modrm.rm & 1); - x->flags |= X86_INSN_HAS_BASE; - break; - - case 6: /* [bp + disp] */ - x->regs[1] = X86_INSN_GP_REG_BP; - x->flags |= X86_INSN_HAS_BASE; - break; - - case 7: /* [bx + disp] */ - x->regs[1] = X86_INSN_GP_REG_BX; - x->flags |= X86_INSN_HAS_BASE; - break; - } - - if (modrm.mode != 0) - log2_disp_bytes = modrm.mode == 1 ? 0 : 1; - break; - } - } - - if (log2_disp_bytes != ~0) - { - i64 disp; - code = x86_insn_parse_number (log2_disp_bytes, code, code_end, - &disp); - if (code) - x->displacement = disp; - } - } - - return code; -} - -u8 * x86_insn_parse (x86_insn_parse_t * p, u8 * code_start) -{ - u8 i, * code, * code_end; - x86_insn_t * insn, * group_insn; - u8 default_operand_bits, effective_operand_bits; - u32 opcode, parse_flags; - - /* Preserve global parse flags. */ - parse_flags = p->flags & (X86_INSN_PARSE_32_BIT | X86_INSN_PARSE_64_BIT); - clib_memset (p, 0, sizeof (p[0])); - p->flags = parse_flags; - - /* 64 implies 32 bit parsing. */ - if (parse_flags & X86_INSN_PARSE_64_BIT) - parse_flags |= X86_INSN_PARSE_32_BIT; - - /* Instruction must be <= 15 bytes. */ - code = code_start; - code_end = code + 15; - - /* Parse legacy prefixes. */ - while (1) - { - if (code >= code_end) - goto insn_too_long; - i = code[0]; - code++; - switch (i) - { - default: goto prefix_done; - - /* Set flags based on prefix. */ -#define _(x,o) case o: p->flags |= X86_INSN_##x; break; - foreach_x86_legacy_prefix; -#undef _ - } - } - prefix_done: - - /* REX prefix. */ - if ((parse_flags & X86_INSN_PARSE_64_BIT) && i >= 0x40 && i <= 0x4f) - { - p->regs[0] |= ((i & (1 << 2)) != 0) << 3; /* r bit */ - p->regs[1] |= ((i & (1 << 0)) != 0) << 3; /* b bit */ - p->regs[2] |= ((i & (1 << 1)) != 0) << 3; /* x bit */ - p->flags |= ((i & (1 << 3)) /* w bit */ - ? X86_INSN_OPERAND_SIZE_64 : 0); - if (code >= code_end) - goto insn_too_long; - i = *code++; - } - - opcode = i; - if (opcode == 0x0f) - { - /* two byte opcode. */; - if (code >= code_end) - goto insn_too_long; - i = *code++; - opcode = (opcode << 8) | i; - insn = x86_insns_two_byte + i; - } - else - { - static x86_insn_t arpl = { - .name = "arpl", - .operands[0].data = "Ew", - .operands[1].data = "Gw", - }; - - if (PREDICT_FALSE (i == 0x63 - && ! (parse_flags & X86_INSN_PARSE_64_BIT))) - insn = &arpl; - else - insn = x86_insns_one_byte + i; - } - - if ((i = X86_INSN_FLAG_GET_SSE_GROUP (insn->flags)) != 0) - { - x86_insn_group8_t * g8; - - if (p->flags & X86_INSN_OPERAND_SIZE) - g8 = x86_insn_sse_groups_operand_size; - else if (p->flags & X86_INSN_REPZ) - g8 = x86_insn_sse_groups_repz; - else if (p->flags & X86_INSN_REPNZ) - g8 = x86_insn_sse_groups_repnz; - else - g8 = 0; - - /* insn flags have 1 + group so != 0 test above can work. */ - ASSERT ((i - 1) < ARRAY_LEN (x86_insn_sse_groups_operand_size)); - if (g8) - insn = g8[i - 1].insns + (opcode & 7); - } - - /* Parse modrm and displacement if present. */ - if (x86_insn_has_modrm_byte (insn)) - { - x86_insn_modrm_byte_t modrm; - - if (code >= code_end) - goto insn_too_long; - modrm.byte = *code++; - - /* Handle special 0x0f01 and 0x0fae encodings. */ - if (PREDICT_FALSE (modrm.mode == 3 - && (opcode == 0x0f01 - || opcode == 0x0fae))) - { - static x86_insn_t x86_insns_0f01_special[] = { - _0 (swapgs), _0 (rdtscp), _0 (bad), _0 (bad), - _0 (bad), _0 (bad), _0 (bad), _0 (bad), - }; - static x86_insn_t x86_insns_0fae_special[] = { - _0 (vmrun), _0 (vmmcall), _0 (vmload), _0 (vmsave), - _0 (stgi), _0 (clgi), _0 (skinit), _0 (invlpga), - }; - - if (opcode == 0x0f01) - insn = x86_insns_0f01_special; - else - insn = x86_insns_0fae_special; - insn += modrm.rm; - opcode = (opcode << 8) | modrm.byte; - } - else - { - code = x86_insn_parse_modrm_byte (p, modrm, parse_flags, - code, code_end); - if (! code) - goto insn_too_long; - } - } - - group_insn = 0; - if ((i = X86_INSN_FLAG_GET_MODRM_REG_GROUP (insn->flags)) != 0) - { - u32 g = i - 1; - ASSERT (g < ARRAY_LEN (x86_insn_modrm_reg_groups)); - group_insn = x86_insn_modrm_reg_groups[g].insns + (p->regs[0] & 7); - } - - p->insn = insn[0]; - if (group_insn) - { - u32 k; - p->insn.name = group_insn->name; - p->insn.flags |= group_insn->flags; - for (k = 0; k < ARRAY_LEN (group_insn->operands); k++) - if (x86_insn_operand_is_valid (group_insn, k)) - p->insn.operands[k] = group_insn->operands[k]; - } - - default_operand_bits - = ((((parse_flags & X86_INSN_PARSE_32_BIT) != 0) - ^ ((p->flags & X86_INSN_OPERAND_SIZE) != 0)) - ? BITS (u32) : BITS (u16)); - - if ((parse_flags & X86_INSN_PARSE_64_BIT) - && (p->insn.flags & X86_INSN_FLAG_DEFAULT_64_BIT)) - default_operand_bits = BITS (u64); - - effective_operand_bits = default_operand_bits; - if (p->flags & X86_INSN_OPERAND_SIZE_64) - effective_operand_bits = BITS (u64); - - p->log2_effective_operand_bytes = 1; - p->log2_effective_operand_bytes += effective_operand_bits > 16; - p->log2_effective_operand_bytes += effective_operand_bits > 32; - - /* Parse immediate if present. */ - { - u32 l = x86_insn_log2_immediate_bytes (p, insn); - if (l <= 3) - { - code = x86_insn_parse_number (l, code, code_end, &p->immediate); - if (! code) - goto insn_too_long; - } - } - - return code; - - insn_too_long: - return 0; -} - -static u8 * format_x86_gp_reg_operand (u8 * s, va_list * va) -{ - u32 r = va_arg (*va, u32); - u32 log2_n_bytes = va_arg (*va, u32); - - const char names8[8] = "acdbsbsd"; - const char names16[8] = "xxxxppii"; - - ASSERT (r < 16); - - /* Add % register prefix. */ - vec_add1 (s, '%'); - - switch (log2_n_bytes) - { - case 0: - { - - if (r < 8) - s = format (s, "%c%c", names8[r & 3], (r >> 2) ? 'l' : 'h'); - else - s = format (s, "r%db", r); - } - break; - - case 2: - case 3: - s = format (s, "%c", log2_n_bytes == 2 ? 'e' : 'r'); - /* fall through */ - case 1: - if (r < 8) - s = format (s, "%c%c", names8[r], names16[r]); - else - { - s = format (s, "%d", r); - if (log2_n_bytes != 3) - s = format (s, "%c", log2_n_bytes == 1 ? 'w' : 'd'); - } - break; - - default: - ASSERT (0); - } - - return s; -} - -static u8 * format_x86_reg_operand (u8 * s, va_list * va) -{ - u32 reg = va_arg (*va, u32); - u32 log2_n_bytes = va_arg (*va, u32); - u32 type = va_arg (*va, u32); - - switch (type) - { - default: - ASSERT (0); - break; - - case 'x': - ASSERT (reg < 16); - return format (s, "%%xmm%d", reg); - - case 'm': - ASSERT (reg < 8); - return format (s, "%%mm%d", reg); - - /* Explicit byte/word/double-word/quad-word */ - case 'b': log2_n_bytes = 0; break; - case 'w': log2_n_bytes = 1; break; - case 'd': log2_n_bytes = 2; break; - case 'q': log2_n_bytes = 3; break; - - /* Use effective operand size. */ - case 'v': break; - - /* word or double-word depending on effective operand size. */ - case 'z': - log2_n_bytes = clib_min (log2_n_bytes, 2); - break; - } - - s = format (s, "%U", format_x86_gp_reg_operand, reg, log2_n_bytes); - return s; -} - -static u8 * format_x86_mem_operand (u8 * s, va_list * va) -{ - x86_insn_parse_t * p = va_arg (*va, x86_insn_parse_t *); - - if (p->displacement != 0) - s = format (s, "0x%x", p->displacement); - - if (p->flags & X86_INSN_HAS_BASE) - { - s = format (s, "(%U", - format_x86_gp_reg_operand, p->regs[1], - p->log2_effective_address_bytes); - if (p->flags & X86_INSN_HAS_INDEX) - { - s = format (s, ",%U", - format_x86_gp_reg_operand, p->regs[2], - p->log2_effective_address_bytes); - if (p->log2_index_scale != 0) - s = format (s, ",%d", 1 << p->log2_index_scale); - } - s = format (s, ")"); - } - - /* [RIP+disp] PC relative addressing in 64 bit mode. */ - else if (p->flags & X86_INSN_PARSE_64_BIT) - s = format (s, "(%%rip)"); - - return s; -} - -static u8 * format_x86_insn_operand (u8 * s, va_list * va) -{ - x86_insn_parse_t * p = va_arg (*va, x86_insn_parse_t *); - x86_insn_t * insn = &p->insn; - u32 o = va_arg (*va, u32); - u8 c, t; - - ASSERT (o < ARRAY_LEN (insn->operands)); - c = insn->operands[o].code; - t = insn->operands[o].type; - - /* Register encoded in instruction. */ - if (c < 8) - return format (s, "%U", - format_x86_gp_reg_operand, c, - p->log2_effective_operand_bytes); - - switch (c) - { - /* Memory or reg field from modrm byte. */ - case 'M': - ASSERT (p->flags & X86_INSN_IS_ADDRESS); - /* FALLTHROUGH */ - case 'E': - if (p->flags & X86_INSN_IS_ADDRESS) - s = format (s, "%U", format_x86_mem_operand, p); - else - s = format (s, "%U", - format_x86_reg_operand, p->regs[1], - p->log2_effective_operand_bytes, t); - break; - - /* reg field from modrm byte. */ - case 'R': - case 'G': - s = format (s, "%U", - format_x86_reg_operand, p->regs[0], - p->log2_effective_operand_bytes, t); - break; - - case 'I': - { - u32 l = x86_insn_log2_immediate_bytes (p, insn); - i64 mask = pow2_mask (8ULL << l); - s = format (s, "$0x%Lx", p->immediate & mask); - } - break; - - case 'J': - if (p->immediate < 0) - s = format (s, "- 0x%Lx", -p->immediate); - else - s = format (s, "+ 0x%Lx", p->immediate); - break; - - case 'O': - s = format (s, "0x%Lx", p->immediate); - break; - - case 'A': - /* AX/AL */ - s = format (s, "%U", - format_x86_gp_reg_operand, X86_INSN_GP_REG_AX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'B': - /* BX/BL/BP */ - s = format (s, "%U", - format_x86_gp_reg_operand, - t == 'P' ? X86_INSN_GP_REG_BP : X86_INSN_GP_REG_BX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'C': - /* CX/CL */ - s = format (s, "%U", - format_x86_gp_reg_operand, X86_INSN_GP_REG_CX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'D': - /* DX/DL/DI */ - s = format (s, "%U", - format_x86_gp_reg_operand, - t == 'I' ? X86_INSN_GP_REG_DI : X86_INSN_GP_REG_DX, - t == 'L' ? 0 : p->log2_effective_operand_bytes); - break; - - case 'S': - /* SI/SP */ - s = format (s, "%U", - format_x86_gp_reg_operand, - t == 'I' ? X86_INSN_GP_REG_SI : X86_INSN_GP_REG_SP, - p->log2_effective_operand_bytes); - break; - - case '1': - s = format (s, "1"); - break; - - default: - ASSERT (0); - } - - return s; -} - -u8 * format_x86_insn_parse (u8 * s, va_list * va) -{ - x86_insn_parse_t * p = va_arg (*va, x86_insn_parse_t *); - x86_insn_t * insn = &p->insn; - u32 o, i, is_src_dst; - - s = format (s, "%s", insn->name); - - if (! x86_insn_operand_is_valid (insn, 0)) - goto done; - - is_src_dst = x86_insn_operand_is_valid (insn, 1); - - /* If instruction has immediate add suffix to opcode to - indicate operand size. */ - if (is_src_dst) - { - u32 b; - - b = x86_insn_log2_immediate_bytes (p, insn); - if (b < p->log2_effective_operand_bytes - && (p->flags & X86_INSN_IS_ADDRESS)) - s = format (s, "%c", "bwlq"[b]); - } - - for (i = 0; i < ARRAY_LEN (insn->operands); i++) - { - o = is_src_dst + i; - if (! x86_insn_operand_is_valid (insn, o)) - break; - s = format (s, "%s%U", - i == 0 ? " " : ", ", - format_x86_insn_operand, p, o); - } - - if (is_src_dst) - s = format (s, ", %U", - format_x86_insn_operand, p, 0); - - done: - return s; -} diff --git a/src/vppinfra/asm_x86.h b/src/vppinfra/asm_x86.h deleted file mode 100644 index dacef61755c..00000000000 --- a/src/vppinfra/asm_x86.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_asm_x86_h -#define included_asm_x86_h - -#include <vppinfra/format.h> - -typedef union -{ - struct - { - u8 code; - u8 type; - }; - u8 data[2]; -} x86_insn_operand_t; - -typedef struct -{ - /* Instruction name. */ - char *name; - - /* X86 instructions may have up to 3 operands. */ - x86_insn_operand_t operands[3]; - - u16 flags; -#define X86_INSN_FLAG_DEFAULT_64_BIT (1 << 0) -#define X86_INSN_FLAG_SET_SSE_GROUP(n) ((n) << 5) -#define X86_INSN_FLAG_GET_SSE_GROUP(f) (((f) >> 5) & 0x1f) -#define X86_INSN_FLAG_SET_MODRM_REG_GROUP(n) (((n) & 0x3f) << 10) -#define X86_INSN_FLAG_GET_MODRM_REG_GROUP(f) (((f) >> 10) & 0x3f) -} x86_insn_t; - -always_inline uword -x86_insn_operand_is_valid (x86_insn_t * i, uword o) -{ - ASSERT (o < ARRAY_LEN (i->operands)); - return i->operands[o].code != '_'; -} - -#define foreach_x86_legacy_prefix \ - _ (OPERAND_SIZE, 0x66) \ - _ (ADDRESS_SIZE, 0x67) \ - _ (SEGMENT_CS, 0x2e) \ - _ (SEGMENT_DS, 0x3e) \ - _ (SEGMENT_ES, 0x26) \ - _ (SEGMENT_FS, 0x64) \ - _ (SEGMENT_GS, 0x65) \ - _ (SEGMENT_SS, 0x36) \ - _ (LOCK, 0xf0) \ - _ (REPZ, 0xf3) \ - _ (REPNZ, 0xf2) - -#define foreach_x86_insn_parse_flag \ - /* Parse in 32/64-bit mode. */ \ - _ (PARSE_32_BIT, 0) \ - _ (PARSE_64_BIT, 0) \ - _ (IS_ADDRESS, 0) \ - /* regs[1/2] is a valid base/index register */ \ - _ (HAS_BASE, 0) \ - _ (HAS_INDEX, 0) \ - /* rex w bit */ \ - _ (OPERAND_SIZE_64, 0) - -typedef enum -{ -#define _(f,o) X86_INSN_FLAG_BIT_##f, - foreach_x86_insn_parse_flag foreach_x86_legacy_prefix -#undef _ -} x86_insn_parse_flag_bit_t; - -typedef enum -{ -#define _(f,o) X86_INSN_##f = 1 << X86_INSN_FLAG_BIT_##f, - foreach_x86_insn_parse_flag foreach_x86_legacy_prefix -#undef _ -} x86_insn_parse_flag_t; - -typedef struct -{ - /* Registers in instruction. - [0] is modrm reg field - [1] is base reg - [2] is index reg. */ - u8 regs[3]; - - /* Scale for index register. */ - u8 log2_index_scale:2; - u8 log2_effective_operand_bytes:3; - u8 log2_effective_address_bytes:3; - - i32 displacement; - - /* Parser flags: set of x86_insn_parse_flag_t enums. */ - u32 flags; - - i64 immediate; - - x86_insn_t insn; -} x86_insn_parse_t; - -u8 *x86_insn_parse (x86_insn_parse_t * p, u8 * code_start); -format_function_t format_x86_insn_parse; - -#endif /* included_asm_x86_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/backtrace.c b/src/vppinfra/backtrace.c deleted file mode 100644 index e713bae6876..00000000000 --- a/src/vppinfra/backtrace.c +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - Copyright (c) 2004 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#include <vppinfra/clib.h> -#include <vppinfra/error.h> - -#ifdef __mips__ - -/* Let code below know we've defined _clib_backtrace */ -#define clib_backtrace_defined - -#include <vppinfra/asm_mips.h> - -__clib_export uword -clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip) -{ - u32 *pc; - void *sp; - uword i, saved_pc; - - /* Figure current PC, saved PC and stack pointer. */ - asm volatile (".set push\n" - ".set noat\n" "move %[saved_pc], $31\n" "move %[sp], $29\n" - /* Fetches current PC. */ - "la $at, 1f\n" - "jalr %[pc], $at\n" - "nop\n" - "1:\n" - ".set pop\n":[pc] "=r" (pc), - [saved_pc] "=r" (saved_pc),[sp] "=r" (sp)); - - /* Also skip current frame. */ - n_frames_to_skip += 1; - - for (i = 0; i < max_callers + n_frames_to_skip; i++) - { - mips_insn_opcode_t op; - mips_insn_special_funct_t funct; - i32 insn, rs, rt, rd, immediate, found_saved_pc; - u32 *start_pc; - - /* Parse instructions until we reach prologue for this - stack frame. We'll need to figure out where saved - PC is and where previous stack frame lives. */ - start_pc = pc; - found_saved_pc = 0; - while (1) - { - insn = *--pc; - op = mips_insn_get_op (insn); - funct = mips_insn_get_funct (insn); - rs = mips_insn_get_rs (insn); - rt = mips_insn_get_rt (insn); - rd = mips_insn_get_rd (insn); - immediate = mips_insn_get_immediate (insn); - - switch (op) - { - default: - break; - - case MIPS_OPCODE_sd: - case MIPS_OPCODE_sw: - /* Trace stores of return address. */ - if (rt == MIPS_REG_RA) - { - void *addr = sp + immediate; - - /* If RA is stored somewhere other than in the - stack frame, give up. */ - if (rs != MIPS_REG_SP) - goto backtrace_done; - - ASSERT (immediate % 4 == 0); - if (op == MIPS_OPCODE_sw) - saved_pc = ((u32 *) addr)[0]; - else - saved_pc = ((u64 *) addr)[0]; - found_saved_pc = 1; - } - break; - - case MIPS_OPCODE_addiu: - case MIPS_OPCODE_daddiu: - case MIPS_OPCODE_addi: - case MIPS_OPCODE_daddi: - if (rt == MIPS_REG_SP) - { - if (rs != MIPS_REG_SP) - goto backtrace_done; - - ASSERT (immediate % 4 == 0); - - /* Assume positive offset is part of the epilogue. - E.g. - jr ra - add sp,sp,100 - */ - if (immediate > 0) - continue; - - /* Negative offset means allocate stack space. - This could either be the prologue or could be due to - alloca. */ - sp -= immediate; - - /* This frame will not save RA. */ - if (i == 0) - goto found_prologue; - - /* Assume that addiu sp,sp,-N without store of ra means - that we have not found the prologue yet. */ - if (found_saved_pc) - goto found_prologue; - } - break; - - case MIPS_OPCODE_slti: - case MIPS_OPCODE_sltiu: - case MIPS_OPCODE_andi: - case MIPS_OPCODE_ori: - case MIPS_OPCODE_xori: - case MIPS_OPCODE_lui: - case MIPS_OPCODE_ldl: - case MIPS_OPCODE_ldr: - case MIPS_OPCODE_lb: - case MIPS_OPCODE_lh: - case MIPS_OPCODE_lwl: - case MIPS_OPCODE_lw: - case MIPS_OPCODE_lbu: - case MIPS_OPCODE_lhu: - case MIPS_OPCODE_lwr: - case MIPS_OPCODE_lwu: - case MIPS_OPCODE_ld: - /* Give up when we find anyone setting the stack pointer. */ - if (rt == MIPS_REG_SP) - goto backtrace_done; - break; - - case MIPS_OPCODE_SPECIAL: - if (rd == MIPS_REG_SP) - switch (funct) - { - default: - /* Give up when we find anyone setting the stack pointer. */ - goto backtrace_done; - - case MIPS_SPECIAL_FUNCT_break: - case MIPS_SPECIAL_FUNCT_jr: - case MIPS_SPECIAL_FUNCT_sync: - case MIPS_SPECIAL_FUNCT_syscall: - case MIPS_SPECIAL_FUNCT_tge: - case MIPS_SPECIAL_FUNCT_tgeu: - case MIPS_SPECIAL_FUNCT_tlt: - case MIPS_SPECIAL_FUNCT_tltu: - case MIPS_SPECIAL_FUNCT_teq: - case MIPS_SPECIAL_FUNCT_tne: - /* These instructions can validly have rd == MIPS_REG_SP */ - break; - } - break; - } - } - - found_prologue: - /* Check sanity of saved pc. */ - if (saved_pc & 3) - goto backtrace_done; - if (saved_pc == 0) - goto backtrace_done; - - if (i >= n_frames_to_skip) - callers[i - n_frames_to_skip] = saved_pc; - pc = uword_to_pointer (saved_pc, u32 *); - } - -backtrace_done: - if (i < n_frames_to_skip) - return 0; - else - return i - n_frames_to_skip; -} -#endif /* __mips__ */ - -#ifndef clib_backtrace_defined -#define clib_backtrace_defined - -/* use glibc backtrace for stack trace */ -#include <execinfo.h> - -__clib_export uword -clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip) -{ - int size; - void *array[20]; - /* Also skip current frame. */ - n_frames_to_skip += 1; - - size = clib_min (ARRAY_LEN (array), max_callers + n_frames_to_skip); - - size = backtrace (array, size); - - uword i; - - for (i = 0; i < max_callers + n_frames_to_skip && i < size; i++) - { - if (i >= n_frames_to_skip) - callers[i - n_frames_to_skip] = pointer_to_uword (array[i]); - } - - if (i < n_frames_to_skip) - return 0; - else - return i - n_frames_to_skip; -} - - -#endif /* clib_backtrace_defined */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/cJSON.c b/src/vppinfra/cJSON.c index 448435de4dc..24e0110ed08 100644 --- a/src/vppinfra/cJSON.c +++ b/src/vppinfra/cJSON.c @@ -20,6 +20,7 @@ THE SOFTWARE. */ /* clang-format off */ + /* cJSON */ /* JSON parser in C. */ @@ -96,9 +97,9 @@ CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) return (const char*) (global_error.json + global_error.position); } -CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) +CJSON_PUBLIC (char *) cJSON_GetStringValue (const cJSON *const item) { - if (!cJSON_IsString(item)) + if (!cJSON_IsString (item)) { return NULL; } @@ -106,9 +107,9 @@ CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) return item->valuestring; } -CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) +CJSON_PUBLIC (double) cJSON_GetNumberValue (const cJSON *const item) { - if (!cJSON_IsNumber(item)) + if (!cJSON_IsNumber (item)) { return (double) NAN; } @@ -117,8 +118,9 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) } /* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 14) - #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || \ + (CJSON_VERSION_PATCH != 17) +#error cJSON.h and cJSON.c have different versions. Make sure that both have the same. #endif CJSON_PUBLIC(const char*) cJSON_Version(void) @@ -157,7 +159,7 @@ typedef struct internal_hooks { void *(CJSON_CDECL *allocate)(size_t size); void (CJSON_CDECL *deallocate)(void *pointer); - void *(CJSON_CDECL *reallocate)(void *pointer, size_t new_size, size_t old_size); + void *(CJSON_CDECL *reallocate) (void *pointer, size_t size); } internal_hooks; #if defined(_MSC_VER) @@ -170,20 +172,17 @@ static void CJSON_CDECL internal_free(void *pointer) { free(pointer); } +static void *CJSON_CDECL +internal_realloc (void *pointer, size_t size) +{ + return realloc (pointer, size); +} #else #define internal_malloc malloc #define internal_free free +#define internal_realloc realloc #endif -static void * CJSON_CDECL internal_realloc(void *pointer, size_t new_size, - size_t old_size) -{ - return realloc(pointer, new_size); -} - -static void * -cjson_realloc_internal (void *ptr, size_t new_size, size_t old_size); - /* strlen of character literals resolved at compile time */ #define static_strlen(string_literal) (sizeof(string_literal) - sizeof("")) @@ -217,8 +216,8 @@ CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) /* Reset hooks */ global_hooks.allocate = malloc; global_hooks.deallocate = free; - global_hooks.reallocate = internal_realloc; - return; + global_hooks.reallocate = realloc; + return; } global_hooks.allocate = malloc; @@ -233,16 +232,11 @@ CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) global_hooks.deallocate = hooks->free_fn; } - /* use realloc only if both free and malloc are used */ - global_hooks.reallocate = NULL; - if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) - { - global_hooks.reallocate = internal_realloc; - } - else - { - global_hooks.reallocate = cjson_realloc_internal; - } + global_hooks.reallocate = realloc; + if (hooks->realloc_fn != NULL) + { + global_hooks.reallocate = hooks->realloc_fn; + } } /* Internal constructor. */ @@ -405,14 +399,22 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) return object->valuedouble = number; } +/* Note: when passing a NULL valuestring, cJSON_SetValuestring treats this as + * an error and return NULL */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { char *copy = NULL; /* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */ - if (!(object->type & cJSON_String) || (object->type & cJSON_IsReference)) - { - return NULL; - } + if ((object == NULL) || !(object->type & cJSON_String) || + (object->type & cJSON_IsReference)) + { + return NULL; + } + /* return NULL if the object is corrupted or valuestring is NULL */ + if (object->valuestring == NULL || valuestring == NULL) + { + return NULL; + } if (strlen(valuestring) <= strlen(object->valuestring)) { strcpy(object->valuestring, valuestring); @@ -443,27 +445,6 @@ typedef struct internal_hooks hooks; } printbuffer; -static void * -cjson_realloc_internal (void *ptr, size_t new_size, size_t old_size) -{ - size_t copy_size; - if (old_size < new_size) - copy_size = old_size; - else - copy_size = new_size; - - unsigned char *newbuffer = global_hooks.allocate(new_size); - if (!newbuffer) - { - global_hooks.deallocate(ptr); - return NULL; - } - - memcpy (newbuffer, ptr, copy_size); - global_hooks.deallocate (ptr); - return newbuffer; -} - /* realloc printbuffer if necessary to have at least "needed" bytes more */ static unsigned char* ensure(printbuffer * const p, size_t needed) { @@ -515,14 +496,35 @@ static unsigned char* ensure(printbuffer * const p, size_t needed) newsize = needed * 2; } - newbuffer = p->hooks.reallocate (p->buffer, newsize, p->length); - if (newbuffer == NULL) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - return NULL; - } + if (p->hooks.reallocate != NULL) + { + /* reallocate with realloc if available */ + newbuffer = (unsigned char *) p->hooks.reallocate (p->buffer, newsize); + if (newbuffer == NULL) + { + p->hooks.deallocate (p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + } + else + { + /* otherwise reallocate manually */ + newbuffer = (unsigned char *) p->hooks.allocate (newsize); + if (!newbuffer) + { + p->hooks.deallocate (p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + + memcpy (newbuffer, p->buffer, p->offset + 1); + p->hooks.deallocate (p->buffer); + } p->length = newsize; p->buffer = newbuffer; @@ -570,6 +572,10 @@ static cJSON_bool print_number(const cJSON * const item, printbuffer * const out { length = sprintf((char*)number_buffer, "null"); } + else if (d == (double) item->valueint) + { + length = sprintf ((char *) number_buffer, "%d", item->valueint); + } else { /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ @@ -1111,7 +1117,7 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer } buffer.content = (const unsigned char*)value; - buffer.length = buffer_length; + buffer.length = buffer_length; buffer.offset = 0; buffer.hooks = global_hooks; @@ -1216,11 +1222,13 @@ static unsigned char *print(const cJSON * const item, cJSON_bool format, const i /* check if reallocate is available */ if (hooks->reallocate != NULL) { - printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->offset + 1, default_buffer_size); - if (printed == NULL) { - goto fail; - } - buffer->buffer = NULL; + printed = (unsigned char *) hooks->reallocate (buffer->buffer, + buffer->offset + 1); + if (printed == NULL) + { + goto fail; + } + buffer->buffer = NULL; } else /* otherwise copy the JSON over to a new buffer */ { @@ -1658,8 +1666,13 @@ static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_bu current_item = new_item; } - /* parse the name of the child */ - input_buffer->offset++; + if (cannot_access_at_index (input_buffer, 1)) + { + goto fail; /* nothing comes after the comma */ + } + + /* parse the name of the child */ + input_buffer->offset++; buffer_skip_whitespace(input_buffer); if (!parse_string(current_item, input_buffer)) { @@ -2268,10 +2281,10 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON { cJSON *after_inserted = NULL; - if (which < 0) - { - return false; - } + if (which < 0 || newitem == NULL) + { + return false; + } after_inserted = get_array_item(array, (size_t)which); if (after_inserted == NULL) @@ -2279,6 +2292,12 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON return add_item_to_array(array, newitem); } + if (after_inserted != array->child && after_inserted->prev == NULL) + { + /* return false if after_inserted is a corrupted array item */ + return false; + } + newitem->next = after_inserted; newitem->prev = after_inserted->prev; after_inserted->prev = newitem; @@ -2295,7 +2314,8 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) { - if ((parent == NULL) || (replacement == NULL) || (item == NULL)) + if ((parent == NULL) || (parent->child == NULL) || (replacement == NULL) || + (item == NULL)) { return false; } @@ -2365,6 +2385,11 @@ static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSO cJSON_free(replacement->string); } replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + if (replacement->string == NULL) + { + return false; + } + replacement->type &= ~cJSON_StringIsConst; return cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); @@ -2639,9 +2664,9 @@ CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) for (i = 0; a && (i < (size_t) count); i++) { - n = cJSON_CreateNumber(numbers[i]); - if(!n) - { + n = cJSON_CreateNumber (numbers[i]); + if (!n) + { cJSON_Delete(a); return NULL; } @@ -2988,7 +3013,7 @@ CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) { - if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a)) + if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF))) { return false; } @@ -3121,7 +3146,7 @@ CJSON_PUBLIC(void) cJSON_free(void *object) global_hooks.deallocate(object); } -CJSON_PUBLIC(void *) cJSON_realloc(void *object, size_t new_size, size_t old_size) +CJSON_PUBLIC (void *) cJSON_realloc (void *object, size_t size) { - return global_hooks.reallocate(object, new_size, old_size); + return global_hooks.reallocate (object, size); } diff --git a/src/vppinfra/cJSON.h b/src/vppinfra/cJSON.h index 1474c4e5c49..1c98dfac70e 100644 --- a/src/vppinfra/cJSON.h +++ b/src/vppinfra/cJSON.h @@ -81,7 +81,7 @@ then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJ /* project version */ #define CJSON_VERSION_MAJOR 1 #define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 14 +#define CJSON_VERSION_PATCH 17 #include <stddef.h> @@ -127,8 +127,7 @@ typedef struct cJSON_Hooks /* malloc/free are CDECL on Windows regardless of the default calling convention of the compiler, so ensure the hooks allow passing those functions directly. */ void *(CJSON_CDECL *malloc_fn)(size_t sz); void (CJSON_CDECL *free_fn)(void *ptr); - void *(CJSON_CDECL *realloc_fn) (void *ptr, size_t new_size, - size_t old_size); + void *(CJSON_CDECL *realloc_fn) (void *ptr, size_t sz); } cJSON_Hooks; typedef int cJSON_bool; @@ -256,9 +255,10 @@ CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); -/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') from strings. - * The input pointer json cannot point to a read-only address area, such as a string constant, - * but should point to a readable and writable adress area. */ +/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') + * from strings. The input pointer json cannot point to a read-only address + * area, such as a string constant, + * but should point to a readable and writable address area. */ CJSON_PUBLIC(void) cJSON_Minify(char *json); /* Helper functions for creating and adding items to an object at the same time. @@ -281,14 +281,21 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); /* Change the valuestring of a cJSON_String object, only takes effect when type of object is cJSON_String */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring); +/* If the object is not a boolean type this does nothing and returns + * cJSON_Invalid else it returns the new type*/ +#define cJSON_SetBoolValue(object, boolValue) \ + ((object != NULL && ((object)->type & (cJSON_False | cJSON_True))) ? \ + (object)->type = ((object)->type & (~(cJSON_False | cJSON_True))) | \ + ((boolValue) ? cJSON_True : cJSON_False) : \ + cJSON_Invalid) + /* Macro for iterating over an array or object */ #define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) /* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ CJSON_PUBLIC(void *) cJSON_malloc(size_t size); CJSON_PUBLIC(void) cJSON_free(void *object); -CJSON_PUBLIC (void *) -cJSON_realloc (void *object, size_t new_size, size_t old_size); +CJSON_PUBLIC (void *) cJSON_realloc (void *object, size_t size); #ifdef __cplusplus } diff --git a/src/vppinfra/cache.h b/src/vppinfra/cache.h index 4229a068486..13778a423fd 100644 --- a/src/vppinfra/cache.h +++ b/src/vppinfra/cache.h @@ -68,6 +68,17 @@ #define CLIB_PREFETCH_WRITE 1 #define CLIB_PREFETCH_STORE 1 /* alias for write */ +/* locality arguments to __builtin_prefetch. */ +#define CLIB_PREFETCH_TO_STREAM 0 // NTA +#define CLIB_PREFETCH_TO_L3 1 // T2 +#define CLIB_PREFETCH_TO_L2 2 // T1 +#define CLIB_PREFETCH_TO_L1 3 // T0 + +#define _CLIB_TARGETED_PREFETCH(n, size, type, loc) \ + if ((size) > (n) *CLIB_CACHE_PREFETCH_BYTES) \ + __builtin_prefetch (_addr + (n) *CLIB_CACHE_PREFETCH_BYTES, \ + CLIB_PREFETCH_##type, CLIB_PREFETCH_TO_##loc); + #define _CLIB_PREFETCH(n, size, type) \ if ((size) > (n) *CLIB_CACHE_PREFETCH_BYTES) \ __builtin_prefetch (_addr + (n) *CLIB_CACHE_PREFETCH_BYTES, \ @@ -86,6 +97,19 @@ } \ while (0) +#define CLIB_TARGETED_PREFETCH(addr, size, type, locality) \ + do \ + { \ + void *_addr = (addr); \ + \ + ASSERT ((size) <= 4 * CLIB_CACHE_PREFETCH_BYTES); \ + _CLIB_TARGETED_PREFETCH (0, size, type, locality); \ + _CLIB_TARGETED_PREFETCH (1, size, type, locality); \ + _CLIB_TARGETED_PREFETCH (2, size, type, locality); \ + _CLIB_TARGETED_PREFETCH (3, size, type, locality); \ + } \ + while (0) + #undef _ static_always_inline void diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h index d14582492d6..75cebc65672 100644 --- a/src/vppinfra/clib.h +++ b/src/vppinfra/clib.h @@ -385,10 +385,6 @@ void qsort (void *base, uword n, uword size, int (*)(const void *, const void *)); #endif -/* Stack backtrace. */ -uword -clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip); - #include <vppinfra/byte_order.h> #endif /* included_clib_h */ diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index 7a1b75fcf7d..b3743d4c26d 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -150,6 +150,7 @@ _CLIB_MARCH_FN_REGISTRATION(fn) _ (movdir64b, 7, ecx, 28) \ _ (enqcmd, 7, ecx, 29) \ _ (avx512_fp16, 7, edx, 23) \ + _ (aperfmperf, 0x00000006, ecx, 0) \ _ (invariant_tsc, 0x80000007, edx, 8) \ _ (monitorx, 0x80000001, ecx, 29) diff --git a/src/vppinfra/crypto/aes_cbc.h b/src/vppinfra/crypto/aes_cbc.h index cb3d0784051..ee9263df260 100644 --- a/src/vppinfra/crypto/aes_cbc.h +++ b/src/vppinfra/crypto/aes_cbc.h @@ -539,4 +539,207 @@ clib_aes256_cbc_decrypt (const aes_cbc_key_data_t *kd, const u8 *ciphertext, clib_aes_cbc_decrypt (kd, ciphertext, len, iv, AES_KEY_256, plaintext); } +#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 +#pragma GCC optimize("O3") +#endif + +#if defined(__VAES__) && defined(__AVX512F__) +#define u8xN u8x64 +#define u32xN u32x16 +#define u32xN_min_scalar u32x16_min_scalar +#define u32xN_is_all_zero u32x16_is_all_zero +#define u32xN_splat u32x16_splat +#elif defined(__VAES__) +#define u8xN u8x32 +#define u32xN u32x8 +#define u32xN_min_scalar u32x8_min_scalar +#define u32xN_is_all_zero u32x8_is_all_zero +#define u32xN_splat u32x8_splat +#else +#define u8xN u8x16 +#define u32xN u32x4 +#define u32xN_min_scalar u32x4_min_scalar +#define u32xN_is_all_zero u32x4_is_all_zero +#define u32xN_splat u32x4_splat +#endif + +static_always_inline u32 +clib_aes_cbc_encrypt_multi (aes_cbc_key_data_t **key_data, + const uword *key_indices, u8 **plaintext, + const uword *oplen, u8 **iv, aes_key_size_t ks, + u8 **ciphertext, uword n_ops) +{ + int rounds = AES_KEY_ROUNDS (ks); + u8 placeholder[8192]; + u32 i, j, count, n_left = n_ops; + u32xN placeholder_mask = {}; + u32xN len = {}; + u32 key_index[4 * N_AES_LANES]; + u8 *src[4 * N_AES_LANES] = {}; + u8 *dst[4 * N_AES_LANES] = {}; + u8xN r[4] = {}; + u8xN k[15][4] = {}; + + for (i = 0; i < 4 * N_AES_LANES; i++) + key_index[i] = ~0; + +more: + for (i = 0; i < 4 * N_AES_LANES; i++) + if (len[i] == 0) + { + if (n_left == 0) + { + /* no more work to enqueue, so we are enqueueing placeholder buffer + */ + src[i] = dst[i] = placeholder; + len[i] = sizeof (placeholder); + placeholder_mask[i] = 0; + } + else + { + u8x16 t = aes_block_load (iv[0]); + ((u8x16 *) r)[i] = t; + + src[i] = plaintext[0]; + dst[i] = ciphertext[0]; + len[i] = oplen[0]; + placeholder_mask[i] = ~0; + if (key_index[i] != key_indices[0]) + { + aes_cbc_key_data_t *kd; + key_index[i] = key_indices[0]; + kd = key_data[key_index[i]]; + for (j = 0; j < rounds + 1; j++) + ((u8x16 *) k[j])[i] = kd->encrypt_key[j]; + } + n_left--; + iv++; + ciphertext++; + plaintext++; + key_indices++; + oplen++; + } + } + + count = u32xN_min_scalar (len); + + ASSERT (count % 16 == 0); + + for (i = 0; i < count; i += 16) + { +#if defined(__VAES__) && defined(__AVX512F__) + r[0] = u8x64_xor3 (r[0], aes_block_load_x4 (src, i), k[0][0]); + r[1] = u8x64_xor3 (r[1], aes_block_load_x4 (src + 4, i), k[0][1]); + r[2] = u8x64_xor3 (r[2], aes_block_load_x4 (src + 8, i), k[0][2]); + r[3] = u8x64_xor3 (r[3], aes_block_load_x4 (src + 12, i), k[0][3]); + + for (j = 1; j < rounds; j++) + { + r[0] = aes_enc_round_x4 (r[0], k[j][0]); + r[1] = aes_enc_round_x4 (r[1], k[j][1]); + r[2] = aes_enc_round_x4 (r[2], k[j][2]); + r[3] = aes_enc_round_x4 (r[3], k[j][3]); + } + r[0] = aes_enc_last_round_x4 (r[0], k[j][0]); + r[1] = aes_enc_last_round_x4 (r[1], k[j][1]); + r[2] = aes_enc_last_round_x4 (r[2], k[j][2]); + r[3] = aes_enc_last_round_x4 (r[3], k[j][3]); + + aes_block_store_x4 (dst, i, r[0]); + aes_block_store_x4 (dst + 4, i, r[1]); + aes_block_store_x4 (dst + 8, i, r[2]); + aes_block_store_x4 (dst + 12, i, r[3]); +#elif defined(__VAES__) + r[0] = u8x32_xor3 (r[0], aes_block_load_x2 (src, i), k[0][0]); + r[1] = u8x32_xor3 (r[1], aes_block_load_x2 (src + 2, i), k[0][1]); + r[2] = u8x32_xor3 (r[2], aes_block_load_x2 (src + 4, i), k[0][2]); + r[3] = u8x32_xor3 (r[3], aes_block_load_x2 (src + 6, i), k[0][3]); + + for (j = 1; j < rounds; j++) + { + r[0] = aes_enc_round_x2 (r[0], k[j][0]); + r[1] = aes_enc_round_x2 (r[1], k[j][1]); + r[2] = aes_enc_round_x2 (r[2], k[j][2]); + r[3] = aes_enc_round_x2 (r[3], k[j][3]); + } + r[0] = aes_enc_last_round_x2 (r[0], k[j][0]); + r[1] = aes_enc_last_round_x2 (r[1], k[j][1]); + r[2] = aes_enc_last_round_x2 (r[2], k[j][2]); + r[3] = aes_enc_last_round_x2 (r[3], k[j][3]); + + aes_block_store_x2 (dst, i, r[0]); + aes_block_store_x2 (dst + 2, i, r[1]); + aes_block_store_x2 (dst + 4, i, r[2]); + aes_block_store_x2 (dst + 6, i, r[3]); +#else +#if __x86_64__ + r[0] = u8x16_xor3 (r[0], aes_block_load (src[0] + i), k[0][0]); + r[1] = u8x16_xor3 (r[1], aes_block_load (src[1] + i), k[0][1]); + r[2] = u8x16_xor3 (r[2], aes_block_load (src[2] + i), k[0][2]); + r[3] = u8x16_xor3 (r[3], aes_block_load (src[3] + i), k[0][3]); + + for (j = 1; j < rounds; j++) + { + r[0] = aes_enc_round_x1 (r[0], k[j][0]); + r[1] = aes_enc_round_x1 (r[1], k[j][1]); + r[2] = aes_enc_round_x1 (r[2], k[j][2]); + r[3] = aes_enc_round_x1 (r[3], k[j][3]); + } + + r[0] = aes_enc_last_round_x1 (r[0], k[j][0]); + r[1] = aes_enc_last_round_x1 (r[1], k[j][1]); + r[2] = aes_enc_last_round_x1 (r[2], k[j][2]); + r[3] = aes_enc_last_round_x1 (r[3], k[j][3]); + + aes_block_store (dst[0] + i, r[0]); + aes_block_store (dst[1] + i, r[1]); + aes_block_store (dst[2] + i, r[2]); + aes_block_store (dst[3] + i, r[3]); +#else + r[0] ^= aes_block_load (src[0] + i); + r[1] ^= aes_block_load (src[1] + i); + r[2] ^= aes_block_load (src[2] + i); + r[3] ^= aes_block_load (src[3] + i); + for (j = 0; j < rounds - 1; j++) + { + r[0] = vaesmcq_u8 (vaeseq_u8 (r[0], k[j][0])); + r[1] = vaesmcq_u8 (vaeseq_u8 (r[1], k[j][1])); + r[2] = vaesmcq_u8 (vaeseq_u8 (r[2], k[j][2])); + r[3] = vaesmcq_u8 (vaeseq_u8 (r[3], k[j][3])); + } + r[0] = vaeseq_u8 (r[0], k[j][0]) ^ k[rounds][0]; + r[1] = vaeseq_u8 (r[1], k[j][1]) ^ k[rounds][1]; + r[2] = vaeseq_u8 (r[2], k[j][2]) ^ k[rounds][2]; + r[3] = vaeseq_u8 (r[3], k[j][3]) ^ k[rounds][3]; + aes_block_store (dst[0] + i, r[0]); + aes_block_store (dst[1] + i, r[1]); + aes_block_store (dst[2] + i, r[2]); + aes_block_store (dst[3] + i, r[3]); +#endif +#endif + } + + len -= u32xN_splat (count); + + for (i = 0; i < 4 * N_AES_LANES; i++) + { + src[i] += count; + dst[i] += count; + } + + if (n_left > 0) + goto more; + + if (!u32xN_is_all_zero (len & placeholder_mask)) + goto more; + + return n_ops; +} + +#undef u8xN +#undef u32xN +#undef u32xN_min_scalar +#undef u32xN_is_all_zero +#undef u32xN_splat + #endif /* __crypto_aes_cbc_h__ */ diff --git a/src/vppinfra/devicetree.c b/src/vppinfra/devicetree.c new file mode 100644 index 00000000000..9bf8eeeac6c --- /dev/null +++ b/src/vppinfra/devicetree.c @@ -0,0 +1,347 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#include <vppinfra/clib.h> +#include <vppinfra/devicetree.h> + +#ifdef __linux +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <dirent.h> + +#endif + +static_always_inline clib_dt_node_t * +clib_dt_node_add_child (clib_dt_main_t *dm, clib_dt_node_t *n, char *name) +{ + clib_dt_node_t *cn; + + cn = clib_mem_alloc (sizeof (clib_dt_node_t)); + *cn = (clib_dt_node_t){ .parent = n, .depth = n ? n->depth + 1 : 0 }; + vec_add1 (dm->nodes, cn); + + if (n == 0) + { + ASSERT (dm->root == 0); + dm->root = cn; + return cn; + } + + vec_add1 (n->child_nodes, cn); + cn->path = format (0, "%v/%s", n->path, name); + cn->dt_main = dm; + hash_set_mem (dm->node_by_path, cn->path, cn); + if (vec_len (n->child_nodes) > 1) + { + clib_dt_node_t *prev = n->child_nodes[vec_len (n->child_nodes) - 2]; + prev->next = cn; + cn->prev = prev; + } + + return cn; +} + +void +clib_dt_main_free (clib_dt_main_t *dm) +{ + vec_foreach_pointer (n, dm->nodes) + { + vec_foreach_pointer (p, n->properties) + clib_mem_free (p); + vec_free (n->child_nodes); + vec_free (n->path); + vec_free (n->properties); + } + + vec_free (dm->nodes); + hash_free (dm->node_by_path); + hash_free (dm->node_by_phandle); +} + +#ifdef __linux +__clib_export clib_error_t * +clib_dt_read_from_sysfs (clib_dt_main_t *dm) +{ + DIR *dir, **dir_stack = 0; + struct dirent *e; + clib_dt_node_t *n; + u8 *path = 0; + u32 path_prefix_len; + clib_error_t *err = 0; + + path = format (0, CLIB_DT_LINUX_PREFIX); + path_prefix_len = vec_len (path); + vec_add1 (path, 0); + + dir = opendir ((char *) path); + if (!dir) + { + err = clib_error_return (0, "'%s' opendir failed", path); + goto done; + } + + dm->node_by_path = hash_create_vec (0, sizeof (u8), sizeof (uword)); + dm->node_by_phandle = hash_create (0, sizeof (uword)); + vec_set_len (path, path_prefix_len); + n = clib_dt_node_add_child (dm, 0, 0); + + while (1) + { + e = readdir (dir); + + if (!e) + { + closedir (dir); + if (vec_len (dir_stack) == 0) + break; + + dir = dir_stack[vec_len (dir_stack) - 1]; + vec_pop (dir_stack); + n = n->parent; + continue; + } + + if (e->d_type == DT_REG) + { + path = format (path, "%v/%s%c", n->path, e->d_name, 0); + int fd = open ((char *) path, 0); + if (fd >= 0) + { + struct stat st; + if (fstat (fd, &st) == 0) + { + u32 sz = sizeof (clib_dt_property_t) + st.st_size; + clib_dt_property_t *p = clib_mem_alloc (sz); + clib_memset (p, 0, sz); + + if (read (fd, p->data, st.st_size) == st.st_size) + { + strncpy (p->name, e->d_name, sizeof (p->name)); + p->size = st.st_size; + vec_add1 (n->properties, p); + if (strncmp ("name", p->name, 5) == 0) + n->name = p; + if ((strncmp ("phandle", p->name, 8) == 0) && + (p->size == 4)) + { + u32 phandle = + clib_net_to_host_u32 (*(u32u *) p->data); + hash_set (dm->node_by_phandle, phandle, n); + } + } + else + { + clib_mem_free (p); + err = clib_error_return (0, "'%s' read failed", path); + close (fd); + goto done; + } + } + else + { + err = clib_error_return (0, "'%s' fstat failed", path); + close (fd); + goto done; + } + close (fd); + } + else + { + err = clib_error_return (0, "'%s' open failed", path); + goto done; + } + + vec_set_len (path, path_prefix_len); + } + else if (e->d_type == DT_DIR) + { + DIR *subdir; + if (strncmp (".", e->d_name, 2) == 0 || + strncmp ("..", e->d_name, 3) == 0) + continue; + + path = format (path, "%v/%s%c", n->path, e->d_name, 0); + subdir = opendir ((char *) path); + vec_set_len (path, path_prefix_len); + if (subdir) + { + vec_add1 (dir_stack, dir); + dir = subdir; + n = clib_dt_node_add_child (dm, n, e->d_name); + } + else + { + err = clib_error_return (0, "'%s' opendir failed", path); + goto done; + } + } + else + err = + clib_error_return (0, "unknown entry %s [%u]", e->d_name, e->d_type); + } + +done: + if (err) + clib_dt_main_free (dm); + while (vec_len (dir_stack)) + closedir (vec_pop (dir_stack)); + vec_free (dir_stack); + vec_free (path); + return err; +} +#endif + +clib_dt_node_t * +clib_dt_get_child_node (clib_dt_node_t *n, char *name) +{ + vec_foreach_pointer (cn, n->child_nodes) + { + u8 *p = cn->path + vec_len (cn->path) - 1; + u32 i = 0; + + while (p > cn->path && p[-1] != '/') + p--; + + if (p[-1] != '/') + continue; + + while (p[i] == name[i] && name[i] != 0) + i++; + + if (name[i] != 0) + continue; + + return cn; + } + + return 0; +} + +__clib_export clib_dt_node_t * +clib_dt_get_node_with_path (clib_dt_main_t *dm, char *fmt, ...) +{ + u8 *s; + uword *p; + + va_list va; + va_start (va, fmt); + s = va_format (0, fmt, &va); + va_end (va); + + if (s[0] != '/') + return 0; + + p = hash_get_mem (dm->node_by_path, s); + if (p) + return (clib_dt_node_t *) p[0]; + + return 0; +} + +__clib_export clib_dt_property_t * +clib_dt_get_node_property_by_name (clib_dt_node_t *n, char *name) +{ + vec_foreach_pointer (p, n->properties) + if (strncmp (name, p->name, sizeof (p->name)) == 0) + return p; + return 0; +} + +__clib_export int +clib_dt_node_is_compatible (clib_dt_node_t *n, char *comp) +{ + clib_dt_property_t *p; + char *s; + + p = clib_dt_get_node_property_by_name (n, "compatible"); + + if (!p) + return 0; + + s = (char *) p->data; + for (u32 i = 1, len = 1; i <= p->size; i++) + { + if (p->data[i - 1] == 0) + { + if (strncmp (comp, s, len) == 0) + return 1; + s = (char *) p->data + i; + len = 1; + } + else + len++; + } + + return 0; +} + +__clib_export u8 * +format_clib_dt_property_data (u8 *s, va_list *args) +{ + clib_dt_property_t *p = va_arg (*args, clib_dt_property_t *); + u32 sz = p->size, is_printable = 0; + u32 n_nulls = 0; + + if (sz > 2 && p->data[sz - 1] == 0 && p->data[0] != 0) + { + is_printable = 1; + for (u32 i = 1; i < sz - 1; i++) + { + u8 c = p->data[i]; + if (c == 0) + { + if (p->data[i - 1] == 0) + { + is_printable = 0; + break; + } + n_nulls++; + } + else if ((c < 0x20) || (c > 0x7f)) + { + is_printable = 0; + break; + } + } + } + + if (is_printable) + { + s = format (s, "'%s'", p->data); + if (n_nulls) + { + for (u32 i = 2; i < p->size; i++) + if (((u8 *) p->data)[i - 1] == 0) + s = format (s, ", '%s'", ((u8 *) p->data) + i); + } + } + else + { + s = format (s, "< %02x", p->data[0]); + for (u32 i = 0; i < p->size; i++) + s = format (s, " %02x", p->data[i]); + s = format (s, " >"); + } + return s; +} + +__clib_export clib_dt_node_t * +clib_dt_dereference_node (clib_dt_node_t *n, char *name) +{ + clib_dt_property_t *p; + uword *h; + + p = clib_dt_get_node_property_by_name (n, name); + if (!p || (p->size != sizeof (u32))) + return 0; + + h = hash_get (n->dt_main->node_by_phandle, + clib_net_to_host_u32 (*(u32u *) p->data)); + + if (h) + return (clib_dt_node_t *) h[0]; + + return 0; +} diff --git a/src/vppinfra/devicetree.h b/src/vppinfra/devicetree.h new file mode 100644 index 00000000000..21c2e0f7006 --- /dev/null +++ b/src/vppinfra/devicetree.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#ifndef CLIB_DEVICETREE_H_ +#define CLIB_DEVICETREE_H_ + +#include <vppinfra/clib.h> +#include <vlib/vlib.h> + +#ifdef __linux +#define CLIB_DT_LINUX_PREFIX "/sys/firmware/devicetree/base" +#endif + +typedef struct +{ + char name[32]; + u32 size; + u8 data[]; +} clib_dt_property_t; + +typedef struct clib_dt_main clib_dt_main_t; + +typedef struct clib_dt_node +{ + u8 *path; + struct clib_dt_node *parent; + struct clib_dt_node *prev; + struct clib_dt_node *next; + struct clib_dt_node **child_nodes; + u8 depth; + clib_dt_property_t *name; + clib_dt_property_t **properties; + clib_dt_main_t *dt_main; +} clib_dt_node_t; + +typedef struct clib_dt_main +{ + clib_dt_node_t **nodes; + clib_dt_node_t *root; + uword *node_by_path; + uword *node_by_phandle; +} clib_dt_main_t; + +clib_dt_node_t *clib_dt_get_node_with_path (clib_dt_main_t *dm, char *fmt, + ...); +clib_dt_property_t *clib_dt_get_node_property_by_name (clib_dt_node_t *, + char *); +int clib_dt_node_is_compatible (clib_dt_node_t *, char *); +clib_dt_node_t *clib_dt_dereference_node (clib_dt_node_t *, char *); +#ifdef __linux +clib_error_t *clib_dt_read_from_sysfs (clib_dt_main_t *dm); +#endif + +format_function_t format_clib_dt_desc; +format_function_t format_clib_dt_property_data; + +static_always_inline int +clib_dt_proprerty_is_u32 (clib_dt_property_t *p) +{ + if (p == 0 || p->size != 4) + return 0; + return 1; +} + +static_always_inline u32 +clib_dt_proprerty_get_u32 (clib_dt_property_t *p) +{ + return clib_net_to_host_u32 (*(u32u *) p->data); +} + +#endif /* CLIB_DEVICETREE_H_ */ diff --git a/src/vppinfra/format.c b/src/vppinfra/format.c index cf17b8a1acb..642d3e20654 100644 --- a/src/vppinfra/format.c +++ b/src/vppinfra/format.c @@ -833,6 +833,16 @@ done: return s; } +__clib_export char * +format_c_string (u8 *s, const char *fmt, ...) +{ + va_list args; + va_start (args, fmt); + s = va_format (s, fmt, &args); + va_end (args); + vec_add1 (s, '\0'); + return (char *) s; +} /* * fd.io coding-style-patch-verification: ON diff --git a/src/vppinfra/format.h b/src/vppinfra/format.h index a1a70a2d64f..14bac869f89 100644 --- a/src/vppinfra/format.h +++ b/src/vppinfra/format.h @@ -372,6 +372,8 @@ int test_unformat_main (unformat_input_t * input); created circular dependency problems. */ int test_vec_main (unformat_input_t * input); +char *format_c_string (u8 *s, const char *fmt, ...); + #endif /* included_format_h */ /* diff --git a/src/vppinfra/format_ansi.h b/src/vppinfra/format_ansi.h new file mode 100644 index 00000000000..c35406aacf7 --- /dev/null +++ b/src/vppinfra/format_ansi.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#ifndef __FORMAT_ANSI_H__ +#define __FORMAT_ANSI_H__ + +#define ANSI_RESET "\x1b[0m" +#define ANSI_BOLD "\x1b[1m" +#define ANSI_ITALIC "\x1b[3m" +#define ANSI_UNDERLINE "\x1b[4m" +#define ANSI_BLINK "\x1b[5m" +#define ANSI_FG_BLACK "\x1b[30m" +#define ANSI_FG_RED "\x1b[31m" +#define ANSI_FG_GREEN "\x1b[32m" +#define ANSI_FG_YELLOW "\x1b[33m" +#define ANSI_FG_BLUE "\x1b[34m" +#define ANSI_FG_MAGENTA "\x1b[35m" +#define ANSI_FG_CYAN "\x1b[36m" +#define ANSI_FG_WHITE "\x1b[37m" +#define ANSI_FG_DEFAULT "\x1b[39m" +#define ANSI_BG_BLACK "\x1b[40m" +#define ANSI_BG_RED "\x1b[41m" +#define ANSI_BG_GREEN "\x1b[42m" +#define ANSI_BG_YELLOW "\x1b[43m" +#define ANSI_BG_BLUE "\x1b[44m" +#define ANSI_BG_MAGENTA "\x1b[45m" +#define ANSI_BG_CYAN "\x1b[46m" +#define ANSI_BG_WHITE "\x1b[47m" +#define ANSI_BG_DEFAULT "\x1b[49m" +#define ANSI_FG_BR_BLACK "\x1b[90m" +#define ANSI_FG_BR_RED "\x1b[91m" +#define ANSI_FG_BR_GREEN "\x1b[92m" +#define ANSI_FG_BR_YELLOW "\x1b[93m" +#define ANSI_FG_BR_BLUE "\x1b[94m" +#define ANSI_FG_BR_MAGENTA "\x1b[95m" +#define ANSI_FG_BR_CYAN "\x1b[96m" +#define ANSI_FG_BR_WHITE "\x1b[97m" +#define ANSI_BG_BR_BLACK "\x1b[100m" +#define ANSI_BG_BR_RED "\x1b[101m" +#define ANSI_BG_BR_GREEN "\x1b[102m" +#define ANSI_BG_BR_YELLOW "\x1b[103m" +#define ANSI_BG_BR_BLUE "\x1b[104m" +#define ANSI_BG_BR_MAGENTA "\x1b[105m" +#define ANSI_BG_BR_CYAN "\x1b[106m" +#define ANSI_BG_BR_WHITE "\x1b[107m" + +#endif /* __FORMAT_ANSI_H__ */ diff --git a/src/vppinfra/heap.c b/src/vppinfra/heap.c index 7db814200f8..9920528732d 100644 --- a/src/vppinfra/heap.c +++ b/src/vppinfra/heap.c @@ -680,6 +680,7 @@ debug_elt (u8 * s, void *v, word i, word n) i = -n / 2; for (e = e0; 1; e = heap_next (e)) { + s = format (s, " "); if (heap_is_free (e)) s = format (s, "index %4d, free\n", e - h->elts); else if (h->format_elt) diff --git a/src/vppinfra/jsonformat.c b/src/vppinfra/jsonformat.c index 1aa3864be04..73cb94769d8 100644 --- a/src/vppinfra/jsonformat.c +++ b/src/vppinfra/jsonformat.c @@ -500,12 +500,13 @@ format_vl_api_mac_address_t (u8 * s, va_list * args) mac->bytes[0], mac->bytes[1], mac->bytes[2], mac->bytes[3], mac->bytes[4], mac->bytes[5]); } -#define _(T) \ - cJSON *vl_api_ ##T## _t_tojson (vl_api_ ##T## _t *a) { \ - u8 *s = format(0, "%U", format_vl_api_ ##T## _t, a); \ - cJSON *o = cJSON_CreateString((char *)s); \ - vec_free(s); \ - return o; \ +#define _(T) \ + cJSON *vl_api_##T##_t_tojson (vl_api_##T##_t *a) \ + { \ + char *s = format_c_string (0, "%U", format_vl_api_##T##_t, a, 0); \ + cJSON *o = cJSON_CreateString (s); \ + vec_free (s); \ + return o; \ } foreach_type_tojson #undef _ diff --git a/src/vppinfra/linux/mem.c b/src/vppinfra/linux/mem.c index 734f5c4788c..17b4412e6c9 100644 --- a/src/vppinfra/linux/mem.c +++ b/src/vppinfra/linux/mem.c @@ -101,11 +101,13 @@ legacy_get_log2_default_hugepage_size (void) void clib_mem_main_init (void) { + unsigned long nodemask = 0, maxnode = CLIB_MAX_NUMAS; + unsigned long flags = MPOL_F_MEMS_ALLOWED; clib_mem_main_t *mm = &clib_mem_main; long sysconf_page_size; uword page_size; - void *va; - int fd; + void *va = 0; + int fd, mode; if (mm->log2_page_sz != CLIB_MEM_PAGE_SZ_UNKNOWN) return; @@ -131,23 +133,8 @@ clib_mem_main_init (void) mm->log2_sys_default_hugepage_sz = mm->log2_default_hugepage_sz; /* numa nodes */ - va = mmap (0, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | - MAP_ANONYMOUS, -1, 0); - if (va == MAP_FAILED) - return; - - if (mlock (va, page_size)) - goto done; - - for (int i = 0; i < CLIB_MAX_NUMAS; i++) - { - int status; - if (syscall (__NR_move_pages, 0, 1, &va, &i, &status, 0) == 0) - mm->numa_node_bitmap |= 1ULL << i; - } - -done: - munmap (va, page_size); + if (syscall (__NR_get_mempolicy, &mode, &nodemask, maxnode, va, flags) == 0) + mm->numa_node_bitmap = nodemask; } __clib_export u64 @@ -530,6 +517,7 @@ clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size, { int i, *status = 0; void **ptr = 0; + unsigned char incore; log2_page_size = clib_mem_log2_page_size_validate (log2_page_size); @@ -551,6 +539,19 @@ clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size, for (i = 0; i < n_pages; i++) { + /* move_pages() returns -ENONET in status for huge pages on 5.19+ kernel. + * Retry with get_mempolicy() to obtain NUMA node info only if the pages + * are allocated and in memory, which is checked by mincore(). */ + if (status[i] == -ENOENT && + syscall (__NR_mincore, ptr[i], 1, &incore) == 0 && (incore & 1) != 0) + { + if (syscall (__NR_get_mempolicy, &status[i], 0, 0, ptr[i], + MPOL_F_NODE | MPOL_F_ADDR) != 0) + { + /* if get_mempolicy fails, keep the original value in status */ + status[i] = -ENONET; + } + } if (status[i] >= 0 && status[i] < CLIB_MAX_NUMAS) { stats->mapped++; diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 75015d59a4a..ab9c5da30ec 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -299,10 +299,27 @@ void *clib_mem_init_thread_safe (void *memory, uword memory_size); void clib_mem_exit (void); +typedef struct +{ + /* Address of callers: outer first, inner last. */ + uword callers[12]; + + /* Count of allocations with this traceback. */ + u32 n_allocations; + + /* Count of bytes allocated with this traceback. */ + u32 n_bytes; + + /* Offset of this item */ + uword offset; +} mheap_trace_t; + void clib_mem_trace (int enable); int clib_mem_is_traced (void); +mheap_trace_t *clib_mem_trace_dup (clib_mem_heap_t *heap); + typedef struct { /* Total number of objects allocated. */ diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c index a188164a7ba..d5ff21e58c0 100644 --- a/src/vppinfra/mem_dlmalloc.c +++ b/src/vppinfra/mem_dlmalloc.c @@ -19,21 +19,7 @@ #include <vppinfra/lock.h> #include <vppinfra/hash.h> #include <vppinfra/elf_clib.h> - -typedef struct -{ - /* Address of callers: outer first, inner last. */ - uword callers[12]; - - /* Count of allocations with this traceback. */ - u32 n_allocations; - - /* Count of bytes allocated with this traceback. */ - u32 n_bytes; - - /* Offset of this item */ - uword offset; -} mheap_trace_t; +#include <vppinfra/stack.h> typedef struct { @@ -65,15 +51,13 @@ mheap_get_trace_internal (const clib_mem_heap_t *heap, uword offset, { mheap_trace_main_t *tm = &mheap_trace_main; mheap_trace_t *t; - uword i, n_callers, trace_index, *p; - mheap_trace_t trace; + uword i, trace_index, *p; + mheap_trace_t trace = {}; + int index; if (heap != tm->current_traced_mheap || mheap_trace_thread_disable) return; - /* Spurious Coverity warnings be gone. */ - clib_memset (&trace, 0, sizeof (trace)); - clib_spinlock_lock (&tm->lock); /* heap could have changed while we were waiting on the lock */ @@ -83,9 +67,19 @@ mheap_get_trace_internal (const clib_mem_heap_t *heap, uword offset, /* Turn off tracing for this thread to avoid embarrassment... */ mheap_trace_thread_disable = 1; - /* Skip our frame and mspace_get_aligned's frame */ - n_callers = clib_backtrace (trace.callers, ARRAY_LEN (trace.callers), 2); - if (n_callers == 0) + index = -2; /* skip first 2 stack frames */ + foreach_clib_stack_frame (sf) + { + if (index >= 0) + { + if (index == ARRAY_LEN (trace.callers)) + break; + trace.callers[index] = sf->ip; + } + index++; + } + + if (index < 1) goto out; if (!tm->trace_by_callers) @@ -565,6 +559,23 @@ clib_mem_trace_enable_disable (uword enable) return rv; } +__clib_export mheap_trace_t * +clib_mem_trace_dup (clib_mem_heap_t *heap) +{ + mheap_trace_main_t *tm = &mheap_trace_main; + mheap_trace_t *traces_copy = 0; + + clib_spinlock_lock (&tm->lock); + if (vec_len (tm->traces) > 0 && heap == tm->current_traced_mheap) + { + traces_copy = vec_dup (tm->traces); + qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]), + mheap_trace_sort); + } + clib_spinlock_unlock (&tm->lock); + return traces_copy; +} + __clib_export clib_mem_heap_t * clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt, ...) { diff --git a/src/vppinfra/mhash.c b/src/vppinfra/mhash.c index f0f1aa470d7..babaaeec726 100644 --- a/src/vppinfra/mhash.c +++ b/src/vppinfra/mhash.c @@ -164,6 +164,8 @@ mhash_sanitize_hash_user (mhash_t * mh) h->user = pointer_to_uword (mh); } +static u8 *mhash_format_pair_default (u8 *s, va_list *args); + __clib_export void mhash_init (mhash_t * h, uword n_value_bytes, uword n_key_bytes) { @@ -208,12 +210,12 @@ mhash_init (mhash_t * h, uword n_value_bytes, uword n_key_bytes) vec_validate (h->key_tmps, os_get_nthreads () - 1); ASSERT (n_key_bytes < ARRAY_LEN (t)); - h->hash = hash_create2 ( /* elts */ 0, + h->hash = hash_create2 (/* elts */ 0, /* user */ pointer_to_uword (h), /* value_bytes */ n_value_bytes, t[n_key_bytes].key_sum, t[n_key_bytes].key_equal, /* format pair/arg */ - 0, 0); + mhash_format_pair_default, 0); } static uword @@ -331,8 +333,8 @@ mhash_set_mem (mhash_t * h, void *key, uword * new_value, uword * old_value) { if (key_alloc_from_free_list) { - h->key_vector_free_indices[l] = i; - vec_set_len (h->key_vector_free_indices, l + 1); + vec_set_len (h->key_vector_free_indices, l); + h->key_vector_free_indices[l - 1] = i; } else vec_dec_len (h->key_vector_or_heap, h->n_key_bytes); @@ -371,8 +373,8 @@ mhash_unset (mhash_t * h, void *key, uword * old_value) return 1; } -u8 * -format_mhash_key (u8 * s, va_list * va) +__clib_export u8 * +format_mhash_key (u8 *s, va_list *va) { mhash_t *h = va_arg (*va, mhash_t *); u32 ki = va_arg (*va, u32); @@ -387,7 +389,43 @@ format_mhash_key (u8 * s, va_list * va) else if (h->format_key) s = format (s, "%U", h->format_key, k); else - s = format (s, "%U", format_hex_bytes, k, h->n_key_bytes); + s = format (s, "0x%U", format_hex_bytes, k, h->n_key_bytes); + + return s; +} + +static u8 * +mhash_format_pair_default (u8 *s, va_list *args) +{ + void *CLIB_UNUSED (user_arg) = va_arg (*args, void *); + void *v = va_arg (*args, void *); + hash_pair_t *p = va_arg (*args, hash_pair_t *); + hash_t *h = hash_header (v); + mhash_t *mh = uword_to_pointer (h->user, mhash_t *); + + s = format (s, "%U", format_mhash_key, mh, (u32) p->key); + if (hash_value_bytes (h) > 0) + s = format (s, " -> 0x%8U", format_hex_bytes, &p->value[0], + hash_value_bytes (h)); + return s; +} + +__clib_export u8 * +format_mhash (u8 *s, va_list *va) +{ + mhash_t *h = va_arg (*va, mhash_t *); + int verbose = va_arg (*va, int); + + s = format (s, "mhash %p, %wd elts, \n", h, mhash_elts (h)); + if (mhash_key_vector_is_heap (h)) + s = format (s, " %U", format_heap, h->key_vector_or_heap, verbose); + else + s = format (s, " keys %wd elts, %wd size, %wd free, %wd bytes used\n", + vec_len (h->key_vector_or_heap) / h->n_key_bytes, + h->n_key_bytes, vec_len (h->key_vector_free_indices), + vec_bytes (h->key_vector_or_heap) + + vec_bytes (h->key_vector_free_indices)); + s = format (s, " %U", format_hash, h->hash, verbose); return s; } diff --git a/src/vppinfra/mhash.h b/src/vppinfra/mhash.h index 7eb1918384e..62aee365fa3 100644 --- a/src/vppinfra/mhash.h +++ b/src/vppinfra/mhash.h @@ -166,8 +166,13 @@ do { \ })); \ } while (0) +u8 *format_mhash (u8 *s, va_list *va); + format_function_t format_mhash_key; +/* Main test routine. */ +int test_mhash_main (unformat_input_t *input); + #endif /* included_clib_mhash_h */ /* diff --git a/src/vppinfra/pmalloc.c b/src/vppinfra/pmalloc.c index 2a27379b573..85b9db9d56c 100644 --- a/src/vppinfra/pmalloc.c +++ b/src/vppinfra/pmalloc.c @@ -17,6 +17,9 @@ #include <stdlib.h> #include <sys/types.h> #include <sys/stat.h> +#ifdef __FreeBSD__ +#include <sys/memrange.h> +#endif /* __FreeBSD__ */ #include <fcntl.h> #include <unistd.h> #include <sched.h> @@ -184,8 +187,9 @@ next_chunk: } static void -pmalloc_update_lookup_table (clib_pmalloc_main_t * pm, u32 first, u32 count) +pmalloc_update_lookup_table (clib_pmalloc_main_t *pm, u32 first, u32 count) { +#ifdef __linux uword seek, va, pa, p; int fd; u32 elts_per_page = 1U << (pm->def_log2_page_sz - pm->lookup_log2_page_sz); @@ -223,6 +227,45 @@ pmalloc_update_lookup_table (clib_pmalloc_main_t * pm, u32 first, u32 count) if (fd != -1) close (fd); +#elif defined(__FreeBSD__) + struct mem_extract meme; + uword p; + int fd; + u32 elts_per_page = 1U << (pm->def_log2_page_sz - pm->lookup_log2_page_sz); + + vec_validate_aligned (pm->lookup_table, + vec_len (pm->pages) * elts_per_page - 1, + CLIB_CACHE_LINE_BYTES); + + p = (uword) first * elts_per_page; + if (pm->flags & CLIB_PMALLOC_F_NO_PAGEMAP) + { + while (p < (uword) elts_per_page * count) + { + pm->lookup_table[p] = + pointer_to_uword (pm->base) + (p << pm->lookup_log2_page_sz); + p++; + } + return; + } + + fd = open ((char *) "/dev/mem", O_RDONLY); + if (fd == -1) + return; + + while (p < (uword) elts_per_page * count) + { + meme.me_vaddr = + pointer_to_uword (pm->base) + (p << pm->lookup_log2_page_sz); + if (ioctl (fd, MEM_EXTRACT_PADDR, &meme) == -1) + continue; + pm->lookup_table[p] = meme.me_vaddr - meme.me_paddr; + p++; + } + return; +#else +#error "Unsupported OS" +#endif } static inline clib_pmalloc_page_t * diff --git a/src/vppinfra/stack.c b/src/vppinfra/stack.c new file mode 100644 index 00000000000..190e880c228 --- /dev/null +++ b/src/vppinfra/stack.c @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#define _GNU_SOURCE +#include <dlfcn.h> + +#include <vppinfra/clib.h> +#include <vppinfra/stack.h> +#include <vppinfra/error.h> + +#if HAVE_LIBUNWIND == 1 + +#define UNW_LOCAL_ONLY +#include <libunwind.h> + +static __thread unw_cursor_t cursor; +static __thread unw_context_t context; + +#endif + +__clib_export clib_stack_frame_t * +clib_stack_frame_get (clib_stack_frame_t *sf) +{ +#if HAVE_LIBUNWIND == 1 + Dl_info info = {}; + + if (sf->index == 0) + { + if (unw_getcontext (&context) < 0) + { + clib_warning ("libunwind: cannot get local machine state\n"); + return 0; + } + if (unw_init_local (&cursor, &context) < 0) + { + clib_warning ( + "libunwind: cannot initialize cursor for local unwinding\n"); + return 0; + } + if (unw_step (&cursor) < 1) + return 0; + } + else if (unw_step (&cursor) < 1) + return 0; + + if (unw_get_reg (&cursor, UNW_REG_IP, &sf->ip)) + { + clib_warning ("libunwind: cannot read IP\n"); + return 0; + } + + if (unw_get_reg (&cursor, UNW_REG_SP, &sf->sp)) + { + clib_warning ("libunwind: cannot read SP\n"); + return 0; + } + + if (unw_get_proc_name (&cursor, sf->name, sizeof (sf->name), &sf->offset) < + 0) + sf->name[0] = sf->offset = 0; + + sf->is_signal_frame = unw_is_signal_frame (&cursor) ? 1 : 0; + + if (dladdr ((void *) sf->ip, &info)) + sf->file_name = info.dli_fname; + else + sf->file_name = 0; + + sf->index++; + return sf; +#else + return 0; +#endif +} diff --git a/src/vppinfra/stack.h b/src/vppinfra/stack.h new file mode 100644 index 00000000000..98a621d4176 --- /dev/null +++ b/src/vppinfra/stack.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Cisco Systems, Inc. + */ + +#ifndef __STACK_H__ +#define __STACK_H__ + +#include <vppinfra/clib.h> + +typedef struct +{ + uword ip, sp; + uword offset; + char name[64]; + const char *file_name; + u32 index; + u8 is_signal_frame; +} clib_stack_frame_t; + +clib_stack_frame_t *clib_stack_frame_get (clib_stack_frame_t *); + +#define foreach_clib_stack_frame(sf) \ + for (clib_stack_frame_t _sf = {}, *sf = clib_stack_frame_get (&_sf); sf; \ + sf = clib_stack_frame_get (sf)) + +#endif /* __STACK_H__ */ diff --git a/src/vppinfra/test_mhash.c b/src/vppinfra/test_mhash.c new file mode 100644 index 00000000000..70be2b9b382 --- /dev/null +++ b/src/vppinfra/test_mhash.c @@ -0,0 +1,403 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2023 Yandex LLC. + */ + +#ifdef CLIB_LINUX_KERNEL +#include <linux/unistd.h> +#endif + +#ifdef CLIB_UNIX +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> +#include <vppinfra/time.h> +#endif + +#include <vppinfra/random.h> +#include <vppinfra/mem.h> +#include <vppinfra/hash.h> +#include <vppinfra/mhash.h> +#include <vppinfra/error.h> +#include <vppinfra/format.h> +#include <vppinfra/bitmap.h> + +static int verbose; +#define if_verbose(format, args...) \ + if (verbose) \ + { \ + clib_warning (format, ##args); \ + } + +typedef struct +{ + int n_iterations; + + int n_iterations_per_print; + + /* Number of pairs to insert into mhash. */ + int n_pairs; + + /* True to validate correctness of mhash functions. */ + int n_iterations_per_validate; + + /* Verbosity level for mhash formats. */ + int verbose; + + /* Random number seed. */ + u32 seed; +} mhash_test_t; + +static clib_error_t * +mhash_next_test (mhash_t *h) +{ + hash_next_t hn = { 0 }; + hash_pair_t *p0, *p1; + clib_error_t *error = 0; + + hash_foreach_pair (p0, h->hash, { + p1 = hash_next (h->hash, &hn); + error = CLIB_ERROR_ASSERT (p0 == p1); + if (error) + break; + }); + + if (!error) + error = CLIB_ERROR_ASSERT (!hash_next (h->hash, &hn)); + + return error; +} + +static clib_error_t * +test_word_key (mhash_test_t *ht) +{ + mhash_t _h = { 0 }, *h = &_h; + word i, j; + + word *keys = 0, *vals = 0; + uword *is_inserted = 0; + + clib_error_t *error = 0; + + vec_resize (keys, ht->n_pairs); + vec_resize (vals, vec_len (keys)); + + mhash_init (h, sizeof (vals[0]), sizeof (keys[0])); + /* borrow 0 elt to make index keys non-zero */ + vec_validate (h->key_vector_or_heap, 0); + + { + uword *unique = 0; + u32 k; + + for (i = 0; i < vec_len (keys); i++) + { + do + { + k = random_u32 (&ht->seed) & 0xfffff; + } + while (clib_bitmap_get (unique, k)); + unique = clib_bitmap_ori (unique, k); + keys[i] = k; + vals[i] = i; + } + + clib_bitmap_free (unique); + } + + for (i = 0; i < ht->n_iterations; i++) + { + u32 vi = random_u32 (&ht->seed) % vec_len (keys); + + if (clib_bitmap_get (is_inserted, vi)) + { + mhash_unset (h, &keys[vi], 0); + mhash_unset (h, &keys[vi], 0); + } + else + { + mhash_set (h, &keys[vi], vals[vi], 0); + mhash_set (h, &keys[vi], vals[vi], 0); + } + + is_inserted = clib_bitmap_xori (is_inserted, vi); + + if (ht->n_iterations_per_print > 0 && + ((i + 1) % ht->n_iterations_per_print) == 0) + if_verbose ("iteration %d\n %U", i + 1, format_mhash, h, ht->verbose); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + { + uword ki, *k, *v; + + mhash_foreach (k, v, h, { + ki = v[0]; + ASSERT (keys[ki] == k[0]); + }); + } + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, &keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + + if ((error = mhash_next_test (h))) + goto done; + + if_verbose ("%U", format_mhash, h, ht->verbose); + + for (i = 0; i < vec_len (keys); i++) + { + if (!clib_bitmap_get (is_inserted, i)) + continue; + + mhash_unset (h, &keys[i], 0); + mhash_unset (h, &keys[i], 0); + is_inserted = clib_bitmap_xori (is_inserted, i); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, &keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + +done: + mhash_free (h); + vec_free (keys); + vec_free (vals); + clib_bitmap_free (is_inserted); + + if (verbose) + fformat (stderr, "%U\n", format_clib_mem_usage, /* verbose */ 0); + + return error; +} + +static u8 * +test2_format (u8 *s, va_list *args) +{ + void *CLIB_UNUSED (user_arg) = va_arg (*args, void *); + void *v = va_arg (*args, void *); + hash_pair_t *p = va_arg (*args, hash_pair_t *); + hash_t *h = hash_header (v); + mhash_t *mh = uword_to_pointer (h->user, mhash_t *); + + return format (s, "0x%8U <- %U", format_hex_bytes, &p->value[0], + hash_value_bytes (h), format_mhash_key, mh, (u32) p->key); +} + +static clib_error_t * +test_string_key (mhash_test_t *ht, uword is_c_string) +{ + mhash_t _h = { 0 }, *h = &_h; + word i, j; + + u8 **keys = 0; + word *vals = 0; + uword *is_inserted = 0; + + clib_error_t *error = 0; + + vec_resize (keys, ht->n_pairs); + vec_resize (vals, vec_len (keys)); + + if (is_c_string) + mhash_init_c_string (h, sizeof (vals[0])); + else + mhash_init_vec_string (h, sizeof (vals[0])); + hash_set_pair_format (h->hash, test2_format, 0); + + for (i = 0; i < vec_len (keys); i++) + { + keys[i] = random_string (&ht->seed, 5 + (random_u32 (&ht->seed) & 0xf)); + keys[i] = format (keys[i], "%x", i); + if (is_c_string) + vec_terminate_c_string (keys[i]); + vals[i] = random_u32 (&ht->seed); + } + + for (i = 0; i < ht->n_iterations; i++) + { + u32 vi = random_u32 (&ht->seed) % vec_len (keys); + + if (clib_bitmap_get (is_inserted, vi)) + { + mhash_unset (h, keys[vi], 0); + mhash_unset (h, keys[vi], 0); + } + else + { + mhash_set (h, keys[vi], vals[vi], 0); + mhash_set (h, keys[vi], vals[vi], 0); + } + + is_inserted = clib_bitmap_xori (is_inserted, vi); + + if (ht->n_iterations_per_print > 0 && + ((i + 1) % ht->n_iterations_per_print) == 0) + if_verbose ("iteration %d\n %U", i + 1, format_mhash, h, ht->verbose); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + + if ((error = mhash_next_test (h))) + goto done; + + if_verbose ("%U", format_mhash, h, ht->verbose); + + for (i = 0; i < vec_len (keys); i++) + { + if (!clib_bitmap_get (is_inserted, i)) + continue; + + mhash_unset (h, keys[i], 0); + mhash_unset (h, keys[i], 0); + is_inserted = clib_bitmap_xori (is_inserted, i); + + if (ht->n_iterations_per_validate == 0 || + (i + 1) % ht->n_iterations_per_validate) + continue; + + if ((error = hash_validate (h->hash))) + goto done; + + for (j = 0; j < vec_len (keys); j++) + { + uword *v; + v = mhash_get (h, keys[j]); + if ((error = CLIB_ERROR_ASSERT (clib_bitmap_get (is_inserted, j) == + (v != 0)))) + goto done; + if (v) + { + if ((error = CLIB_ERROR_ASSERT (v[0] == vals[j]))) + goto done; + } + } + } + +done: + mhash_free (h); + vec_free (vals); + clib_bitmap_free (is_inserted); + + for (i = 0; i < vec_len (keys); i++) + vec_free (keys[i]); + vec_free (keys); + + if (verbose) + fformat (stderr, "%U\n", format_clib_mem_usage, /* verbose */ 0); + + return error; +} + +int +test_mhash_main (unformat_input_t *input) +{ + mhash_test_t _ht = { 0 }, *ht = &_ht; + clib_error_t *error; + + ht->n_iterations = 100; + ht->n_pairs = 10; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (0 == unformat (input, "iter %d", &ht->n_iterations) && + 0 == unformat (input, "print %d", &ht->n_iterations_per_print) && + 0 == unformat (input, "elts %d", &ht->n_pairs) && + 0 == unformat (input, "seed %d", &ht->seed) && + 0 == unformat (input, "verbose %=", &ht->verbose, 1) && + 0 == unformat (input, "valid %d", &ht->n_iterations_per_validate)) + { + clib_warning ("unknown input `%U'", format_unformat_error, input); + return 1; + } + } + + if (!ht->seed) + ht->seed = random_default_seed (); + + if_verbose ("testing %d iterations, seed %d", ht->n_iterations, ht->seed); + + error = test_word_key (ht); + if (error) + clib_error_report (error); + + error = test_string_key (ht, 0); + if (error) + clib_error_report (error); + + error = test_string_key (ht, 1); + if (error) + clib_error_report (error); + + return 0; +} + +#ifdef CLIB_UNIX +int +main (int argc, char *argv[]) +{ + unformat_input_t i; + int ret; + + clib_mem_init (0, 3ULL << 30); + + verbose = (argc > 1); + unformat_init_command_line (&i, argv); + ret = test_mhash_main (&i); + unformat_free (&i); + + return ret; +} +#endif /* CLIB_UNIX */ diff --git a/src/vppinfra/time.c b/src/vppinfra/time.c index 5a6aaf182e4..f1736499a0a 100644 --- a/src/vppinfra/time.c +++ b/src/vppinfra/time.c @@ -76,8 +76,11 @@ clock_frequency_from_proc_filesystem (void) f64 ppc_timebase = 0; /* warnings be gone */ unformat_input_t input; -/* $$$$ aarch64 kernel doesn't report "cpu MHz" */ -#if defined(__aarch64__) +#if defined(__x86_64__) + if (clib_cpu_supports_aperfmperf ()) + return 0.0; +#elif defined(__aarch64__) + /* $$$$ aarch64 kernel doesn't report "cpu MHz" */ return 0.0; #endif diff --git a/src/vppinfra/time_range.c b/src/vppinfra/time_range.c index 4b5e1303763..54f5629641a 100644 --- a/src/vppinfra/time_range.c +++ b/src/vppinfra/time_range.c @@ -264,11 +264,10 @@ format_clib_timebase_time (u8 * s, va_list * args) clib_timebase_time_to_components (now, cp); - s = format (s, "%s, %u %s %u %u:%02u:%02u", - day_names_epoch_order[cp->day_name_index], - cp->day, - month_short_names[cp->month], - cp->year, cp->hour, cp->minute, cp->second); + s = format (s, "%s, %02u %s %u %02u:%02u:%02u", + day_names_epoch_order[cp->day_name_index], cp->day, + month_short_names[cp->month], cp->year, cp->hour, cp->minute, + cp->second); return (s); } diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index 4dbc5ce98ce..05ca2f901c6 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -35,6 +35,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + #include <vppinfra/error.h> #include <vppinfra/os.h> #include <vppinfra/bitmap.h> @@ -42,11 +46,19 @@ #include <vppinfra/format.h> #ifdef __linux__ #include <vppinfra/linux/sysfs.h> +#include <sched.h> +#elif defined(__FreeBSD__) +#define _WANT_FREEBSD_BITSET +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/types.h> +#include <sys/cpuset.h> +#include <sys/domainset.h> +#include <sys/sysctl.h> #endif #include <sys/stat.h> #include <sys/types.h> -#include <sys/syscall.h> #include <sys/uio.h> /* writev */ #include <fcntl.h> #include <stdio.h> /* for sprintf */ @@ -55,6 +67,8 @@ __clib_export __thread uword __os_thread_index = 0; __clib_export __thread uword __os_numa_index = 0; +__clib_export clib_bitmap_t *os_get_cpu_affinity_bitmap (int pid); + clib_error_t * clib_file_n_bytes (char *file, uword * result) { @@ -207,27 +221,20 @@ unix_proc_file_contents (char *file, u8 ** result) return 0; } -void os_panic (void) __attribute__ ((weak)); - -__clib_export void +__clib_export __clib_weak void os_panic (void) { abort (); } -void os_exit (int) __attribute__ ((weak)); - -void +__clib_export __clib_weak void os_exit (int code) { exit (code); } -void os_puts (u8 * string, uword string_length, uword is_error) - __attribute__ ((weak)); - -void -os_puts (u8 * string, uword string_length, uword is_error) +__clib_export __clib_weak void +os_puts (u8 *string, uword string_length, uword is_error) { int cpu = os_get_thread_index (); int nthreads = os_get_nthreads (); @@ -270,6 +277,8 @@ os_get_online_cpu_core_bitmap () { #if __linux__ return clib_sysfs_read_bitmap ("/sys/devices/system/cpu/online"); +#elif defined(__FreeBSD__) + return os_get_cpu_affinity_bitmap (0); #else return 0; #endif @@ -286,9 +295,9 @@ os_get_cpu_affinity_bitmap (int pid) clib_bitmap_alloc (affinity_cpus, sizeof (cpu_set_t)); clib_bitmap_zero (affinity_cpus); - __CPU_ZERO_S (sizeof (cpu_set_t), &cpuset); + CPU_ZERO_S (sizeof (cpu_set_t), &cpuset); - ret = syscall (SYS_sched_getaffinity, 0, sizeof (cpu_set_t), &cpuset); + ret = sched_getaffinity (0, sizeof (cpu_set_t), &cpuset); if (ret < 0) { @@ -297,11 +306,29 @@ os_get_cpu_affinity_bitmap (int pid) } for (index = 0; index < sizeof (cpu_set_t); index++) - if (__CPU_ISSET_S (index, sizeof (cpu_set_t), &cpuset)) + if (CPU_ISSET_S (index, sizeof (cpu_set_t), &cpuset)) clib_bitmap_set (affinity_cpus, index, 1); return affinity_cpus; +#elif defined(__FreeBSD__) + cpuset_t mask; + uword *r = NULL; + + clib_bitmap_alloc (r, sizeof (CPU_SETSIZE)); + clib_bitmap_zero (r); + + if (cpuset_getaffinity (CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, -1, + sizeof (mask), &mask) != 0) + { + clib_bitmap_free (r); + return NULL; + } + + for (int bit = 0; bit < CPU_SETSIZE; bit++) + clib_bitmap_set (r, bit, CPU_ISSET (bit, &mask)); + + return r; #else - return 0; + return NULL; #endif } @@ -358,6 +385,28 @@ os_get_cpu_phys_core_id (int cpu_id) #endif } +__clib_export u8 * +os_get_exec_path () +{ + u8 *rv = 0; +#ifdef __linux__ + char tmp[PATH_MAX]; + ssize_t sz = readlink ("/proc/self/exe", tmp, sizeof (tmp)); + + if (sz <= 0) + return 0; +#else + char tmp[MAXPATHLEN]; + int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; + size_t sz = MAXPATHLEN; + + if (sysctl (mib, 4, tmp, &sz, NULL, 0) == -1) + return 0; +#endif + vec_add (rv, tmp, sz); + return rv; +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vppinfra/unix.h b/src/vppinfra/unix.h index 3ad57b05e72..d0ddb93a46f 100644 --- a/src/vppinfra/unix.h +++ b/src/vppinfra/unix.h @@ -56,9 +56,6 @@ clib_error_t *unix_proc_file_contents (char *file, u8 ** result); /* Retrieve bitmap of online cpu cures */ clib_bitmap_t *os_get_online_cpu_core_bitmap (); -/* Retrieve bitmap of cpus vpp has affinity to */ -clib_bitmap_t *os_get_cpu_affinity_bitmap (int pid); - /* Retrieve bitmap of online cpu nodes (sockets) */ clib_bitmap_t *os_get_online_cpu_node_bitmap (); @@ -71,6 +68,10 @@ clib_bitmap_t *os_get_cpu_on_node_bitmap (int node); /* Retrieve physical core id of specific cpu, -1 if not available */ int os_get_cpu_phys_core_id (int cpu); +/* Retrieve the path of the current executable as a vector (not + * null-terminated). */ +u8 *os_get_exec_path (); + #endif /* included_clib_unix_h */ /* |