16 #ifndef included_clib_cpu_h 17 #define included_clib_cpu_h 19 #include <sys/syscall.h> 29 #if __x86_64__ && CLIB_DEBUG == 0 30 #define foreach_march_variant(macro, x) \ 31 macro(avx2, x, "arch=core-avx2") 33 #define foreach_march_variant(macro, x) 37 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 38 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3"))) 40 #define CLIB_CPU_OPTIMIZED 44 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \ 45 if (clib_cpu_supports_ ## arch()) \ 46 return & fn ## _ ##arch; 49 #define CLIB_MULTIARCH_SELECT_FN(fn,...) 51 #ifdef CLIB_MARCH_VARIANT 52 #define __CLIB_MULTIARCH_FN(a,b) a##_##b 53 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b) 54 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT) 56 #define CLIB_MULTIARCH_FN(fn) fn 59 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN 61 typedef struct _clib_march_fn_registration
65 struct _clib_march_fn_registration *next;
79 last_prio = r->priority;
87 #define CLIB_MARCH_FN_POINTER(fn) \ 88 clib_march_select_fn_ptr (fn##_march_fn_registrations); 90 #define _CLIB_MARCH_FN_REGISTRATION(fn) \ 91 static clib_march_fn_registration \ 92 CLIB_MARCH_SFX(fn##_march_fn_registration) = \ 94 .name = CLIB_MARCH_VARIANT_STR \ 97 static void __clib_constructor \ 98 fn##_march_register () \ 100 clib_march_fn_registration *r; \ 101 r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \ 102 r->priority = CLIB_MARCH_FN_PRIORITY(); \ 103 r->next = fn##_march_fn_registrations; \ 104 r->function = CLIB_MARCH_SFX (fn); \ 105 fn##_march_fn_registrations = r; \ 108 #ifdef CLIB_MARCH_VARIANT 109 #define CLIB_MARCH_FN_REGISTRATION(fn) \ 110 extern clib_march_fn_registration *fn##_march_fn_registrations; \ 111 _CLIB_MARCH_FN_REGISTRATION(fn) 113 #define CLIB_MARCH_FN_REGISTRATION(fn) \ 114 clib_march_fn_registration *fn##_march_fn_registrations = 0; \ 115 _CLIB_MARCH_FN_REGISTRATION(fn) 117 #define foreach_x86_64_flags \ 118 _ (sse3, 1, ecx, 0) \ 119 _ (pclmulqdq, 1, ecx, 1) \ 120 _ (ssse3, 1, ecx, 9) \ 121 _ (sse41, 1, ecx, 19) \ 122 _ (sse42, 1, ecx, 20) \ 123 _ (avx, 1, ecx, 28) \ 124 _ (rdrand, 1, ecx, 30) \ 125 _ (avx2, 7, ebx, 5) \ 126 _ (rtm, 7, ebx, 11) \ 127 _ (pqm, 7, ebx, 12) \ 128 _ (pqe, 7, ebx, 15) \ 129 _ (avx512f, 7, ebx, 16) \ 130 _ (rdseed, 7, ebx, 18) \ 131 _ (x86_aes, 1, ecx, 25) \ 132 _ (sha, 7, ebx, 29) \ 133 _ (vaes, 7, ecx, 9) \ 134 _ (vpclmulqdq, 7, ecx, 10) \ 135 _ (avx512_vnni, 7, ecx, 11) \ 136 _ (avx512_bitalg, 7, ecx, 12) \ 137 _ (avx512_vpopcntdq, 7, ecx, 14) \ 138 _ (invariant_tsc, 0x80000007, edx, 8) 141 #define foreach_aarch64_flags \ 170 syscall (__NR_getcpu, &cpu, &node, 0);
178 syscall (__NR_getcpu, &cpu, &node, 0);
182 #if defined(__x86_64__) 188 if ((
u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
191 __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
193 __cpuid (lev, *eax, *ebx, *ecx, *edx);
198 #define _(flag, func, reg, bit) \ 200 clib_cpu_supports_ ## flag() \ 202 u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \ 203 clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \ 205 return ((reg & (1 << bit)) != 0); \ 211 #define _(flag, func, reg, bit) \ 212 static inline int clib_cpu_supports_ ## flag() { return 0; } 216 #if defined(__aarch64__) 217 #include <sys/auxv.h> 218 #define _(flag, bit) \ 220 clib_cpu_supports_ ## flag() \ 222 unsigned long hwcap = getauxval(AT_HWCAP); \ 223 return (hwcap & (1 << bit)); \ 228 #define _(flag, bit) \ 229 static inline int clib_cpu_supports_ ## flag() { return 0; } 241 #if defined(__x86_64__) 242 return clib_cpu_supports_x86_aes ();
243 #elif defined (__aarch64__) 244 return clib_cpu_supports_aarch64_aes ();
253 if (clib_cpu_supports_avx512_bitalg ())
261 if (clib_cpu_supports_avx512f ())
269 if (clib_cpu_supports_avx2 ())
278 static u32 implementer = -1;
280 if (-1 != implementer)
283 FILE *fp = fopen (
"/proc/cpuinfo",
"r");
289 if (!fgets (buf,
sizeof (buf), fp))
292 if (strstr (buf,
"CPU implementer"))
293 implementer = (
u32) strtol (memchr (buf,
':', 128) + 2, NULL, 0);
294 if (-1 != implementer)
306 static u32 part = -1;
311 FILE *fp = fopen (
"/proc/cpuinfo",
"r");
317 if (!fgets (buf,
sizeof (buf), fp))
320 if (strstr (buf,
"CPU part"))
321 part = (
u32) strtol (memchr (buf,
':', 128) + 2, NULL, 0);
330 #define AARCH64_CPU_IMPLEMENTER_CAVIUM 0x43 331 #define AARCH64_CPU_PART_THUNDERX2 0x0af 332 #define AARCH64_CPU_PART_OCTEONTX2T96 0x0b2 333 #define AARCH64_CPU_PART_OCTEONTX2T98 0x0b1 334 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51 335 #define AARCH64_CPU_PART_QDF24XX 0xc00 336 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41 337 #define AARCH64_CPU_PART_CORTEXA72 0xd08 338 #define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 0x41 339 #define AARCH64_CPU_PART_NEOVERSEN1 0xd0c 387 #ifdef CLIB_MARCH_VARIANT 388 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)() 390 #define CLIB_MARCH_FN_PRIORITY() 0 394 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \ 395 static void __clib_constructor \ 396 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \ 398 if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \ 400 fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \ 401 fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \ 405 #ifndef CLIB_MARCH_VARIANT 406 #define CLIB_MARCH_FN(fn, rtype, _args...) \ 407 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ 408 rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \ 409 int fn ## _selected_priority = 0; \ 410 static inline rtype CLIB_CPU_OPTIMIZED \ 411 CLIB_MARCH_SFX (fn ## _ma)(_args) 413 #define CLIB_MARCH_FN(fn, rtype, _args...) \ 414 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ 415 extern rtype (*fn ## _selected) (_args); \ 416 extern int fn ## _selected_priority; \ 417 CLIB_MARCH_FN_CONSTRUCTOR (fn) \ 418 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) 421 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected) static int clib_cpu_march_priority_cortexa72()
#define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1
static int clib_cpu_march_priority_octeontx2()
static u32 clib_get_current_numa_node()
static int clib_cpu_march_priority_neoversen1()
#define AARCH64_CPU_PART_OCTEONTX2T96
static u32 clib_get_current_cpu_id()
format_function_t format_cpu_flags
static int clib_cpu_march_priority_hsw()
#define foreach_aarch64_flags
static int clib_cpu_march_priority_skx()
#define static_always_inline
#define AARCH64_CPU_IMPLEMENTER_CAVIUM
static u32 clib_cpu_part()
#define AARCH64_CPU_PART_QDF24XX
#define AARCH64_CPU_PART_NEOVERSEN1
static int clib_cpu_march_priority_thunderx2t99()
static int clib_get_cpuid(const u32 lev, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
static u32 clib_cpu_implementer()
#define AARCH64_CPU_PART_OCTEONTX2T98
#define foreach_x86_64_flags
vlib_main_t vlib_node_runtime_t * node
struct _clib_march_fn_registration clib_march_fn_registration
static_always_inline void * clib_march_select_fn_ptr(clib_march_fn_registration *r)
static foreach_aarch64_flags int clib_cpu_supports_aes()
#define AARCH64_CPU_IMPLEMENTER_CORTEXA72
static int clib_cpu_march_priority_qdf24xx()
format_function_t format_cpu_uarch
#define AARCH64_CPU_PART_THUNDERX2
format_function_t format_cpu_model_name
#define AARCH64_CPU_PART_CORTEXA72
static int clib_cpu_march_priority_icl()
#define AARCH64_CPU_IMPLEMENTER_QDF24XX