16 #ifndef included_clib_cpu_h 17 #define included_clib_cpu_h 19 #include <sys/syscall.h> 29 #if __x86_64__ && CLIB_DEBUG == 0 30 #define foreach_march_variant(macro, x) \ 31 macro(avx2, x, "arch=core-avx2") 33 #define foreach_march_variant(macro, x) 37 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 38 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3"))) 40 #define CLIB_CPU_OPTIMIZED 44 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \ 45 if (clib_cpu_supports_ ## arch()) \ 46 return & fn ## _ ##arch; 49 #define CLIB_MULTIARCH_SELECT_FN(fn,...) 51 #ifdef CLIB_MARCH_VARIANT 52 #define __CLIB_MULTIARCH_FN(a,b) a##_##b 53 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b) 54 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT) 56 #define CLIB_MULTIARCH_FN(fn) fn 59 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN 61 typedef struct _clib_march_fn_registration
65 struct _clib_march_fn_registration *next;
79 last_prio = r->priority;
94 if (strncmp (name, r->name,
vec_len (r->name) - 1) == 0)
104 #define CLIB_MARCH_FN_POINTER(fn) \ 105 clib_march_select_fn_ptr (fn##_march_fn_registrations); 107 #define CLIB_MARCH_FN_POINTER_BY_NAME(fn, name) \ 108 clib_march_select_fn_ptr_by_name (fn##_march_fn_registrations, name); 110 #define _CLIB_MARCH_FN_REGISTRATION(fn) \ 111 static clib_march_fn_registration \ 112 CLIB_MARCH_SFX(fn##_march_fn_registration) = \ 114 .name = CLIB_MARCH_VARIANT_STR \ 117 static void __clib_constructor \ 118 fn##_march_register () \ 120 clib_march_fn_registration *r; \ 121 r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \ 122 r->priority = CLIB_MARCH_FN_PRIORITY(); \ 123 r->next = fn##_march_fn_registrations; \ 124 r->function = CLIB_MARCH_SFX (fn); \ 125 fn##_march_fn_registrations = r; \ 128 #ifdef CLIB_MARCH_VARIANT 129 #define CLIB_MARCH_FN_REGISTRATION(fn) \ 130 extern clib_march_fn_registration *fn##_march_fn_registrations; \ 131 _CLIB_MARCH_FN_REGISTRATION(fn) 133 #define CLIB_MARCH_FN_REGISTRATION(fn) \ 134 clib_march_fn_registration *fn##_march_fn_registrations = 0; \ 135 _CLIB_MARCH_FN_REGISTRATION(fn) 137 #define foreach_x86_64_flags \ 138 _ (sse3, 1, ecx, 0) \ 139 _ (pclmulqdq, 1, ecx, 1) \ 140 _ (ssse3, 1, ecx, 9) \ 141 _ (sse41, 1, ecx, 19) \ 142 _ (sse42, 1, ecx, 20) \ 143 _ (avx, 1, ecx, 28) \ 144 _ (rdrand, 1, ecx, 30) \ 145 _ (avx2, 7, ebx, 5) \ 146 _ (rtm, 7, ebx, 11) \ 147 _ (pqm, 7, ebx, 12) \ 148 _ (pqe, 7, ebx, 15) \ 149 _ (avx512f, 7, ebx, 16) \ 150 _ (rdseed, 7, ebx, 18) \ 151 _ (x86_aes, 1, ecx, 25) \ 152 _ (sha, 7, ebx, 29) \ 153 _ (vaes, 7, ecx, 9) \ 154 _ (vpclmulqdq, 7, ecx, 10) \ 155 _ (avx512_vnni, 7, ecx, 11) \ 156 _ (avx512_bitalg, 7, ecx, 12) \ 157 _ (avx512_vpopcntdq, 7, ecx, 14) \ 158 _ (invariant_tsc, 0x80000007, edx, 8) 161 #define foreach_aarch64_flags \ 190 syscall (__NR_getcpu, &cpu, &node, 0);
198 syscall (__NR_getcpu, &cpu, &node, 0);
202 #if defined(__x86_64__) 208 if ((
u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
211 __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
213 __cpuid (lev, *eax, *ebx, *ecx, *edx);
218 #define _(flag, func, reg, bit) \ 220 clib_cpu_supports_ ## flag() \ 222 u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \ 223 clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \ 225 return ((reg & (1 << bit)) != 0); \ 231 #define _(flag, func, reg, bit) \ 232 static inline int clib_cpu_supports_ ## flag() { return 0; } 236 #if defined(__aarch64__) 237 #include <sys/auxv.h> 238 #define _(flag, bit) \ 240 clib_cpu_supports_ ## flag() \ 242 unsigned long hwcap = getauxval(AT_HWCAP); \ 243 return (hwcap & (1 << bit)); \ 248 #define _(flag, bit) \ 249 static inline int clib_cpu_supports_ ## flag() { return 0; } 261 #if defined(__x86_64__) 262 return clib_cpu_supports_x86_aes ();
263 #elif defined (__aarch64__) 264 return clib_cpu_supports_aarch64_aes ();
273 if (clib_cpu_supports_avx512_bitalg ())
281 if (clib_cpu_supports_avx512f ())
289 if (clib_cpu_supports_avx2 ())
298 static u32 implementer = -1;
300 if (-1 != implementer)
303 FILE *fp = fopen (
"/proc/cpuinfo",
"r");
309 if (!fgets (buf,
sizeof (buf), fp))
312 if (strstr (buf,
"CPU implementer"))
313 implementer = (
u32) strtol (memchr (buf,
':', 128) + 2, NULL, 0);
314 if (-1 != implementer)
326 static u32 part = -1;
331 FILE *fp = fopen (
"/proc/cpuinfo",
"r");
337 if (!fgets (buf,
sizeof (buf), fp))
340 if (strstr (buf,
"CPU part"))
341 part = (
u32) strtol (memchr (buf,
':', 128) + 2, NULL, 0);
350 #define AARCH64_CPU_IMPLEMENTER_CAVIUM 0x43 351 #define AARCH64_CPU_PART_THUNDERX2 0x0af 352 #define AARCH64_CPU_PART_OCTEONTX2T96 0x0b2 353 #define AARCH64_CPU_PART_OCTEONTX2T98 0x0b1 354 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51 355 #define AARCH64_CPU_PART_QDF24XX 0xc00 356 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41 357 #define AARCH64_CPU_PART_CORTEXA72 0xd08 358 #define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 0x41 359 #define AARCH64_CPU_PART_NEOVERSEN1 0xd0c 407 #ifdef CLIB_MARCH_VARIANT 408 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)() 410 #define CLIB_MARCH_FN_PRIORITY() 0 414 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \ 415 static void __clib_constructor \ 416 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \ 418 if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \ 420 fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \ 421 fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \ 425 #ifndef CLIB_MARCH_VARIANT 426 #define CLIB_MARCH_FN(fn, rtype, _args...) \ 427 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ 428 rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \ 429 int fn ## _selected_priority = 0; \ 430 static inline rtype CLIB_CPU_OPTIMIZED \ 431 CLIB_MARCH_SFX (fn ## _ma)(_args) 433 #define CLIB_MARCH_FN(fn, rtype, _args...) \ 434 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ 435 extern rtype (*fn ## _selected) (_args); \ 436 extern int fn ## _selected_priority; \ 437 CLIB_MARCH_FN_CONSTRUCTOR (fn) \ 438 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) 441 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected) static int clib_cpu_march_priority_cortexa72()
#define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1
static int clib_cpu_march_priority_octeontx2()
static u32 clib_get_current_numa_node()
static_always_inline void * clib_march_select_fn_ptr_by_name(clib_march_fn_registration *r, char *name)
static int clib_cpu_march_priority_neoversen1()
#define AARCH64_CPU_PART_OCTEONTX2T96
static u32 clib_get_current_cpu_id()
format_function_t format_cpu_flags
static int clib_cpu_march_priority_hsw()
#define foreach_aarch64_flags
static int clib_cpu_march_priority_skx()
#define static_always_inline
#define AARCH64_CPU_IMPLEMENTER_CAVIUM
static u32 clib_cpu_part()
#define AARCH64_CPU_PART_QDF24XX
#define AARCH64_CPU_PART_NEOVERSEN1
static int clib_cpu_march_priority_thunderx2t99()
static int clib_get_cpuid(const u32 lev, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
static u32 clib_cpu_implementer()
#define AARCH64_CPU_PART_OCTEONTX2T98
#define foreach_x86_64_flags
vlib_main_t vlib_node_runtime_t * node
struct _clib_march_fn_registration clib_march_fn_registration
static_always_inline void * clib_march_select_fn_ptr(clib_march_fn_registration *r)
static foreach_aarch64_flags int clib_cpu_supports_aes()
#define AARCH64_CPU_IMPLEMENTER_CORTEXA72
static int clib_cpu_march_priority_qdf24xx()
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
format_function_t format_cpu_uarch
#define AARCH64_CPU_PART_THUNDERX2
format_function_t format_cpu_model_name
#define AARCH64_CPU_PART_CORTEXA72
static int clib_cpu_march_priority_icl()
#define AARCH64_CPU_IMPLEMENTER_QDF24XX