FD.io VPP  v21.10.1-2-g0a485f517
Vector Packet Processing
cpu.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
18 
19 #include <sys/syscall.h>
20 #include <vppinfra/format.h>
21 
22 #if defined(__x86_64__)
23 #define foreach_march_variant \
24  _ (hsw, "Intel Haswell") \
25  _ (trm, "Intel Tremont") \
26  _ (skx, "Intel Skylake (server) / Cascade Lake") \
27  _ (icl, "Intel Ice Lake")
28 #elif defined(__aarch64__)
29 #define foreach_march_variant \
30  _ (octeontx2, "Marvell Octeon TX2") \
31  _ (thunderx2t99, "Marvell ThunderX2 T99") \
32  _ (qdf24xx, "Qualcomm CentriqTM 2400") \
33  _ (cortexa72, "ARM Cortex-A72") \
34  _ (neoversen1, "ARM Neoverse N1")
35 #else
36 #define foreach_march_variant
37 #endif
38 
39 typedef enum
40 {
42 #define _(s, n) CLIB_MARCH_VARIANT_TYPE_##s,
44 #undef _
47 
48 #ifdef CLIB_MARCH_VARIANT
49 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
50 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
51 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
52 #else
53 #define CLIB_MULTIARCH_FN(fn) fn
54 #endif
55 
56 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
57 
58 typedef struct _clib_march_fn_registration
59 {
60  void *function;
61  int priority;
62  struct _clib_march_fn_registration *next;
63  char *name;
65 
68 {
69  void *rv = 0;
70  int last_prio = -1;
71 
72  while (r)
73  {
74  if (last_prio < r->priority)
75  {
76  last_prio = r->priority;
77  rv = r->function;
78  }
79  r = r->next;
80  }
81  return rv;
82 }
83 
84 #define CLIB_MARCH_FN_POINTER(fn) \
85  (__typeof__ (fn) *) clib_march_select_fn_ptr (fn##_march_fn_registrations);
86 
87 #define _CLIB_MARCH_FN_REGISTRATION(fn) \
88 static clib_march_fn_registration \
89 CLIB_MARCH_SFX(fn##_march_fn_registration) = \
90 { \
91  .name = CLIB_MARCH_VARIANT_STR \
92 }; \
93 \
94 static void __clib_constructor \
95 fn##_march_register () \
96 { \
97  clib_march_fn_registration *r; \
98  r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
99  r->priority = CLIB_MARCH_FN_PRIORITY(); \
100  r->next = fn##_march_fn_registrations; \
101  r->function = CLIB_MARCH_SFX (fn); \
102  fn##_march_fn_registrations = r; \
103 }
104 
105 #ifdef CLIB_MARCH_VARIANT
106 #define CLIB_MARCH_FN_REGISTRATION(fn) \
107 extern clib_march_fn_registration *fn##_march_fn_registrations; \
108 _CLIB_MARCH_FN_REGISTRATION(fn)
109 #else
110 #define CLIB_MARCH_FN_REGISTRATION(fn) \
111 clib_march_fn_registration *fn##_march_fn_registrations = 0; \
112 _CLIB_MARCH_FN_REGISTRATION(fn)
113 #endif
114 #define foreach_x86_64_flags \
115  _ (sse3, 1, ecx, 0) \
116  _ (pclmulqdq, 1, ecx, 1) \
117  _ (ssse3, 1, ecx, 9) \
118  _ (sse41, 1, ecx, 19) \
119  _ (sse42, 1, ecx, 20) \
120  _ (avx, 1, ecx, 28) \
121  _ (rdrand, 1, ecx, 30) \
122  _ (avx2, 7, ebx, 5) \
123  _ (rtm, 7, ebx, 11) \
124  _ (pqm, 7, ebx, 12) \
125  _ (pqe, 7, ebx, 15) \
126  _ (avx512f, 7, ebx, 16) \
127  _ (rdseed, 7, ebx, 18) \
128  _ (x86_aes, 1, ecx, 25) \
129  _ (sha, 7, ebx, 29) \
130  _ (vaes, 7, ecx, 9) \
131  _ (vpclmulqdq, 7, ecx, 10) \
132  _ (avx512_vnni, 7, ecx, 11) \
133  _ (avx512_bitalg, 7, ecx, 12) \
134  _ (avx512_vpopcntdq, 7, ecx, 14) \
135  _ (movdiri, 7, ecx, 27) \
136  _ (movdir64b, 7, ecx, 28) \
137  _ (invariant_tsc, 0x80000007, edx, 8)
138 
139 #define foreach_aarch64_flags \
140 _ (fp, 0) \
141 _ (asimd, 1) \
142 _ (evtstrm, 2) \
143 _ (aarch64_aes, 3) \
144 _ (pmull, 4) \
145 _ (sha1, 5) \
146 _ (sha2, 6) \
147 _ (crc32, 7) \
148 _ (atomics, 8) \
149 _ (fphp, 9) \
150 _ (asimdhp, 10) \
151 _ (cpuid, 11) \
152 _ (asimdrdm, 12) \
153 _ (jscvt, 13) \
154 _ (fcma, 14) \
155 _ (lrcpc, 15) \
156 _ (dcpop, 16) \
157 _ (sha3, 17) \
158 _ (sm3, 18) \
159 _ (sm4, 19) \
160 _ (asimddp, 20) \
161 _ (sha512, 21) \
162 _ (sve, 22)
163 
166 
167 #if defined(__x86_64__)
168 #include "cpuid.h"
169 
170 static inline int
171 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
172 {
173  if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
174  return 0;
175  if (lev == 7)
176  __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
177  else
178  __cpuid (lev, *eax, *ebx, *ecx, *edx);
179  return 1;
180 }
181 
182 typedef int (*clib_cpu_supports_func_t) ();
183 
184 #define _(flag, func, reg, bit) \
185 static inline int \
186 clib_cpu_supports_ ## flag() \
187 { \
188  u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
189  clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
190  \
191  return ((reg & (1 << bit)) != 0); \
192 }
194 #undef _
195 #else /* __x86_64__ */
196 
197 #define _(flag, func, reg, bit) \
198 static inline int clib_cpu_supports_ ## flag() { return 0; }
200 #undef _
201 #endif /* __x86_64__ */
202 #if defined(__aarch64__)
203 #include <sys/auxv.h>
204 #define _(flag, bit) \
205 static inline int \
206 clib_cpu_supports_ ## flag() \
207 { \
208  unsigned long hwcap = getauxval(AT_HWCAP); \
209  return (hwcap & (1 << bit)); \
210 }
212 #undef _
213 #else /* ! __x86_64__ && !__aarch64__ */
214 #define _(flag, bit) \
215 static inline int clib_cpu_supports_ ## flag() { return 0; }
217 #undef _
218 #endif /* __x86_64__, __aarch64__ */
219 /*
220  * aes is the only feature with the same name in both flag lists
221  * handle this by prefixing it with the arch name, and handling it
222  * with the custom function below
223  */
224  static inline int
226 {
227 #if defined(__x86_64__)
228  return clib_cpu_supports_x86_aes ();
229 #elif defined (__aarch64__)
230  return clib_cpu_supports_aarch64_aes ();
231 #else
232  return 0;
233 #endif
234 }
235 
236 static inline int
238 {
239  if (clib_cpu_supports_avx512_bitalg ())
240  return 200;
241  return -1;
242 }
243 
244 static inline int
246 {
247  if (clib_cpu_supports_avx512f ())
248  return 100;
249  return -1;
250 }
251 
252 static inline int
254 {
255  if (clib_cpu_supports_movdiri ())
256  return 60;
257  return -1;
258 }
259 
260 static inline int
262 {
263  if (clib_cpu_supports_avx2 ())
264  return 50;
265  return -1;
266 }
267 
268 static inline u32
270 {
271  char buf[128];
272  static u32 implementer = -1;
273 
274  if (-1 != implementer)
275  return implementer;
276 
277  FILE *fp = fopen ("/proc/cpuinfo", "r");
278  if (!fp)
279  return implementer;
280 
281  while (!feof (fp))
282  {
283  if (!fgets (buf, sizeof (buf), fp))
284  break;
285  buf[127] = '\0';
286  if (strstr (buf, "CPU implementer"))
287  implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
288  if (-1 != implementer)
289  break;
290  }
291  fclose (fp);
292 
293  return implementer;
294 }
295 
296 static inline u32
298 {
299  char buf[128];
300  static u32 part = -1;
301 
302  if (-1 != part)
303  return part;
304 
305  FILE *fp = fopen ("/proc/cpuinfo", "r");
306  if (!fp)
307  return part;
308 
309  while (!feof (fp))
310  {
311  if (!fgets (buf, sizeof (buf), fp))
312  break;
313  buf[127] = '\0';
314  if (strstr (buf, "CPU part"))
315  part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
316  if (-1 != part)
317  break;
318  }
319  fclose (fp);
320 
321  return part;
322 }
323 
324 #define AARCH64_CPU_IMPLEMENTER_CAVIUM 0x43
325 #define AARCH64_CPU_PART_THUNDERX2 0x0af
326 #define AARCH64_CPU_PART_OCTEONTX2T96 0x0b2
327 #define AARCH64_CPU_PART_OCTEONTX2T98 0x0b1
328 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
329 #define AARCH64_CPU_PART_QDF24XX 0xc00
330 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
331 #define AARCH64_CPU_PART_CORTEXA72 0xd08
332 #define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 0x41
333 #define AARCH64_CPU_PART_NEOVERSEN1 0xd0c
334 
335 static inline int
337 {
341  return 20;
342  return -1;
343 }
344 
345 static inline int
347 {
350  return 20;
351  return -1;
352 }
353 
354 static inline int
356 {
359  return 20;
360  return -1;
361 }
362 
363 static inline int
365 {
368  return 10;
369  return -1;
370 }
371 
372 static inline int
374 {
377  return 10;
378  return -1;
379 }
380 
381 #ifdef CLIB_MARCH_VARIANT
382 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
383 #else
384 #define CLIB_MARCH_FN_PRIORITY() 0
385 #endif
386 #endif /* included_clib_cpu_h */
387 
388 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
389 static void __clib_constructor \
390 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
391 { \
392  if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
393  { \
394  fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
395  fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
396  } \
397 } \
398 
399 #ifndef CLIB_MARCH_VARIANT
400 #define CLIB_MARCH_FN(fn, rtype, _args...) \
401  static rtype CLIB_MARCH_SFX (fn##_ma) (_args); \
402  rtype (*fn##_selected) (_args) = &CLIB_MARCH_SFX (fn##_ma); \
403  int fn##_selected_priority = 0; \
404  static inline rtype CLIB_MARCH_SFX (fn##_ma) (_args)
405 #else
406 #define CLIB_MARCH_FN(fn, rtype, _args...) \
407  static rtype CLIB_MARCH_SFX (fn##_ma) (_args); \
408  extern rtype (*fn##_selected) (_args); \
409  extern int fn##_selected_priority; \
410  CLIB_MARCH_FN_CONSTRUCTOR (fn) \
411  static rtype CLIB_MARCH_SFX (fn##_ma) (_args)
412 #endif
413 
414 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
415 
420 
421 /*
422  * fd.io coding-style-patch-verification: ON
423  *
424  * Local Variables:
425  * eval: (c-set-style "gnu")
426  * End:
427  */
clib_cpu_march_priority_hsw
static int clib_cpu_march_priority_hsw()
Definition: cpu.h:261
clib_get_current_numa_node
u32 clib_get_current_numa_node()
Definition: cpu.c:234
AARCH64_CPU_IMPLEMENTER_CORTEXA72
#define AARCH64_CPU_IMPLEMENTER_CORTEXA72
Definition: cpu.h:330
clib_cpu_part
static u32 clib_cpu_part()
Definition: cpu.h:297
clib_cpu_march_priority_octeontx2
static int clib_cpu_march_priority_octeontx2()
Definition: cpu.h:336
AARCH64_CPU_PART_QDF24XX
#define AARCH64_CPU_PART_QDF24XX
Definition: cpu.h:329
clib_cpu_supports_func_t
int(* clib_cpu_supports_func_t)()
Definition: cpu.h:182
clib_get_current_cpu_id
u32 clib_get_current_cpu_id()
Definition: cpu.c:226
clib_cpu_march_priority_qdf24xx
static int clib_cpu_march_priority_qdf24xx()
Definition: cpu.h:355
name
string name[64]
Definition: fib.api:25
clib_cpu_march_priority_cortexa72
static int clib_cpu_march_priority_cortexa72()
Definition: cpu.h:364
next
u16 * next
Definition: nat44_ei_out2in.c:718
AARCH64_CPU_PART_OCTEONTX2T98
#define AARCH64_CPU_PART_OCTEONTX2T98
Definition: cpu.h:327
AARCH64_CPU_PART_CORTEXA72
#define AARCH64_CPU_PART_CORTEXA72
Definition: cpu.h:331
CLIB_MARCH_VARIANT_TYPE
@ CLIB_MARCH_VARIANT_TYPE
Definition: cpu.h:41
clib_march_fn_registration
struct _clib_march_fn_registration clib_march_fn_registration
r
vnet_hw_if_output_node_runtime_t * r
Definition: interface_output.c:1089
clib_cpu_march_priority_trm
static int clib_cpu_march_priority_trm()
Definition: cpu.h:253
clib_cpu_implementer
static u32 clib_cpu_implementer()
Definition: cpu.h:269
clib_march_select_fn_ptr
static_always_inline void * clib_march_select_fn_ptr(clib_march_fn_registration *r)
Definition: cpu.h:67
clib_cpu_march_priority_neoversen1
static int clib_cpu_march_priority_neoversen1()
Definition: cpu.h:373
format_cpu_model_name
format_function_t format_cpu_model_name
Definition: cpu.h:417
foreach_aarch64_flags
#define foreach_aarch64_flags
Definition: cpu.h:139
priority
i32 priority
Definition: ipsec.api:95
static_always_inline
#define static_always_inline
Definition: clib.h:112
clib_cpu_march_priority_icl
static int clib_cpu_march_priority_icl()
Definition: cpu.h:237
format_cpu_flags
format_function_t format_cpu_flags
Definition: cpu.h:418
AARCH64_CPU_PART_OCTEONTX2T96
#define AARCH64_CPU_PART_OCTEONTX2T96
Definition: cpu.h:326
AARCH64_CPU_IMPLEMENTER_NEOVERSEN1
#define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1
Definition: cpu.h:332
clib_get_cpuid
static int clib_get_cpuid(const u32 lev, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
Definition: cpu.h:171
format.h
AARCH64_CPU_IMPLEMENTER_CAVIUM
#define AARCH64_CPU_IMPLEMENTER_CAVIUM
Definition: cpu.h:324
CLIB_MARCH_TYPE_N_VARIANTS
@ CLIB_MARCH_TYPE_N_VARIANTS
Definition: cpu.h:45
format_function_t
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
clib_march_variant_type_t
clib_march_variant_type_t
Definition: cpu.h:39
foreach_march_variant
#define foreach_march_variant
Definition: cpu.h:23
buf
u64 buf
Definition: application.c:493
u32
unsigned int u32
Definition: types.h:88
foreach_x86_64_flags
#define foreach_x86_64_flags
Definition: cpu.h:114
clib_cpu_supports_aes
static foreach_aarch64_flags int clib_cpu_supports_aes()
Definition: cpu.h:225
format_cpu_uarch
format_function_t format_cpu_uarch
Definition: cpu.h:416
format_march_variant
format_function_t format_march_variant
Definition: cpu.h:419
clib_cpu_march_priority_thunderx2t99
static int clib_cpu_march_priority_thunderx2t99()
Definition: cpu.h:346
rv
int __clib_unused rv
Definition: application.c:491
AARCH64_CPU_IMPLEMENTER_QDF24XX
#define AARCH64_CPU_IMPLEMENTER_QDF24XX
Definition: cpu.h:328
AARCH64_CPU_PART_THUNDERX2
#define AARCH64_CPU_PART_THUNDERX2
Definition: cpu.h:325
clib_cpu_march_priority_skx
static int clib_cpu_march_priority_skx()
Definition: cpu.h:245
AARCH64_CPU_PART_NEOVERSEN1
#define AARCH64_CPU_PART_NEOVERSEN1
Definition: cpu.h:333