FD.io VPP  v21.01.1
Vector Packet Processing
cpu.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
18 
19 #include <sys/syscall.h>
20 #include <vppinfra/format.h>
21 
22 /*
23  * multiarchitecture support. Adding new entry will produce
24  * new graph node function variant optimized for specific cpu
25  * microarchitecture.
26  * Order is important for runtime selection, as 1st match wins...
27  */
28 
29 #if __x86_64__ && CLIB_DEBUG == 0
30 #define foreach_march_variant(macro, x) \
31  macro(avx2, x, "arch=core-avx2")
32 #else
33 #define foreach_march_variant(macro, x)
34 #endif
35 
36 
37 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
38 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
39 #else
40 #define CLIB_CPU_OPTIMIZED
41 #endif
42 
43 
44 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
45  if (clib_cpu_supports_ ## arch()) \
46  return & fn ## _ ##arch;
47 
48 /* FIXME to be removed */
49 #define CLIB_MULTIARCH_SELECT_FN(fn,...)
50 
51 #ifdef CLIB_MARCH_VARIANT
52 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
53 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
54 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
55 #else
56 #define CLIB_MULTIARCH_FN(fn) fn
57 #endif
58 
59 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
60 
61 typedef struct _clib_march_fn_registration
62 {
63  void *function;
64  int priority;
65  struct _clib_march_fn_registration *next;
66  char *name;
68 
71 {
72  void *rv = 0;
73  int last_prio = -1;
74 
75  while (r)
76  {
77  if (last_prio < r->priority)
78  {
79  last_prio = r->priority;
80  rv = r->function;
81  }
82  r = r->next;
83  }
84  return rv;
85 }
86 
89 {
90  void *rv = 0;
91 
92  while (r)
93  {
94  if (strncmp (name, r->name, vec_len (r->name) - 1) == 0)
95  {
96  rv = r->function;
97  break;
98  }
99  r = r->next;
100  }
101  return rv;
102 }
103 
104 #define CLIB_MARCH_FN_POINTER(fn) \
105  clib_march_select_fn_ptr (fn##_march_fn_registrations);
106 
107 #define CLIB_MARCH_FN_POINTER_BY_NAME(fn, name) \
108  clib_march_select_fn_ptr_by_name (fn##_march_fn_registrations, name);
109 
110 #define _CLIB_MARCH_FN_REGISTRATION(fn) \
111 static clib_march_fn_registration \
112 CLIB_MARCH_SFX(fn##_march_fn_registration) = \
113 { \
114  .name = CLIB_MARCH_VARIANT_STR \
115 }; \
116 \
117 static void __clib_constructor \
118 fn##_march_register () \
119 { \
120  clib_march_fn_registration *r; \
121  r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
122  r->priority = CLIB_MARCH_FN_PRIORITY(); \
123  r->next = fn##_march_fn_registrations; \
124  r->function = CLIB_MARCH_SFX (fn); \
125  fn##_march_fn_registrations = r; \
126 }
127 
128 #ifdef CLIB_MARCH_VARIANT
129 #define CLIB_MARCH_FN_REGISTRATION(fn) \
130 extern clib_march_fn_registration *fn##_march_fn_registrations; \
131 _CLIB_MARCH_FN_REGISTRATION(fn)
132 #else
133 #define CLIB_MARCH_FN_REGISTRATION(fn) \
134 clib_march_fn_registration *fn##_march_fn_registrations = 0; \
135 _CLIB_MARCH_FN_REGISTRATION(fn)
136 #endif
137 #define foreach_x86_64_flags \
138 _ (sse3, 1, ecx, 0) \
139 _ (pclmulqdq, 1, ecx, 1) \
140 _ (ssse3, 1, ecx, 9) \
141 _ (sse41, 1, ecx, 19) \
142 _ (sse42, 1, ecx, 20) \
143 _ (avx, 1, ecx, 28) \
144 _ (rdrand, 1, ecx, 30) \
145 _ (avx2, 7, ebx, 5) \
146 _ (rtm, 7, ebx, 11) \
147 _ (pqm, 7, ebx, 12) \
148 _ (pqe, 7, ebx, 15) \
149 _ (avx512f, 7, ebx, 16) \
150 _ (rdseed, 7, ebx, 18) \
151 _ (x86_aes, 1, ecx, 25) \
152 _ (sha, 7, ebx, 29) \
153 _ (vaes, 7, ecx, 9) \
154 _ (vpclmulqdq, 7, ecx, 10) \
155 _ (avx512_vnni, 7, ecx, 11) \
156 _ (avx512_bitalg, 7, ecx, 12) \
157 _ (avx512_vpopcntdq, 7, ecx, 14) \
158 _ (invariant_tsc, 0x80000007, edx, 8)
159 
160 
161 #define foreach_aarch64_flags \
162 _ (fp, 0) \
163 _ (asimd, 1) \
164 _ (evtstrm, 2) \
165 _ (aarch64_aes, 3) \
166 _ (pmull, 4) \
167 _ (sha1, 5) \
168 _ (sha2, 6) \
169 _ (crc32, 7) \
170 _ (atomics, 8) \
171 _ (fphp, 9) \
172 _ (asimdhp, 10) \
173 _ (cpuid, 11) \
174 _ (asimdrdm, 12) \
175 _ (jscvt, 13) \
176 _ (fcma, 14) \
177 _ (lrcpc, 15) \
178 _ (dcpop, 16) \
179 _ (sha3, 17) \
180 _ (sm3, 18) \
181 _ (sm4, 19) \
182 _ (asimddp, 20) \
183 _ (sha512, 21) \
184 _ (sve, 22)
185 
186 static inline u32
188 {
189  unsigned cpu, node;
190  syscall (__NR_getcpu, &cpu, &node, 0);
191  return cpu;
192 }
193 
194 static inline u32
196 {
197  unsigned cpu, node;
198  syscall (__NR_getcpu, &cpu, &node, 0);
199  return node;
200 }
201 
202 #if defined(__x86_64__)
203 #include "cpuid.h"
204 
205 static inline int
206 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
207 {
208  if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
209  return 0;
210  if (lev == 7)
211  __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
212  else
213  __cpuid (lev, *eax, *ebx, *ecx, *edx);
214  return 1;
215 }
216 
217 
218 #define _(flag, func, reg, bit) \
219 static inline int \
220 clib_cpu_supports_ ## flag() \
221 { \
222  u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
223  clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
224  \
225  return ((reg & (1 << bit)) != 0); \
226 }
228 #undef _
229 #else /* __x86_64__ */
230 
231 #define _(flag, func, reg, bit) \
232 static inline int clib_cpu_supports_ ## flag() { return 0; }
234 #undef _
235 #endif /* __x86_64__ */
236 #if defined(__aarch64__)
237 #include <sys/auxv.h>
238 #define _(flag, bit) \
239 static inline int \
240 clib_cpu_supports_ ## flag() \
241 { \
242  unsigned long hwcap = getauxval(AT_HWCAP); \
243  return (hwcap & (1 << bit)); \
244 }
246 #undef _
247 #else /* ! __x86_64__ && !__aarch64__ */
248 #define _(flag, bit) \
249 static inline int clib_cpu_supports_ ## flag() { return 0; }
251 #undef _
252 #endif /* __x86_64__, __aarch64__ */
253 /*
254  * aes is the only feature with the same name in both flag lists
255  * handle this by prefixing it with the arch name, and handling it
256  * with the custom function below
257  */
258  static inline int
260 {
261 #if defined(__x86_64__)
262  return clib_cpu_supports_x86_aes ();
263 #elif defined (__aarch64__)
264  return clib_cpu_supports_aarch64_aes ();
265 #else
266  return 0;
267 #endif
268 }
269 
270 static inline int
272 {
273  if (clib_cpu_supports_avx512_bitalg ())
274  return 200;
275  return -1;
276 }
277 
278 static inline int
280 {
281  if (clib_cpu_supports_avx512f ())
282  return 100;
283  return -1;
284 }
285 
286 static inline int
288 {
289  if (clib_cpu_supports_avx2 ())
290  return 50;
291  return -1;
292 }
293 
294 static inline u32
296 {
297  char buf[128];
298  static u32 implementer = -1;
299 
300  if (-1 != implementer)
301  return implementer;
302 
303  FILE *fp = fopen ("/proc/cpuinfo", "r");
304  if (!fp)
305  return implementer;
306 
307  while (!feof (fp))
308  {
309  if (!fgets (buf, sizeof (buf), fp))
310  break;
311  buf[127] = '\0';
312  if (strstr (buf, "CPU implementer"))
313  implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
314  if (-1 != implementer)
315  break;
316  }
317  fclose (fp);
318 
319  return implementer;
320 }
321 
322 static inline u32
324 {
325  char buf[128];
326  static u32 part = -1;
327 
328  if (-1 != part)
329  return part;
330 
331  FILE *fp = fopen ("/proc/cpuinfo", "r");
332  if (!fp)
333  return part;
334 
335  while (!feof (fp))
336  {
337  if (!fgets (buf, sizeof (buf), fp))
338  break;
339  buf[127] = '\0';
340  if (strstr (buf, "CPU part"))
341  part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
342  if (-1 != part)
343  break;
344  }
345  fclose (fp);
346 
347  return part;
348 }
349 
350 #define AARCH64_CPU_IMPLEMENTER_CAVIUM 0x43
351 #define AARCH64_CPU_PART_THUNDERX2 0x0af
352 #define AARCH64_CPU_PART_OCTEONTX2T96 0x0b2
353 #define AARCH64_CPU_PART_OCTEONTX2T98 0x0b1
354 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
355 #define AARCH64_CPU_PART_QDF24XX 0xc00
356 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
357 #define AARCH64_CPU_PART_CORTEXA72 0xd08
358 #define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 0x41
359 #define AARCH64_CPU_PART_NEOVERSEN1 0xd0c
360 
361 static inline int
363 {
367  return 20;
368  return -1;
369 }
370 
371 static inline int
373 {
376  return 20;
377  return -1;
378 }
379 
380 static inline int
382 {
385  return 20;
386  return -1;
387 }
388 
389 static inline int
391 {
394  return 10;
395  return -1;
396 }
397 
398 static inline int
400 {
403  return 10;
404  return -1;
405 }
406 
407 #ifdef CLIB_MARCH_VARIANT
408 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
409 #else
410 #define CLIB_MARCH_FN_PRIORITY() 0
411 #endif
412 #endif /* included_clib_cpu_h */
413 
414 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
415 static void __clib_constructor \
416 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
417 { \
418  if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
419  { \
420  fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
421  fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
422  } \
423 } \
424 
425 #ifndef CLIB_MARCH_VARIANT
426 #define CLIB_MARCH_FN(fn, rtype, _args...) \
427  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
428  rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
429  int fn ## _selected_priority = 0; \
430  static inline rtype CLIB_CPU_OPTIMIZED \
431  CLIB_MARCH_SFX (fn ## _ma)(_args)
432 #else
433 #define CLIB_MARCH_FN(fn, rtype, _args...) \
434  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
435  extern rtype (*fn ## _selected) (_args); \
436  extern int fn ## _selected_priority; \
437  CLIB_MARCH_FN_CONSTRUCTOR (fn) \
438  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
439 #endif
440 
441 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
442 
446 
447 /*
448  * fd.io coding-style-patch-verification: ON
449  *
450  * Local Variables:
451  * eval: (c-set-style "gnu")
452  * End:
453  */
static int clib_cpu_march_priority_cortexa72()
Definition: cpu.h:390
#define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1
Definition: cpu.h:358
static int clib_cpu_march_priority_octeontx2()
Definition: cpu.h:362
static u32 clib_get_current_numa_node()
Definition: cpu.h:195
static_always_inline void * clib_march_select_fn_ptr_by_name(clib_march_fn_registration *r, char *name)
Definition: cpu.h:88
static int clib_cpu_march_priority_neoversen1()
Definition: cpu.h:399
#define AARCH64_CPU_PART_OCTEONTX2T96
Definition: cpu.h:352
static u32 clib_get_current_cpu_id()
Definition: cpu.h:187
format_function_t format_cpu_flags
Definition: cpu.h:445
static int clib_cpu_march_priority_hsw()
Definition: cpu.h:287
#define foreach_aarch64_flags
Definition: cpu.h:161
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
static int clib_cpu_march_priority_skx()
Definition: cpu.h:279
#define static_always_inline
Definition: clib.h:109
unsigned int u32
Definition: types.h:88
#define AARCH64_CPU_IMPLEMENTER_CAVIUM
Definition: cpu.h:350
i32 priority
Definition: ipsec.api:95
static u32 clib_cpu_part()
Definition: cpu.h:323
#define AARCH64_CPU_PART_QDF24XX
Definition: cpu.h:355
#define AARCH64_CPU_PART_NEOVERSEN1
Definition: cpu.h:359
static int clib_cpu_march_priority_thunderx2t99()
Definition: cpu.h:372
static int clib_get_cpuid(const u32 lev, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
Definition: cpu.h:206
static u32 clib_cpu_implementer()
Definition: cpu.h:295
#define AARCH64_CPU_PART_OCTEONTX2T98
Definition: cpu.h:353
string name[64]
Definition: ip.api:44
#define foreach_x86_64_flags
Definition: cpu.h:137
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1580
struct _clib_march_fn_registration clib_march_fn_registration
static_always_inline void * clib_march_select_fn_ptr(clib_march_fn_registration *r)
Definition: cpu.h:70
static foreach_aarch64_flags int clib_cpu_supports_aes()
Definition: cpu.h:259
#define AARCH64_CPU_IMPLEMENTER_CORTEXA72
Definition: cpu.h:356
static int clib_cpu_march_priority_qdf24xx()
Definition: cpu.h:381
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
format_function_t format_cpu_uarch
Definition: cpu.h:443
#define AARCH64_CPU_PART_THUNDERX2
Definition: cpu.h:351
format_function_t format_cpu_model_name
Definition: cpu.h:444
#define AARCH64_CPU_PART_CORTEXA72
Definition: cpu.h:357
static int clib_cpu_march_priority_icl()
Definition: cpu.h:271
#define AARCH64_CPU_IMPLEMENTER_QDF24XX
Definition: cpu.h:354