FD.io VPP  v20.01-48-g3e0dafb74
Vector Packet Processing
cpu.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
18 
19 #include <sys/syscall.h>
20 #include <vppinfra/format.h>
21 
22 /*
23  * multiarchitecture support. Adding new entry will produce
24  * new graph node function variant optimized for specific cpu
25  * microarchitecture.
26  * Order is important for runtime selection, as 1st match wins...
27  */
28 
29 #if __x86_64__ && CLIB_DEBUG == 0
30 #define foreach_march_variant(macro, x) \
31  macro(avx2, x, "arch=core-avx2")
32 #else
33 #define foreach_march_variant(macro, x)
34 #endif
35 
36 
37 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
38 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
39 #else
40 #define CLIB_CPU_OPTIMIZED
41 #endif
42 
43 
44 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
45  if (clib_cpu_supports_ ## arch()) \
46  return & fn ## _ ##arch;
47 
48 /* FIXME to be removed */
49 #define CLIB_MULTIARCH_SELECT_FN(fn,...)
50 
51 #ifdef CLIB_MARCH_VARIANT
52 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
53 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
54 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
55 #else
56 #define CLIB_MULTIARCH_FN(fn) fn
57 #endif
58 
59 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
60 
61 typedef struct _clib_march_fn_registration
62 {
63  void *function;
64  int priority;
65  struct _clib_march_fn_registration *next;
66  char *name;
68 
71 {
72  void *rv = 0;
73  int last_prio = -1;
74 
75  while (r)
76  {
77  if (last_prio < r->priority)
78  {
79  last_prio = r->priority;
80  rv = r->function;
81  }
82  r = r->next;
83  }
84  return rv;
85 }
86 
87 #define CLIB_MARCH_FN_POINTER(fn) \
88  clib_march_select_fn_ptr (fn##_march_fn_registrations);
89 
90 #define _CLIB_MARCH_FN_REGISTRATION(fn) \
91 static clib_march_fn_registration \
92 CLIB_MARCH_SFX(fn##_march_fn_registration) = \
93 { \
94  .name = CLIB_MARCH_VARIANT_STR \
95 }; \
96 \
97 static void __clib_constructor \
98 fn##_march_register () \
99 { \
100  clib_march_fn_registration *r; \
101  r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
102  r->priority = CLIB_MARCH_FN_PRIORITY(); \
103  r->next = fn##_march_fn_registrations; \
104  r->function = CLIB_MARCH_SFX (fn); \
105  fn##_march_fn_registrations = r; \
106 }
107 
108 #ifdef CLIB_MARCH_VARIANT
109 #define CLIB_MARCH_FN_REGISTRATION(fn) \
110 extern clib_march_fn_registration *fn##_march_fn_registrations; \
111 _CLIB_MARCH_FN_REGISTRATION(fn)
112 #else
113 #define CLIB_MARCH_FN_REGISTRATION(fn) \
114 clib_march_fn_registration *fn##_march_fn_registrations = 0; \
115 _CLIB_MARCH_FN_REGISTRATION(fn)
116 #endif
117 #define foreach_x86_64_flags \
118 _ (sse3, 1, ecx, 0) \
119 _ (pclmulqdq, 1, ecx, 1) \
120 _ (ssse3, 1, ecx, 9) \
121 _ (sse41, 1, ecx, 19) \
122 _ (sse42, 1, ecx, 20) \
123 _ (avx, 1, ecx, 28) \
124 _ (rdrand, 1, ecx, 30) \
125 _ (avx2, 7, ebx, 5) \
126 _ (rtm, 7, ebx, 11) \
127 _ (pqm, 7, ebx, 12) \
128 _ (pqe, 7, ebx, 15) \
129 _ (avx512f, 7, ebx, 16) \
130 _ (rdseed, 7, ebx, 18) \
131 _ (x86_aes, 1, ecx, 25) \
132 _ (sha, 7, ebx, 29) \
133 _ (vaes, 7, ecx, 9) \
134 _ (vpclmulqdq, 7, ecx, 10) \
135 _ (invariant_tsc, 0x80000007, edx, 8)
136 
137 
138 #define foreach_aarch64_flags \
139 _ (fp, 0) \
140 _ (asimd, 1) \
141 _ (evtstrm, 2) \
142 _ (aarch64_aes, 3) \
143 _ (pmull, 4) \
144 _ (sha1, 5) \
145 _ (sha2, 6) \
146 _ (crc32, 7) \
147 _ (atomics, 8) \
148 _ (fphp, 9) \
149 _ (asimdhp, 10) \
150 _ (cpuid, 11) \
151 _ (asimdrdm, 12) \
152 _ (jscvt, 13) \
153 _ (fcma, 14) \
154 _ (lrcpc, 15) \
155 _ (dcpop, 16) \
156 _ (sha3, 17) \
157 _ (sm3, 18) \
158 _ (sm4, 19) \
159 _ (asimddp, 20) \
160 _ (sha512, 21) \
161 _ (sve, 22)
162 
163 static inline u32
165 {
166  unsigned cpu, node;
167  syscall (__NR_getcpu, &cpu, &node, 0);
168  return cpu;
169 }
170 
171 static inline u32
173 {
174  unsigned cpu, node;
175  syscall (__NR_getcpu, &cpu, &node, 0);
176  return node;
177 }
178 
179 #if defined(__x86_64__)
180 #include "cpuid.h"
181 
182 static inline int
183 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
184 {
185  if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
186  return 0;
187  if (lev == 7)
188  __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
189  else
190  __cpuid (lev, *eax, *ebx, *ecx, *edx);
191  return 1;
192 }
193 
194 
195 #define _(flag, func, reg, bit) \
196 static inline int \
197 clib_cpu_supports_ ## flag() \
198 { \
199  u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
200  clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
201  \
202  return ((reg & (1 << bit)) != 0); \
203 }
205 #undef _
206 #else /* __x86_64__ */
207 
208 #define _(flag, func, reg, bit) \
209 static inline int clib_cpu_supports_ ## flag() { return 0; }
211 #undef _
212 #endif /* __x86_64__ */
213 #if defined(__aarch64__)
214 #include <sys/auxv.h>
215 #define _(flag, bit) \
216 static inline int \
217 clib_cpu_supports_ ## flag() \
218 { \
219  unsigned long hwcap = getauxval(AT_HWCAP); \
220  return (hwcap & (1 << bit)); \
221 }
223 #undef _
224 #else /* ! __x86_64__ && !__aarch64__ */
225 #define _(flag, bit) \
226 static inline int clib_cpu_supports_ ## flag() { return 0; }
228 #undef _
229 #endif /* __x86_64__, __aarch64__ */
230 /*
231  * aes is the only feature with the same name in both flag lists
232  * handle this by prefixing it with the arch name, and handling it
233  * with the custom function below
234  */
235  static inline int
237 {
238 #if defined(__x86_64__)
239  return clib_cpu_supports_x86_aes ();
240 #elif defined (__aarch64__)
241  return clib_cpu_supports_aarch64_aes ();
242 #else
243  return 0;
244 #endif
245 }
246 
247 static inline int
249 {
250  if (clib_cpu_supports_avx512f ())
251  return 20;
252  return -1;
253 }
254 
255 static inline int
257 {
258  if (clib_cpu_supports_avx2 ())
259  return 50;
260  return -1;
261 }
262 
263 static inline u32
265 {
266  char buf[128];
267  static u32 implementer = -1;
268 
269  if (-1 != implementer)
270  return implementer;
271 
272  FILE *fp = fopen ("/proc/cpuinfo", "r");
273  if (!fp)
274  return implementer;
275 
276  while (!feof (fp))
277  {
278  if (!fgets (buf, sizeof (buf), fp))
279  break;
280  buf[127] = '\0';
281  if (strstr (buf, "CPU implementer"))
282  implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
283  if (-1 != implementer)
284  break;
285  }
286  fclose (fp);
287 
288  return implementer;
289 }
290 
291 static inline u32
293 {
294  char buf[128];
295  static u32 part = -1;
296 
297  if (-1 != part)
298  return part;
299 
300  FILE *fp = fopen ("/proc/cpuinfo", "r");
301  if (!fp)
302  return part;
303 
304  while (!feof (fp))
305  {
306  if (!fgets (buf, sizeof (buf), fp))
307  break;
308  buf[127] = '\0';
309  if (strstr (buf, "CPU part"))
310  part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
311  if (-1 != part)
312  break;
313  }
314  fclose (fp);
315 
316  return part;
317 }
318 
319 #define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43
320 #define AARCH64_CPU_PART_THUNERDERX2 0x0af
321 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
322 #define AARCH64_CPU_PART_QDF24XX 0xc00
323 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
324 #define AARCH64_CPU_PART_CORTEXA72 0xd08
325 
326 static inline int
328 {
331  return 20;
332  return -1;
333 }
334 
335 static inline int
337 {
340  return 20;
341  return -1;
342 }
343 
344 static inline int
346 {
349  return 10;
350  return -1;
351 }
352 
353 #ifdef CLIB_MARCH_VARIANT
354 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
355 #else
356 #define CLIB_MARCH_FN_PRIORITY() 0
357 #endif
358 #endif /* included_clib_cpu_h */
359 
360 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
361 static void __clib_constructor \
362 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
363 { \
364  if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
365  { \
366  fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
367  fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
368  } \
369 } \
370 
371 #ifndef CLIB_MARCH_VARIANT
372 #define CLIB_MARCH_FN(fn, rtype, _args...) \
373  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
374  rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
375  int fn ## _selected_priority = 0; \
376  static inline rtype CLIB_CPU_OPTIMIZED \
377  CLIB_MARCH_SFX (fn ## _ma)(_args)
378 #else
379 #define CLIB_MARCH_FN(fn, rtype, _args...) \
380  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
381  extern rtype (*fn ## _selected) (_args); \
382  extern int fn ## _selected_priority; \
383  CLIB_MARCH_FN_CONSTRUCTOR (fn) \
384  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
385 #endif
386 
387 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
388 
392 
393 /*
394  * fd.io coding-style-patch-verification: ON
395  *
396  * Local Variables:
397  * eval: (c-set-style "gnu")
398  * End:
399  */
static int clib_cpu_march_priority_cortexa72()
Definition: cpu.h:345
#define AARCH64_CPU_IMPLEMENTER_THUNERDERX2
Definition: cpu.h:319
static u32 clib_get_current_numa_node()
Definition: cpu.h:172
#define NULL
Definition: clib.h:58
static u32 clib_get_current_cpu_id()
Definition: cpu.h:164
format_function_t format_cpu_flags
Definition: cpu.h:391
#define foreach_aarch64_flags
Definition: cpu.h:138
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
#define static_always_inline
Definition: clib.h:99
unsigned int u32
Definition: types.h:88
i32 priority
Definition: ipsec.api:92
static u32 clib_cpu_part()
Definition: cpu.h:292
static int clib_cpu_march_priority_avx512()
Definition: cpu.h:248
#define AARCH64_CPU_PART_QDF24XX
Definition: cpu.h:322
static int clib_cpu_march_priority_thunderx2t99()
Definition: cpu.h:327
static int clib_get_cpuid(const u32 lev, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
Definition: cpu.h:183
static u32 clib_cpu_implementer()
Definition: cpu.h:264
static int clib_cpu_march_priority_avx2()
Definition: cpu.h:256
string name[64]
Definition: ip.api:44
#define foreach_x86_64_flags
Definition: cpu.h:117
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1810
struct _clib_march_fn_registration clib_march_fn_registration
#define AARCH64_CPU_PART_THUNERDERX2
Definition: cpu.h:320
static_always_inline void * clib_march_select_fn_ptr(clib_march_fn_registration *r)
Definition: cpu.h:70
static foreach_aarch64_flags int clib_cpu_supports_aes()
Definition: cpu.h:236
#define AARCH64_CPU_IMPLEMENTER_CORTEXA72
Definition: cpu.h:323
static int clib_cpu_march_priority_qdf24xx()
Definition: cpu.h:336
format_function_t format_cpu_uarch
Definition: cpu.h:389
format_function_t format_cpu_model_name
Definition: cpu.h:390
#define AARCH64_CPU_PART_CORTEXA72
Definition: cpu.h:324
#define AARCH64_CPU_IMPLEMENTER_QDF24XX
Definition: cpu.h:321