FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
cpu.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
18 
19 #include <vppinfra/format.h>
20 
21 /*
22  * multiarchitecture support. Adding new entry will produce
23  * new graph node function variant optimized for specific cpu
24  * microarchitecture.
25  * Order is important for runtime selection, as 1st match wins...
26  */
27 
28 #if __x86_64__ && CLIB_DEBUG == 0
29 #define foreach_march_variant(macro, x) \
30  macro(avx2, x, "arch=core-avx2")
31 #else
32 #define foreach_march_variant(macro, x)
33 #endif
34 
35 
36 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
37 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
38 #else
39 #define CLIB_CPU_OPTIMIZED
40 #endif
41 
42 
43 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
44  if (clib_cpu_supports_ ## arch()) \
45  return & fn ## _ ##arch;
46 
47 #define CLIB_MULTIARCH_SELECT_FN(fn,...) \
48  __VA_ARGS__ void * fn ## _multiarch_select(void) \
49 { \
50  foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \
51  return & fn; \
52 }
53 
54 #ifdef CLIB_MARCH_VARIANT
55 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
56 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
57 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
58 #else
59 #define CLIB_MULTIARCH_FN(fn) fn
60 #endif
61 
62 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
63 
64 #define foreach_x86_64_flags \
65 _ (sse3, 1, ecx, 0) \
66 _ (ssse3, 1, ecx, 9) \
67 _ (sse41, 1, ecx, 19) \
68 _ (sse42, 1, ecx, 20) \
69 _ (avx, 1, ecx, 28) \
70 _ (avx2, 7, ebx, 5) \
71 _ (avx512f, 7, ebx, 16) \
72 _ (x86_aes, 1, ecx, 25) \
73 _ (sha, 7, ebx, 29) \
74 _ (invariant_tsc, 0x80000007, edx, 8)
75 
76 
77 #define foreach_aarch64_flags \
78 _ (fp, 0) \
79 _ (asimd, 1) \
80 _ (evtstrm, 2) \
81 _ (aarch64_aes, 3) \
82 _ (pmull, 4) \
83 _ (sha1, 5) \
84 _ (sha2, 6) \
85 _ (crc32, 7) \
86 _ (atomics, 8) \
87 _ (fphp, 9) \
88 _ (asimdhp, 10) \
89 _ (cpuid, 11) \
90 _ (asimdrdm, 12) \
91 _ (jscvt, 13) \
92 _ (fcma, 14) \
93 _ (lrcpc, 15) \
94 _ (dcpop, 16) \
95 _ (sha3, 17) \
96 _ (sm3, 18) \
97 _ (sm4, 19) \
98 _ (asimddp, 20) \
99 _ (sha512, 21) \
100 _ (sve, 22)
101 
102 #if defined(__x86_64__)
103 #include "cpuid.h"
104 
105 static inline int
106 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
107 {
108  if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
109  return 0;
110  if (lev == 7)
111  __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
112  else
113  __cpuid (lev, *eax, *ebx, *ecx, *edx);
114  return 1;
115 }
116 
117 
118 #define _(flag, func, reg, bit) \
119 static inline int \
120 clib_cpu_supports_ ## flag() \
121 { \
122  u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
123  clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
124  \
125  return ((reg & (1 << bit)) != 0); \
126 }
128 #undef _
129 #else /* __x86_64__ */
130 
131 #define _(flag, func, reg, bit) \
132 static inline int clib_cpu_supports_ ## flag() { return 0; }
134 #undef _
135 #endif /* __x86_64__ */
136 #if defined(__aarch64__)
137 #include <sys/auxv.h>
138 #define _(flag, bit) \
139 static inline int \
140 clib_cpu_supports_ ## flag() \
141 { \
142  unsigned long hwcap = getauxval(AT_HWCAP); \
143  return (hwcap & (1 << bit)); \
144 }
146 #undef _
147 #else /* ! __x86_64__ && !__aarch64__ */
148 #define _(flag, bit) \
149 static inline int clib_cpu_supports_ ## flag() { return 0; }
151 #undef _
152 #endif /* __x86_64__, __aarch64__ */
153 /*
154  * aes is the only feature with the same name in both flag lists
155  * handle this by prefixing it with the arch name, and handling it
156  * with the custom function below
157  */
158  static inline int
160 {
161 #if defined (__aarch64__)
162  return clib_cpu_supports_x86_aes ();
163 #elif defined (__aarch64__)
164  return clib_cpu_supports_aarch64_aes ();
165 #else
166  return 0;
167 #endif
168 }
169 
170 static inline int
172 {
173  if (clib_cpu_supports_avx512f ())
174  return 20;
175  return -1;
176 }
177 
178 static inline int
180 {
181  if (clib_cpu_supports_avx2 ())
182  return 10;
183  return -1;
184 }
185 
186 static inline u32
188 {
189  char buf[128];
190  static u32 implementer = -1;
191 
192  if (-1 != implementer)
193  return implementer;
194 
195  FILE *fp = fopen ("/proc/cpuinfo", "r");
196  if (!fp)
197  return implementer;
198 
199  while (!feof (fp))
200  {
201  if (!fgets (buf, sizeof (buf), fp))
202  break;
203  buf[127] = '\0';
204  if (strstr (buf, "CPU implementer"))
205  implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
206  if (-1 != implementer)
207  break;
208  }
209  fclose (fp);
210 
211  return implementer;
212 }
213 
214 static inline u32
216 {
217  char buf[128];
218  static u32 part = -1;
219 
220  if (-1 != part)
221  return part;
222 
223  FILE *fp = fopen ("/proc/cpuinfo", "r");
224  if (!fp)
225  return part;
226 
227  while (!feof (fp))
228  {
229  if (!fgets (buf, sizeof (buf), fp))
230  break;
231  buf[127] = '\0';
232  if (strstr (buf, "CPU part"))
233  part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
234  if (-1 != part)
235  break;
236  }
237  fclose (fp);
238 
239  return part;
240 }
241 
242 #define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43
243 #define AARCH64_CPU_PART_THUNERDERX2 0x0af
244 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
245 #define AARCH64_CPU_PART_QDF24XX 0xc00
246 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
247 #define AARCH64_CPU_PART_CORTEXA72 0xd08
248 
249 static inline int
251 {
254  return 20;
255  return -1;
256 }
257 
258 static inline int
260 {
263  return 20;
264  return -1;
265 }
266 
267 static inline int
269 {
272  return 10;
273  return -1;
274 }
275 
276 #ifdef CLIB_MARCH_VARIANT
277 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
278 #else
279 #define CLIB_MARCH_FN_PRIORITY() 0
280 #endif
281 #endif /* included_clib_cpu_h */
282 
283 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
284 static void __clib_constructor \
285 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
286 { \
287  if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
288  { \
289  fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
290  fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
291  } \
292 } \
293 
294 #ifndef CLIB_MARCH_VARIANT
295 #define CLIB_MARCH_FN(fn, rtype, _args...) \
296  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
297  rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
298  int fn ## _selected_priority = 0; \
299  static inline rtype CLIB_CPU_OPTIMIZED \
300  CLIB_MARCH_SFX (fn ## _ma)(_args)
301 #else
302 #define CLIB_MARCH_FN(fn, rtype, _args...) \
303  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
304  extern int (*fn ## _selected) (_args); \
305  extern int fn ## _selected_priority; \
306  CLIB_MARCH_FN_CONSTRUCTOR (fn) \
307  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
308 #endif
309 
310 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
311 
315 
316 /*
317  * fd.io coding-style-patch-verification: ON
318  *
319  * Local Variables:
320  * eval: (c-set-style "gnu")
321  * End:
322  */
static int clib_cpu_march_priority_cortexa72()
Definition: cpu.h:268
#define AARCH64_CPU_IMPLEMENTER_THUNERDERX2
Definition: cpu.h:242
#define NULL
Definition: clib.h:58
format_function_t format_cpu_flags
Definition: cpu.h:314
#define foreach_aarch64_flags
Definition: cpu.h:77
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
unsigned int u32
Definition: types.h:88
static u32 clib_cpu_part()
Definition: cpu.h:215
static int clib_cpu_march_priority_avx512()
Definition: cpu.h:171
#define AARCH64_CPU_PART_QDF24XX
Definition: cpu.h:245
static int clib_cpu_march_priority_thunderx2t99()
Definition: cpu.h:250
static int clib_get_cpuid(const u32 lev, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
Definition: cpu.h:106
static u32 clib_cpu_implementer()
Definition: cpu.h:187
static int clib_cpu_march_priority_avx2()
Definition: cpu.h:179
#define foreach_x86_64_flags
Definition: cpu.h:64
#define AARCH64_CPU_PART_THUNERDERX2
Definition: cpu.h:243
static foreach_aarch64_flags int clib_cpu_supports_aes()
Definition: cpu.h:159
#define AARCH64_CPU_IMPLEMENTER_CORTEXA72
Definition: cpu.h:246
static int clib_cpu_march_priority_qdf24xx()
Definition: cpu.h:259
format_function_t format_cpu_uarch
Definition: cpu.h:312
format_function_t format_cpu_model_name
Definition: cpu.h:313
#define AARCH64_CPU_PART_CORTEXA72
Definition: cpu.h:247
#define AARCH64_CPU_IMPLEMENTER_QDF24XX
Definition: cpu.h:244