FD.io VPP  v21.10.1-2-g0a485f517
Vector Packet Processing
clib.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 #ifndef included_clib_h
39 #define included_clib_h
40 
41 #include <stddef.h>
42 
43 #if __has_include(<vppinfra/config.h>)
44 #include <vppinfra/config.h>
45 #endif
46 
47 #ifdef __x86_64__
48 #include <x86intrin.h>
49 #endif
50 
51 /* Standalone means to not assume we are running on a Unix box. */
52 #if ! defined (CLIB_STANDALONE) && ! defined (CLIB_LINUX_KERNEL)
53 #define CLIB_UNIX
54 #endif
55 
56 #include <vppinfra/types.h>
57 #include <vppinfra/atomics.h>
58 
59 /* Global DEBUG flag. Setting this to 1 or 0 turns off
60  ASSERT (see vppinfra/error.h) & other debugging code. */
61 #ifndef CLIB_DEBUG
62 #define CLIB_DEBUG 0
63 #endif
64 
65 #ifndef NULL
66 #define NULL ((void *) 0)
67 #endif
68 
69 #define BITS(x) (8*sizeof(x))
70 #define ARRAY_LEN(x) (sizeof (x)/sizeof (x[0]))
71 
72 #define _STRUCT_FIELD(t,f) (((t *) 0)->f)
73 #define STRUCT_OFFSET_OF(t,f) offsetof(t, f)
74 #define STRUCT_BIT_OFFSET_OF(t,f) (BITS(u8) * STRUCT_OFFSET_OF (t, f))
75 #define STRUCT_SIZE_OF(t,f) (sizeof (_STRUCT_FIELD (t, f)))
76 #define STRUCT_BITS_OF(t,f) (BITS (_STRUCT_FIELD (t, f)))
77 #define STRUCT_ARRAY_LEN(t,f) ARRAY_LEN (_STRUCT_FIELD (t, f))
78 #define STRUCT_MARK(mark) u8 mark[0]
79 #define STRUCT_MARK_PTR(v, f) &(v)->f
80 
81 /* Stride in bytes between struct array elements. */
82 #define STRUCT_STRIDE_OF(t,f) \
83  ( ((uword) & (((t *) 0)[1].f)) \
84  - ((uword) & (((t *) 0)[0].f)))
85 
86 #define STRUCT_OFFSET_OF_VAR(v,f) ((uword) (&(v)->f) - (uword) (v))
87 
88 /* Used to pack structure elements. */
89 #define CLIB_PACKED(x) x __attribute__ ((packed))
90 #define CLIB_UNUSED(x) x __attribute__ ((unused))
91 
92 /* similar to CLIB_CACHE_LINE_ALIGN_MARK() but with arbitrary alignment */
93 #define CLIB_ALIGN_MARK(name, alignment) u8 name[0] __attribute__((aligned(alignment)))
94 
95 /* Make a string from the macro's argument */
96 #define CLIB_STRING_MACRO(x) #x
97 
98 #define __clib_unused __attribute__ ((unused))
99 #define __clib_weak __attribute__ ((weak))
100 #define __clib_packed __attribute__ ((packed))
101 #define __clib_constructor __attribute__ ((constructor))
102 #define __clib_noinline __attribute__ ((noinline))
103 #define __clib_aligned(x) __attribute__ ((aligned(x)))
104 #define __clib_section(s) __attribute__ ((section(s)))
105 #define __clib_warn_unused_result __attribute__ ((warn_unused_result))
106 #define __clib_export __attribute__ ((visibility("default")))
107 
108 #define never_inline __attribute__ ((__noinline__))
109 
110 #if CLIB_DEBUG > 0
111 #define always_inline static inline
112 #define static_always_inline static inline
113 #else
114 #define always_inline static inline __attribute__ ((__always_inline__))
115 #define static_always_inline static inline __attribute__ ((__always_inline__))
116 #endif
117 
118 
119 /* Reserved (unused) structure element with address offset between
120  from and to. */
121 #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)]
122 
123 /* Hints to compiler about hot/cold code. */
124 #define PREDICT_FALSE(x) __builtin_expect((x),0)
125 #define PREDICT_TRUE(x) __builtin_expect((x),1)
126 #define COMPILE_TIME_CONST(x) __builtin_constant_p (x)
127 
128 /*
129  * Compiler barrier
130  * prevent compiler to reorder memory access accross this boundary
131  * prevent compiler to cache values in register (force reload)
132  * Not to be confused with CPU memory barrier below
133  */
134 #define CLIB_COMPILER_BARRIER() asm volatile ("":::"memory")
135 
136 /* Full memory barrier (read and write). */
137 #define CLIB_MEMORY_BARRIER() __sync_synchronize ()
138 
139 #if __x86_64__
140 #define CLIB_MEMORY_STORE_BARRIER() __builtin_ia32_sfence ()
141 #else
142 #define CLIB_MEMORY_STORE_BARRIER() __sync_synchronize ()
143 #endif
144 
145 /* Arranges for function to be called before main. */
146 #define INIT_FUNCTION(decl) \
147  decl __attribute ((constructor)); \
148  decl
149 
150 /* Arranges for function to be called before exit. */
151 #define EXIT_FUNCTION(decl) \
152  decl __attribute ((destructor)); \
153  decl
154 
155 /* Use __builtin_clz if available. */
156 #if uword_bits == 64
157 #define count_leading_zeros(x) __builtin_clzll (x)
158 #define count_trailing_zeros(x) __builtin_ctzll (x)
159 #else
160 #define count_leading_zeros(x) __builtin_clzl (x)
161 #define count_trailing_zeros(x) __builtin_ctzl (x)
162 #endif
163 
164 #if defined (count_leading_zeros)
167 {
168 #ifdef __BMI2__
169  return _blsr_u64 (x);
170 #else
171  return x ^ (1ULL << count_trailing_zeros (x));
172 #endif
173 }
174 
177 {
178  uword n;
179  n = count_leading_zeros (x);
180  return BITS (uword) - n - 1;
181 }
182 #else
184 min_log2 (uword x)
185 {
186  uword a = x, b = BITS (uword) / 2, c = 0, r = 0;
187 
188  /* Reduce x to 4 bit result. */
189 #define _ \
190 { \
191  c = a >> b; \
192  if (c) a = c; \
193  if (c) r += b; \
194  b /= 2; \
195 }
196 
197  if (BITS (uword) > 32)
198  _;
199  _;
200  _;
201  _;
202 #undef _
203 
204  /* Do table lookup on 4 bit partial. */
205  if (BITS (uword) > 32)
206  {
207  const u64 table = 0x3333333322221104LL;
208  uword t = (table >> (4 * a)) & 0xf;
209  r = t < 4 ? r + t : ~0;
210  }
211  else
212  {
213  const u32 table = 0x22221104;
214  uword t = (a & 8) ? 3 : ((table >> (4 * a)) & 0xf);
215  r = t < 4 ? r + t : ~0;
216  }
217 
218  return r;
219 }
220 #endif
221 
224 {
225  uword l = min_log2 (x);
226  if (x > ((uword) 1 << l))
227  l++;
228  return l;
229 }
230 
233 {
234  if (BITS (uword) == 64)
235  return min_log2 (x);
236  else
237  {
238  uword l, y;
239  y = x;
240  l = 0;
241  if (y == 0)
242  {
243  l += 32;
244  x >>= 32;
245  }
246  l += min_log2 (x);
247  return l;
248  }
249 }
250 
253 {
254  return ((uword) 1 << x) - (uword) 1;
255 }
256 
259 {
260  word y = (word) 1 << min_log2 (x);
261  if (x > y)
262  y *= 2;
263  return y;
264 }
265 
268 {
269  return 0 == (x & (x - 1));
270 }
271 
274 {
275  return (x) & ~(pow2 - 1);
276 }
277 
280 {
281  return (x + pow2 - 1) & ~(pow2 - 1);
282 }
283 
286 {
287  return (x + pow2 - 1) & ~(pow2 - 1);
288 }
289 
292 {
293  return x & -x;
294 }
295 
298 {
299  uword result;
300 #ifdef count_trailing_zeros
301  result = count_trailing_zeros (x);
302 #else
303  result = min_log2 (first_set (x));
304 #endif
305  return result;
306 }
307 
310 {
311  return (int) x;
312 }
313 
316 {
317  return (word) (x + .5);
318 }
319 
322 {
323  return f * flt_round_nearest (x / f);
324 }
325 
327 extract_bits (uword x, int start, int count)
328 {
329 #ifdef __BMI__
330  return _bextr_u64 (x, start, count);
331 #endif
332  return (x >> start) & pow2_mask (count);
333 }
334 
335 #define clib_max(x,y) \
336 ({ \
337  __typeof__ (x) _x = (x); \
338  __typeof__ (y) _y = (y); \
339  _x > _y ? _x : _y; \
340 })
341 
342 #define clib_min(x,y) \
343 ({ \
344  __typeof__ (x) _x = (x); \
345  __typeof__ (y) _y = (y); \
346  _x < _y ? _x : _y; \
347 })
348 
349 #define clib_clamp(x,lo,hi) \
350 ({ \
351  __typeof__ (x) _x = (x); \
352  __typeof__ (lo) _lo = (lo); \
353  __typeof__ (hi) _hi = (hi); \
354  _x < _lo ? _lo : (_x > _hi ? _hi : _x); \
355 })
356 
357 #define clib_abs(x) \
358 ({ \
359  __typeof__ (x) _x = (x); \
360  _x < 0 ? -_x : _x; \
361 })
362 
363 /* Standard standalone-only function declarations. */
364 #ifndef CLIB_UNIX
365 void clib_standalone_init (void *memory, uword memory_bytes);
366 
367 void qsort (void *base, uword n, uword size,
368  int (*)(const void *, const void *));
369 #endif
370 
371 /* Stack backtrace. */
372 uword
373 clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip);
374 
375 #include <vppinfra/byte_order.h>
376 #endif /* included_clib_h */
377 
378 /*
379  * fd.io coding-style-patch-verification: ON
380  *
381  * Local Variables:
382  * eval: (c-set-style "gnu")
383  * End:
384  */
min_log2_u64
static u64 min_log2_u64(u64 x)
Definition: clib.h:232
flt_round_down
static f64 flt_round_down(f64 x)
Definition: clib.h:309
count_trailing_zeros
#define count_trailing_zeros(x)
Definition: clib.h:161
types.h
pow2_mask
static uword pow2_mask(uword x)
Definition: clib.h:252
f
vlib_frame_t * f
Definition: interface_output.c:1098
count_leading_zeros
#define count_leading_zeros(x)
Definition: clib.h:160
byte_order.h
r
vnet_hw_if_output_node_runtime_t * r
Definition: interface_output.c:1089
max_log2
static uword max_log2(uword x)
Definition: clib.h:223
round_pow2
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:279
count
u8 count
Definition: dhcp.api:208
flt_round_to_multiple
static f64 flt_round_to_multiple(f64 x, f64 f)
Definition: clib.h:321
memory
vhost_user_memory_t memory
Definition: vhost_user.h:131
BITS
#define BITS(x)
Definition: clib.h:69
c
svmdb_client_t * c
Definition: vpp_get_metrics.c:48
uword
u64 uword
Definition: types.h:112
round_pow2_u64
static u64 round_pow2_u64(u64 x, u64 pow2)
Definition: clib.h:285
f64
double f64
Definition: types.h:142
size
u32 size
Definition: vhost_user.h:125
always_inline
#define always_inline
Definition: clib.h:111
u64
unsigned long u64
Definition: types.h:89
u32
unsigned int u32
Definition: types.h:88
extract_bits
static uword extract_bits(uword x, int start, int count)
Definition: clib.h:327
flt_round_nearest
static word flt_round_nearest(f64 x)
Definition: clib.h:315
log2_first_set
static uword log2_first_set(uword x)
Definition: clib.h:297
clear_lowest_set_bit
static uword clear_lowest_set_bit(uword x)
Definition: clib.h:166
round_down_pow2
static uword round_down_pow2(uword x, uword pow2)
Definition: clib.h:273
clib_backtrace
uword clib_backtrace(uword *callers, uword max_callers, uword n_frames_to_skip)
Definition: backtrace.c:226
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
a
a
Definition: bitmap.h:525
word
i64 word
Definition: types.h:111
max_pow2
static uword max_pow2(uword x)
Definition: clib.h:258
atomics.h
qsort
void qsort(void *base, uword n, uword size, int(*compar)(const void *, const void *))
Definition: qsort.c:56
min_log2
static uword min_log2(uword x)
Definition: clib.h:176
is_pow2
static uword is_pow2(uword x)
Definition: clib.h:267
first_set
static uword first_set(uword x)
Definition: clib.h:291