FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
mem.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 #ifndef _included_clib_mem_h
39 #define _included_clib_mem_h
40 
41 #include <stdarg.h>
42 #include <unistd.h>
43 #include <sys/mman.h>
44 
45 #include <vppinfra/clib.h> /* uword, etc */
46 #include <vppinfra/clib_error.h>
47 
48 #include <vppinfra/os.h>
49 #include <vppinfra/string.h> /* memcpy, clib_memset */
50 #include <vppinfra/sanitizer.h>
51 
52 #define CLIB_MAX_MHEAPS 256
53 #define CLIB_MAX_NUMAS 16
54 #define CLIB_MEM_VM_MAP_FAILED ((void *) ~0)
55 #define CLIB_MEM_ERROR (-1)
56 
57 typedef enum
58 {
73 
74 typedef struct _clib_mem_vm_map_hdr
75 {
76  /* base address */
77  uword base_addr;
78 
79  /* number of pages */
80  uword num_pages;
81 
82  /* page size (log2) */
83  clib_mem_page_sz_t log2_page_sz;
84 
85  /* file descriptor, -1 if memory is not shared */
86  int fd;
87 
88  /* allocation mame */
89 #define CLIB_VM_MAP_HDR_NAME_MAX_LEN 64
91 
92  /* linked list */
93  struct _clib_mem_vm_map_hdr *prev, *next;
95 
96 #define foreach_clib_mem_heap_flag \
97  _(0, LOCKED, "locked") \
98  _(1, UNMAP_ON_DESTROY, "unmap-on-destroy")
99 
100 typedef enum
101 {
102 #define _(i, v, s) CLIB_MEM_HEAP_F_##v = (1 << i),
104 #undef _
106 
107 typedef struct
108 {
109  /* base address */
110  void *base;
111 
112  /* dlmalloc mspace */
113  void *mspace;
114 
115  /* heap size */
117 
118  /* page size (log2) */
120 
121  /* flags */
123 
124  /* name - _MUST_ be last */
125  char name[0];
127 
128 typedef struct
129 {
130  /* log2 system page size */
132 
133  /* log2 system default hugepage size */
135 
136  /* bitmap of available numa nodes */
138 
139  /* per CPU heaps */
140  void *per_cpu_mheaps[CLIB_MAX_MHEAPS];
141 
142  /* per NUMA heaps */
143  void *per_numa_mheaps[CLIB_MAX_NUMAS];
144 
145  /* memory maps */
147 
148  /* map lock */
150 
151  /* last error */
154 
156 
157 /* Unspecified NUMA socket */
158 #define VEC_NUMA_UNSPECIFIED (0xFF)
159 
162 {
163  int cpu = os_get_thread_index ();
164  return clib_mem_main.per_cpu_mheaps[cpu];
165 }
166 
167 always_inline void *
169 {
170  int cpu = os_get_thread_index ();
171  void *old = clib_mem_main.per_cpu_mheaps[cpu];
172  clib_mem_main.per_cpu_mheaps[cpu] = new_heap;
173  return old;
174 }
175 
176 always_inline void *
178 {
180  return clib_mem_main.per_numa_mheaps[numa_id];
181 }
182 
183 always_inline void *
185 {
186  int numa = os_get_numa_index ();
187  void *old = clib_mem_main.per_numa_mheaps[numa];
188  clib_mem_main.per_numa_mheaps[numa] = new_heap;
189  return old;
190 }
191 
192 always_inline void
194 {
195  /*
196  * Find an unused slot in the per-cpu-mheaps array,
197  * and grab it for this thread. We need to be able to
198  * push/pop the thread heap without affecting other thread(s).
199  */
200  int i;
201  if (__os_thread_index != 0)
202  return;
203  for (i = 0; i < ARRAY_LEN (clib_mem_main.per_cpu_mheaps); i++)
206  {
208  break;
209  }
210  ASSERT (__os_thread_index > 0);
211 }
212 
215 {
216  size_t mspace_usable_size_with_delta (const void *p);
218 }
219 
220 /* Memory allocator which may call os_out_of_memory() if it fails */
221 always_inline void *
223  int os_out_of_memory_on_failure)
224 {
225  void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes,
226  unsigned long align, unsigned long align_offset);
228  void *p;
229 
230  if (align_offset > align)
231  {
232  if (align > 0)
233  align_offset %= align;
234  else
235  align_offset = align;
236  }
237 
238  p = mspace_get_aligned (h->mspace, size, align, align_offset);
239 
240  if (PREDICT_FALSE (0 == p))
241  {
242  if (os_out_of_memory_on_failure)
243  os_out_of_memory ();
244  return 0;
245  }
246 
247  CLIB_MEM_UNPOISON (p, size);
248  return p;
249 }
250 
251 /* Memory allocator which calls os_out_of_memory() when it fails */
252 always_inline void *
254 {
255  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
256  /* align_offset */ 0,
257  /* os_out_of_memory */ 1);
258 }
259 
260 always_inline void *
262 {
263  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
264  /* os_out_of_memory */ 1);
265 }
266 
267 /* Memory allocator which calls os_out_of_memory() when it fails */
268 always_inline void *
270 {
271  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
272  /* align_offset */ 0,
273  /* os_out_of_memory */ 0);
274 }
275 
276 always_inline void *
278 {
279  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
280  /* os_out_of_memory */ 0);
281 }
282 
283 
284 
285 /* Memory allocator which panics when it fails.
286  Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
287 #define clib_mem_alloc_aligned_no_fail(size,align) \
288 ({ \
289  uword _clib_mem_alloc_size = (size); \
290  void * _clib_mem_alloc_p; \
291  _clib_mem_alloc_p = clib_mem_alloc_aligned (_clib_mem_alloc_size, (align)); \
292  if (! _clib_mem_alloc_p) \
293  clib_panic ("failed to allocate %d bytes", _clib_mem_alloc_size); \
294  _clib_mem_alloc_p; \
295 })
296 
297 #define clib_mem_alloc_no_fail(size) clib_mem_alloc_aligned_no_fail(size,1)
298 
299 /* Alias to stack allocator for naming consistency. */
300 #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
301 
304 {
305  int mspace_is_heap_object (void *msp, void *p);
307  return mspace_is_heap_object (h->mspace, p);
308 }
309 
310 always_inline void
311 clib_mem_free (void *p)
312 {
313  void mspace_put (void *msp, void *p_arg);
315 
316  /* Make sure object is in the correct heap. */
318 
320 
321  mspace_put (h->mspace, p);
322 }
323 
324 always_inline void *
325 clib_mem_realloc (void *p, uword new_size, uword old_size)
326 {
327  /* By default use alloc, copy and free to emulate realloc. */
328  void *q = clib_mem_alloc (new_size);
329  if (q)
330  {
331  uword copy_size;
332  if (old_size < new_size)
333  copy_size = old_size;
334  else
335  copy_size = new_size;
336  clib_memcpy_fast (q, p, copy_size);
337  clib_mem_free (p);
338  }
339  return q;
340 }
341 
343 clib_mem_size (void *p)
344 {
346  return clib_mem_size_nocheck (p);
347 }
348 
349 always_inline void
351 {
352  uword size = clib_mem_size (p);
353  CLIB_MEM_UNPOISON (p, size);
354  memset_s_inline (p, size, 0, size);
355  clib_mem_free (p);
356 }
357 
360 {
361  return clib_mem_get_per_cpu_heap ();
362 }
363 
366 {
367  return clib_mem_set_per_cpu_heap (heap);
368 }
369 
371 clib_mem_heap_t *clib_mem_create_heap (void *base, uword size, int is_locked,
372  char *fmt, ...);
373 
374 void clib_mem_main_init ();
375 void *clib_mem_init (void *base, uword size);
377  clib_mem_page_sz_t log2_page_sz);
379 
380 void clib_mem_exit (void);
381 
382 void clib_mem_trace (int enable);
383 
384 int clib_mem_is_traced (void);
385 
386 typedef struct
387 {
388  /* Total number of objects allocated. */
390 
391  /* Total allocated bytes. Bytes used and free.
392  used + free = total */
393  uword bytes_total, bytes_used, bytes_free;
394 
395  /* Number of bytes used by mheap data structure overhead
396  (e.g. free lists, mheap header). */
398 
399  /* Amount of free space returned to operating system. */
401 
402  /* For malloc which puts small objects in sbrk region and
403  large objects in mmap'ed regions. */
406 
407  /* Max. number of bytes in this heap. */
410 
413 
417 
418 u8 *format_clib_mem_usage (u8 * s, va_list * args);
419 u8 *format_clib_mem_heap (u8 * s, va_list * va);
420 u8 *format_clib_mem_page_stats (u8 * s, va_list * va);
421 
422 /* Allocate virtual address space. */
423 always_inline void *
425 {
426  void *mmap_addr;
427  uword flags = MAP_PRIVATE;
428 
429 #ifdef MAP_ANONYMOUS
430  flags |= MAP_ANONYMOUS;
431 #endif
432 
433  mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0);
434  if (mmap_addr == (void *) -1)
435  mmap_addr = 0;
436  else
437  CLIB_MEM_UNPOISON (mmap_addr, size);
438 
439  return mmap_addr;
440 }
441 
442 always_inline void
444 {
445  munmap (addr, size);
446 }
447 
448 void *clib_mem_vm_map_internal (void *base, clib_mem_page_sz_t log2_page_sz,
449  uword size, int fd, uword offset, char *name);
450 
451 void *clib_mem_vm_map (void *start, uword size,
452  clib_mem_page_sz_t log2_page_size, char *fmt, ...);
453 void *clib_mem_vm_map_stack (uword size, clib_mem_page_sz_t log2_page_size,
454  char *fmt, ...);
455 void *clib_mem_vm_map_shared (void *start, uword size, int fd, uword offset,
456  char *fmt, ...);
457 int clib_mem_vm_unmap (void *base);
459  hdr);
460 
463 {
465 }
466 
469 {
470  return 1ULL << clib_mem_main.log2_page_sz;
471 }
472 
475 {
477 }
478 
479 int clib_mem_vm_create_fd (clib_mem_page_sz_t log2_page_size, char *fmt, ...);
484  clib_mem_page_sz_t log2_page_sz);
485 u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size,
486  int n_pages);
487 void clib_mem_destroy (void);
488 int clib_mem_set_numa_affinity (u8 numa_node, int force);
490 void clib_mem_vm_randomize_va (uword * requested_va,
491  clib_mem_page_sz_t log2_page_size);
492 void mheap_trace (clib_mem_heap_t * v, int enable);
494 void clib_mem_trace (int enable);
495 
498 {
499  ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN);
500 
501  if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT)
502  log2_page_size = clib_mem_get_log2_page_size ();
503  else if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE)
504  log2_page_size = clib_mem_get_log2_default_hugepage_size ();
505 
506  return round_pow2 (size, 1ULL << log2_page_size);
507 }
508 
509 typedef struct
510 {
518 
519 void clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
520  uword n_pages, clib_mem_page_stats_t * stats);
521 
524 {
526  u32 bitmap = mm->numa_node_bitmap;
527 
528  if (numa >= 0)
529  bitmap &= ~pow2_mask (numa + 1);
530  if (bitmap == 0)
531  return -1;
532 
533  return count_trailing_zeros (bitmap);
534 }
535 
538 {
539  if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT)
540  return clib_mem_get_log2_page_size ();
541  if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE)
543  return log2_page_size;
544 }
545 
548 {
549  return 1ULL << clib_mem_log2_page_size_validate (log2_page_size);
550 }
551 
554 {
555  return clib_mem_main.error;
556 }
557 
558 /* bulk allocator */
559 
562  u32 min_elts_per_chunk);
566 u8 *format_clib_mem_bulk (u8 *s, va_list *args);
567 
568 #include <vppinfra/error.h> /* clib_panic */
569 
570 #endif /* _included_clib_mem_h */
571 
572 /*
573  * fd.io coding-style-patch-verification: ON
574  *
575  * Local Variables:
576  * eval: (c-set-style "gnu")
577  * End:
578  */
clib_mem_main_t::error
clib_error_t * error
Definition: mem.h:152
CLIB_MAX_MHEAPS
#define CLIB_MAX_MHEAPS
Definition: mem.h:52
os.h
clib_mem_usage_t::bytes_used_sbrk
uword bytes_used_sbrk
Definition: mem.h:404
clib_mem_vm_map_stack
void * clib_mem_vm_map_stack(uword size, clib_mem_page_sz_t log2_page_size, char *fmt,...)
Definition: mem.c:42
CLIB_MEM_PAGE_SZ_64K
@ CLIB_MEM_PAGE_SZ_64K
Definition: mem.h:64
clib_mem_round_to_page_size
static uword clib_mem_round_to_page_size(uword size, clib_mem_page_sz_t log2_page_size)
Definition: mem.h:497
clib_mem_vm_unmap
int clib_mem_vm_unmap(void *base)
Definition: mem.c:510
CLIB_MEM_PAGE_SZ_32M
@ CLIB_MEM_PAGE_SZ_32M
Definition: mem.h:68
count_trailing_zeros
#define count_trailing_zeros(x)
Definition: clib.h:161
os_set_thread_index
static_always_inline void os_set_thread_index(uword thread_index)
Definition: os.h:69
clib_mem_vm_map_hdr_t
struct _clib_mem_vm_map_hdr clib_mem_vm_map_hdr_t
clib_mem_page_stats_t
Definition: mem.h:509
clib_mem_page_stats_t::mapped
uword mapped
Definition: mem.h:513
CLIB_MEM_PAGE_SZ_16G
@ CLIB_MEM_PAGE_SZ_16G
Definition: mem.h:71
clib_mem_log2_page_size_validate
static_always_inline clib_mem_page_sz_t clib_mem_log2_page_size_validate(clib_mem_page_sz_t log2_page_size)
Definition: mem.h:537
clib_mem_get_heap
static clib_mem_heap_t * clib_mem_get_heap(void)
Definition: mem.h:359
clib_mem_vm_reserve
uword clib_mem_vm_reserve(uword start, uword size, clib_mem_page_sz_t log2_page_sz)
Definition: mem.c:331
clib_mem_get_heap_base
void * clib_mem_get_heap_base(clib_mem_heap_t *heap)
Definition: mem_dlmalloc.c:586
clib_mem_bulk_handle_t
void * clib_mem_bulk_handle_t
Definition: mem.h:560
clib_mem_realloc
static void * clib_mem_realloc(void *p, uword new_size, uword old_size)
Definition: mem.h:325
CLIB_MEM_PAGE_SZ_DEFAULT_HUGE
@ CLIB_MEM_PAGE_SZ_DEFAULT_HUGE
Definition: mem.h:61
clib_mem_usage_t::bytes_max
uword bytes_max
Definition: mem.h:408
clib_mem_usage_t
Definition: mem.h:386
CLIB_MEM_POISON
#define CLIB_MEM_POISON(a, s)
Definition: sanitizer.h:46
pow2_mask
static uword pow2_mask(uword x)
Definition: clib.h:252
name
string name[64]
Definition: fib.api:25
clib.h
clib_mem_heap_t::mspace
void * mspace
Definition: mem.h:113
usage
static void usage(void)
Definition: health_check.c:14
next
u16 * next
Definition: nat44_ei_out2in.c:718
string.h
clib_mem_free
static void clib_mem_free(void *p)
Definition: mem.h:311
clib_mem_heap_t::base
void * base
Definition: mem.h:110
clib_mem_get_heap_free_space
uword clib_mem_get_heap_free_space(clib_mem_heap_t *heap)
Definition: mem_dlmalloc.c:579
mspace_usable_size_with_delta
DLMALLOC_EXPORT size_t mspace_usable_size_with_delta(const void *p)
format_clib_mem_heap
u8 * format_clib_mem_heap(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:422
os_out_of_memory
void os_out_of_memory(void)
Definition: unix-misc.c:219
clib_mem_get_heap_size
uword clib_mem_get_heap_size(clib_mem_heap_t *heap)
Definition: mem_dlmalloc.c:592
clib_mem_bulk_free
void clib_mem_bulk_free(clib_mem_bulk_handle_t h, void *p)
Definition: mem_bulk.c:179
clib_mem_alloc_aligned_or_null
static void * clib_mem_alloc_aligned_or_null(uword size, uword align)
Definition: mem.h:277
clib_mem_page_bytes
static_always_inline uword clib_mem_page_bytes(clib_mem_page_sz_t log2_page_size)
Definition: mem.h:547
mem
void * mem
Definition: flowhash_template.h:361
clib_mem_vm_get_next_map_hdr
clib_mem_vm_map_hdr_t * clib_mem_vm_get_next_map_hdr(clib_mem_vm_map_hdr_t *hdr)
Definition: mem.c:387
stats
vl_api_ikev2_sa_stats_t stats
Definition: ikev2_types.api:162
clib_mem_trace_enable_disable
uword clib_mem_trace_enable_disable(uword enable)
Definition: mem_dlmalloc.c:525
addr
vhost_vring_addr_t addr
Definition: vhost_user.h:130
clib_mem_set_thread_index
static void clib_mem_set_thread_index(void)
Definition: mem.h:193
clib_mem_set_per_numa_heap
static void * clib_mem_set_per_numa_heap(void *new_heap)
Definition: mem.h:184
clib_memcpy_fast
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
Definition: string.h:92
memory_size
u64 memory_size
Definition: vhost_user.h:124
h
h
Definition: flowhash_template.h:372
clib_mem_set_default_numa_affinity
int clib_mem_set_default_numa_affinity()
Definition: mem.c:674
clib_mem_main_t
Definition: mem.h:128
clib_mem_alloc_or_null
static void * clib_mem_alloc_or_null(uword size)
Definition: mem.h:269
clib_mem_main_t::per_cpu_mheaps
void * per_cpu_mheaps[CLIB_MAX_MHEAPS]
Definition: mem.h:140
round_pow2
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:279
clib_mem_heap_t::size
uword size
Definition: mem.h:116
clib_mem_get_per_numa_heap
static void * clib_mem_get_per_numa_heap(u32 numa_id)
Definition: mem.h:177
mspace_get_aligned
DLMALLOC_EXPORT void * mspace_get_aligned(mspace msp, unsigned long n_user_data_bytes, unsigned long align, unsigned long align_offset)
clib_mem_set_numa_affinity
int clib_mem_set_numa_affinity(u8 numa_node, int force)
Definition: mem.c:638
clib_mem_page_stats_t::unknown
uword unknown
Definition: mem.h:516
clib_mem_get_per_cpu_heap
static clib_mem_heap_t * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:161
clib_mem_create_heap
clib_mem_heap_t * clib_mem_create_heap(void *base, uword size, int is_locked, char *fmt,...)
Definition: mem_dlmalloc.c:536
error.h
clib_mem_size
static uword clib_mem_size(void *p)
Definition: mem.h:343
clib_mem_page_stats_t::total
uword total
Definition: mem.h:512
memory
vhost_user_memory_t memory
Definition: vhost_user.h:131
CLIB_MEM_PAGE_SZ_1G
@ CLIB_MEM_PAGE_SZ_1G
Definition: mem.h:70
clib_mem_vm_map
void * clib_mem_vm_map(void *start, uword size, clib_mem_page_sz_t log2_page_size, char *fmt,...)
Definition: mem.c:25
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
clib_mem_vm_map_shared
void * clib_mem_vm_map_shared(void *start, uword size, int fd, uword offset, char *fmt,...)
Definition: mem.c:59
clib_mem_main_t::per_numa_mheaps
void * per_numa_mheaps[CLIB_MAX_NUMAS]
Definition: mem.h:143
ARRAY_LEN
#define ARRAY_LEN(x)
Definition: clib.h:70
static_always_inline
#define static_always_inline
Definition: clib.h:112
CLIB_MEM_PAGE_SZ_16K
@ CLIB_MEM_PAGE_SZ_16K
Definition: mem.h:63
clib_mem_destroy_heap
void clib_mem_destroy_heap(clib_mem_heap_t *heap)
Definition: mem_dlmalloc.c:566
foreach_clib_mem_heap_flag
#define foreach_clib_mem_heap_flag
Definition: mem.h:96
CLIB_MEM_PAGE_SZ_4K
@ CLIB_MEM_PAGE_SZ_4K
Definition: mem.h:62
clib_mem_free_s
static void clib_mem_free_s(void *p)
Definition: mem.h:350
uword
u64 uword
Definition: types.h:112
clib_mem_main_t::numa_node_bitmap
u32 numa_node_bitmap
Definition: mem.h:137
clib_mem_destroy
void clib_mem_destroy(void)
Definition: mem_dlmalloc.c:287
CLIB_MEM_PAGE_SZ_512M
@ CLIB_MEM_PAGE_SZ_512M
Definition: mem.h:69
CLIB_MEM_PAGE_SZ_1M
@ CLIB_MEM_PAGE_SZ_1M
Definition: mem.h:65
memset_s_inline
static errno_t memset_s_inline(void *s, rsize_t smax, int c, rsize_t n)
Definition: string.h:202
CLIB_VM_MAP_HDR_NAME_MAX_LEN
#define CLIB_VM_MAP_HDR_NAME_MAX_LEN
Definition: mem.h:89
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
clib_mem_vm_free
static void clib_mem_vm_free(void *addr, uword size)
Definition: mem.h:443
clib_mem_bulk_init
clib_mem_bulk_handle_t clib_mem_bulk_init(u32 elt_sz, u32 align, u32 min_elts_per_chunk)
Definition: mem_bulk.c:54
CLIB_MEM_PAGE_SZ_2M
@ CLIB_MEM_PAGE_SZ_2M
Definition: mem.h:66
CLIB_MEM_PAGE_SZ_16M
@ CLIB_MEM_PAGE_SZ_16M
Definition: mem.h:67
clib_mem_exit
void clib_mem_exit(void)
vlib_mem_get_next_numa_node
static_always_inline int vlib_mem_get_next_numa_node(int numa)
Definition: mem.h:523
clib_mem_heap_t::log2_page_sz
clib_mem_page_sz_t log2_page_sz
Definition: mem.h:119
align_offset
#define align_offset(A)
Definition: dlmalloc.c:200
clib_mem_vm_get_paddr
u64 * clib_mem_vm_get_paddr(void *mem, clib_mem_page_sz_t log2_page_size, int n_pages)
Definition: mem.c:596
fmt
int cJSON_bool fmt
Definition: cJSON.h:160
format_clib_mem_page_stats
u8 * format_clib_mem_page_stats(u8 *s, va_list *va)
Definition: mem.c:75
os_get_thread_index
static_always_inline uword os_get_thread_index(void)
Definition: os.h:63
clib_mem_init_thread_safe
void * clib_mem_init_thread_safe(void *memory, uword memory_size)
Definition: mem_dlmalloc.c:280
size
u32 size
Definition: vhost_user.h:125
clib_mem_main_t::log2_page_sz
clib_mem_page_sz_t log2_page_sz
Definition: mem.h:131
format_clib_mem_bulk
u8 * format_clib_mem_bulk(u8 *s, va_list *args)
Definition: mem_bulk.c:213
clib_mem_vm_create_fd
int clib_mem_vm_create_fd(clib_mem_page_sz_t log2_page_size, char *fmt,...)
Definition: mem.c:263
always_inline
#define always_inline
Definition: rdma_mlx5dv.h:23
clib_bihash_value
template key/value backing page structure
Definition: bihash_doc.h:44
clib_mem_heap_flag_t
clib_mem_heap_flag_t
Definition: mem.h:100
u64
unsigned long u64
Definition: types.h:89
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
clib_mem_get_log2_default_hugepage_size
static_always_inline clib_mem_page_sz_t clib_mem_get_log2_default_hugepage_size()
Definition: mem.h:474
clib_mem_vm_alloc
static void * clib_mem_vm_alloc(uword size)
Definition: mem.h:424
clib_mem_bulk_alloc
void * clib_mem_bulk_alloc(clib_mem_bulk_handle_t h)
Definition: mem_bulk.c:141
clib_mem_main
clib_mem_main_t clib_mem_main
Definition: mem.c:22
clib_mem_get_page_stats
void clib_mem_get_page_stats(void *start, clib_mem_page_sz_t log2_page_size, uword n_pages, clib_mem_page_stats_t *stats)
Definition: mem.c:552
u32
unsigned int u32
Definition: types.h:88
clib_mem_get_fd_page_size
uword clib_mem_get_fd_page_size(int fd)
Definition: mem.c:180
clib_mem_usage_t::bytes_used_mmap
uword bytes_used_mmap
Definition: mem.h:405
CLIB_MEM_PAGE_SZ_DEFAULT
@ CLIB_MEM_PAGE_SZ_DEFAULT
Definition: mem.h:60
CLIB_MEM_UNPOISON
#define CLIB_MEM_UNPOISON(a, s)
Definition: sanitizer.h:47
clib_mem_set_per_cpu_heap
static void * clib_mem_set_per_cpu_heap(void *new_heap)
Definition: mem.h:168
clib_mem_main_t::map_lock
u8 map_lock
Definition: mem.h:149
CLIB_MAX_NUMAS
#define CLIB_MAX_NUMAS
Definition: mem.h:53
clib_mem_vm_randomize_va
void clib_mem_vm_randomize_va(uword *requested_va, clib_mem_page_sz_t log2_page_size)
Definition: mem.c:196
clib_mem_get_page_size
static_always_inline uword clib_mem_get_page_size(void)
Definition: mem.h:468
mspace_put
DLMALLOC_EXPORT void mspace_put(mspace msp, void *p)
clib_mem_main_t::log2_default_hugepage_sz
clib_mem_page_sz_t log2_default_hugepage_sz
Definition: mem.h:134
clib_mem_get_log2_page_size
static_always_inline clib_mem_page_sz_t clib_mem_get_log2_page_size(void)
Definition: mem.h:462
clib_mem_vm_map_internal
void * clib_mem_vm_map_internal(void *base, clib_mem_page_sz_t log2_page_sz, uword size, int fd, uword offset, char *name)
Definition: mem.c:407
clib_mem_usage_t::bytes_used
uword bytes_used
Definition: mem.h:393
clib_mem_init
void * clib_mem_init(void *base, uword size)
Definition: mem_dlmalloc.c:266
clib_mem_heap_t::flags
clib_mem_heap_flag_t flags
Definition: mem.h:122
clib_mem_page_sz_t
clib_mem_page_sz_t
Definition: mem.h:57
clib_mem_get_default_hugepage_size
uword clib_mem_get_default_hugepage_size(void)
Definition: mem.c:79
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
mheap_trace
void mheap_trace(clib_mem_heap_t *v, int enable)
Definition: mem_dlmalloc.c:494
clib_mem_main_t::last_map
clib_mem_vm_map_hdr_t * last_map
Definition: mem.h:146
clib_error.h
os_get_numa_index
static_always_inline uword os_get_numa_index(void)
Definition: os.h:75
clib_mem_alloc_aligned_at_offset
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
Definition: mem.h:222
clib_mem_is_traced
int clib_mem_is_traced(void)
Definition: mem_dlmalloc.c:518
clib_mem_alloc_aligned
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:261
mspace_is_heap_object
DLMALLOC_EXPORT int mspace_is_heap_object(mspace msp, void *p)
clib_mem_get_fd_log2_page_size
clib_mem_page_sz_t clib_mem_get_fd_log2_page_size(int fd)
Definition: mem.c:189
clib_mem_init_with_page_size
void * clib_mem_init_with_page_size(uword memory_size, clib_mem_page_sz_t log2_page_sz)
Definition: mem_dlmalloc.c:273
clib_mem_usage_t::object_count
uword object_count
Definition: mem.h:389
clib_mem_bulk_destroy
void clib_mem_bulk_destroy(clib_mem_bulk_handle_t h)
Definition: mem_bulk.c:83
clib_mem_get_heap_usage
void clib_mem_get_heap_usage(clib_mem_heap_t *heap, clib_mem_usage_t *usage)
Definition: mem_dlmalloc.c:473
sanitizer.h
clib_mem_get_last_error
static_always_inline clib_error_t * clib_mem_get_last_error(void)
Definition: mem.h:553
clib_mem_page_stats_t::not_mapped
uword not_mapped
Definition: mem.h:514
CLIB_MEM_PAGE_SZ_UNKNOWN
@ CLIB_MEM_PAGE_SZ_UNKNOWN
Definition: mem.h:59
clib_mem_main_init
void clib_mem_main_init()
Definition: mem.c:136
clib_mem_usage_t::bytes_free_reclaimed
uword bytes_free_reclaimed
Definition: mem.h:400
clib_mem_set_heap
static clib_mem_heap_t * clib_mem_set_heap(clib_mem_heap_t *heap)
Definition: mem.h:365
clib_mem_heap_t
Definition: mem.h:107
clib_mem_page_stats_t::log2_page_sz
clib_mem_page_sz_t log2_page_sz
Definition: mem.h:511
clib_atomic_bool_cmp_and_swap
#define clib_atomic_bool_cmp_and_swap(addr, old, new)
Definition: atomics.h:38
clib_mem_size_nocheck
static uword clib_mem_size_nocheck(void *p)
Definition: mem.h:214
clib_mem_alloc
static void * clib_mem_alloc(uword size)
Definition: mem.h:253
clib_mem_trace
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:503
clib_mem_is_heap_object
static uword clib_mem_is_heap_object(void *p)
Definition: mem.h:303
format_clib_mem_usage
u8 * format_clib_mem_usage(u8 *s, va_list *args)
Definition: mem_dlmalloc.c:301
clib_mem_usage_t::bytes_overhead
uword bytes_overhead
Definition: mem.h:397
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105