FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
mem.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define _GNU_SOURCE
17 #include <stdlib.h>
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <unistd.h>
21 #include <sys/mount.h>
22 #include <sys/mman.h>
23 #include <fcntl.h>
24 #include <linux/mempolicy.h>
25 #include <linux/memfd.h>
26 
27 #include <vppinfra/clib.h>
28 #include <vppinfra/mem.h>
29 #include <vppinfra/lock.h>
30 #include <vppinfra/time.h>
31 #include <vppinfra/format.h>
32 #include <vppinfra/clib_error.h>
33 #include <vppinfra/linux/sysfs.h>
34 
35 #ifndef F_LINUX_SPECIFIC_BASE
36 #define F_LINUX_SPECIFIC_BASE 1024
37 #endif
38 
39 #ifndef F_ADD_SEALS
40 #define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
41 #define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
42 
43 #define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */
44 #define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */
45 #define F_SEAL_GROW 0x0004 /* prevent file from growing */
46 #define F_SEAL_WRITE 0x0008 /* prevent writes */
47 #endif
48 
49 #ifndef MFD_HUGETLB
50 #define MFD_HUGETLB 0x0004U
51 #endif
52 
53 #ifndef MAP_HUGE_SHIFT
54 #define MAP_HUGE_SHIFT 26
55 #endif
56 
57 #ifndef MFD_HUGE_SHIFT
58 #define MFD_HUGE_SHIFT 26
59 #endif
60 
61 #ifndef MAP_FIXED_NOREPLACE
62 #define MAP_FIXED_NOREPLACE 0x100000
63 #endif
64 
65 static void
67 {
69  CLIB_PAUSE ();
70 }
71 
72 static void
74 {
76 }
77 
78 __clib_export uword
80 {
81  unformat_input_t input;
82  static u32 size = 0;
83  int fd;
84 
85  if (size)
86  goto done;
87 
88  /*
89  * If the kernel doesn't support hugepages, /proc/meminfo won't
90  * say anything about it. Use the regular page size as a default.
91  */
92  size = clib_mem_get_page_size () / 1024;
93 
94  if ((fd = open ("/proc/meminfo", 0)) == -1)
95  return 0;
96 
97  unformat_init_clib_file (&input, fd);
98 
100  {
101  if (unformat (&input, "Hugepagesize:%_%u kB", &size))
102  ;
103  else
104  unformat_skip_line (&input);
105  }
106  unformat_free (&input);
107  close (fd);
108 done:
109  return 1024ULL * size;
110 }
111 
112 static clib_mem_page_sz_t
114 {
116  FILE *fp;
117  char tmp[33] = { };
118 
119  if ((fp = fopen ("/proc/meminfo", "r")) == NULL)
121 
122  while (fscanf (fp, "%32s", tmp) > 0)
123  if (strncmp ("Hugepagesize:", tmp, 13) == 0)
124  {
125  u32 size;
126  if (fscanf (fp, "%u", &size) > 0)
127  log2_page_size = 10 + min_log2 (size);
128  break;
129  }
130 
131  fclose (fp);
132  return log2_page_size;
133 }
134 
135 void
137 {
139  uword page_size;
140  void *va;
141  int fd;
142 
144  return;
145 
146  /* system page size */
147  page_size = sysconf (_SC_PAGESIZE);
148  mm->log2_page_sz = min_log2 (page_size);
149 
150  /* default system hugeppage size */
151  if ((fd = syscall (__NR_memfd_create, "test", MFD_HUGETLB)) != -1)
152  {
154  close (fd);
155  }
156  else /* likely kernel older than 4.14 */
158 
159  /* numa nodes */
160  va = mmap (0, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
161  MAP_ANONYMOUS, -1, 0);
162  if (va == MAP_FAILED)
163  return;
164 
165  if (mlock (va, page_size))
166  goto done;
167 
168  for (int i = 0; i < CLIB_MAX_NUMAS; i++)
169  {
170  int status;
171  if (syscall (__NR_move_pages, 0, 1, &va, &i, &status, 0) == 0)
172  mm->numa_node_bitmap |= 1ULL << i;
173  }
174 
175 done:
176  munmap (va, page_size);
177 }
178 
179 __clib_export u64
181 {
182  struct stat st = { 0 };
183  if (fstat (fd, &st) == -1)
184  return 0;
185  return st.st_blksize;
186 }
187 
188 __clib_export clib_mem_page_sz_t
190 {
191  uword page_size = clib_mem_get_fd_page_size (fd);
192  return page_size ? min_log2 (page_size) : CLIB_MEM_PAGE_SZ_UNKNOWN;
193 }
194 
195 __clib_export void
197  clib_mem_page_sz_t log2_page_size)
198 {
199  u8 bit_mask = 15;
200 
201  if (log2_page_size <= 12)
202  bit_mask = 15;
203  else if (log2_page_size > 12 && log2_page_size <= 16)
204  bit_mask = 3;
205  else
206  bit_mask = 0;
207 
208  *requested_va +=
209  (clib_cpu_time_now () & bit_mask) * (1ull << log2_page_size);
210 }
211 
212 static int
214 {
216  int fd = -1;
217  char *mount_dir;
218  u8 *temp;
219  u8 *filename;
220 
221  /*
222  * Since mkdtemp will modify template string "/tmp/hugepage_mount.XXXXXX",
223  * it must not be a string constant, but should be declared as
224  * a character array.
225  */
226  temp = format (0, "/tmp/hugepage_mount.XXXXXX%c", 0);
227 
228  /* create mount directory */
229  if ((mount_dir = mkdtemp ((char *) temp)) == 0)
230  {
231  vec_free (temp);
232  vec_reset_length (mm->error);
233  mm->error = clib_error_return_unix (mm->error, "mkdtemp");
234  return CLIB_MEM_ERROR;
235  }
236 
237  if (mount ("none", mount_dir, "hugetlbfs", 0, NULL))
238  {
239  vec_free (temp);
240  rmdir ((char *) mount_dir);
241  vec_reset_length (mm->error);
242  mm->error = clib_error_return_unix (mm->error, "mount");
243  return CLIB_MEM_ERROR;
244  }
245 
246  filename = format (0, "%s/%s%c", mount_dir, name, 0);
247 
248  if ((fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1)
249  {
250  vec_reset_length (mm->error);
251  mm->error = clib_error_return_unix (mm->error, "mkdtemp");
252  }
253 
254  umount2 ((char *) mount_dir, MNT_DETACH);
255  rmdir ((char *) mount_dir);
256  vec_free (filename);
257  vec_free (temp);
258 
259  return fd;
260 }
261 
262 __clib_export int
263 clib_mem_vm_create_fd (clib_mem_page_sz_t log2_page_size, char *fmt, ...)
264 {
266  int fd;
267  unsigned int memfd_flags;
268  va_list va;
269  u8 *s = 0;
270 
271  if (log2_page_size == mm->log2_page_sz)
272  log2_page_size = CLIB_MEM_PAGE_SZ_DEFAULT;
273  else if (log2_page_size == mm->log2_default_hugepage_sz)
274  log2_page_size = CLIB_MEM_PAGE_SZ_DEFAULT_HUGE;
275 
276  switch (log2_page_size)
277  {
279  return CLIB_MEM_ERROR;
281  memfd_flags = MFD_ALLOW_SEALING;
282  break;
284  memfd_flags = MFD_HUGETLB;
285  break;
286  default:
287  memfd_flags = MFD_HUGETLB | log2_page_size << MFD_HUGE_SHIFT;
288  }
289 
290  va_start (va, fmt);
291  s = va_format (0, fmt, &va);
292  va_end (va);
293 
294  /* memfd_create maximum string size is 249 chars without trailing zero */
295  if (vec_len (s) > 249)
296  _vec_len (s) = 249;
297  vec_add1 (s, 0);
298 
299  /* memfd_create introduced in kernel 3.17, we don't support older kernels */
300  fd = syscall (__NR_memfd_create, (char *) s, memfd_flags);
301 
302  /* kernel versions < 4.14 does not support memfd_create for huge pages */
303  if (fd == -1 && errno == EINVAL &&
304  log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE)
305  {
306  fd = legacy_memfd_create (s);
307  }
308  else if (fd == -1)
309  {
310  vec_reset_length (mm->error);
311  mm->error = clib_error_return_unix (mm->error, "memfd_create");
312  vec_free (s);
313  return CLIB_MEM_ERROR;
314  }
315 
316  vec_free (s);
317 
318  if ((memfd_flags & MFD_ALLOW_SEALING) &&
319  ((fcntl (fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1))
320  {
321  vec_reset_length (mm->error);
322  mm->error = clib_error_return_unix (mm->error, "fcntl (F_ADD_SEALS)");
323  close (fd);
324  return CLIB_MEM_ERROR;
325  }
326 
327  return fd;
328 }
329 
330 uword
332 {
334  uword pagesize = 1ULL << log2_page_sz;
335  uword sys_page_sz = 1ULL << mm->log2_page_sz;
336  uword n_bytes;
337  void *base = 0, *p;
338 
339  size = round_pow2 (size, pagesize);
340 
341  /* in adition of requested reservation, we also rserve one system page
342  * (typically 4K) adjacent to the start off reservation */
343 
344  if (start)
345  {
346  /* start address is provided, so we just need to make sure we are not
347  * replacing existing map */
348  if (start & pow2_mask (log2_page_sz))
349  return ~0;
350 
351  base = (void *) start - sys_page_sz;
352  base = mmap (base, size + sys_page_sz, PROT_NONE,
353  MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0);
354  return (base == MAP_FAILED) ? ~0 : start;
355  }
356 
357  /* to make sure that we get reservation aligned to page_size we need to
358  * request one additional page as mmap will return us address which is
359  * aligned only to system page size */
360  base = mmap (0, size + pagesize, PROT_NONE,
361  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
362 
363  if (base == MAP_FAILED)
364  return ~0;
365 
366  /* return additional space at the end of allocation */
367  p = base + size + pagesize;
368  n_bytes = (uword) p & pow2_mask (log2_page_sz);
369  if (n_bytes)
370  {
371  p -= n_bytes;
372  munmap (p, n_bytes);
373  }
374 
375  /* return additional space at the start of allocation */
376  n_bytes = pagesize - sys_page_sz - n_bytes;
377  if (n_bytes)
378  {
379  munmap (base, n_bytes);
380  base += n_bytes;
381  }
382 
383  return (uword) base + sys_page_sz;
384 }
385 
386 __clib_export clib_mem_vm_map_hdr_t *
388 {
390  uword sys_page_sz = 1ULL << mm->log2_page_sz;
392  if (hdr == 0)
393  {
394  hdr = mm->first_map;
395  if (hdr)
396  mprotect (hdr, sys_page_sz, PROT_READ);
397  return hdr;
398  }
399  next = hdr->next;
400  mprotect (hdr, sys_page_sz, PROT_NONE);
401  if (next)
402  mprotect (next, sys_page_sz, PROT_READ);
403  return next;
404 }
405 
406 void *
408  uword size, int fd, uword offset, char *name)
409 {
412  uword sys_page_sz = 1ULL << mm->log2_page_sz;
413  int mmap_flags = MAP_FIXED, is_huge = 0;
414 
415  if (fd != -1)
416  {
417  mmap_flags |= MAP_SHARED;
418  log2_page_sz = clib_mem_get_fd_log2_page_size (fd);
419  if (log2_page_sz > mm->log2_page_sz)
420  is_huge = 1;
421  }
422  else
423  {
424  mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
425 
426  if (log2_page_sz == mm->log2_page_sz)
427  log2_page_sz = CLIB_MEM_PAGE_SZ_DEFAULT;
428 
429  switch (log2_page_sz)
430  {
432  /* will fail later */
433  break;
435  log2_page_sz = mm->log2_page_sz;
436  break;
438  mmap_flags |= MAP_HUGETLB;
439  log2_page_sz = mm->log2_default_hugepage_sz;
440  is_huge = 1;
441  break;
442  default:
443  mmap_flags |= MAP_HUGETLB;
444  mmap_flags |= log2_page_sz << MAP_HUGE_SHIFT;
445  is_huge = 1;
446  }
447  }
448 
449  if (log2_page_sz == CLIB_MEM_PAGE_SZ_UNKNOWN)
450  return CLIB_MEM_VM_MAP_FAILED;
451 
452  size = round_pow2 (size, 1ULL << log2_page_sz);
453 
454  base = (void *) clib_mem_vm_reserve ((uword) base, size, log2_page_sz);
455 
456  if (base == (void *) ~0)
457  return CLIB_MEM_VM_MAP_FAILED;
458 
459  base = mmap (base, size, PROT_READ | PROT_WRITE, mmap_flags, fd, offset);
460 
461  if (base == MAP_FAILED)
462  return CLIB_MEM_VM_MAP_FAILED;
463 
464  if (is_huge && (mlock (base, size) != 0))
465  {
466  munmap (base, size);
467  return CLIB_MEM_VM_MAP_FAILED;
468  }
469 
470  hdr = mmap (base - sys_page_sz, sys_page_sz, PROT_READ | PROT_WRITE,
471  MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
472 
473  if (hdr != base - sys_page_sz)
474  {
475  munmap (base, size);
476  return CLIB_MEM_VM_MAP_FAILED;
477  }
478 
479  map_lock ();
480 
481  if (mm->last_map)
482  {
483  mprotect (mm->last_map, sys_page_sz, PROT_READ | PROT_WRITE);
484  mm->last_map->next = hdr;
485  mprotect (mm->last_map, sys_page_sz, PROT_NONE);
486  }
487  else
488  mm->first_map = hdr;
489 
490  CLIB_MEM_UNPOISON (hdr, sys_page_sz);
491  hdr->next = 0;
492  hdr->prev = mm->last_map;
493  snprintf (hdr->name, CLIB_VM_MAP_HDR_NAME_MAX_LEN - 1, "%s", (char *) name);
494  mm->last_map = hdr;
495 
496  map_unlock ();
497 
498  hdr->base_addr = (uword) base;
499  hdr->log2_page_sz = log2_page_sz;
500  hdr->num_pages = size >> log2_page_sz;
501  hdr->fd = fd;
502  hdr->name[CLIB_VM_MAP_HDR_NAME_MAX_LEN - 1] = 0;
503  mprotect (hdr, sys_page_sz, PROT_NONE);
504 
505  CLIB_MEM_UNPOISON (base, size);
506  return base;
507 }
508 
509 __clib_export int
510 clib_mem_vm_unmap (void *base)
511 {
513  uword size, sys_page_sz = 1ULL << mm->log2_page_sz;
514  clib_mem_vm_map_hdr_t *hdr = base - sys_page_sz;;
515 
516  if (mprotect (hdr, sys_page_sz, PROT_READ | PROT_WRITE) != 0)
517  return CLIB_MEM_ERROR;
518 
519  size = hdr->num_pages << hdr->log2_page_sz;
520  if (munmap ((void *) hdr->base_addr, size) != 0)
521  return CLIB_MEM_ERROR;
522 
523  map_lock ();
524 
525  if (hdr->next)
526  {
527  mprotect (hdr->next, sys_page_sz, PROT_READ | PROT_WRITE);
528  hdr->next->prev = hdr->prev;
529  mprotect (hdr->next, sys_page_sz, PROT_NONE);
530  }
531  else
532  mm->last_map = hdr->prev;
533 
534  if (hdr->prev)
535  {
536  mprotect (hdr->prev, sys_page_sz, PROT_READ | PROT_WRITE);
537  hdr->prev->next = hdr->next;
538  mprotect (hdr->prev, sys_page_sz, PROT_NONE);
539  }
540  else
541  mm->first_map = hdr->next;
542 
543  map_unlock ();
544 
545  if (munmap (hdr, sys_page_sz) != 0)
546  return CLIB_MEM_ERROR;
547 
548  return 0;
549 }
550 
551 __clib_export void
552 clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
553  uword n_pages, clib_mem_page_stats_t * stats)
554 {
555  int i, *status = 0;
556  void **ptr = 0;
557 
558  log2_page_size = clib_mem_log2_page_size_validate (log2_page_size);
559 
560  vec_validate (status, n_pages - 1);
561  vec_validate (ptr, n_pages - 1);
562 
563  for (i = 0; i < n_pages; i++)
564  ptr[i] = start + (i << log2_page_size);
565 
566  clib_memset (stats, 0, sizeof (clib_mem_page_stats_t));
567  stats->total = n_pages;
568  stats->log2_page_sz = log2_page_size;
569 
570  if (syscall (__NR_move_pages, 0, n_pages, ptr, 0, status, 0) != 0)
571  {
572  stats->unknown = n_pages;
573  goto done;
574  }
575 
576  for (i = 0; i < n_pages; i++)
577  {
578  if (status[i] >= 0 && status[i] < CLIB_MAX_NUMAS)
579  {
580  stats->mapped++;
581  stats->per_numa[status[i]]++;
582  }
583  else if (status[i] == -EFAULT)
584  stats->not_mapped++;
585  else
586  stats->unknown++;
587  }
588 
589 done:
590  vec_free (status);
591  vec_free (ptr);
592 }
593 
594 
595 __clib_export u64 *
597  int n_pages)
598 {
599  int pagesize = sysconf (_SC_PAGESIZE);
600  int fd;
601  int i;
602  u64 *r = 0;
603 
604  log2_page_size = clib_mem_log2_page_size_validate (log2_page_size);
605 
606  if ((fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1)
607  return 0;
608 
609  for (i = 0; i < n_pages; i++)
610  {
611  u64 seek, pagemap = 0;
612  uword vaddr = pointer_to_uword (mem) + (((u64) i) << log2_page_size);
613  seek = ((u64) vaddr / pagesize) * sizeof (u64);
614  if (lseek (fd, seek, SEEK_SET) != seek)
615  goto done;
616 
617  if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap)))
618  goto done;
619 
620  if ((pagemap & (1ULL << 63)) == 0)
621  goto done;
622 
623  pagemap &= pow2_mask (55);
624  vec_add1 (r, pagemap * pagesize);
625  }
626 
627 done:
628  close (fd);
629  if (vec_len (r) != n_pages)
630  {
631  vec_free (r);
632  return 0;
633  }
634  return r;
635 }
636 
637 __clib_export int
638 clib_mem_set_numa_affinity (u8 numa_node, int force)
639 {
641  long unsigned int mask[16] = { 0 };
642  int mask_len = sizeof (mask) * 8 + 1;
643 
644  /* no numa support */
645  if (mm->numa_node_bitmap == 0)
646  {
647  if (numa_node)
648  {
649  vec_reset_length (mm->error);
650  mm->error = clib_error_return (mm->error, "%s: numa not supported",
651  (char *) __func__);
652  return CLIB_MEM_ERROR;
653  }
654  else
655  return 0;
656  }
657 
658  mask[0] = 1 << numa_node;
659 
660  if (syscall (__NR_set_mempolicy, force ? MPOL_BIND : MPOL_PREFERRED, mask,
661  mask_len))
662  goto error;
663 
664  vec_reset_length (mm->error);
665  return 0;
666 
667 error:
668  vec_reset_length (mm->error);
669  mm->error = clib_error_return_unix (mm->error, (char *) __func__);
670  return CLIB_MEM_ERROR;
671 }
672 
673 __clib_export int
675 {
677 
678  if (syscall (__NR_set_mempolicy, MPOL_DEFAULT, 0, 0))
679  {
680  vec_reset_length (mm->error);
681  mm->error = clib_error_return_unix (mm->error, (char *) __func__);
682  return CLIB_MEM_ERROR;
683  }
684  return 0;
685 }
686 
687 /*
688  * fd.io coding-style-patch-verification: ON
689  *
690  * Local Variables:
691  * eval: (c-set-style "gnu")
692  * End:
693  */
vec_reset_length
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
Definition: vec_bootstrap.h:194
CLIB_MEM_ERROR
#define CLIB_MEM_ERROR
Definition: mem.h:55
clib_mem_main_t::error
clib_error_t * error
Definition: mem.h:152
tmp
u32 * tmp
Definition: interface_output.c:1078
unformat_init_clib_file
void unformat_init_clib_file(unformat_input_t *input, int file_descriptor)
Definition: unformat.c:1064
clib_mem_set_default_numa_affinity
__clib_export int clib_mem_set_default_numa_affinity()
Definition: mem.c:674
clib_mem_vm_map_hdr_t
struct _clib_mem_vm_map_hdr clib_mem_vm_map_hdr_t
clib_mem_page_stats_t
Definition: mem.h:509
clib_mem_vm_get_next_map_hdr
__clib_export clib_mem_vm_map_hdr_t * clib_mem_vm_get_next_map_hdr(clib_mem_vm_map_hdr_t *hdr)
Definition: mem.c:387
clib_mem_log2_page_size_validate
static_always_inline clib_mem_page_sz_t clib_mem_log2_page_size_validate(clib_mem_page_sz_t log2_page_size)
Definition: mem.h:537
CLIB_MEM_PAGE_SZ_DEFAULT_HUGE
@ CLIB_MEM_PAGE_SZ_DEFAULT_HUGE
Definition: mem.h:61
F_SEAL_SHRINK
#define F_SEAL_SHRINK
Definition: mem.c:44
pow2_mask
static uword pow2_mask(uword x)
Definition: clib.h:252
pointer_to_uword
static uword pointer_to_uword(const void *p)
Definition: types.h:131
name
string name[64]
Definition: fib.api:25
clib.h
next
u16 * next
Definition: nat44_ei_out2in.c:718
clib_mem_vm_unmap
__clib_export int clib_mem_vm_unmap(void *base)
Definition: mem.c:510
clib_error_return
#define clib_error_return(e, args...)
Definition: error.h:99
mem
void * mem
Definition: flowhash_template.h:361
va_format
__clib_export u8 * va_format(u8 *s, const char *fmt, va_list *va)
Definition: format.c:391
stats
vl_api_ikev2_sa_stats_t stats
Definition: ikev2_types.api:162
unformat_input_t
struct _unformat_input_t unformat_input_t
CLIB_MEM_VM_MAP_FAILED
#define CLIB_MEM_VM_MAP_FAILED
Definition: mem.h:54
r
vnet_hw_if_output_node_runtime_t * r
Definition: interface_output.c:1071
clib_mem_get_page_stats
__clib_export void clib_mem_get_page_stats(void *start, clib_mem_page_sz_t log2_page_size, uword n_pages, clib_mem_page_stats_t *stats)
Definition: mem.c:552
clib_atomic_release
#define clib_atomic_release(a)
Definition: atomics.h:46
error
Definition: cJSON.c:88
sysfs.h
clib_mem_vm_create_fd
__clib_export int clib_mem_vm_create_fd(clib_mem_page_sz_t log2_page_size, char *fmt,...)
Definition: mem.c:263
clib_mem_main_t
Definition: mem.h:128
unformat
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
legacy_memfd_create
static int legacy_memfd_create(u8 *name)
Definition: mem.c:213
round_pow2
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:279
clib_mem_set_numa_affinity
__clib_export int clib_mem_set_numa_affinity(u8 numa_node, int force)
Definition: mem.c:638
clib_mem_main_init
void clib_mem_main_init()
Definition: mem.c:136
vec_len
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
Definition: vec_bootstrap.h:142
unformat_free
static void unformat_free(unformat_input_t *i)
Definition: format.h:155
vec_add1
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:606
lock.h
unformat_check_input
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:163
clib_atomic_test_and_set
#define clib_atomic_test_and_set(a)
Definition: atomics.h:45
uword
u64 uword
Definition: types.h:112
clib_mem_main_t::numa_node_bitmap
u32 numa_node_bitmap
Definition: mem.h:137
F_ADD_SEALS
#define F_ADD_SEALS
Definition: mem.c:40
CLIB_VM_MAP_HDR_NAME_MAX_LEN
#define CLIB_VM_MAP_HDR_NAME_MAX_LEN
Definition: mem.h:89
time.h
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
mask
vl_api_pnat_mask_t mask
Definition: pnat.api:45
vec_validate
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment)
Definition: vec.h:523
format.h
clib_mem_vm_map_internal
void * clib_mem_vm_map_internal(void *base, clib_mem_page_sz_t log2_page_sz, uword size, int fd, uword offset, char *name)
Definition: mem.c:407
MFD_ALLOW_SEALING
#define MFD_ALLOW_SEALING
Definition: main.c:104
clib_mem_vm_reserve
uword clib_mem_vm_reserve(uword start, uword size, clib_mem_page_sz_t log2_page_sz)
Definition: mem.c:331
fmt
int cJSON_bool fmt
Definition: cJSON.h:160
legacy_get_log2_default_hugepage_size
static clib_mem_page_sz_t legacy_get_log2_default_hugepage_size(void)
Definition: mem.c:113
clib_mem_get_fd_log2_page_size
__clib_export clib_mem_page_sz_t clib_mem_get_fd_log2_page_size(int fd)
Definition: mem.c:189
clib_mem_main
__clib_export clib_mem_main_t clib_mem_main
Definition: mem.c:22
vec_free
#define vec_free(V)
Free vector's memory (no header).
Definition: vec.h:395
clib_mem_get_fd_page_size
__clib_export u64 clib_mem_get_fd_page_size(int fd)
Definition: mem.c:180
size
u32 size
Definition: vhost_user.h:125
clib_mem_main_t::log2_page_sz
clib_mem_page_sz_t log2_page_sz
Definition: mem.h:131
clib_bihash_value
template key/value backing page structure
Definition: bihash_doc.h:44
u64
unsigned long u64
Definition: types.h:89
format
description fragment has unexpected format
Definition: map.api:433
u32
unsigned int u32
Definition: types.h:88
unformat_skip_line
static void unformat_skip_line(unformat_input_t *i)
Definition: format.h:214
CLIB_MEM_PAGE_SZ_DEFAULT
@ CLIB_MEM_PAGE_SZ_DEFAULT
Definition: mem.h:60
CLIB_MEM_UNPOISON
#define CLIB_MEM_UNPOISON(a, s)
Definition: sanitizer.h:47
n_bytes
u32 n_bytes
Definition: interface_output.c:401
clib_mem_main_t::map_lock
u8 map_lock
Definition: mem.h:149
CLIB_MAX_NUMAS
#define CLIB_MAX_NUMAS
Definition: mem.h:53
CLIB_PAUSE
#define CLIB_PAUSE()
Definition: lock.h:23
clib_mem_get_page_size
static_always_inline uword clib_mem_get_page_size(void)
Definition: mem.h:468
clib_mem_main_t::log2_default_hugepage_sz
clib_mem_page_sz_t log2_default_hugepage_sz
Definition: mem.h:134
clib_error_return_unix
#define clib_error_return_unix(e, args...)
Definition: error.h:102
clib_memset
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
clib_mem_page_sz_t
clib_mem_page_sz_t
Definition: mem.h:57
u8
unsigned char u8
Definition: types.h:56
clib_mem_main_t::last_map
clib_mem_vm_map_hdr_t * last_map
Definition: mem.h:146
clib_error.h
clib_mem_vm_get_paddr
__clib_export u64 * clib_mem_vm_get_paddr(void *mem, clib_mem_page_sz_t log2_page_size, int n_pages)
Definition: mem.c:596
clib_mem_get_default_hugepage_size
__clib_export uword clib_mem_get_default_hugepage_size(void)
Definition: mem.c:79
mem.h
min_log2
static uword min_log2(uword x)
Definition: clib.h:176
clib_mem_main_t::first_map
clib_mem_vm_map_hdr_t * first_map
Definition: mem.h:146
MAP_HUGE_SHIFT
#define MAP_HUGE_SHIFT
Definition: ip4_mtrie.c:787
clib_cpu_time_now
static u64 clib_cpu_time_now(void)
Definition: time.h:81
map_lock
static void map_lock()
Definition: mem.c:66
CLIB_MEM_PAGE_SZ_UNKNOWN
@ CLIB_MEM_PAGE_SZ_UNKNOWN
Definition: mem.h:59
clib_mem_vm_randomize_va
__clib_export void clib_mem_vm_randomize_va(uword *requested_va, clib_mem_page_sz_t log2_page_size)
Definition: mem.c:196
UNFORMAT_END_OF_INPUT
#define UNFORMAT_END_OF_INPUT
Definition: format.h:137
map_unlock
static void map_unlock()
Definition: mem.c:73