18 #include <sys/types.h> 21 #include <sys/mount.h> 24 #include <linux/mempolicy.h> 25 #include <linux/memfd.h> 35 #ifndef F_LINUX_SPECIFIC_BASE 36 #define F_LINUX_SPECIFIC_BASE 1024 40 #define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) 41 #define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) 43 #define F_SEAL_SEAL 0x0001 44 #define F_SEAL_SHRINK 0x0002 45 #define F_SEAL_GROW 0x0004 46 #define F_SEAL_WRITE 0x0008 53 return getpagesize ();
72 if ((fd = open (
"/proc/meminfo", 0)) == -1)
79 if (
unformat (&input,
"Hugepagesize:%_%u kB", &size))
87 return 1024ULL *
size;
93 struct stat st = { 0 };
94 if (fstat (fd, &st) == -1)
110 if (log2_page_size <= 12)
112 else if (log2_page_size > 12 && log2_page_size <= 16)
122 #define MFD_HUGETLB 0x0004U 150 static int memfd_hugetlb_supported = 1;
152 char template[] =
"/tmp/hugepage_mount.XXXXXX";
157 if (memfd_hugetlb_supported)
163 if (errno ==
EINVAL && strnlen (name, 256) <= 249)
164 memfd_hugetlb_supported = 0;
167 mount_dir = mkdtemp (
template);
171 if (mount (
"none", (
char *) mount_dir,
"hugetlbfs", 0,
NULL))
173 rmdir ((
char *) mount_dir);
178 filename =
format (0,
"%s/%s%c", mount_dir, name, 0);
179 fd = open ((
char *) filename, O_CREAT | O_RDWR, 0755);
180 umount2 ((
char *) mount_dir, MNT_DETACH);
181 rmdir ((
char *) mount_dir);
203 long unsigned int old_mask[16] = { 0 };
209 rv =
get_mempolicy (&old_mpol, old_mask,
sizeof (old_mask) * 8 + 1,
225 mmap_flags |= MAP_LOCKED;
230 mmap_flags |= MAP_SHARED;
237 mmap_flags |= MAP_LOCKED;
246 if (log2_page_size == 0)
255 mmap_flags |= MAP_FIXED;
260 mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
263 mmap_flags |= MAP_HUGETLB;
268 log2_page_size =
min_log2 (sysconf (_SC_PAGESIZE));
272 n_pages = ((a->
size - 1) >> log2_page_size) + 1;
284 if ((ftruncate (fd, (
u64) n_pages * (1 << log2_page_size))) == -1)
293 long unsigned int mask[16] = { 0 };
305 (PROT_READ | PROT_WRITE), mmap_flags, fd, 0);
306 if (addr == MAP_FAILED)
313 if (old_mpol != -1 &&
314 set_mempolicy (old_mpol, old_mask,
sizeof (old_mask) * 8 + 1) == -1)
349 int pagesize = sysconf (_SC_PAGESIZE);
354 if ((fd = open ((
char *)
"/proc/self/pagemap", O_RDONLY)) == -1)
357 for (i = 0; i < n_pages; i++)
359 u64 seek, pagemap = 0;
361 seek = ((
u64) vaddr / pagesize) *
sizeof (
u64);
362 if (lseek (fd, seek, SEEK_SET) != seek)
365 if (read (fd, &pagemap,
sizeof (pagemap)) != (
sizeof (pagemap)))
368 if ((pagemap & (1ULL << 63)) == 0)
388 int mmap_flags = MAP_SHARED;
392 mmap_flags |= MAP_FIXED;
395 PROT_READ | PROT_WRITE, mmap_flags, a->
fd, 0);
397 if (addr == MAP_FAILED)
void clib_mem_vm_ext_free(clib_mem_vm_alloc_t *a)
#define CLIB_MEM_VM_F_HUGETLB
clib_error_t * clib_sysfs_prealloc_hugepages(int numa_node, int log2_page_size, int nr)
#define CLIB_MEM_VM_F_NUMA_PREFER
void * addr
Pointer to allocated memory, set on successful allocation.
uword requested_va
Request fixed position mapping.
static int memfd_create(const char *name, unsigned int flags)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static u64 clib_cpu_time_now(void)
int numa_node
numa node preference.
static uword min_log2(uword x)
#define MFD_ALLOW_SEALING
clib_error_t * clib_mem_vm_ext_map(clib_mem_vm_map_t *a)
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
uword requested_va
Request fixed position mapping.
static long set_mempolicy(int mode, const unsigned long *nodemask, unsigned long maxnode)
static uword pow2_mask(uword x)
char * name
Name for memory allocation, set by caller.
uword size
Allocation size, set by caller.
#define CLIB_MEM_VM_F_SHARED
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
uword clib_mem_get_page_size(void)
#define clib_error_return_unix(e, args...)
#define CLIB_MEM_VM_F_NUMA_FORCE
static int get_mempolicy(int *mode, unsigned long *nodemask, unsigned long maxnode, void *addr, unsigned long flags)
int clib_mem_get_fd_log2_page_size(int fd)
u64 clib_mem_get_fd_page_size(int fd)
#define vec_free(V)
Free vector's memory (no header).
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
#define CLIB_MEM_VM_F_HUGETLB_PREALLOC
#define uword_to_pointer(u, type)
#define CLIB_MEM_VM_F_LOCKED
uword clib_mem_get_default_hugepage_size(void)
static uword pointer_to_uword(const void *p)
clib_error_t * clib_mem_create_hugetlb_fd(char *name, int *fdp)
clib_error_t * clib_mem_create_fd(char *name, int *fdp)
static void clib_mem_vm_free(void *addr, uword size)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
void clib_mem_vm_randomize_va(uword *requested_va, u32 log2_page_size)
int fd
File descriptor to be mapped.
void * addr
Pointer to mapped memory, if successful.