FD.io VPP  v18.07-rc0-415-g6c78436
Vector Packet Processing
mem.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define _GNU_SOURCE
17 #include <stdlib.h>
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <unistd.h>
21 #include <sys/mount.h>
22 #include <sys/mman.h>
23 #include <fcntl.h>
24 #include <linux/mempolicy.h>
25 #include <linux/memfd.h>
26 
27 #include <vppinfra/clib.h>
28 #include <vppinfra/mem.h>
29 #include <vppinfra/time.h>
30 #include <vppinfra/format.h>
31 #include <vppinfra/clib_error.h>
32 #include <vppinfra/linux/syscall.h>
33 #include <vppinfra/linux/sysfs.h>
34 
35 #ifndef F_LINUX_SPECIFIC_BASE
36 #define F_LINUX_SPECIFIC_BASE 1024
37 #endif
38 
39 #ifndef F_ADD_SEALS
40 #define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
41 #define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
42 
43 #define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */
44 #define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */
45 #define F_SEAL_GROW 0x0004 /* prevent file from growing */
46 #define F_SEAL_WRITE 0x0008 /* prevent writes */
47 #endif
48 
49 uword
51 {
52  struct stat st = { 0 };
53  if (fstat (fd, &st) == -1)
54  return 0;
55  return st.st_blksize;
56 }
57 
58 int
60 {
61  return min_log2 (clib_mem_vm_get_page_size (fd));
62 }
63 
64 void
65 clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size)
66 {
67  u8 bit_mask = 15;
68 
69  if (log2_page_size <= 12)
70  bit_mask = 15;
71  else if (log2_page_size > 12 && log2_page_size <= 16)
72  bit_mask = 3;
73  else
74  bit_mask = 0;
75 
76  *requested_va += (clib_cpu_time_now () & bit_mask) * (1 << log2_page_size);
77 }
78 
81 {
82  int fd = -1;
83  clib_error_t *err = 0;
84  void *addr = 0;
85  u8 *filename = 0;
86  int mmap_flags = 0;
87  int log2_page_size;
88  int n_pages;
89  int old_mpol = -1;
90  u64 old_mask[16] = { 0 };
91 
92  /* save old numa mem policy if needed */
94  {
95  int rv;
96  rv = get_mempolicy (&old_mpol, old_mask, sizeof (old_mask) * 8 + 1,
97  0, 0);
98 
99  if (rv == -1)
100  {
101  if (a->numa_node != 0 && (a->flags & CLIB_MEM_VM_F_NUMA_FORCE) != 0)
102  {
103  err = clib_error_return_unix (0, "get_mempolicy");
104  goto error;
105  }
106  else
107  old_mpol = -1;
108  }
109  }
110 
111  if (a->flags & CLIB_MEM_VM_F_LOCKED)
112  mmap_flags |= MAP_LOCKED;
113 
114  /* if we are creating shared segment, we need file descriptor */
115  if (a->flags & CLIB_MEM_VM_F_SHARED)
116  {
117  mmap_flags |= MAP_SHARED;
118  /* if hugepages are needed we need to create mount point */
119  if (a->flags & CLIB_MEM_VM_F_HUGETLB)
120  {
121  char *mount_dir;
122  char template[] = "/tmp/hugepage_mount.XXXXXX";
123 
124  mount_dir = mkdtemp (template);
125  if (mount_dir == 0)
126  return clib_error_return_unix (0, "mkdtemp \'%s\'", template);
127 
128  if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL))
129  {
130  rmdir ((char *) mount_dir);
131  err = clib_error_return_unix (0, "mount hugetlb directory '%s'",
132  mount_dir);
133  goto error;
134  }
135 
136  filename = format (0, "%s/%s%c", mount_dir, a->name, 0);
137 
138  if ((fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1)
139  {
140  umount2 ((char *) mount_dir, MNT_DETACH);
141  rmdir ((char *) mount_dir);
142  err = clib_error_return_unix (0, "open");
143  goto error;
144  }
145  umount2 ((char *) mount_dir, MNT_DETACH);
146  rmdir ((char *) mount_dir);
147  mmap_flags |= MAP_LOCKED;
148  }
149  else
150  {
151  if ((fd = memfd_create (a->name, MFD_ALLOW_SEALING)) == -1)
152  {
153  err = clib_error_return_unix (0, "memfd_create");
154  goto error;
155  }
156 
157  if ((fcntl (fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1)
158  {
159  err = clib_error_return_unix (0, "fcntl (F_ADD_SEALS)");
160  goto error;
161  }
162  }
163 
164  log2_page_size = clib_mem_vm_get_log2_page_size (fd);
165  if (log2_page_size == 0)
166  {
167  err = clib_error_return_unix (0, "cannot determine page size");
168  goto error;
169  }
170 
171  if (a->requested_va)
172  {
173  clib_mem_vm_randomize_va (&a->requested_va, log2_page_size);
174  mmap_flags |= MAP_FIXED;
175  }
176  }
177  else /* not CLIB_MEM_VM_F_SHARED */
178  {
179  mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
180  if (a->flags & CLIB_MEM_VM_F_HUGETLB)
181  {
182  mmap_flags |= MAP_HUGETLB;
183  log2_page_size = 21;
184  }
185  else
186  {
187  log2_page_size = min_log2 (sysconf (_SC_PAGESIZE));
188  }
189  }
190 
191  n_pages = ((a->size - 1) >> log2_page_size) + 1;
192 
194  {
196  1 << (log2_page_size - 10),
197  n_pages);
198  if (err)
199  goto error;
200 
201  }
202 
203  if (fd != -1)
204  if ((ftruncate (fd, (u64) n_pages * (1 << log2_page_size))) == -1)
205  {
206  err = clib_error_return_unix (0, "ftruncate");
207  goto error;
208  }
209 
210  if (old_mpol != -1)
211  {
212  int rv;
213  u64 mask[16] = { 0 };
214  mask[0] = 1 << a->numa_node;
215  rv = set_mempolicy (MPOL_BIND, mask, sizeof (mask) * 8 + 1);
216  if (rv == -1 && a->numa_node != 0 &&
217  (a->flags & CLIB_MEM_VM_F_NUMA_FORCE) != 0)
218  {
219  err = clib_error_return_unix (0, "set_mempolicy");
220  goto error;
221  }
222  }
223 
224  addr = mmap (uword_to_pointer (a->requested_va, void *), a->size,
225  (PROT_READ | PROT_WRITE), mmap_flags, fd, 0);
226  if (addr == MAP_FAILED)
227  {
228  err = clib_error_return_unix (0, "mmap");
229  goto error;
230  }
231 
232  /* re-apply old numa memory policy */
233  if (old_mpol != -1 &&
234  set_mempolicy (old_mpol, old_mask, sizeof (old_mask) * 8 + 1) == -1)
235  {
236  err = clib_error_return_unix (0, "set_mempolicy");
237  goto error;
238  }
239 
240  a->log2_page_size = log2_page_size;
241  a->n_pages = n_pages;
242  a->addr = addr;
243  a->fd = fd;
244  goto done;
245 
246 error:
247  if (fd != -1)
248  close (fd);
249 
250 done:
251  vec_free (filename);
252  return err;
253 }
254 
255 u64 *
256 clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages)
257 {
258  int pagesize = sysconf (_SC_PAGESIZE);
259  int fd;
260  int i;
261  u64 *r = 0;
262 
263  if ((fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1)
264  return 0;
265 
266  for (i = 0; i < n_pages; i++)
267  {
268  u64 seek, pagemap = 0;
269  uword vaddr = pointer_to_uword (mem) + (((u64) i) << log2_page_size);
270  seek = ((u64) vaddr / pagesize) * sizeof (u64);
271  if (lseek (fd, seek, SEEK_SET) != seek)
272  goto done;
273 
274  if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap)))
275  goto done;
276 
277  if ((pagemap & (1ULL << 63)) == 0)
278  goto done;
279 
280  pagemap &= pow2_mask (55);
281  vec_add1 (r, pagemap * pagesize);
282  }
283 
284 done:
285  close (fd);
286  if (vec_len (r) != n_pages)
287  {
288  vec_free (r);
289  return 0;
290  }
291  return r;
292 }
293 
294 clib_error_t *
296 {
297  int mmap_flags = MAP_SHARED;
298  void *addr;
299 
300  if (a->requested_va)
301  mmap_flags |= MAP_FIXED;
302 
303  addr = (void *) mmap (uword_to_pointer (a->requested_va, void *), a->size,
304  PROT_READ | PROT_WRITE, mmap_flags, a->fd, 0);
305 
306  if (addr == MAP_FAILED)
307  return clib_error_return_unix (0, "mmap");
308 
309  a->addr = addr;
310  return 0;
311 }
312 
313 /*
314  * fd.io coding-style-patch-verification: ON
315  *
316  * Local Variables:
317  * eval: (c-set-style "gnu")
318  * End:
319  */
#define CLIB_MEM_VM_F_HUGETLB
Definition: mem.h:327
a
Definition: bitmap.h:537
#define CLIB_MEM_VM_F_NUMA_PREFER
Definition: mem.h:328
unsigned long u64
Definition: types.h:89
#define F_ADD_SEALS
Definition: mem.c:40
#define NULL
Definition: clib.h:55
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:346
uword requested_va
Request fixed position mapping.
Definition: mem.h:350
static int memfd_create(const char *name, unsigned int flags)
Definition: syscall.h:44
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
static u64 clib_cpu_time_now(void)
Definition: time.h:73
int i
int numa_node
numa node preference.
Definition: mem.h:345
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
unsigned char u8
Definition: types.h:56
static uword min_log2(uword x)
Definition: clib.h:138
clib_error_t * clib_mem_vm_ext_map(clib_mem_vm_map_t *a)
Definition: mem.c:295
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:80
uword requested_va
Request fixed position mapping.
Definition: mem.h:362
static long set_mempolicy(int mode, const unsigned long *nodemask, unsigned long maxnode)
Definition: syscall.h:23
static uword pow2_mask(uword x)
Definition: clib.h:214
char * name
Name for memory allocation, set by caller.
Definition: mem.h:343
uword size
Allocation size, set by caller.
Definition: mem.h:344
unsigned int u32
Definition: types.h:88
#define F_SEAL_SHRINK
Definition: mem.c:44
#define CLIB_MEM_VM_F_SHARED
Definition: mem.h:326
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:347
#define clib_error_return_unix(e, args...)
Definition: error.h:102
#define CLIB_MEM_VM_F_NUMA_FORCE
Definition: mem.h:329
uword clib_mem_vm_get_page_size(int fd)
Definition: mem.c:50
#define MFD_ALLOW_SEALING
static int get_mempolicy(int *mode, unsigned long *nodemask, unsigned long maxnode, void *addr, unsigned long flags)
Definition: syscall.h:29
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:332
int clib_mem_vm_get_log2_page_size(int fd)
Definition: mem.c:59
#define CLIB_MEM_VM_F_HUGETLB_PREALLOC
Definition: mem.h:330
#define uword_to_pointer(u, type)
Definition: types.h:136
#define CLIB_MEM_VM_F_LOCKED
Definition: mem.h:331
uword size
Map size.
Definition: mem.h:360
static uword pointer_to_uword(const void *p)
Definition: types.h:131
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u64 uword
Definition: types.h:112
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
Definition: mem.c:256
void clib_mem_vm_randomize_va(uword *requested_va, u32 log2_page_size)
Definition: mem.c:65
void * mem
int fd
File descriptor to be mapped.
Definition: mem.h:361
int log2_page_size
Definition: mem.h:348
vhost_vring_addr_t addr
Definition: vhost-user.h:83
void * addr
Pointer to mapped memory, if successful.
Definition: mem.h:363
clib_error_t * clib_sysfs_prealloc_hugepages(int numa_node, int page_size, int nr)
Definition: sysfs.c:224