FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
buffer.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.c: allocate/free network buffers.
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 /**
41  * @file
42  *
43  * Allocate/free network buffers.
44  */
45 
46 #include <unistd.h>
47 #include <linux/vfio.h>
48 #include <sys/ioctl.h>
49 
50 #include <rte_config.h>
51 
52 #include <rte_common.h>
53 #include <rte_log.h>
54 #include <rte_memory.h>
55 #include <rte_memzone.h>
56 #include <rte_tailq.h>
57 #include <rte_eal.h>
58 #include <rte_per_lcore.h>
59 #include <rte_launch.h>
60 #include <rte_atomic.h>
61 #include <rte_cycles.h>
62 #include <rte_prefetch.h>
63 #include <rte_lcore.h>
64 #include <rte_per_lcore.h>
65 #include <rte_branch_prediction.h>
66 #include <rte_interrupts.h>
67 #include <rte_pci.h>
68 #include <rte_random.h>
69 #include <rte_debug.h>
70 #include <rte_ether.h>
71 #include <rte_ethdev.h>
72 #include <rte_ring.h>
73 #include <rte_mempool.h>
74 #include <rte_mbuf.h>
75 #include <rte_version.h>
76 
77 #include <vlib/vlib.h>
78 #include <vlib/unix/unix.h>
79 #include <vlib/pci/pci.h>
80 #include <vlib/linux/vfio.h>
81 #include <vnet/vnet.h>
82 #include <dpdk/device/dpdk.h>
83 #include <dpdk/device/dpdk_priv.h>
84 
85 STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
86  "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
87 
88 typedef struct
89 {
90  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
91  struct rte_mbuf **mbuf_alloc_list;
93 
94 typedef struct
95 {
99 
101 
104  int maybe_next)
105 {
106  struct rte_mbuf *mb;
107  u32 next, flags;
108 
109 next:
110  flags = b->flags;
111  next = b->next_buffer;
112  mb = rte_mbuf_from_vlib_buffer (b);
113 
114  if (PREDICT_FALSE (b->n_add_refs))
115  {
116  rte_mbuf_refcnt_update (mb, b->n_add_refs);
117  b->n_add_refs = 0;
118  }
119 
120  if ((mb = rte_pktmbuf_prefree_seg (mb)))
121  rte_mempool_put (mb->pool, mb);
122 
123  if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
124  {
125  b = vlib_get_buffer (vm, next);
126  goto next;
127  }
128 }
129 
130 #ifndef CLIB_MARCH_VARIANT
131 static void
133 {
134  u32 i;
135  vlib_buffer_t *b;
136  u32 thread_index = vlib_get_thread_index ();
137 
138  for (i = 0; i < vec_len (f->buffers); i++)
139  {
140  b = vlib_get_buffer (vm, f->buffers[i]);
141  dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
142  }
143 
144  vec_free (f->name);
145  vec_free (f->buffers);
146  /* Poison it. */
147  memset (f, 0xab, sizeof (f[0]));
148 }
149 
150 /* Add buffer free list. */
151 static void
153  vlib_buffer_free_list_index_t free_list_index)
154 {
156  int i;
157 
158  ASSERT (vlib_get_thread_index () == 0);
159 
160  f = vlib_buffer_get_free_list (vm, free_list_index);
161 
162  del_free_list (vm, f);
163 
165 
166  for (i = 1; i < vec_len (vlib_mains); i++)
167  {
168  vlib_main_t *wvm = vlib_mains[i];
169  f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);
170  del_free_list (wvm, f);
172  }
173 }
174 #endif
175 
176 /* Make sure free list has at least given number of free buffers. */
177 uword
180  uword min_free_buffers)
181 {
182  dpdk_main_t *dm = &dpdk_main;
184  struct rte_mbuf **mb;
185  uword n_left, first;
186  word n_alloc;
187  unsigned socket_id = rte_socket_id ();
188  u32 thread_index = vlib_get_thread_index ();
189  dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
190  struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id];
191  dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp);
192  vlib_buffer_t bt;
193  u32 *bi;
194 
195  /* Too early? */
196  if (PREDICT_FALSE (rmp == 0))
197  return 0;
198 
199  /* Already have enough free buffers on free list? */
200  n_alloc = min_free_buffers - vec_len (fl->buffers);
201  if (n_alloc <= 0)
202  return min_free_buffers;
203 
204  /* Always allocate round number of buffers. */
205  n_alloc = round_pow2 (n_alloc, CLIB_CACHE_LINE_BYTES / sizeof (u32));
206 
207  /* Always allocate new buffers in reasonably large sized chunks. */
208  n_alloc = clib_max (n_alloc, fl->min_n_buffers_each_alloc);
209 
210  vec_validate_aligned (d->mbuf_alloc_list, n_alloc - 1,
212 
213  if (rte_mempool_get_bulk (rmp, (void *) d->mbuf_alloc_list, n_alloc) < 0)
214  return 0;
215 
216  memset (&bt, 0, sizeof (vlib_buffer_t));
219 
220  _vec_len (d->mbuf_alloc_list) = n_alloc;
221 
222  first = vec_len (fl->buffers);
223  vec_resize_aligned (fl->buffers, n_alloc, CLIB_CACHE_LINE_BYTES);
224 
225  n_left = n_alloc;
226  mb = d->mbuf_alloc_list;
227  bi = fl->buffers + first;
228 
229  ASSERT (n_left % 8 == 0);
230 
231  while (n_left >= 8)
232  {
233  if (PREDICT_FALSE (n_left < 24))
234  goto no_prefetch;
235 
244 
245  no_prefetch:
246  vlib_get_buffer_indices_with_offset (vm, (void **) mb, bi, 8,
247  sizeof (struct rte_mbuf));
251  vlib_buffer_from_rte_mbuf (mb[3]), &bt);
255  vlib_buffer_from_rte_mbuf (mb[7]), &bt);
256 
257  n_left -= 8;
258  mb += 8;
259  bi += 8;
260  }
261 
262  if (fl->buffer_init_function)
263  fl->buffer_init_function (vm, fl, fl->buffers + first, n_alloc);
264 
265  fl->n_alloc += n_alloc;
266 
267  return n_alloc;
268 }
269 
272 {
273  struct rte_mbuf *mb;
274  mb = rte_mbuf_from_vlib_buffer (b);
275  CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
277 }
278 
281  vlib_buffer_t * b)
282 {
283  u32 thread_index = vlib_get_thread_index ();
284 
285  dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
286 }
287 
290  u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
291 {
293  vlib_buffer_t *bufp[n_buffers], **b = bufp;
294  u32 thread_index = vlib_get_thread_index ();
295  int i = 0;
296  u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST |
297  VLIB_BUFFER_NEXT_PRESENT);
298  u32 n_left, *bi;
299  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
300  u32 follow_buffer_next);
301 
302  cb = bm->buffer_free_callback;
303 
304  if (PREDICT_FALSE (cb != 0))
305  n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
306 
307  if (!n_buffers)
308  return;
309 
310  n_left = n_buffers;
311  bi = buffers;
312  b = bufp;
313  vlib_get_buffers (vm, bi, b, n_buffers);
314 
315  while (n_left >= 4)
316  {
317  u32 or_flags;
318  vlib_buffer_t **p;
319 
320  if (n_left < 16)
321  goto no_prefetch;
322 
323  p = b + 12;
324  dpdk_prefetch_buffer (p[0]);
325  dpdk_prefetch_buffer (p[1]);
326  dpdk_prefetch_buffer (p[2]);
327  dpdk_prefetch_buffer (p[3]);
328  no_prefetch:
329 
330  for (i = 0; i < 4; i++)
332 
333  or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
334 
335  if (or_flags & simple_mask)
336  {
337  recycle_or_free (vm, bm, bi[0], b[0]);
338  recycle_or_free (vm, bm, bi[1], b[1]);
339  recycle_or_free (vm, bm, bi[2], b[2]);
340  recycle_or_free (vm, bm, bi[3], b[3]);
341  }
342  else
343  {
344  dpdk_rte_pktmbuf_free (vm, thread_index, b[0], 0);
345  dpdk_rte_pktmbuf_free (vm, thread_index, b[1], 0);
346  dpdk_rte_pktmbuf_free (vm, thread_index, b[2], 0);
347  dpdk_rte_pktmbuf_free (vm, thread_index, b[3], 0);
348  }
349  bi += 4;
350  b += 4;
351  n_left -= 4;
352  }
353  while (n_left)
354  {
356  recycle_or_free (vm, bm, bi[0], b[0]);
357  bi += 1;
358  b += 1;
359  n_left -= 1;
360  }
361 }
362 
363 void
365  u32 n_buffers)
366 {
367  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
368  1);
369 }
370 
371 void
373  u32 n_buffers)
374 {
375  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
376  0);
377 }
378 
379 #ifndef CLIB_MARCH_VARIANT
380 static void
382  void *vt,
383  void *packet_data,
384  uword n_packet_data_bytes,
385  uword min_n_buffers_each_alloc, u8 * name)
386 {
388 
390  memset (t, 0, sizeof (t[0]));
391 
392  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
393 
395 }
396 
397 static clib_error_t *
398 scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name)
399 {
402  const char fn[] = "/dev/vfio/vfio";
403  char buff[sizeof (fn)] = { 0 };
404  int fd;
405  u8 *path = format (0, "%v%c", path_name, 0);
406 
407  if (readlink ((char *) path, buff, sizeof (fn)) + 1 != sizeof (fn))
408  goto done;
409 
410  if (strncmp (fn, buff, sizeof (fn)))
411  goto done;
412 
413  fd = atoi ((char *) file_name);
414  if (fd != lvm->container_fd)
415  dbm->vfio_container_fd = fd;
416 
417 done:
418  vec_free (path);
419  return 0;
420 }
421 
422 clib_error_t *
423 dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size,
424  u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa,
425  struct rte_mempool ** _mp,
427 {
429  struct rte_mempool *mp;
432  clib_error_t *error = 0;
433  size_t min_chunk_size, align;
434  u32 size;
435  i32 ret;
436  uword i;
437 
438 
439  mp = rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size,
440  512, pool_priv_size, numa, 0);
441  if (!mp)
442  return clib_error_return (0, "failed to create %s", pool_name);
443 
444  rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL);
445 
446  size = rte_mempool_op_calc_mem_size_default (mp, num_elts, 21,
447  &min_chunk_size, &align);
448 
449  error = vlib_physmem_region_alloc (vm, (char *) pool_name, size, numa,
451  VLIB_PHYSMEM_F_SHARED, pri);
452  if (error)
453  {
454  rte_mempool_free (mp);
455  return error;
456  }
457 
458  pr = vlib_physmem_get_region (vm, pri[0]);
459 
460  /* Call the mempool priv initializer */
461  priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE +
463  priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE;
464  rte_pktmbuf_pool_init (mp, &priv);
465 
466  for (i = 0; i < pr->n_pages; i++)
467  {
468  size_t page_size = 1 << pr->log2_page_size;
469  ret = rte_mempool_populate_iova (mp, ((char *) pr->mem) + i * page_size,
470  pr->page_table[i], page_size, 0, 0);
471  if (ret < 0)
472  {
473  rte_mempool_free (mp);
474  return clib_error_return (0, "failed to populate %s", pool_name);
475  }
476  }
477 
478  _mp[0] = mp;
479 
480  /* DPDK currently doesn't provide API to map DMA memory for empty mempool
481  so we are using this hack, will be nice to have at least API to get
482  VFIO container FD */
483  if (dbm->vfio_container_fd == -1)
484  foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0);
485 
486  if (dbm->vfio_container_fd != -1)
487  {
488  struct vfio_iommu_type1_dma_map dm = { 0 };
489  int i, rv = 0;
490  dm.argsz = sizeof (struct vfio_iommu_type1_dma_map);
491  dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
492 
493  /* *INDENT-OFF* */
495  {
496  dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size);
497  dm.size = 1 << pr->log2_page_size;
498  dm.iova = pr->page_table[i];
499  if ((rv = ioctl (dbm->vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dm)))
500  break;
501  }
502  /* *INDENT-ON* */
503  if (rv != 0 && errno != EINVAL)
504  clib_unix_warning ("ioctl(VFIO_IOMMU_MAP_DMA) pool '%s'", pool_name);
505  }
506 
507  return 0;
508 }
509 
510 clib_error_t *
511 dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
512  unsigned socket_id)
513 {
514  dpdk_main_t *dm = &dpdk_main;
515  struct rte_mempool *rmp;
517  clib_error_t *error = 0;
518  u8 *pool_name;
519  u32 elt_size, i;
520 
522 
523  /* pool already exists, nothing to do */
524  if (dm->pktmbuf_pools[socket_id])
525  return 0;
526 
527  pool_name = format (0, "dpdk_mbuf_pool_socket%u%c", socket_id, 0);
528 
529  elt_size = sizeof (struct rte_mbuf) +
530  VLIB_BUFFER_HDR_SIZE /* priv size */ +
531  VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE; /*data room size */
532 
533  error =
534  dpdk_pool_create (vm, pool_name, elt_size, num_mbufs,
535  sizeof (dpdk_mempool_private_t), 512, socket_id,
536  &rmp, &pri);
537 
538  vec_free (pool_name);
539 
540  if (!error)
541  {
542  /* call the object initializers */
543  rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0);
544 
545  dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp);
546  privp->buffer_pool_index = vlib_buffer_pool_create (vm, pri, 0);
547 
548  dm->pktmbuf_pools[socket_id] = rmp;
549 
550  return 0;
551  }
552 
553  clib_error_report (error);
554 
555  /* no usable pool for this socket, try to use pool from another one */
556  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
557  {
558  if (dm->pktmbuf_pools[i])
559  {
560  clib_warning ("WARNING: Failed to allocate mempool for CPU socket "
561  "%u. Threads running on socket %u will use socket %u "
562  "mempool.", socket_id, socket_id, i);
563  dm->pktmbuf_pools[socket_id] = dm->pktmbuf_pools[i];
564  return 0;
565  }
566  }
567 
568  return clib_error_return (0, "failed to allocate mempool on socket %u",
569  socket_id);
570 }
571 
572 #if CLIB_DEBUG > 0
573 
577 
578 static clib_error_t *
580 {
581  void *oldheap;
582 
584  mheap_alloc_with_lock (0, 10 << 20, 0 /* locked */ );
586 
590  clib_mem_set_heap (oldheap);
591  return 0;
592 }
593 
595 #endif
596 
597 #if CLI_DEBUG
598 struct dpdk_validate_buf_result
599 {
600  u32 invalid;
601  u32 uninitialized;
602 };
603 
604 #define DPDK_TRAJECTORY_POISON 31
605 
606 static void
607 dpdk_buffer_validate_trajectory (struct rte_mempool *mp, void *opaque,
608  void *obj, unsigned obj_idx)
609 {
610  vlib_buffer_t *b;
611  struct dpdk_validate_buf_result *counter = opaque;
612  b = vlib_buffer_from_rte_mbuf ((struct rte_mbuf *) obj);
613  if (b->pre_data[0] != 0)
614  {
615  if (b->pre_data[0] == DPDK_TRAJECTORY_POISON)
616  counter->uninitialized++;
617  else
618  counter->invalid++;
619  }
620 }
621 
622 int
623 dpdk_buffer_validate_trajectory_all (u32 * uninitialized)
624 {
625  dpdk_main_t *dm = &dpdk_main;
626  struct dpdk_validate_buf_result counter = { 0 };
627  int i;
628 
629  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
630  rte_mempool_obj_iter (dm->pktmbuf_pools[i],
631  dpdk_buffer_validate_trajectory, &counter);
632  if (uninitialized)
633  *uninitialized = counter.uninitialized;
634  return counter.invalid;
635 }
636 
637 static void
638 dpdk_buffer_poison_trajectory (struct rte_mempool *mp, void *opaque,
639  void *obj, unsigned obj_idx)
640 {
641  vlib_buffer_t *b;
642  b = vlib_buffer_from_rte_mbuf ((struct rte_mbuf *) obj);
643  b->pre_data[0] = DPDK_TRAJECTORY_POISON;
644 }
645 
646 void
647 dpdk_buffer_poison_trajectory_all (void)
648 {
649  dpdk_main_t *dm = &dpdk_main;
650  int i;
651 
652  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
653  rte_mempool_obj_iter (dm->pktmbuf_pools[i], dpdk_buffer_poison_trajectory,
654  0);
655 }
656 #endif
657 
658 static clib_error_t *
660 {
663 
664  vec_validate_aligned (dbm->ptd, tm->n_vlib_mains - 1,
666 
667  dbm->vfio_container_fd = -1;
668 
669  return 0;
670 }
671 
673 
674 /* *INDENT-OFF* */
675 VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
676  .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list,
677  .vlib_buffer_free_cb = &dpdk_buffer_free,
678  .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
679  .vlib_packet_template_init_cb = &dpdk_packet_template_init,
680  .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list,
681 };
682 /* *INDENT-ON* */
683 
684 #if __x86_64__
691 
692 static void __clib_constructor
694 {
695  vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
696  if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ())
697  {
701  }
702  else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ())
703  {
707  }
708 }
709 #endif
710 #endif
711 
712 /** @endcond */
713 /*
714  * fd.io coding-style-patch-verification: ON
715  *
716  * Local Variables:
717  * eval: (c-set-style "gnu")
718  * End:
719  */
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define vlib_buffer_from_rte_mbuf(x)
Definition: dpdk_priv.h:17
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
dpdk_buffer_main_t dpdk_buffer_main
Definition: buffer.c:100
static clib_error_t * dpdk_buffer_init(vlib_main_t *vm)
Definition: buffer.c:659
static void dpdk_packet_template_init(vlib_main_t *vm, void *vt, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, u8 *name)
Definition: buffer.c:381
#define VLIB_PHYSMEM_F_HUGETLB
Definition: physmem.h:58
dpdk_main_t dpdk_main
Definition: init.c:42
static_always_inline void dpdk_prefetch_buffer(vlib_buffer_t *b)
Definition: buffer.c:271
u32 * vlib_buffer_state_validation_lock
Definition: buffer.c:574
#define NULL
Definition: clib.h:57
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512
Definition: buffer.c:687
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:143
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
int i
static void dpdk_buffer_delete_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer.c:152
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
void( vlib_buffer_free_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:385
vlib_main_t ** vlib_mains
Definition: buffer.c:303
unsigned char u8
Definition: types.h:56
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:600
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2
Definition: buffer.c:686
#define VLIB_PHYSMEM_F_SHARED
Definition: physmem.h:59
memset(h->entries, 0, sizeof(h->entries[0])*entries)
#define static_always_inline
Definition: clib.h:95
i64 word
Definition: types.h:111
uword * vlib_buffer_state_validation_hash
Definition: buffer.c:575
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
u8 vlib_buffer_pool_create(vlib_main_t *vm, vlib_physmem_region_index_t pri, u16 buffer_size)
Definition: buffer.c:869
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512
Definition: buffer.c:689
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:187
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:394
static void __clib_constructor dpdk_input_multiarch_select(void)
Definition: buffer.c:693
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:167
unsigned int u32
Definition: types.h:88
#define fl(x, y)
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define rte_mbuf_from_vlib_buffer(x)
Definition: dpdk_priv.h:16
uword size
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
Definition: buffer.c:511
static clib_error_t * buffer_state_validation_init(vlib_main_t *vm)
Definition: buffer.c:579
static_always_inline void dpdk_rte_pktmbuf_free(vlib_main_t *vm, u32 thread_index, vlib_buffer_t *b, int maybe_next)
Definition: buffer.c:103
unsigned short u16
Definition: types.h:57
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
static clib_error_t * scan_vfio_fd(void *arg, u8 *path_name, u8 *file_name)
Definition: buffer.c:398
uword CLIB_MULTIARCH_FN() dpdk_buffer_fill_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.c:178
#define PREDICT_FALSE(x)
Definition: clib.h:107
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:113
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
Definition: buffer_funcs.h:173
u8 name[64]
Definition: memclnt.api:151
u32 flags
Definition: vhost_user.h:115
clib_error_t * dpdk_pool_create(vlib_main_t *vm, u8 *pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, struct rte_mempool **_mp, vlib_physmem_region_index_t *pri)
Definition: buffer.c:423
struct rte_pktmbuf_pool_private mbp_priv
Definition: dpdk_priv.h:57
STATIC_ASSERT(VLIB_BUFFER_PRE_DATA_SIZE==RTE_PKTMBUF_HEADROOM,"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM")
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:392
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.c:599
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:211
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.h:437
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:79
vlib_main_t * vm
Definition: buffer.c:294
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512
Definition: buffer.c:685
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
static_always_inline void clib_memcpy64_x4(void *d0, void *d1, void *d2, void *d3, void *s)
Definition: string.h:89
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:261
#define clib_warning(format, args...)
Definition: error.h:59
void CLIB_MULTIARCH_FN() dpdk_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:372
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
Definition: buffer.c:132
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:237
void * vlib_buffer_state_heap
Definition: buffer.c:576
#define VLIB_BUFFER_DATA_SIZE
Definition: buffer.h:51
#define VLIB_BUFFER_HDR_SIZE
Definition: buffer.h:178
void CLIB_MULTIARCH_FN() dpdk_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:364
void( vlib_buffer_free_no_next_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:387
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2
Definition: buffer.c:688
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:393
int vfio_container_fd
Definition: buffer.c:96
void * mheap_alloc_with_lock(void *memory, uword size, int locked)
Definition: mem_dlmalloc.c:456
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:129
dpdk_buffer_per_thread_data * ptd
Definition: buffer.c:97
struct rte_mempool ** pktmbuf_pools
Definition: dpdk.h:439
#define clib_error_report(e)
Definition: error.h:113
VLIB_BUFFER_REGISTER_CALLBACKS(dpdk, static)
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:141
#define clib_max(x, y)
Definition: clib.h:284
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b, u32 follow_buffer_next)
Definition: buffer.c:571
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
Definition: util.c:49
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:547
u64 uword
Definition: types.h:112
#define clib_unix_warning(format, args...)
Definition: error.h:68
int container_fd
Definition: vfio.h:31
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1455
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
u8 vlib_buffer_free_list_index_t
Definition: buffer.h:62
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx2
Definition: buffer.c:690
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:675
uword( vlib_buffer_fill_free_list_cb_t)(struct vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.h:382
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:141
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:116
u8 vlib_physmem_region_index_t
Definition: physmem.h:43
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:58
linux_vfio_main_t vfio_main
Definition: vfio.c:35
struct rte_mbuf ** mbuf_alloc_list
Definition: buffer.c:91
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:59
#define vec_resize_aligned(V, N, A)
Resize a vector (no header, alignment specified).
Definition: vec.h:253