FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
buffer.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.c: allocate/free network buffers.
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 /**
41  * @file
42  *
43  * Allocate/free network buffers.
44  */
45 
46 #include <vlib/vlib.h>
47 #include <vlib/unix/unix.h>
48 
49 vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
50 static u32 vlib_buffer_physmem_sz = 32 << 20;
51 
53 
54 uword
56  vlib_buffer_t * b_first)
57 {
58  vlib_buffer_t *b = b_first;
59  uword l_first = b_first->current_length;
60  uword l = 0;
61  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
62  {
63  b = vlib_get_buffer (vm, b->next_buffer);
64  l += b->current_length;
65  }
67  b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
68  return l + l_first;
69 }
70 
71 u8 *
72 format_vlib_buffer (u8 * s, va_list * args)
73 {
74  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
75  u32 indent = format_get_indent (s);
76  u8 *a = 0;
77 
78 #define _(bit, name, v) \
79  if (v && (b->flags & VLIB_BUFFER_##name)) \
80  a = format (a, "%s ", v);
82 #undef _
83  s = format (s, "current data %d, length %d, free-list %d, clone-count %u",
86 
87  if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
88  s = format (s, ", totlen-nifb %d",
90 
91  if (b->flags & VLIB_BUFFER_IS_TRACED)
92  s = format (s, ", trace 0x%x", b->trace_index);
93 
94  if (a)
95  s = format (s, "\n%U%v", format_white_space, indent, a);
96  vec_free (a);
97 
98  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
99  {
101  u32 next_buffer = b->next_buffer;
102  b = vlib_get_buffer (vm, next_buffer);
103 
104  s =
105  format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u",
106  format_white_space, indent, next_buffer, b->current_length,
107  b->n_add_refs);
108  }
109 
110  return s;
111 }
112 
113 u8 *
114 format_vlib_buffer_and_data (u8 * s, va_list * args)
115 {
116  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
117 
118  s = format (s, "%U, %U",
121 
122  return s;
123 }
124 
125 static u8 *
126 format_vlib_buffer_known_state (u8 * s, va_list * args)
127 {
129  char *t;
130 
131  switch (state)
132  {
133  case VLIB_BUFFER_UNKNOWN:
134  t = "unknown";
135  break;
136 
138  t = "known-allocated";
139  break;
140 
142  t = "known-free";
143  break;
144 
145  default:
146  t = "invalid";
147  break;
148  }
149 
150  return format (s, "%s", t);
151 }
152 
153 u8 *
154 format_vlib_buffer_contents (u8 * s, va_list * va)
155 {
156  vlib_main_t *vm = va_arg (*va, vlib_main_t *);
157  vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
158 
159  while (1)
160  {
162  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
163  break;
164  b = vlib_get_buffer (vm, b->next_buffer);
165  }
166 
167  return s;
168 }
169 
170 static u8 *
172  u32 bi,
173  uword follow_buffer_next, uword ** unique_hash)
174 {
175  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
177 
180  return format (0, "unknown free list 0x%x",
182 
183  fl =
186 
187  if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
188  return format (0, "current data %d before pre-data", b->current_data);
189 
190  if (b->current_data + b->current_length > fl->n_data_bytes)
191  return format (0, "%d-%d beyond end of buffer %d",
193 
194  if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
195  {
197  u8 *msg, *result;
198 
201  return format (0, "next 0x%x: %U",
203 
204  if (unique_hash)
205  {
206  if (hash_get (*unique_hash, b->next_buffer))
207  return format (0, "duplicate buffer 0x%x", b->next_buffer);
208 
209  hash_set1 (*unique_hash, b->next_buffer);
210  }
211 
212  msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
213  if (msg)
214  {
215  result = format (0, "next 0x%x: %v", b->next_buffer, msg);
216  vec_free (msg);
217  return result;
218  }
219  }
220 
221  return 0;
222 }
223 
224 u8 *
225 vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
226 {
227  return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
228  /* unique_hash */ 0);
229 }
230 
231 u8 *
233  u32 * buffers,
234  uword next_buffer_stride,
235  uword n_buffers,
236  vlib_buffer_known_state_t known_state,
237  uword follow_buffer_next)
238 {
239  uword i, *hash;
240  u32 bi, *b = buffers;
242  u8 *msg = 0, *result = 0;
243 
244  hash = hash_create (0, 0);
245  for (i = 0; i < n_buffers; i++)
246  {
247  bi = b[0];
248  b += next_buffer_stride;
249 
250  /* Buffer is not unique. */
251  if (hash_get (hash, bi))
252  {
253  msg = format (0, "not unique");
254  goto done;
255  }
256 
257  k = vlib_buffer_is_known (bi);
258  if (k != known_state)
259  {
260  msg = format (0, "is %U; expected %U",
262  format_vlib_buffer_known_state, known_state);
263  goto done;
264  }
265 
266  msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
267  if (msg)
268  goto done;
269 
270  hash_set1 (hash, bi);
271  }
272 
273 done:
274  if (msg)
275  {
276  result = format (0, "0x%x: %v", bi, msg);
277  vec_free (msg);
278  }
279  hash_free (hash);
280  return result;
281 }
282 
283 /*
284  * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
285  * and vlib_mains[0] = &vlib_global_main from the beginning of time.
286  *
287  * The only place which should ever expand vlib_mains is start_workers()
288  * in threads.c. It knows about the bootstrap vector.
289  */
290 /* *INDENT-OFF* */
291 static struct
292 {
295 } __attribute__ ((packed)) __bootstrap_vlib_main_vector
296  __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
297 {
298  .h.len = 1,
299  .vm = &vlib_global_main,
300 };
301 /* *INDENT-ON* */
302 
303 vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
304 
305 
306 /* When dubugging validate that given buffers are either known allocated
307  or known free. */
308 void
310  u32 * buffers,
311  uword n_buffers,
312  vlib_buffer_known_state_t expected_state)
313 {
314  u32 *b;
315  uword i, bi, is_free;
316 
317  if (CLIB_DEBUG == 0)
318  return;
319 
320  if (vlib_buffer_callbacks)
321  return;
322 
323  is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
324  b = buffers;
325  for (i = 0; i < n_buffers; i++)
326  {
328 
329  bi = b[0];
330  b += 1;
331  known = vlib_buffer_is_known (bi);
332  if (known != expected_state)
333  {
334  ASSERT (0);
336  (vm, "%s %U buffer 0x%x",
337  is_free ? "freeing" : "allocating",
338  format_vlib_buffer_known_state, known, bi);
339  }
340 
343  }
344 }
345 
346 /* Add buffer free list. */
349  u32 n_data_bytes,
350  u32 is_public, u32 is_default, u8 * name)
351 {
354  int i;
355 
356  ASSERT (vlib_get_thread_index () == 0);
357 
358  if (!is_default && pool_elts (vm->buffer_free_list_pool) == 0)
359  {
360  vlib_buffer_free_list_index_t default_free_free_list_index;
361 
362  /* *INDENT-OFF* */
363  default_free_free_list_index =
365  (vm,
366  /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
367  /* is_public */ 1,
368  /* is_default */ 1,
369  (u8 *) "default");
370  /* *INDENT-ON* */
371  ASSERT (default_free_free_list_index ==
373 
374  if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
375  return default_free_free_list_index;
376  }
377 
379 
380  memset (f, 0, sizeof (f[0]));
381  f->index = f - vm->buffer_free_list_pool;
382  f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
384  f->buffer_pool_index = 0;
385  f->name = clib_mem_is_vec (name) ? name : format (0, "%s", name);
386 
387  /* Setup free buffer template. */
390 
391  if (is_public)
392  {
394  if (!p)
396  }
397 
398  for (i = 1; i < vec_len (vlib_mains); i++)
399  {
400  vlib_main_t *wvm = vlib_mains[i];
404  ASSERT (f - vm->buffer_free_list_pool ==
405  wf - wvm->buffer_free_list_pool);
406  wf[0] = f[0];
407  wf->buffers = 0;
408  wf->n_alloc = 0;
409  }
410 
411  return f->index;
412 }
413 
416  char *fmt, ...)
417 {
418  va_list va;
419  u8 *name;
420 
421  va_start (va, fmt);
422  name = va_format (0, fmt, &va);
423  va_end (va);
424 
425  return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
426  /* is_public */ 0,
427  /* is_default */ 0,
428  name);
429 }
430 
431 static void
433 {
435 
438  vec_free (f->name);
439  vec_free (f->buffers);
440 
441  /* Poison it. */
442  memset (f, 0xab, sizeof (f[0]));
443 }
444 
445 /* Add buffer free list. */
446 void
449 {
451  int i;
452 
453  ASSERT (vlib_get_thread_index () == 0);
454 
455  f = vlib_buffer_get_free_list (vm, index);
456 
457  ASSERT (vec_len (f->buffers) == f->n_alloc);
458 
459  del_free_list (vm, f);
460 
462 
463  for (i = 1; i < vec_len (vlib_mains); i++)
464  {
465  vlib_main_t *wvm = vlib_mains[i];
466  f = vlib_buffer_get_free_list (vlib_mains[i], index);
467  del_free_list (wvm, f);
469  }
470 }
471 
474 {
475  uword slot, page, addr;
476 
477  if (PREDICT_FALSE (bp->n_elts == bp->n_used))
478  {
479  clib_spinlock_unlock (&bp->lock);
480  return 0;
481  }
482  slot = bp->next_clear;
483  bp->bitmap = clib_bitmap_set (bp->bitmap, slot, 1);
484  bp->next_clear = clib_bitmap_next_clear (bp->bitmap, slot + 1);
485  bp->n_used++;
486 
487  page = slot / bp->buffers_per_page;
488  slot -= page * bp->buffers_per_page;
489 
490  addr = bp->start + (page << bp->log2_page_size) + slot * bp->buffer_size;
491 
492  return uword_to_pointer (addr, void *);
493 }
494 
495 /* Make sure free list has at least given number of free buffers. */
496 static uword
499  uword min_free_buffers)
500 {
501  vlib_buffer_t *b;
503  int n;
504  u32 *bi;
505  u32 n_alloc = 0;
506 
507  /* Already have enough free buffers on free list? */
508  n = min_free_buffers - vec_len (fl->buffers);
509  if (n <= 0)
510  return min_free_buffers;
511 
512  if (vec_len (bp->buffers) > 0)
513  {
514  int n_copy, n_left;
515  clib_spinlock_lock (&bp->lock);
516  n_copy = clib_min (vec_len (bp->buffers), n);
517  n_left = vec_len (bp->buffers) - n_copy;
518  vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy,
520  _vec_len (bp->buffers) = n_left;
521  clib_spinlock_unlock (&bp->lock);
522  n = min_free_buffers - vec_len (fl->buffers);
523  if (n <= 0)
524  return min_free_buffers;
525  }
526 
527  /* Always allocate round number of buffers. */
528  n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
529 
530  /* Always allocate new buffers in reasonably large sized chunks. */
531  n = clib_max (n, fl->min_n_buffers_each_alloc);
532 
533  clib_spinlock_lock (&bp->lock);
534  while (n_alloc < n)
535  {
536  if ((b = vlib_buffer_pool_get_buffer (bp)) == 0)
537  goto done;
538 
539  n_alloc += 1;
540 
542  bi[0] = vlib_get_buffer_index (vm, b);
543 
544  if (CLIB_DEBUG > 0)
546 
547  memset (b, 0, sizeof (vlib_buffer_t));
549 
550  if (fl->buffer_init_function)
551  fl->buffer_init_function (vm, fl, bi, 1);
552  }
553 
554 done:
555  clib_spinlock_unlock (&bp->lock);
556  fl->n_alloc += n_alloc;
557  return n_alloc;
558 }
559 
560 void *
562 {
564  void *rv = bm->buffer_free_callback;
565 
566  bm->buffer_free_callback = fp;
567  return rv;
568 }
569 
572  vlib_buffer_t * b, u32 follow_buffer_next)
573 {
576  u32 flags, next;
577 
578  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
579 
580  do
581  {
582  vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
583  flags = nb->flags;
584  next = nb->next_buffer;
585  if (nb->n_add_refs)
586  nb->n_add_refs--;
587  else
588  {
591  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
592  }
593  bi = next;
594  }
595  while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
596 }
597 
600  u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
601 {
603  vlib_buffer_t *p, *b0, *b1, *b2, *b3;
604  int i = 0;
605  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
606  u32 follow_buffer_next);
607 
608  cb = bm->buffer_free_callback;
609 
610  if (PREDICT_FALSE (cb != 0))
611  n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
612 
613  if (!n_buffers)
614  return;
615 
616  while (i + 11 < n_buffers)
617  {
618  p = vlib_get_buffer (vm, buffers[i + 8]);
619  vlib_prefetch_buffer_header (p, LOAD);
620  p = vlib_get_buffer (vm, buffers[i + 9]);
621  vlib_prefetch_buffer_header (p, LOAD);
622  p = vlib_get_buffer (vm, buffers[i + 10]);
623  vlib_prefetch_buffer_header (p, LOAD);
624  p = vlib_get_buffer (vm, buffers[i + 11]);
625  vlib_prefetch_buffer_header (p, LOAD);
626 
627  b0 = vlib_get_buffer (vm, buffers[i]);
628  b1 = vlib_get_buffer (vm, buffers[i + 1]);
629  b2 = vlib_get_buffer (vm, buffers[i + 2]);
630  b3 = vlib_get_buffer (vm, buffers[i + 3]);
631 
636 
637  recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
638  recycle_or_free (vm, bm, buffers[i + 1], b1, follow_buffer_next);
639  recycle_or_free (vm, bm, buffers[i + 2], b2, follow_buffer_next);
640  recycle_or_free (vm, bm, buffers[i + 3], b3, follow_buffer_next);
641 
642  i += 4;
643  }
644 
645  while (i < n_buffers)
646  {
647  b0 = vlib_get_buffer (vm, buffers[i]);
649  recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
650  i++;
651  }
652 }
653 
654 static void
655 vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
656 {
657  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
658  1);
659 }
660 
661 static void
663  u32 n_buffers)
664 {
665  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
666  0);
667 }
668 
669 /* Copy template packet data into buffers as they are allocated. */
670 static void __attribute__ ((unused))
673  u32 * buffers, u32 n_buffers)
674 {
676  uword_to_pointer (fl->buffer_init_function_opaque,
678  uword i;
679 
680  for (i = 0; i < n_buffers; i++)
681  {
682  vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
685  b->current_length);
686  }
687 }
688 
689 void
692  void *packet_data,
693  uword n_packet_data_bytes,
694  uword min_n_buffers_each_alloc, char *fmt, ...)
695 {
697  va_list va;
698  u8 *name;
700 
701  va_start (va, fmt);
702  name = va_format (0, fmt, &va);
703  va_end (va);
704 
706  bm->cb.vlib_packet_template_init_cb (vm, (void *) t, packet_data,
707  n_packet_data_bytes,
708  min_n_buffers_each_alloc, name);
709 
711 
712  memset (t, 0, sizeof (t[0]));
713 
714  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
715  t->min_n_buffers_each_alloc = min_n_buffers_each_alloc;
716 
718  (vm, n_packet_data_bytes,
719  /* is_public */ 1,
720  /* is_default */ 0,
721  name);
722 
723  ASSERT (t->free_list_index != 0);
726 
729 
731  fl->buffer_init_template.current_length = n_packet_data_bytes;
732  fl->buffer_init_template.flags = 0;
735 }
736 
737 void *
739  vlib_packet_template_t * t, u32 * bi_result)
740 {
741  u32 bi;
742  vlib_buffer_t *b;
743 
744  if (vlib_buffer_alloc (vm, &bi, 1) != 1)
745  return 0;
746 
747  *bi_result = bi;
748 
749  b = vlib_get_buffer (vm, bi);
751  t->packet_data, vec_len (t->packet_data));
753 
754  return b->data;
755 }
756 
757 void
760 {
762  word l = vec_len (t->packet_data);
763  word n_alloc;
764 
765  ASSERT (l > 0);
766  ASSERT (vec_len (t->free_buffers) == 0);
767 
768  vec_validate (t->free_buffers, n - 1);
770  n, t->free_list_index);
771  _vec_len (t->free_buffers) = n_alloc;
772 }
773 
774 /* Append given data to end of buffer, possibly allocating new buffers. */
775 u32
777  vlib_buffer_free_list_index_t free_list_index,
778  u32 buffer_index, void *data, u32 n_data_bytes)
779 {
780  u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
781  vlib_buffer_t *b;
782  void *d;
783 
784  bi = buffer_index;
785  if (bi == ~0
786  && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index))
787  goto out_of_buffers;
788 
789  d = data;
790  n_left = n_data_bytes;
791  n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
792 
793  b = vlib_get_buffer (vm, bi);
794  b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
795 
796  /* Get to the end of the chain before we try to append data... */
797  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
798  b = vlib_get_buffer (vm, b->next_buffer);
799 
800  while (1)
801  {
802  u32 n;
803 
804  ASSERT (n_buffer_bytes >= b->current_length);
805  n_left_this_buffer =
806  n_buffer_bytes - (b->current_data + b->current_length);
807  n = clib_min (n_left_this_buffer, n_left);
809  b->current_length += n;
810  n_left -= n;
811  if (n_left == 0)
812  break;
813 
814  d += n;
815  if (1 !=
817  free_list_index))
818  goto out_of_buffers;
819 
820  b->flags |= VLIB_BUFFER_NEXT_PRESENT;
821 
822  b = vlib_get_buffer (vm, b->next_buffer);
823  }
824 
825  return bi;
826 
827 out_of_buffers:
828  clib_error ("out of buffers");
829  return bi;
830 }
831 
832 u16
835  free_list_index,
837  vlib_buffer_t ** last, void *data,
838  u16 data_len)
839 {
840  vlib_buffer_t *l = *last;
841  u32 n_buffer_bytes =
842  vlib_buffer_free_list_buffer_size (vm, free_list_index);
843  u16 copied = 0;
844  ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
845  while (data_len)
846  {
847  u16 max = n_buffer_bytes - l->current_length - l->current_data;
848  if (max == 0)
849  {
850  if (1 !=
852  free_list_index))
853  return copied;
854  *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer);
855  max = n_buffer_bytes - l->current_length - l->current_data;
856  }
857 
858  u16 len = (data_len > max) ? max : data_len;
860  data + copied, len);
861  vlib_buffer_chain_increase_length (first, l, len);
862  data_len -= len;
863  copied += len;
864  }
865  return copied;
866 }
867 
868 u8
870  u16 buffer_size)
871 {
875  uword start = pointer_to_uword (pr->mem);
876  uword size = pr->size;
877 
878  if (bm->buffer_mem_size == 0)
879  {
880  bm->buffer_mem_start = start;
881  bm->buffer_mem_size = size;
882  }
883  else if (start < bm->buffer_mem_start)
884  {
885  bm->buffer_mem_size += bm->buffer_mem_start - start;
886  bm->buffer_mem_start = start;
887  if (size > bm->buffer_mem_size)
888  bm->buffer_mem_size = size;
889  }
890  else if (start > bm->buffer_mem_start)
891  {
892  uword new_size = start - bm->buffer_mem_start + size;
893  if (new_size > bm->buffer_mem_size)
894  bm->buffer_mem_size = new_size;
895  }
896 
897  if ((u64) bm->buffer_mem_size >
898  ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
899  {
900  clib_panic ("buffer memory size out of range!");
901  }
902 
903  vec_add2 (bm->buffer_pools, p, 1);
904  p->start = start;
905  p->size = size;
906  p->physmem_region = pri;
907 
908  if (buffer_size == 0)
909  goto done;
910 
912  p->buffer_size = buffer_size;
913  p->buffers_per_page = (1 << pr->log2_page_size) / p->buffer_size;
914  p->n_elts = p->buffers_per_page * pr->n_pages;
915  p->n_used = 0;
916  clib_spinlock_init (&p->lock);
917 done:
918  ASSERT (p - bm->buffer_pools < 256);
919  return p - bm->buffer_pools;
920 }
921 
922 static u8 *
923 format_vlib_buffer_free_list (u8 * s, va_list * va)
924 {
926  u32 threadnum = va_arg (*va, u32);
927  uword bytes_alloc, bytes_free, n_free, size;
928 
929  if (!f)
930  return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
931  "Thread", "Name", "Index", "Size", "Alloc", "Free",
932  "#Alloc", "#Free");
933 
934  size = sizeof (vlib_buffer_t) + f->n_data_bytes;
935  n_free = vec_len (f->buffers);
936  bytes_alloc = size * f->n_alloc;
937  bytes_free = size * n_free;
938 
939  s = format (s, "%7d%30v%12d%12d%=12U%=12U%=12d%=12d", threadnum,
940  f->name, f->index, f->n_data_bytes,
941  format_memory_size, bytes_alloc,
942  format_memory_size, bytes_free, f->n_alloc, n_free);
943 
944  return s;
945 }
946 
947 static clib_error_t *
949  unformat_input_t * input, vlib_cli_command_t * cmd)
950 {
952  vlib_main_t *curr_vm;
953  u32 vm_index = 0;
954 
956 
957  do
958  {
959  curr_vm = vlib_mains[vm_index];
960 
961  /* *INDENT-OFF* */
962  pool_foreach (f, curr_vm->buffer_free_list_pool, ({
963  vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index);
964  }));
965  /* *INDENT-ON* */
966 
967  vm_index++;
968  }
969  while (vm_index < vec_len (vlib_mains));
970 
971  return 0;
972 }
973 
974 /* *INDENT-OFF* */
975 VLIB_CLI_COMMAND (show_buffers_command, static) = {
976  .path = "show buffers",
977  .short_help = "Show packet buffer allocation",
978  .function = show_buffers,
979 };
980 /* *INDENT-ON* */
981 
982 clib_error_t *
984 {
987  clib_error_t *error;
988 
989  if (vlib_buffer_callbacks)
990  {
991  /* external plugin has registered own buffer callbacks
992  so we just copy them and quit */
993  clib_memcpy (&bm->cb, vlib_buffer_callbacks,
994  sizeof (vlib_buffer_callbacks_t));
995  bm->callbacks_registered = 1;
996  return 0;
997  }
998 
1005 
1006  /* allocate default region */
1007  error = vlib_physmem_region_alloc (vm, "buffers",
1010  VLIB_PHYSMEM_F_HUGETLB, &pri);
1011 
1012  if (error == 0)
1013  goto done;
1014 
1015  clib_error_free (error);
1016 
1017  error = vlib_physmem_region_alloc (vm, "buffers",
1019  VLIB_PHYSMEM_F_SHARED, &pri);
1020 done:
1021  if (error == 0)
1022  vlib_buffer_pool_create (vm, pri, sizeof (vlib_buffer_t) +
1024  return error;
1025 }
1026 
1027 static clib_error_t *
1029 {
1030  u32 size_in_mb;
1031 
1032  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1033  {
1034  if (unformat (input, "memory-size-in-mb %d", &size_in_mb))
1035  vlib_buffer_physmem_sz = size_in_mb << 20;
1036  else
1037  return unformat_parse_error (input);
1038  }
1039 
1040  unformat_free (input);
1041  return 0;
1042 }
1043 
1045 
1046 
1047 /** @endcond */
1048 /*
1049  * fd.io coding-style-patch-verification: ON
1050  *
1051  * Local Variables:
1052  * eval: (c-set-style "gnu")
1053  * End:
1054  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
vlib_physmem_region_index_t physmem_region
Definition: buffer.h:414
vlib_main_t vlib_global_main
Definition: main.c:1638
u32 vlib_buffer_add_data(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, u32 buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:776
#define hash_set(h, key, value)
Definition: hash.h:255
#define clib_min(x, y)
Definition: clib.h:291
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
a
Definition: bitmap.h:538
#define VLIB_PHYSMEM_F_HUGETLB
Definition: physmem.h:58
static clib_error_t * vlib_buffers_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: buffer.c:1028
static vlib_buffer_free_list_index_t vlib_buffer_create_free_list_helper(vlib_main_t *vm, u32 n_data_bytes, u32 is_public, u32 is_default, u8 *name)
Definition: buffer.c:348
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:833
#define clib_error(format, args...)
Definition: error.h:62
unsigned long u64
Definition: types.h:89
vlib_buffer_callbacks_t cb
Definition: buffer.h:455
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:947
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:574
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:964
vlib_buffer_t buffer_init_template
Definition: buffer.h:350
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
static u32 format_get_indent(u8 *s)
Definition: format.h:72
static void vlib_buffer_free_no_next_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:662
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
u8 * va_format(u8 *s, const char *fmt, va_list *va)
Definition: format.c:387
uword log2_page_size
Definition: buffer.h:413
static clib_error_t * show_buffers(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: buffer.c:948
vlib_buffer_free_list_index_t index
Definition: buffer.h:353
vhost_vring_addr_t addr
Definition: vhost_user.h:121
vlib_main_t ** vlib_mains
Definition: buffer.c:303
unsigned char u8
Definition: types.h:56
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:55
vlib_buffer_free_list_index_t free_list_index
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer.c:758
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:600
#define VLIB_PHYSMEM_F_SHARED
Definition: physmem.h:59
static u8 * format_vlib_buffer_free_list(u8 *s, va_list *va)
Definition: buffer.c:923
memset(h->entries, 0, sizeof(h->entries[0])*entries)
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:109
#define static_always_inline
Definition: clib.h:95
static u32 vlib_buffer_physmem_sz
Definition: buffer.c:50
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:738
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
i64 word
Definition: types.h:111
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.h:375
u8 vlib_buffer_pool_create(vlib_main_t *vm, vlib_physmem_region_index_t pri, u16 buffer_size)
Definition: buffer.c:869
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:187
uword buffer_mem_size
Definition: buffer.h:433
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:193
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:394
u8 * format_hex_bytes(u8 *s, va_list *va)
Definition: std-formats.c:84
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:441
vhost_vring_state_t state
Definition: vhost_user.h:120
unsigned int u32
Definition: types.h:88
u8 * format_vlib_buffer(u8 *s, va_list *args)
Definition: buffer.c:72
#define VLIB_FRAME_SIZE
Definition: node.h:382
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:154
#define fl(x, y)
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
int callbacks_registered
Definition: buffer.h:456
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define hash_get(h, key)
Definition: hash.h:249
uword next_clear
Definition: buffer.h:422
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
uword size
void * vlib_set_buffer_free_callback(vlib_main_t *vm, void *fp)
Definition: buffer.c:561
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
Definition: buffer.c:690
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer.h:401
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:205
#define hash_free(h)
Definition: hash.h:310
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:440
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:434
static u8 * vlib_validate_buffer_helper(vlib_main_t *vm, u32 bi, uword follow_buffer_next, uword **unique_hash)
Definition: buffer.c:171
#define PREDICT_FALSE(x)
Definition: clib.h:107
static_always_inline void * vlib_buffer_pool_get_buffer(vlib_buffer_pool_t *bp)
Definition: buffer.c:473
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:113
static void vlib_packet_template_buffer_init(vlib_main_t *vm, vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.c:671
u8 name[64]
Definition: memclnt.api:151
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
u32 flags
Definition: vhost_user.h:115
static void vlib_buffer_set_known_state(u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:392
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P (general version).
Definition: pool.h:188
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:610
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:452
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:216
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:392
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.c:599
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:211
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.h:437
vlib_main_t * vm
Definition: buffer.c:294
vec_header_t h
Definition: buffer.c:293
u8 * format_vlib_buffer_contents(u8 *s, va_list *va)
Definition: buffer.c:154
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 bi, uword follow_buffer_next)
Definition: buffer.c:225
#define clib_memcpy(a, b, c)
Definition: string.h:75
clib_spinlock_t lock
Definition: buffer.h:424
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:983
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:271
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:237
vlib_buffer_known_state_t
Definition: buffer_funcs.h:365
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:155
uword * bitmap
Definition: buffer.h:423
#define hash_set1(h, key)
Definition: hash.h:258
#define hash_create(elts, value_bytes)
Definition: hash.h:696
void vlib_buffer_delete_free_list_internal(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer.c:447
#define uword_to_pointer(u, type)
Definition: types.h:136
#define ASSERT(truth)
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:393
static vlib_buffer_free_list_index_t vlib_buffer_get_free_list_index(vlib_buffer_t *b)
Definition: buffer_funcs.h:414
static u8 * format_vlib_buffer_known_state(u8 *s, va_list *args)
Definition: buffer.c:126
void(* vlib_packet_template_init_cb)(struct vlib_main_t *vm, void *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, u8 *name)
Definition: buffer.h:395
uword buffers_per_page
Definition: buffer.h:419
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:129
u8 * format_vlib_buffer_and_data(u8 *s, va_list *args)
Definition: buffer.c:114
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, vlib_buffer_free_list_index_t *index)
Definition: buffer_funcs.h:665
vector header structure
Definition: vec_bootstrap.h:55
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:141
#define clib_max(x, y)
Definition: clib.h:284
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, vlib_buffer_free_list_index_t index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:444
#define unformat_parse_error(input)
Definition: format.h:268
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:155
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b, u32 follow_buffer_next)
Definition: buffer.c:571
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:547
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
uword buffer_mem_start
Definition: buffer.h:432
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
Definition: buffer.c:232
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:689
#define clib_error_free(e)
Definition: error.h:86
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1455
static vlib_buffer_known_state_t vlib_buffer_is_known(u32 buffer_index)
Definition: buffer_funcs.h:381
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
static uword clib_mem_is_vec(void *v)
Predicate function, says whether the supplied vector is a clib heap object.
Definition: vec.h:204
u8 data[0]
Packet data.
Definition: buffer.h:175
static void vlib_buffer_set_free_list_index(vlib_buffer_t *b, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:423
static_always_inline vlib_buffer_pool_t * vlib_buffer_pool_get(u8 buffer_pool_index)
Definition: buffer.h:462
u8 vlib_buffer_free_list_index_t
Definition: buffer.h:62
uword buffer_init_function_opaque
Definition: buffer.h:379
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:675
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:116
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:503
u8 vlib_physmem_region_index_t
Definition: physmem.h:43
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
Definition: buffer.c:432
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:725
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:58
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:150
uword * free_list_by_size
Definition: buffer.h:445
#define clib_panic(format, args...)
Definition: error.h:72
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:309
#define vlib_panic_with_msg(vm, args...)
Definition: main.h:278
vlib_buffer_free_list_index_t vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:415
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
static uword vlib_buffer_fill_free_list_internal(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.c:497
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
static void vlib_buffer_free_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:655
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:408
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128