FD.io VPP  v18.07.1-19-g511ce25
Vector Packet Processing
buffer_funcs.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_funcs.h: VLIB buffer related functions/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
42 
43 #include <vppinfra/hash.h>
44 
45 /** \file
46  vlib buffer access methods.
47 */
48 
49 
50 /** \brief Translate buffer index into buffer pointer
51 
52  @param vm - (vlib_main_t *) vlib main data structure pointer
53  @param buffer_index - (u32) buffer index
54  @return - (vlib_buffer_t *) buffer pointer
55 */
57 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
58 {
60  uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
61  ASSERT (offset < bm->buffer_mem_size);
62 
63  return uword_to_pointer (bm->buffer_mem_start + offset, void *);
64 }
65 
66 /** \brief Translate array of buffer indices into buffer pointers with offset
67 
68  @param vm - (vlib_main_t *) vlib main data structure pointer
69  @param bi - (u32 *) array of buffer indices
70  @param b - (void **) array to store buffer pointers
71  @param count - (uword) number of elements
72  @param offset - (i32) offset applied to each pointer
73 */
76  i32 offset)
77 {
78 #ifdef CLIB_HAVE_VEC256
79  u64x4 off = u64x4_splat (buffer_main.buffer_mem_start + offset);
80  /* if count is not const, compiler will not unroll while loop
81  se we maintain two-in-parallel variant */
82  while (count >= 8)
83  {
84  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
85  u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
86  /* shift and add to get vlib_buffer_t pointer */
87  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
88  u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
89  b += 8;
90  bi += 8;
91  count -= 8;
92  }
93 #endif
94  while (count >= 4)
95  {
96 #ifdef CLIB_HAVE_VEC256
97  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
98  /* shift and add to get vlib_buffer_t pointer */
99  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
100 #else
101  b[0] = ((u8 *) vlib_get_buffer (vm, bi[0])) + offset;
102  b[1] = ((u8 *) vlib_get_buffer (vm, bi[1])) + offset;
103  b[2] = ((u8 *) vlib_get_buffer (vm, bi[2])) + offset;
104  b[3] = ((u8 *) vlib_get_buffer (vm, bi[3])) + offset;
105 #endif
106  b += 4;
107  bi += 4;
108  count -= 4;
109  }
110  while (count)
111  {
112  b[0] = ((u8 *) vlib_get_buffer (vm, bi[0])) + offset;
113  b += 1;
114  bi += 1;
115  count -= 1;
116  }
117 }
118 
119 /** \brief Translate array of buffer indices into buffer pointers
120 
121  @param vm - (vlib_main_t *) vlib main data structure pointer
122  @param bi - (u32 *) array of buffer indices
123  @param b - (vlib_buffer_t **) array to store buffer pointers
124  @param count - (uword) number of elements
125 */
126 
129 {
130  vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
131 }
132 
133 /** \brief Translate buffer pointer into buffer index
134 
135  @param vm - (vlib_main_t *) vlib main data structure pointer
136  @param p - (void *) buffer pointer
137  @return - (u32) buffer index
138 */
139 
142 {
146  ASSERT (offset < bm->buffer_mem_size);
147  ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
148  return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
149 }
150 
151 /** \brief Translate array of buffer pointers into buffer indices with offset
152 
153  @param vm - (vlib_main_t *) vlib main data structure pointer
154  @param b - (void **) array of buffer pointers
155  @param bi - (u32 *) array to store buffer indices
156  @param count - (uword) number of elements
157  @param offset - (i32) offset applied to each pointer
158 */
162 {
163 #ifdef CLIB_HAVE_VEC256
164  u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
165  u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - offset);
166 
167  while (count >= 8)
168  {
169  /* load 4 pointers into 256-bit register */
170  u64x4 v0 = u64x4_load_unaligned (b);
171  u64x4 v1 = u64x4_load_unaligned (b + 4);
172  u32x8 v2, v3;
173 
174  v0 -= off4;
175  v1 -= off4;
176 
179 
180  /* permute 256-bit register so lower u32s of each buffer index are
181  * placed into lower 128-bits */
182  v2 = u32x8_permute ((u32x8) v0, mask);
183  v3 = u32x8_permute ((u32x8) v1, mask);
184 
185  /* extract lower 128-bits and save them to the array of buffer indices */
186  u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
187  u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
188  bi += 8;
189  b += 8;
190  count -= 8;
191  }
192 #endif
193  while (count >= 4)
194  {
195  /* equivalent non-nector implementation */
196  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
197  bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
198  bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
199  bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
200  bi += 4;
201  b += 4;
202  count -= 4;
203  }
204  while (count)
205  {
206  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
207  bi += 1;
208  b += 1;
209  count -= 1;
210  }
211 }
212 
213 /** \brief Translate array of buffer pointers into buffer indices
214 
215  @param vm - (vlib_main_t *) vlib main data structure pointer
216  @param b - (vlib_buffer_t **) array of buffer pointers
217  @param bi - (u32 *) array to store buffer indices
218  @param count - (uword) number of elements
219 */
222  uword count)
223 {
224  vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
225 }
226 
227 /** \brief Get next buffer in buffer linklist, or zero for end of list.
228 
229  @param vm - (vlib_main_t *) vlib main data structure pointer
230  @param b - (void *) buffer pointer
231  @return - (vlib_buffer_t *) next buffer, or NULL
232 */
235 {
236  return (b->flags & VLIB_BUFFER_NEXT_PRESENT
237  ? vlib_get_buffer (vm, b->next_buffer) : 0);
238 }
239 
241  vlib_buffer_t * b_first);
242 
243 /** \brief Get length in bytes of the buffer chain
244 
245  @param vm - (vlib_main_t *) vlib main data structure pointer
246  @param b - (void *) buffer pointer
247  @return - (uword) length of buffer chain
248 */
251 {
252  uword len = b->current_length;
253 
254  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
255  return len;
256 
257  if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
259 
261 }
262 
263 /** \brief Get length in bytes of the buffer index buffer chain
264 
265  @param vm - (vlib_main_t *) vlib main data structure pointer
266  @param bi - (u32) buffer index
267  @return - (uword) length of buffer chain
268 */
271 {
272  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
273  return vlib_buffer_length_in_chain (vm, b);
274 }
275 
276 /** \brief Copy buffer contents to memory
277 
278  @param vm - (vlib_main_t *) vlib main data structure pointer
279  @param buffer_index - (u32) buffer index
280  @param contents - (u8 *) memory, <strong>must be large enough</strong>
281  @return - (uword) length of buffer chain
282 */
284 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
285 {
286  uword content_len = 0;
287  uword l;
288  vlib_buffer_t *b;
289 
290  while (1)
291  {
292  b = vlib_get_buffer (vm, buffer_index);
293  l = b->current_length;
294  clib_memcpy (contents + content_len, b->data + b->current_data, l);
295  content_len += l;
296  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
297  break;
298  buffer_index = b->next_buffer;
299  }
300 
301  return content_len;
302 }
303 
304 /* Return physical address of buffer->data start. */
307 {
309  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index);
311  b->buffer_pool_index);
312 
314 }
315 
316 /** \brief Prefetch buffer metadata by buffer index
317  The first 64 bytes of buffer contains most header information
318 
319  @param vm - (vlib_main_t *) vlib main data structure pointer
320  @param bi - (u32) buffer index
321  @param type - LOAD, STORE. In most cases, STORE is the right answer
322 */
323 /* Prefetch buffer header given index. */
324 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
325  do { \
326  vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
327  vlib_prefetch_buffer_header (_b, type); \
328  } while (0)
329 
330 #if 0
331 /* Iterate over known allocated vlib bufs. You probably do not want
332  * to do this!
333  @param vm the vlib_main_t
334  @param bi found allocated buffer index
335  @param body operation to perform on buffer index
336  function executes body for each allocated buffer index
337  */
338 #define vlib_buffer_foreach_allocated(vm,bi,body) \
339 do { \
340  vlib_main_t * _vmain = (vm); \
341  vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
342  hash_pair_t * _vbpair; \
343  hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
344  if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
345  (bi) = _vbpair->key; \
346  body; \
347  } \
348  })); \
349 } while (0)
350 #endif
351 
352 typedef enum
353 {
354  /* Index is unknown. */
356 
357  /* Index is known and free/allocated. */
361 
362 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
363  uword n_buffers,
365  expected_state);
366 
368 vlib_buffer_is_known (u32 buffer_index)
369 {
371 
373  uword *p = hash_get (bm->buffer_known_hash, buffer_index);
375  return p ? p[0] : VLIB_BUFFER_UNKNOWN;
376 }
377 
378 always_inline void
381 {
383 
385  hash_set (bm->buffer_known_hash, buffer_index, state);
387 }
388 
389 /* Validates sanity of a single buffer.
390  Returns format'ed vector with error message if any. */
391 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
392  uword follow_chain);
393 
396 {
397  return round_pow2 (size, sizeof (vlib_buffer_t));
398 }
399 
402 {
403  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NON_DEFAULT_FREELIST))
404  return b->free_list_index;
405 
406  return 0;
407 }
408 
409 always_inline void
412 {
413  if (PREDICT_FALSE (index))
414  {
415  b->flags |= VLIB_BUFFER_NON_DEFAULT_FREELIST;
416  b->free_list_index = index;
417  }
418  else
419  b->flags &= ~VLIB_BUFFER_NON_DEFAULT_FREELIST;
420 }
421 
422 /** \brief Allocate buffers from specific freelist into supplied array
423 
424  @param vm - (vlib_main_t *) vlib main data structure pointer
425  @param buffers - (u32 * ) buffer index array
426  @param n_buffers - (u32) number of buffers requested
427  @return - (u32) number of buffers actually allocated, may be
428  less than the number requested or zero
429 */
432  u32 * buffers,
433  u32 n_buffers,
435 {
438  u32 *src;
439  uword len;
440 
442 
443  fl = pool_elt_at_index (vm->buffer_free_list_pool, index);
444 
445  len = vec_len (fl->buffers);
446 
447  if (PREDICT_FALSE (len < n_buffers))
448  {
449  bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
450  if (PREDICT_FALSE ((len = vec_len (fl->buffers)) == 0))
451  return 0;
452 
453  /* even if fill free list didn't manage to refill free list
454  we should give what we have */
455  n_buffers = clib_min (len, n_buffers);
456 
457  /* following code is intentionaly duplicated to allow compiler
458  to optimize fast path when n_buffers is constant value */
459  src = fl->buffers + len - n_buffers;
460  clib_memcpy (buffers, src, n_buffers * sizeof (u32));
461  _vec_len (fl->buffers) -= n_buffers;
462 
463  /* Verify that buffers are known free. */
464  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
466 
467  return n_buffers;
468  }
469 
470  src = fl->buffers + len - n_buffers;
471  clib_memcpy (buffers, src, n_buffers * sizeof (u32));
472  _vec_len (fl->buffers) -= n_buffers;
473 
474  /* Verify that buffers are known free. */
475  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
477 
478  return n_buffers;
479 }
480 
481 /** \brief Allocate buffers into supplied array
482 
483  @param vm - (vlib_main_t *) vlib main data structure pointer
484  @param buffers - (u32 * ) buffer index array
485  @param n_buffers - (u32) number of buffers requested
486  @return - (u32) number of buffers actually allocated, may be
487  less than the number requested or zero
488 */
490 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
491 {
492  return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
494 }
495 
496 /** \brief Allocate buffers into ring
497 
498  @param vm - (vlib_main_t *) vlib main data structure pointer
499  @param buffers - (u32 * ) buffer index ring
500  @param start - (u32) first slot in the ring
501  @param ring_size - (u32) ring size
502  @param n_buffers - (u32) number of buffers requested
503  @return - (u32) number of buffers actually allocated, may be
504  less than the number requested or zero
505 */
508  u32 ring_size, u32 n_buffers)
509 {
510  u32 n_alloc;
511 
512  ASSERT (n_buffers <= ring_size);
513 
514  if (PREDICT_TRUE (start + n_buffers <= ring_size))
515  return vlib_buffer_alloc (vm, ring + start, n_buffers);
516 
517  n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
518 
519  if (PREDICT_TRUE (n_alloc == ring_size - start))
520  n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
521 
522  return n_alloc;
523 }
524 
525 /** \brief Free buffers
526  Frees the entire buffer chain for each buffer
527 
528  @param vm - (vlib_main_t *) vlib main data structure pointer
529  @param buffers - (u32 * ) buffer index array
530  @param n_buffers - (u32) number of buffers to free
531 
532 */
533 always_inline void
535  /* pointer to first buffer */
536  u32 * buffers,
537  /* number of buffers to free */
538  u32 n_buffers)
539 {
541 
543 
544  return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers);
545 }
546 
547 /** \brief Free buffers, does not free the buffer chain for each buffer
548 
549  @param vm - (vlib_main_t *) vlib main data structure pointer
550  @param buffers - (u32 * ) buffer index array
551  @param n_buffers - (u32) number of buffers to free
552 
553 */
554 always_inline void
556  /* pointer to first buffer */
557  u32 * buffers,
558  /* number of buffers to free */
559  u32 n_buffers)
560 {
562 
564 
565  return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers);
566 }
567 
568 /** \brief Free one buffer
569  Shorthand to free a single buffer chain.
570 
571  @param vm - (vlib_main_t *) vlib main data structure pointer
572  @param buffer_index - (u32) buffer index to free
573 */
574 always_inline void
576 {
577  vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1);
578 }
579 
580 /** \brief Free buffers from ring
581 
582  @param vm - (vlib_main_t *) vlib main data structure pointer
583  @param buffers - (u32 * ) buffer index ring
584  @param start - (u32) first slot in the ring
585  @param ring_size - (u32) ring size
586  @param n_buffers - (u32) number of buffers
587 */
588 always_inline void
590  u32 ring_size, u32 n_buffers)
591 {
592  ASSERT (n_buffers <= ring_size);
593 
594  if (PREDICT_TRUE (start + n_buffers <= ring_size))
595  {
596  vlib_buffer_free (vm, ring + start, n_buffers);
597  }
598  else
599  {
600  vlib_buffer_free (vm, ring + start, ring_size - start);
601  vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
602  }
603 }
604 
605 /** \brief Free buffers from ring without freeing tail buffers
606 
607  @param vm - (vlib_main_t *) vlib main data structure pointer
608  @param buffers - (u32 * ) buffer index ring
609  @param start - (u32) first slot in the ring
610  @param ring_size - (u32) ring size
611  @param n_buffers - (u32) number of buffers
612 */
613 always_inline void
615  u32 ring_size, u32 n_buffers)
616 {
617  ASSERT (n_buffers <= ring_size);
618 
619  if (PREDICT_TRUE (start + n_buffers <= ring_size))
620  {
621  vlib_buffer_free_no_next (vm, ring + start, n_buffers);
622  }
623  else
624  {
625  vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
626  vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
627  }
628 }
629 
630 /* Add/delete buffer free lists. */
632  u32 n_data_bytes,
633  char *fmt, ...);
634 always_inline void
636  vlib_buffer_free_list_index_t free_list_index)
637 {
639 
641 
642  bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index);
643 }
644 
645 /* Make sure we have at least given number of unaligned buffers. */
648  free_list,
649  uword n_unaligned_buffers);
650 
654 {
656 
657  *index = i = vlib_buffer_get_free_list_index (b);
659 }
660 
663  vlib_buffer_free_list_index_t free_list_index)
664 {
666 
667  f = pool_elt_at_index (vm->buffer_free_list_pool, free_list_index);
668 
669  /* Sanity: indices must match. */
670  ASSERT (f->index == free_list_index);
671 
672  return f;
673 }
674 
678 {
680  return f->n_data_bytes;
681 }
682 
683 void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes);
684 
685 /* Reasonably fast buffer copy routine. */
686 always_inline void
687 vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
688 {
689  while (n >= 4)
690  {
691  dst[0] = src[0];
692  dst[1] = src[1];
693  dst[2] = src[2];
694  dst[3] = src[3];
695  dst += 4;
696  src += 4;
697  n -= 4;
698  }
699  while (n > 0)
700  {
701  dst[0] = src[0];
702  dst += 1;
703  src += 1;
704  n -= 1;
705  }
706 }
707 
708 /* Append given data to end of buffer, possibly allocating new buffers. */
710  vlib_buffer_free_list_index_t free_list_index,
711  u32 buffer_index, void *data, u32 n_data_bytes);
712 
713 /* duplicate all buffers in chain */
716 {
717  vlib_buffer_t *s, *d, *fd;
718  uword n_alloc, n_buffers = 1;
719  u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
720  int i;
721 
722  s = b;
723  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
724  {
725  n_buffers++;
726  s = vlib_get_buffer (vm, s->next_buffer);
727  }
728  u32 new_buffers[n_buffers];
729 
730  n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
731 
732  /* No guarantee that we'll get all the buffers we asked for */
733  if (PREDICT_FALSE (n_alloc < n_buffers))
734  {
735  if (n_alloc > 0)
736  vlib_buffer_free (vm, new_buffers, n_alloc);
737  return 0;
738  }
739 
740  /* 1st segment */
741  s = b;
742  fd = d = vlib_get_buffer (vm, new_buffers[0]);
743  d->current_data = s->current_data;
745  d->flags = s->flags & flag_mask;
748  clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
751 
752  /* next segments */
753  for (i = 1; i < n_buffers; i++)
754  {
755  /* previous */
756  d->next_buffer = new_buffers[i];
757  /* current */
758  s = vlib_get_buffer (vm, s->next_buffer);
759  d = vlib_get_buffer (vm, new_buffers[i]);
760  d->current_data = s->current_data;
764  d->flags = s->flags & flag_mask;
765  }
766 
767  return fd;
768 }
769 
770 /** \brief Create a maximum of 256 clones of buffer and store them
771  in the supplied array
772 
773  @param vm - (vlib_main_t *) vlib main data structure pointer
774  @param src_buffer - (u32) source buffer index
775  @param buffers - (u32 * ) buffer index array
776  @param n_buffers - (u16) number of buffer clones requested (<=256)
777  @param head_end_offset - (u16) offset relative to current position
778  where packet head ends
779  @return - (u16) number of buffers actually cloned, may be
780  less than the number requested or zero
781 */
783 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
784  u16 n_buffers, u16 head_end_offset)
785 {
786  u16 i;
787  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
788 
789  ASSERT (s->n_add_refs == 0);
790  ASSERT (n_buffers);
791  ASSERT (n_buffers <= 256);
792 
793  if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
794  {
795  buffers[0] = src_buffer;
796  for (i = 1; i < n_buffers; i++)
797  {
798  vlib_buffer_t *d;
799  d = vlib_buffer_copy (vm, s);
800  if (d == 0)
801  return i;
802  buffers[i] = vlib_get_buffer_index (vm, d);
803 
804  }
805  return n_buffers;
806  }
807 
808  if (PREDICT_FALSE (n_buffers == 1))
809  {
810  buffers[0] = src_buffer;
811  return 1;
812  }
813 
814  n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
816  (s));
817 
818  for (i = 0; i < n_buffers; i++)
819  {
820  vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
821  d->current_data = s->current_data;
822  d->current_length = head_end_offset;
825 
827  head_end_offset;
828  if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
829  {
832  }
833  d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
834  d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
835  clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
837  head_end_offset);
838  d->next_buffer = src_buffer;
839  }
840  vlib_buffer_advance (s, head_end_offset);
841  s->n_add_refs = n_buffers - 1;
842  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
843  {
844  s = vlib_get_buffer (vm, s->next_buffer);
845  s->n_add_refs = n_buffers - 1;
846  }
847 
848  return n_buffers;
849 }
850 
851 /** \brief Create multiple clones of buffer and store them
852  in the supplied array
853 
854  @param vm - (vlib_main_t *) vlib main data structure pointer
855  @param src_buffer - (u32) source buffer index
856  @param buffers - (u32 * ) buffer index array
857  @param n_buffers - (u16) number of buffer clones requested (<=256)
858  @param head_end_offset - (u16) offset relative to current position
859  where packet head ends
860  @return - (u16) number of buffers actually cloned, may be
861  less than the number requested or zero
862 */
864 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
865  u16 n_buffers, u16 head_end_offset)
866 {
867  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
868  u16 n_cloned = 0;
869 
870  while (n_buffers > 256)
871  {
872  vlib_buffer_t *copy;
873  copy = vlib_buffer_copy (vm, s);
874  n_cloned += vlib_buffer_clone_256 (vm,
875  vlib_get_buffer_index (vm, copy),
876  (buffers + n_cloned),
877  256, head_end_offset);
878  n_buffers -= 256;
879  }
880  n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
881  buffers + n_cloned,
882  n_buffers, head_end_offset);
883 
884  return n_cloned;
885 }
886 
887 /** \brief Attach cloned tail to the buffer
888 
889  @param vm - (vlib_main_t *) vlib main data structure pointer
890  @param head - (vlib_buffer_t *) head buffer
891  @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
892 */
893 
894 always_inline void
896  vlib_buffer_t * tail)
897 {
898  ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
901 
902  head->flags |= VLIB_BUFFER_NEXT_PRESENT;
903  head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
904  head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
905  head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
906  head->next_buffer = vlib_get_buffer_index (vm, tail);
909 
910 next_segment:
911  __sync_add_and_fetch (&tail->n_add_refs, 1);
912 
913  if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
914  {
915  tail = vlib_get_buffer (vm, tail->next_buffer);
916  goto next_segment;
917  }
918 }
919 
920 /* Initializes the buffer as an empty packet with no chained buffers. */
921 always_inline void
923 {
925  first->current_length = 0;
926  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
927  first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
928 }
929 
930 /* The provided next_bi buffer index is appended to the end of the packet. */
933  vlib_buffer_t * first,
934  vlib_buffer_t * last, u32 next_bi)
935 {
936  vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
937  last->next_buffer = next_bi;
938  last->flags |= VLIB_BUFFER_NEXT_PRESENT;
939  next_buffer->current_length = 0;
940  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
941  return next_buffer;
942 }
943 
944 /* Increases or decreases the packet length.
945  * It does not allocate or deallocate new buffers.
946  * Therefore, the added length must be compatible
947  * with the last buffer. */
948 always_inline void
950  vlib_buffer_t * last, i32 len)
951 {
952  last->current_length += len;
953  if (first != last)
955 }
956 
957 /* Copy data to the end of the packet and increases its length.
958  * It does not allocate new buffers.
959  * Returns the number of copied bytes. */
962  vlib_buffer_free_list_index_t free_list_index,
963  vlib_buffer_t * first,
964  vlib_buffer_t * last, void *data, u16 data_len)
965 {
966  u32 n_buffer_bytes =
967  vlib_buffer_free_list_buffer_size (vm, free_list_index);
968  ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
969  u16 len = clib_min (data_len,
970  n_buffer_bytes - last->current_length -
971  last->current_data);
972  clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
973  len);
974  vlib_buffer_chain_increase_length (first, last, len);
975  return len;
976 }
977 
978 /* Copy data to the end of the packet and increases its length.
979  * Allocates additional buffers from the free list if necessary.
980  * Returns the number of copied bytes.
981  * 'last' value is modified whenever new buffers are allocated and
982  * chained and points to the last buffer in the chain. */
983 u16
986  free_list_index,
987  vlib_buffer_t * first,
988  vlib_buffer_t ** last, void *data,
989  u16 data_len);
991 
994 
995 typedef struct
996 {
997  /* Vector of packet data. */
999 
1000  /* Number of buffers to allocate in each call to allocator. */
1002 
1003  /* Buffer free list for this template. */
1005 
1008 
1011 
1014  void *packet_data,
1015  uword n_packet_data_bytes,
1016  uword min_n_buffers_each_alloc,
1017  char *fmt, ...);
1018 
1021  u32 * bi_result);
1022 
1023 always_inline void
1025 {
1026  vec_free (t->packet_data);
1027 }
1028 
1031 {
1032  serialize_stream_t *s = &m->stream;
1036  vlib_main_t *vm = sm->vlib_main;
1037  u32 n, *f;
1038 
1040  if (sm->last_buffer != ~0)
1041  {
1042  vlib_buffer_t *b = vlib_get_buffer (vm, sm->last_buffer);
1043  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1044  {
1045  b = vlib_get_buffer (vm, b->next_buffer);
1046  n += b->current_length;
1047  }
1048  }
1049 
1050  /* *INDENT-OFF* */
1051  clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
1052  n += vlib_buffer_index_length_in_chain (vm, f[0]);
1053  }));
1054 /* *INDENT-ON* */
1055 
1056  return n;
1057 }
1058 
1059 /* Set a buffer quickly into "uninitialized" state. We want this to
1060  be extremely cheap and arrange for all fields that need to be
1061  initialized to be in the first 128 bits of the buffer. */
1062 always_inline void
1065 {
1066  vlib_buffer_t *src = &fl->buffer_init_template;
1067 
1068  /* Make sure vlib_buffer_t is cacheline aligned and sized */
1069  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
1070  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) ==
1072  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) ==
1073  CLIB_CACHE_LINE_BYTES * 2);
1074 
1075  /* Make sure buffer template is sane. */
1077 
1078  clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
1079  STRUCT_MARK_PTR (src, template_start),
1080  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
1081  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
1082 
1083  /* Not in the first 16 octets. */
1084  dst->n_add_refs = src->n_add_refs;
1086 
1087  /* Make sure it really worked. */
1088 #define _(f) ASSERT (dst->f == src->f);
1089  _(current_data);
1090  _(current_length);
1091  _(flags);
1092 #undef _
1093  /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
1094  /* total_length_not_including_first_buffer is not in the template anymore
1095  * so it may actually not zeroed for some buffers. One option is to
1096  * uncomment the line lower (comes at a cost), the other, is to just not
1097  * care */
1098  /* dst->total_length_not_including_first_buffer = 0; */
1099  ASSERT (dst->n_add_refs == 0);
1100 }
1101 
1102 always_inline void
1105  u32 buffer_index, u8 do_init)
1106 {
1108  vlib_buffer_t *b;
1109  b = vlib_get_buffer (vm, buffer_index);
1110  if (PREDICT_TRUE (do_init))
1112  vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
1113 
1114  if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE)
1115  {
1116  clib_spinlock_lock (&bp->lock);
1117  /* keep last stored buffers, as they are more likely hot in the cache */
1121  f->n_alloc -= VLIB_FRAME_SIZE;
1122  clib_spinlock_unlock (&bp->lock);
1123  }
1124 }
1125 
1126 #if CLIB_DEBUG > 0
1129 extern void *vlib_buffer_state_heap;
1130 #endif
1131 
1132 static inline void
1134 {
1135 #if CLIB_DEBUG > 0
1136  uword *p;
1137  void *oldheap;
1138 
1140 
1141  while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
1142  ;
1143 
1145 
1146  /* If we don't know about b, declare it to be in the expected state */
1147  if (!p)
1148  {
1150  goto out;
1151  }
1152 
1153  if (p[0] != expected)
1154  {
1155  void cj_stop (void);
1156  u32 bi;
1158 
1159  cj_stop ();
1160 
1161  bi = vlib_get_buffer_index (vm, b);
1162 
1163  clib_mem_set_heap (oldheap);
1164  clib_warning ("%.6f buffer %llx (%d): %s, not %s",
1165  vlib_time_now (vm), bi,
1166  p[0] ? "busy" : "free", expected ? "busy" : "free");
1167  os_panic ();
1168  }
1169 out:
1172  clib_mem_set_heap (oldheap);
1173 #endif
1174 }
1175 
1176 static inline void
1178 {
1179 #if CLIB_DEBUG > 0
1180  void *oldheap;
1181 
1183 
1184  while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
1185  ;
1186 
1188 
1191  clib_mem_set_heap (oldheap);
1192 #endif
1193 }
1194 
1195 /** minimum data size of first buffer in a buffer chain */
1196 #define VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE (256)
1197 
1198 /**
1199  * @brief compress buffer chain in a way where the first buffer is at least
1200  * VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE long
1201  *
1202  * @param[in] vm - vlib_main
1203  * @param[in,out] first - first buffer in chain
1204  * @param[in,out] discard_vector - vector of buffer indexes which were removed
1205  * from the chain
1206  */
1207 always_inline void
1209  vlib_buffer_t * first, u32 ** discard_vector)
1210 {
1212  !(first->flags & VLIB_BUFFER_NEXT_PRESENT))
1213  {
1214  /* this is already big enough or not a chain */
1215  return;
1216  }
1217  /* probe free list to find allocated buffer size to avoid overfill */
1219  vlib_buffer_free_list_t *free_list =
1220  vlib_buffer_get_buffer_free_list (vm, first, &index);
1221 
1223  free_list->n_data_bytes -
1224  first->current_data);
1225  do
1226  {
1227  vlib_buffer_t *second = vlib_get_buffer (vm, first->next_buffer);
1228  u32 need = want_first_size - first->current_length;
1229  u32 amount_to_copy = clib_min (need, second->current_length);
1230  clib_memcpy (((u8 *) vlib_buffer_get_current (first)) +
1231  first->current_length,
1232  vlib_buffer_get_current (second), amount_to_copy);
1233  first->current_length += amount_to_copy;
1234  vlib_buffer_advance (second, amount_to_copy);
1235  if (first->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
1236  {
1237  first->total_length_not_including_first_buffer -= amount_to_copy;
1238  }
1239  if (!second->current_length)
1240  {
1241  vec_add1 (*discard_vector, first->next_buffer);
1242  if (second->flags & VLIB_BUFFER_NEXT_PRESENT)
1243  {
1244  first->next_buffer = second->next_buffer;
1245  }
1246  else
1247  {
1248  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1249  }
1250  second->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1251  }
1252  }
1253  while ((first->current_length < want_first_size) &&
1254  (first->flags & VLIB_BUFFER_NEXT_PRESENT));
1255 }
1256 
1257 #endif /* included_vlib_buffer_funcs_h */
1258 
1259 /*
1260  * fd.io coding-style-patch-verification: ON
1261  *
1262  * Local Variables:
1263  * eval: (c-set-style "gnu")
1264  * End:
1265  */
vlib_physmem_region_index_t physmem_region
Definition: buffer.h:413
vlib_main_t vlib_global_main
Definition: main.c:1644
#define hash_set(h, key, value)
Definition: hash.h:255
#define clib_min(x, y)
Definition: clib.h:289
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:55
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:534
#define STRUCT_MARK_PTR(v, f)
Definition: clib.h:68
format_function_t format_vlib_buffer_contents
Definition: buffer_funcs.h:992
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:142
#define PREDICT_TRUE(x)
Definition: clib.h:106
format_function_t format_vlib_buffer
Definition: buffer_funcs.h:992
unsigned long u64
Definition: types.h:89
static void vlib_buffer_chain_compress(vlib_main_t *vm, vlib_buffer_t *first, u32 **discard_vector)
compress buffer chain in a way where the first buffer is at least VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SI...
vlib_buffer_callbacks_t cb
Definition: buffer.h:454
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
Definition: buffer_funcs.h:895
static void vlib_validate_buffer_in_use(vlib_buffer_t *b, u32 expected)
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:932
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:228
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:949
vlib_buffer_t buffer_init_template
Definition: buffer.h:344
void os_panic(void)
Definition: unix-misc.c:174
struct vlib_main_t * vlib_main
Definition: buffer.h:475
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:53
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:140
int i
foreach_avx2_vec256i static foreach_avx2_vec256u u32x8 u32x8_permute(u32x8 v, u32x8 idx)
Definition: vector_avx2.h:62
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:62
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:75
static void vlib_buffer_delete_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:635
u8 *( format_function_t)(u8 *s, va_list *args)
Definition: format.h:48
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:868
uword * vlib_buffer_state_validation_hash
Definition: buffer.c:606
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:250
vlib_buffer_free_list_index_t index
Definition: buffer.h:347
static u64 vlib_get_buffer_data_physical_address(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:306
unsigned char u8
Definition: types.h:56
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:715
vlib_buffer_free_list_index_t free_list_index
static_always_inline void vlib_get_buffer_indices(vlib_main_t *vm, vlib_buffer_t **b, u32 *bi, uword count)
Translate array of buffer pointers into buffer indices.
Definition: buffer_funcs.h:221
void cj_stop(void)
Definition: cj.c:58
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:533
static u16 vlib_buffer_clone_256(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create a maximum of 256 clones of buffer and store them in the supplied array.
Definition: buffer_funcs.h:783
u32 vlib_buffer_add_data(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, u32 buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:811
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:104
#define static_always_inline
Definition: clib.h:93
#define always_inline
Definition: clib.h:92
void vlib_aligned_memcpy(void *_dst, void *_src, int n_bytes)
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:393
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE
minimum data size of first buffer in a buffer chain
u32 * vlib_buffer_state_validation_lock
Definition: buffer.c:605
vhost_vring_state_t state
Definition: vhost_user.h:115
unsigned int u32
Definition: types.h:88
static void vlib_copy_buffers(u32 *dst, u32 *src, u32 n)
Definition: buffer_funcs.h:687
#define VLIB_FRAME_SIZE
Definition: node.h:364
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:141
#define fl(x, y)
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:589
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
uword size
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
Definition: buffer.c:225
format_function_t format_vlib_buffer_and_data
Definition: buffer_funcs.h:992
unsigned short u16
Definition: types.h:57
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer.h:400
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:202
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:439
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:433
#define PREDICT_FALSE(x)
Definition: clib.h:105
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:111
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
Definition: buffer_funcs.h:160
struct vlib_serialize_buffer_main_t::@32::@35 rx
#define clib_fifo_foreach(v, f, body)
Definition: fifo.h:279
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
Definition: buffer_funcs.h:284
u32 flags
Definition: vhost_user.h:110
static void vlib_buffer_set_known_state(u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:379
static void vlib_buffer_chain_init(vlib_buffer_t *first)
Definition: buffer_funcs.h:922
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:610
serialize_stream_t stream
Definition: serialize.h:147
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:451
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:391
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:555
static u16 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
Definition: buffer_funcs.h:864
u32 current_buffer_index
Definition: serialize.h:62
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:226
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
clib_spinlock_t lock
Definition: buffer.h:423
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
Definition: buffer_funcs.h:234
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:309
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:235
vlib_buffer_known_state_t
Definition: buffer_funcs.h:352
uword data_function_opaque
Definition: serialize.h:74
static void vlib_validate_buffer_set_in_use(vlib_buffer_t *b, u32 expected)
signed int i32
Definition: types.h:81
#define uword_to_pointer(u, type)
Definition: types.h:136
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:786
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:392
static vlib_buffer_free_list_index_t vlib_buffer_get_free_list_index(vlib_buffer_t *b)
Definition: buffer_funcs.h:401
void vlib_buffer_free_list_fill_unaligned(vlib_main_t *vm, vlib_buffer_free_list_t *free_list, uword n_unaligned_buffers)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, vlib_buffer_free_list_index_t *index)
Definition: buffer_funcs.h:652
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:215
size_t count
Definition: vapi.c:46
vlib_buffer_free_list_index_t vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:415
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
Definition: buffer_funcs.h:270
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:138
void * vlib_buffer_state_heap
Definition: buffer.c:607
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, vlib_buffer_free_list_index_t index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:431
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:152
template key/value backing page structure
Definition: bihash_doc.h:44
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
Definition: buffer_funcs.h:961
static u32 vlib_buffer_alloc_to_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Allocate buffers into ring.
Definition: buffer_funcs.h:507
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static u64 vlib_physmem_virtual_to_physical(vlib_main_t *vm, vlib_physmem_region_index_t idx, void *mem)
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:431
uword * buffer_known_hash
Definition: buffer.h:450
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:676
u64x4
Definition: vector_avx2.h:110
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:773
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
static vlib_buffer_known_state_t vlib_buffer_is_known(u32 buffer_index)
Definition: buffer_funcs.h:368
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
u8 data[0]
Packet data.
Definition: buffer.h:172
static void vlib_buffer_set_free_list_index(vlib_buffer_t *b, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:410
static_always_inline vlib_buffer_pool_t * vlib_buffer_pool_get(u8 buffer_pool_index)
Definition: buffer.h:461
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:109
u8 vlib_buffer_free_list_index_t
Definition: buffer.h:54
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:575
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer.c:793
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:662
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:128
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
Definition: buffer.c:725
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:490
static void vlib_buffer_free_from_ring_no_next(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring without freeing tail buffers.
Definition: buffer_funcs.h:614
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:395
static u32 unserialize_vlib_buffer_n_bytes(serialize_main_t *m)
vlib_buffer_free_list_index_t free_list_index
Definition: buffer.h:156