FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
buffer_funcs.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_funcs.h: VLIB buffer related functions/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
42 
43 #include <vppinfra/hash.h>
44 #include <vppinfra/fifo.h>
45 #include <vlib/buffer.h>
46 #include <vlib/physmem_funcs.h>
47 #include <vlib/main.h>
48 #include <vlib/node.h>
49 
50 /** \file
51  vlib buffer access methods.
52 */
53 
54 always_inline void
56 {
59 
60  /* reference count in allocated buffer always must be 1 or higher */
61  ASSERT (b->ref_count > 0);
62 
63  /* verify that buffer pool index is valid */
65  ASSERT (pointer_to_uword (b) >= bp->start);
66  ASSERT (pointer_to_uword (b) < bp->start + bp->size -
67  (bp->data_size + sizeof (vlib_buffer_t)));
68 }
69 
70 always_inline void *
71 vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index,
72  uword offset)
73 {
74  offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
75  return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *);
76 }
77 
78 /** \brief Translate buffer index into buffer pointer
79 
80  @param vm - (vlib_main_t *) vlib main data structure pointer
81  @param buffer_index - (u32) buffer index
82  @return - (vlib_buffer_t *) buffer pointer
83 */
85 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
86 {
88  vlib_buffer_t *b;
89 
90  b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0);
91  vlib_buffer_validate (vm, b);
92  return b;
93 }
94 
97 {
98  return vm->buffer_main->default_data_size;
99 }
100 
103 {
104 #if defined(CLIB_HAVE_VEC512)
105  while (n_indices >= 16)
106  {
107  u32x16_store_unaligned (u32x16_load_unaligned (src), dst);
108  dst += 16;
109  src += 16;
110  n_indices -= 16;
111  }
112 #endif
113 
114 #if defined(CLIB_HAVE_VEC256)
115  while (n_indices >= 8)
116  {
117  u32x8_store_unaligned (u32x8_load_unaligned (src), dst);
118  dst += 8;
119  src += 8;
120  n_indices -= 8;
121  }
122 #endif
123 
124 #if defined(CLIB_HAVE_VEC128)
125  while (n_indices >= 4)
126  {
127  u32x4_store_unaligned (u32x4_load_unaligned (src), dst);
128  dst += 4;
129  src += 4;
130  n_indices -= 4;
131  }
132 #endif
133 
134  while (n_indices)
135  {
136  dst[0] = src[0];
137  dst += 1;
138  src += 1;
139  n_indices -= 1;
140  }
141 }
142 
143 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
146 {
147 #if defined CLIB_HAVE_VEC512
148  b->as_u8x64[0] = bt->as_u8x64[0];
149 #elif defined (CLIB_HAVE_VEC256)
150  b->as_u8x32[0] = bt->as_u8x32[0];
151  b->as_u8x32[1] = bt->as_u8x32[1];
152 #elif defined (CLIB_HAVE_VEC128)
153  b->as_u8x16[0] = bt->as_u8x16[0];
154  b->as_u8x16[1] = bt->as_u8x16[1];
155  b->as_u8x16[2] = bt->as_u8x16[2];
156  b->as_u8x16[3] = bt->as_u8x16[3];
157 #else
158  clib_memcpy_fast (b, bt, 64);
159 #endif
160 }
161 
164 {
165  ASSERT (numa_node < VLIB_BUFFER_MAX_NUMA_NODES);
166  return vm->buffer_main->default_buffer_pool_index_for_numa[numa_node];
167 }
168 
169 /** \brief Translate array of buffer indices into buffer pointers with offset
170 
171  @param vm - (vlib_main_t *) vlib main data structure pointer
172  @param bi - (u32 *) array of buffer indices
173  @param b - (void **) array to store buffer pointers
174  @param count - (uword) number of elements
175  @param offset - (i32) offset applied to each pointer
176 */
179  i32 offset)
180 {
181  uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
182 #ifdef CLIB_HAVE_VEC256
183  u64x4 off = u64x4_splat (buffer_mem_start + offset);
184  /* if count is not const, compiler will not unroll while loop
185  se we maintain two-in-parallel variant */
186  while (count >= 8)
187  {
188  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
189  u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
190  /* shift and add to get vlib_buffer_t pointer */
191  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
192  u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
193  b += 8;
194  bi += 8;
195  count -= 8;
196  }
197 #endif
198  while (count >= 4)
199  {
200 #ifdef CLIB_HAVE_VEC256
201  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
202  /* shift and add to get vlib_buffer_t pointer */
203  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
204 #elif defined (CLIB_HAVE_VEC128)
205  u64x2 off = u64x2_splat (buffer_mem_start + offset);
206  u32x4 bi4 = u32x4_load_unaligned (bi);
207  u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
208 #if defined (__aarch64__)
210 #else
211  bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
212  u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
213 #endif
214  u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
215  u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
216 #else
217  b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
218  b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset);
219  b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset);
220  b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset);
221 #endif
222  b += 4;
223  bi += 4;
224  count -= 4;
225  }
226  while (count)
227  {
228  b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
229  b += 1;
230  bi += 1;
231  count -= 1;
232  }
233 }
234 
235 /** \brief Translate array of buffer indices into buffer pointers
236 
237  @param vm - (vlib_main_t *) vlib main data structure pointer
238  @param bi - (u32 *) array of buffer indices
239  @param b - (vlib_buffer_t **) array to store buffer pointers
240  @param count - (uword) number of elements
241 */
242 
245 {
246  vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
247 }
248 
249 /** \brief Translate buffer pointer into buffer index
250 
251  @param vm - (vlib_main_t *) vlib main data structure pointer
252  @param p - (void *) buffer pointer
253  @return - (u32) buffer index
254 */
255 
258 {
262  ASSERT (offset < bm->buffer_mem_size);
263  ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
264  return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
265 }
266 
267 /** \brief Translate array of buffer pointers into buffer indices with offset
268 
269  @param vm - (vlib_main_t *) vlib main data structure pointer
270  @param b - (void **) array of buffer pointers
271  @param bi - (u32 *) array to store buffer indices
272  @param count - (uword) number of elements
273  @param offset - (i32) offset applied to each pointer
274 */
278 {
279 #ifdef CLIB_HAVE_VEC256
280  u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
281  u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
282 
283  while (count >= 8)
284  {
285  /* load 4 pointers into 256-bit register */
286  u64x4 v0 = u64x4_load_unaligned (b);
287  u64x4 v1 = u64x4_load_unaligned (b + 4);
288  u32x8 v2, v3;
289 
290  v0 -= off4;
291  v1 -= off4;
292 
295 
296  /* permute 256-bit register so lower u32s of each buffer index are
297  * placed into lower 128-bits */
298  v2 = u32x8_permute ((u32x8) v0, mask);
299  v3 = u32x8_permute ((u32x8) v1, mask);
300 
301  /* extract lower 128-bits and save them to the array of buffer indices */
302  u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
303  u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
304  bi += 8;
305  b += 8;
306  count -= 8;
307  }
308 #endif
309  while (count >= 4)
310  {
311  /* equivalent non-nector implementation */
312  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
313  bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
314  bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
315  bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
316  bi += 4;
317  b += 4;
318  count -= 4;
319  }
320  while (count)
321  {
322  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
323  bi += 1;
324  b += 1;
325  count -= 1;
326  }
327 }
328 
329 /** \brief Translate array of buffer pointers into buffer indices
330 
331  @param vm - (vlib_main_t *) vlib main data structure pointer
332  @param b - (vlib_buffer_t **) array of buffer pointers
333  @param bi - (u32 *) array to store buffer indices
334  @param count - (uword) number of elements
335 */
338  uword count)
339 {
340  vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
341 }
342 
343 /** \brief Get next buffer in buffer linklist, or zero for end of list.
344 
345  @param vm - (vlib_main_t *) vlib main data structure pointer
346  @param b - (void *) buffer pointer
347  @return - (vlib_buffer_t *) next buffer, or NULL
348 */
351 {
352  return (b->flags & VLIB_BUFFER_NEXT_PRESENT
353  ? vlib_get_buffer (vm, b->next_buffer) : 0);
354 }
355 
357  vlib_buffer_t * b_first);
358 
359 /** \brief Get length in bytes of the buffer chain
360 
361  @param vm - (vlib_main_t *) vlib main data structure pointer
362  @param b - (void *) buffer pointer
363  @return - (uword) length of buffer chain
364 */
367 {
368  uword len = b->current_length;
369 
370  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
371  return len;
372 
373  if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
375 
377 }
378 
379 /** \brief Get length in bytes of the buffer index buffer chain
380 
381  @param vm - (vlib_main_t *) vlib main data structure pointer
382  @param bi - (u32) buffer index
383  @return - (uword) length of buffer chain
384 */
387 {
388  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
389  return vlib_buffer_length_in_chain (vm, b);
390 }
391 
392 /** \brief Copy buffer contents to memory
393 
394  @param vm - (vlib_main_t *) vlib main data structure pointer
395  @param buffer_index - (u32) buffer index
396  @param contents - (u8 *) memory, <strong>must be large enough</strong>
397  @return - (uword) length of buffer chain
398 */
400 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
401 {
402  uword content_len = 0;
403  uword l;
404  vlib_buffer_t *b;
405 
406  while (1)
407  {
408  b = vlib_get_buffer (vm, buffer_index);
409  l = b->current_length;
410  clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
411  content_len += l;
412  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
413  break;
414  buffer_index = b->next_buffer;
415  }
416 
417  return content_len;
418 }
419 
422 {
423  return vlib_physmem_get_pa (vm, b->data);
424 }
425 
428 {
429  return vlib_buffer_get_pa (vm, b) + b->current_data;
430 }
431 
432 /** \brief Prefetch buffer metadata by buffer index
433  The first 64 bytes of buffer contains most header information
434 
435  @param vm - (vlib_main_t *) vlib main data structure pointer
436  @param bi - (u32) buffer index
437  @param type - LOAD, STORE. In most cases, STORE is the right answer
438 */
439 /* Prefetch buffer header given index. */
440 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
441  do { \
442  vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
443  vlib_prefetch_buffer_header (_b, type); \
444  } while (0)
445 
446 typedef enum
447 {
448  /* Index is unknown. */
450 
451  /* Index is known and free/allocated. */
455 
456 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
457  uword n_buffers,
459  expected_state);
460 
463 {
465 
467  uword *p = hash_get (bm->buffer_known_hash, buffer_index);
469  return p ? p[0] : VLIB_BUFFER_UNKNOWN;
470 }
471 
472 /* Validates sanity of a single buffer.
473  Returns format'ed vector with error message if any. */
474 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
475  uword follow_chain);
476 
478 vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index)
479 {
481  return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
482 }
483 
485 vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers,
486  u32 n_buffers)
487 {
488  vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
489  u32 len;
490 
491  ASSERT (bp->buffers);
492 
493  clib_spinlock_lock (&bp->lock);
494  len = vec_len (bp->buffers);
495  if (PREDICT_TRUE (n_buffers < len))
496  {
497  len -= n_buffers;
498  vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers);
499  _vec_len (bp->buffers) = len;
500  clib_spinlock_unlock (&bp->lock);
501  return n_buffers;
502  }
503  else
504  {
505  vlib_buffer_copy_indices (buffers, bp->buffers, len);
506  _vec_len (bp->buffers) = 0;
507  clib_spinlock_unlock (&bp->lock);
508  return len;
509  }
510 }
511 
512 
513 /** \brief Allocate buffers from specific pool into supplied array
514 
515  @param vm - (vlib_main_t *) vlib main data structure pointer
516  @param buffers - (u32 * ) buffer index array
517  @param n_buffers - (u32) number of buffers requested
518  @return - (u32) number of buffers actually allocated, may be
519  less than the number requested or zero
520 */
521 
523 vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
524  u8 buffer_pool_index)
525 {
527  vlib_buffer_pool_t *bp;
529  u32 *src, *dst, len, n_left;
530 
531  bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
532  bpt = vec_elt_at_index (bp->threads, vm->thread_index);
533 
534  dst = buffers;
535  n_left = n_buffers;
536  len = vec_len (bpt->cached_buffers);
537 
538  /* per-thread cache contains enough buffers */
539  if (len >= n_buffers)
540  {
541  src = bpt->cached_buffers + len - n_buffers;
542  vlib_buffer_copy_indices (dst, src, n_buffers);
543  _vec_len (bpt->cached_buffers) -= n_buffers;
544 
545  if (CLIB_DEBUG > 0)
546  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
548  return n_buffers;
549  }
550 
551  /* take everything available in the cache */
552  if (len)
553  {
554  vlib_buffer_copy_indices (dst, bpt->cached_buffers, len);
555  _vec_len (bpt->cached_buffers) = 0;
556  dst += len;
557  n_left -= len;
558  }
559 
560  len = round_pow2 (n_left, 32);
562  len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers,
563  len);
564  _vec_len (bpt->cached_buffers) = len;
565 
566  if (len)
567  {
568  u32 n_copy = clib_min (len, n_left);
569  src = bpt->cached_buffers + len - n_copy;
570  vlib_buffer_copy_indices (dst, src, n_copy);
571  _vec_len (bpt->cached_buffers) -= n_copy;
572  n_left -= n_copy;
573  }
574 
575  n_buffers -= n_left;
576 
577  /* Verify that buffers are known free. */
578  if (CLIB_DEBUG > 0)
579  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
581 
582  return n_buffers;
583 }
584 
585 /** \brief Allocate buffers from specific numa node into supplied array
586 
587  @param vm - (vlib_main_t *) vlib main data structure pointer
588  @param buffers - (u32 * ) buffer index array
589  @param n_buffers - (u32) number of buffers requested
590  @param numa_node - (u32) numa node
591  @return - (u32) number of buffers actually allocated, may be
592  less than the number requested or zero
593 */
595 vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
596  u32 numa_node)
597 {
598  u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node);
599  return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index);
600 }
601 
602 /** \brief Allocate buffers into supplied array
603 
604  @param vm - (vlib_main_t *) vlib main data structure pointer
605  @param buffers - (u32 * ) buffer index array
606  @param n_buffers - (u32) number of buffers requested
607  @return - (u32) number of buffers actually allocated, may be
608  less than the number requested or zero
609 */
610 
612 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
613 {
614  return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node);
615 }
616 
617 /** \brief Allocate buffers into ring
618 
619  @param vm - (vlib_main_t *) vlib main data structure pointer
620  @param buffers - (u32 * ) buffer index ring
621  @param start - (u32) first slot in the ring
622  @param ring_size - (u32) ring size
623  @param n_buffers - (u32) number of buffers requested
624  @return - (u32) number of buffers actually allocated, may be
625  less than the number requested or zero
626 */
629  u32 ring_size, u32 n_buffers)
630 {
631  u32 n_alloc;
632 
633  ASSERT (n_buffers <= ring_size);
634 
635  if (PREDICT_TRUE (start + n_buffers <= ring_size))
636  return vlib_buffer_alloc (vm, ring + start, n_buffers);
637 
638  n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
639 
640  if (PREDICT_TRUE (n_alloc == ring_size - start))
641  n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
642 
643  return n_alloc;
644 }
645 
646 /** \brief Allocate buffers into ring from specific buffer pool
647 
648  @param vm - (vlib_main_t *) vlib main data structure pointer
649  @param buffers - (u32 * ) buffer index ring
650  @param start - (u32) first slot in the ring
651  @param ring_size - (u32) ring size
652  @param n_buffers - (u32) number of buffers requested
653  @return - (u32) number of buffers actually allocated, may be
654  less than the number requested or zero
655 */
658  u32 ring_size, u32 n_buffers,
659  u8 buffer_pool_index)
660 {
661  u32 n_alloc;
662 
663  ASSERT (n_buffers <= ring_size);
664 
665  if (PREDICT_TRUE (start + n_buffers <= ring_size))
666  return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers,
667  buffer_pool_index);
668 
669  n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start,
670  buffer_pool_index);
671 
672  if (PREDICT_TRUE (n_alloc == ring_size - start))
673  n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc,
674  buffer_pool_index);
675 
676  return n_alloc;
677 }
678 
680 vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
681  u32 * buffers, u32 n_buffers)
682 {
683  vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
686 
687  if (CLIB_DEBUG > 0)
688  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
690 
691  vec_add_aligned (bpt->cached_buffers, buffers, n_buffers,
693 
694  if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE)
695  {
696  clib_spinlock_lock (&bp->lock);
697  /* keep last stored buffers, as they are more likely hot in the cache */
701  bpt->n_alloc -= VLIB_FRAME_SIZE;
702  clib_spinlock_unlock (&bp->lock);
703  }
704 }
705 
707 vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
708  int maybe_next)
709 {
710  const int queue_size = 128;
711  vlib_buffer_pool_t *bp = 0;
712  u8 buffer_pool_index = ~0;
713  u32 n_queue = 0, queue[queue_size + 4];
714  vlib_buffer_t bt = { };
715 #if defined(CLIB_HAVE_VEC128)
716  vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
717  vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 };
718  vlib_buffer_t flags_refs_mask = {
719  .flags = VLIB_BUFFER_NEXT_PRESENT,
720  .ref_count = ~1
721  };
722 #endif
723 
724  while (n_buffers)
725  {
726  vlib_buffer_t *b[8];
727  u32 bi, sum = 0, flags, next;
728 
729  if (n_buffers < 12)
730  goto one_by_one;
731 
732  vlib_get_buffers (vm, buffers, b, 4);
733  vlib_get_buffers (vm, buffers + 8, b + 4, 4);
734 
735  vlib_prefetch_buffer_header (b[4], LOAD);
736  vlib_prefetch_buffer_header (b[5], LOAD);
737  vlib_prefetch_buffer_header (b[6], LOAD);
738  vlib_prefetch_buffer_header (b[7], LOAD);
739 
740 #if defined(CLIB_HAVE_VEC128)
741  u8x16 p0, p1, p2, p3, r;
742  p0 = u8x16_load_unaligned (b[0]);
743  p1 = u8x16_load_unaligned (b[1]);
744  p2 = u8x16_load_unaligned (b[2]);
745  p3 = u8x16_load_unaligned (b[3]);
746 
747  r = p0 ^ bpi_vec.as_u8x16[0];
748  r |= p1 ^ bpi_vec.as_u8x16[0];
749  r |= p2 ^ bpi_vec.as_u8x16[0];
750  r |= p3 ^ bpi_vec.as_u8x16[0];
751  r &= bpi_mask.as_u8x16[0];
752  r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
753 
754  sum = !u8x16_is_all_zero (r);
755 #else
756  sum |= b[0]->flags;
757  sum |= b[1]->flags;
758  sum |= b[2]->flags;
759  sum |= b[3]->flags;
760  sum &= VLIB_BUFFER_NEXT_PRESENT;
761  sum += b[0]->ref_count - 1;
762  sum += b[1]->ref_count - 1;
763  sum += b[2]->ref_count - 1;
764  sum += b[3]->ref_count - 1;
765  sum |= b[0]->buffer_pool_index ^ buffer_pool_index;
766  sum |= b[1]->buffer_pool_index ^ buffer_pool_index;
767  sum |= b[2]->buffer_pool_index ^ buffer_pool_index;
768  sum |= b[3]->buffer_pool_index ^ buffer_pool_index;
769 #endif
770 
771  if (sum)
772  goto one_by_one;
773 
774  vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
775  vlib_buffer_copy_template (b[0], &bt);
776  vlib_buffer_copy_template (b[1], &bt);
777  vlib_buffer_copy_template (b[2], &bt);
778  vlib_buffer_copy_template (b[3], &bt);
779  n_queue += 4;
780 
781  vlib_buffer_validate (vm, b[0]);
782  vlib_buffer_validate (vm, b[1]);
783  vlib_buffer_validate (vm, b[2]);
784  vlib_buffer_validate (vm, b[3]);
785 
790 
791  if (n_queue >= queue_size)
792  {
793  vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
794  n_queue = 0;
795  }
796  buffers += 4;
797  n_buffers -= 4;
798  continue;
799 
800  one_by_one:
801  bi = buffers[0];
802 
803  next_in_chain:
804  b[0] = vlib_get_buffer (vm, bi);
805  flags = b[0]->flags;
806  next = b[0]->next_buffer;
807 
808  if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
809  {
810 
811  if (n_queue)
812  {
813  vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
814  n_queue = 0;
815  }
816 
817  buffer_pool_index = b[0]->buffer_pool_index;
818 #if defined(CLIB_HAVE_VEC128)
819  bpi_vec.buffer_pool_index = buffer_pool_index;
820 #endif
821  bp = vlib_get_buffer_pool (vm, buffer_pool_index);
823  }
824 
825  vlib_buffer_validate (vm, b[0]);
826 
828 
829  if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
830  {
831  vlib_buffer_copy_template (b[0], &bt);
832  queue[n_queue++] = bi;
833  }
834 
835  if (n_queue == queue_size)
836  {
837  vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size);
838  n_queue = 0;
839  }
840 
841  if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
842  {
843  bi = next;
844  goto next_in_chain;
845  }
846 
847  buffers++;
848  n_buffers--;
849  }
850 
851  if (n_queue)
852  vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
853 }
854 
855 
856 /** \brief Free buffers
857  Frees the entire buffer chain for each buffer
858 
859  @param vm - (vlib_main_t *) vlib main data structure pointer
860  @param buffers - (u32 * ) buffer index array
861  @param n_buffers - (u32) number of buffers to free
862 
863 */
864 always_inline void
866  /* pointer to first buffer */
867  u32 * buffers,
868  /* number of buffers to free */
869  u32 n_buffers)
870 {
871  vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1);
872 }
873 
874 /** \brief Free buffers, does not free the buffer chain for each buffer
875 
876  @param vm - (vlib_main_t *) vlib main data structure pointer
877  @param buffers - (u32 * ) buffer index array
878  @param n_buffers - (u32) number of buffers to free
879 
880 */
881 always_inline void
883  /* pointer to first buffer */
884  u32 * buffers,
885  /* number of buffers to free */
886  u32 n_buffers)
887 {
888  vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0);
889 }
890 
891 /** \brief Free one buffer
892  Shorthand to free a single buffer chain.
893 
894  @param vm - (vlib_main_t *) vlib main data structure pointer
895  @param buffer_index - (u32) buffer index to free
896 */
897 always_inline void
899 {
900  vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1);
901 }
902 
903 /** \brief Free buffers from ring
904 
905  @param vm - (vlib_main_t *) vlib main data structure pointer
906  @param buffers - (u32 * ) buffer index ring
907  @param start - (u32) first slot in the ring
908  @param ring_size - (u32) ring size
909  @param n_buffers - (u32) number of buffers
910 */
911 always_inline void
913  u32 ring_size, u32 n_buffers)
914 {
915  ASSERT (n_buffers <= ring_size);
916 
917  if (PREDICT_TRUE (start + n_buffers <= ring_size))
918  {
919  vlib_buffer_free (vm, ring + start, n_buffers);
920  }
921  else
922  {
923  vlib_buffer_free (vm, ring + start, ring_size - start);
924  vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
925  }
926 }
927 
928 /** \brief Free buffers from ring without freeing tail buffers
929 
930  @param vm - (vlib_main_t *) vlib main data structure pointer
931  @param buffers - (u32 * ) buffer index ring
932  @param start - (u32) first slot in the ring
933  @param ring_size - (u32) ring size
934  @param n_buffers - (u32) number of buffers
935 */
936 always_inline void
938  u32 ring_size, u32 n_buffers)
939 {
940  ASSERT (n_buffers <= ring_size);
941 
942  if (PREDICT_TRUE (start + n_buffers <= ring_size))
943  {
944  vlib_buffer_free_no_next (vm, ring + start, n_buffers);
945  }
946  else
947  {
948  vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
949  vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
950  }
951 }
952 
953 /* Append given data to end of buffer, possibly allocating new buffers. */
954 int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
955  u32 n_data_bytes);
956 
957 /* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */
958 #define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \
959  (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \
960  VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL)
961 
962 /* duplicate all buffers in chain */
965 {
966  vlib_buffer_t *s, *d, *fd;
967  uword n_alloc, n_buffers = 1;
969  int i;
970 
971  s = b;
972  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
973  {
974  n_buffers++;
975  s = vlib_get_buffer (vm, s->next_buffer);
976  }
977  u32 new_buffers[n_buffers];
978 
979  n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
980 
981  /* No guarantee that we'll get all the buffers we asked for */
982  if (PREDICT_FALSE (n_alloc < n_buffers))
983  {
984  if (n_alloc > 0)
985  vlib_buffer_free (vm, new_buffers, n_alloc);
986  return 0;
987  }
988 
989  /* 1st segment */
990  s = b;
991  fd = d = vlib_get_buffer (vm, new_buffers[0]);
992  d->current_data = s->current_data;
994  d->flags = s->flags & flag_mask;
995  d->trace_handle = s->trace_handle;
998  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
999  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1002 
1003  /* next segments */
1004  for (i = 1; i < n_buffers; i++)
1005  {
1006  /* previous */
1007  d->next_buffer = new_buffers[i];
1008  /* current */
1009  s = vlib_get_buffer (vm, s->next_buffer);
1010  d = vlib_get_buffer (vm, new_buffers[i]);
1011  d->current_data = s->current_data;
1015  d->flags = s->flags & flag_mask;
1016  }
1017 
1018  return fd;
1019 }
1020 
1021 /* duplicate first buffer in chain */
1024 {
1025  vlib_buffer_t *d;
1026 
1027  if ((vlib_buffer_alloc (vm, di, 1)) != 1)
1028  return 0;
1029 
1030  d = vlib_get_buffer (vm, *di);
1031  /* 1st segment */
1032  d->current_data = b->current_data;
1034  clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque));
1035  clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2));
1038 
1039  return d;
1040 }
1041 
1042 /* \brief Move packet from current position to offset position in buffer.
1043  Only work for small packet using one buffer with room to fit the move
1044  @param vm - (vlib_main_t *) vlib main data structure pointer
1045  @param b - (vlib_buffer_t *) pointer to buffer
1046  @param offset - (i16) position to move the packet in buffer
1047  */
1048 always_inline void
1050 {
1051  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1052  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1053  ASSERT (offset + b->current_length <
1055 
1056  u8 *source = vlib_buffer_get_current (b);
1057  b->current_data = offset;
1058  u8 *destination = vlib_buffer_get_current (b);
1059  u16 length = b->current_length;
1060 
1061  if (source + length <= destination) /* no overlap */
1062  clib_memcpy_fast (destination, source, length);
1063  else
1064  memmove (destination, source, length);
1065 }
1066 
1067 /** \brief Create a maximum of 256 clones of buffer and store them
1068  in the supplied array
1069 
1070  @param vm - (vlib_main_t *) vlib main data structure pointer
1071  @param src_buffer - (u32) source buffer index
1072  @param buffers - (u32 * ) buffer index array
1073  @param n_buffers - (u16) number of buffer clones requested (<=256)
1074  @param head_end_offset - (u16) offset relative to current position
1075  where packet head ends
1076  @param offset - (i16) copy packet head at current position if 0,
1077  else at offset position to change headroom space as specified
1078  @return - (u16) number of buffers actually cloned, may be
1079  less than the number requested or zero
1080 */
1082 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1083  u16 n_buffers, u16 head_end_offset, i16 offset)
1084 {
1085  u16 i;
1086  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1087 
1088  ASSERT (s->ref_count == 1);
1089  ASSERT (n_buffers);
1090  ASSERT (n_buffers <= 256);
1091  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1092  ASSERT ((offset + head_end_offset) <
1094 
1095  if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
1096  {
1097  buffers[0] = src_buffer;
1098  if (offset)
1099  vlib_buffer_move (vm, s, offset);
1100 
1101  for (i = 1; i < n_buffers; i++)
1102  {
1103  vlib_buffer_t *d;
1104  d = vlib_buffer_copy (vm, s);
1105  if (d == 0)
1106  return i;
1107  buffers[i] = vlib_get_buffer_index (vm, d);
1108 
1109  }
1110  return n_buffers;
1111  }
1112 
1113  if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0)))
1114  {
1115  buffers[0] = src_buffer;
1116  return 1;
1117  }
1118 
1119  n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers,
1120  s->buffer_pool_index);
1121 
1122  for (i = 0; i < n_buffers; i++)
1123  {
1124  vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
1125  if (offset)
1126  d->current_data = offset;
1127  else
1128  d->current_data = s->current_data;
1129 
1130  d->current_length = head_end_offset;
1132 
1134  head_end_offset;
1135  if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
1136  {
1139  }
1141  VLIB_BUFFER_NEXT_PRESENT;
1142  d->trace_handle = s->trace_handle;
1143  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
1144  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1146  vlib_buffer_get_current (s), head_end_offset);
1147  d->next_buffer = src_buffer;
1148  }
1149  vlib_buffer_advance (s, head_end_offset);
1150  s->ref_count = n_buffers ? n_buffers : s->ref_count;
1151  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
1152  {
1153  s = vlib_get_buffer (vm, s->next_buffer);
1154  s->ref_count = n_buffers ? n_buffers : s->ref_count;
1155  }
1156 
1157  return n_buffers;
1158 }
1159 
1160 /** \brief Create multiple clones of buffer and store them
1161  in the supplied array
1162 
1163  @param vm - (vlib_main_t *) vlib main data structure pointer
1164  @param src_buffer - (u32) source buffer index
1165  @param buffers - (u32 * ) buffer index array
1166  @param n_buffers - (u16) number of buffer clones requested (<=256)
1167  @param head_end_offset - (u16) offset relative to current position
1168  where packet head ends
1169  @param offset - (i16) copy packet head at current position if 0,
1170  else at offset position to change headroom space as specified
1171  @return - (u16) number of buffers actually cloned, may be
1172  less than the number requested or zero
1173 */
1175 vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1176  u16 n_buffers, u16 head_end_offset, i16 offset)
1177 {
1178  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1179  u16 n_cloned = 0;
1180 
1181  while (n_buffers > 256)
1182  {
1183  vlib_buffer_t *copy;
1184  copy = vlib_buffer_copy (vm, s);
1185  n_cloned += vlib_buffer_clone_256 (vm,
1186  vlib_get_buffer_index (vm, copy),
1187  (buffers + n_cloned),
1188  256, head_end_offset, offset);
1189  n_buffers -= 256;
1190  }
1191  n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
1192  buffers + n_cloned,
1193  n_buffers, head_end_offset, offset);
1194 
1195  return n_cloned;
1196 }
1197 
1198 /** \brief Create multiple clones of buffer and store them
1199  in the supplied array
1200 
1201  @param vm - (vlib_main_t *) vlib main data structure pointer
1202  @param src_buffer - (u32) source buffer index
1203  @param buffers - (u32 * ) buffer index array
1204  @param n_buffers - (u16) number of buffer clones requested (<=256)
1205  @param head_end_offset - (u16) offset relative to current position
1206  where packet head ends
1207  @return - (u16) number of buffers actually cloned, may be
1208  less than the number requested or zero
1209 */
1211 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1212  u16 n_buffers, u16 head_end_offset)
1213 {
1214  return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers,
1215  head_end_offset, 0);
1216 }
1217 
1218 /** \brief Attach cloned tail to the buffer
1219 
1220  @param vm - (vlib_main_t *) vlib main data structure pointer
1221  @param head - (vlib_buffer_t *) head buffer
1222  @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
1223 */
1224 
1225 always_inline void
1227  vlib_buffer_t * tail)
1228 {
1229  ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1230  ASSERT (head->buffer_pool_index == tail->buffer_pool_index);
1231 
1232  head->flags |= VLIB_BUFFER_NEXT_PRESENT;
1233  head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1234  head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1235  head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
1236  head->next_buffer = vlib_get_buffer_index (vm, tail);
1239 
1240 next_segment:
1241  clib_atomic_add_fetch (&tail->ref_count, 1);
1242 
1243  if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
1244  {
1245  tail = vlib_get_buffer (vm, tail->next_buffer);
1246  goto next_segment;
1247  }
1248 }
1249 
1250 /* Initializes the buffer as an empty packet with no chained buffers. */
1251 always_inline void
1253 {
1255  first->current_length = 0;
1256  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1257  first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1258 }
1259 
1260 /* The provided next_bi buffer index is appended to the end of the packet. */
1263 {
1264  vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
1265  last->next_buffer = next_bi;
1266  last->flags |= VLIB_BUFFER_NEXT_PRESENT;
1267  next_buffer->current_length = 0;
1268  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1269  return next_buffer;
1270 }
1271 
1272 /* Increases or decreases the packet length.
1273  * It does not allocate or deallocate new buffers.
1274  * Therefore, the added length must be compatible
1275  * with the last buffer. */
1276 always_inline void
1278  vlib_buffer_t * last, i32 len)
1279 {
1280  last->current_length += len;
1281  if (first != last)
1283 }
1284 
1285 /* Copy data to the end of the packet and increases its length.
1286  * It does not allocate new buffers.
1287  * Returns the number of copied bytes. */
1290  vlib_buffer_t * first,
1291  vlib_buffer_t * last, void *data, u16 data_len)
1292 {
1293  u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
1294  ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
1295  u16 len = clib_min (data_len,
1296  n_buffer_bytes - last->current_length -
1297  last->current_data);
1299  data, len);
1300  vlib_buffer_chain_increase_length (first, last, len);
1301  return len;
1302 }
1303 
1304 /* Copy data to the end of the packet and increases its length.
1305  * Allocates additional buffers from the free list if necessary.
1306  * Returns the number of copied bytes.
1307  * 'last' value is modified whenever new buffers are allocated and
1308  * chained and points to the last buffer in the chain. */
1309 u16
1311  vlib_buffer_t * first,
1312  vlib_buffer_t ** last, void *data,
1313  u16 data_len);
1315 
1318 
1319 typedef struct
1320 {
1321  /* Vector of packet data. */
1323 
1324  /* Number of buffers to allocate in each call to allocator. */
1326 
1329 
1332  void *packet_data,
1333  uword n_packet_data_bytes,
1334  uword min_n_buffers_each_alloc,
1335  char *fmt, ...);
1336 
1339  u32 * bi_result);
1340 
1341 always_inline void
1343 {
1344  vec_free (t->packet_data);
1345 }
1346 
1349 {
1350  return b->data + vlib_buffer_get_default_data_size (vm) -
1351  ((u8 *) vlib_buffer_get_current (b) + b->current_length);
1352 }
1353 
1356 {
1357  vlib_buffer_t *db = b, *sb, *first = b;
1358  int is_cloned = 0;
1359  u32 bytes_left = 0, data_size;
1360  u16 src_left, dst_left, n_buffers = 1;
1361  u8 *dp, *sp;
1362  u32 to_free = 0;
1363 
1364  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
1365  return 1;
1366 
1367  data_size = vlib_buffer_get_default_data_size (vm);
1368 
1369  dst_left = vlib_buffer_space_left_at_end (vm, b);
1370 
1371  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1372  {
1373  b = vlib_get_buffer (vm, b->next_buffer);
1374  if (b->ref_count > 1)
1375  is_cloned = 1;
1376  bytes_left += b->current_length;
1377  n_buffers++;
1378  }
1379 
1380  /* if buffer is cloned, create completely new chain - unless everything fits
1381  * into one buffer */
1382  if (is_cloned && bytes_left >= dst_left)
1383  {
1384  u32 len = 0;
1385  u32 space_needed = bytes_left - dst_left;
1386  u32 tail;
1387 
1388  if (vlib_buffer_alloc (vm, &tail, 1) == 0)
1389  return 0;
1390 
1391  ++n_buffers;
1392  len += data_size;
1393  b = vlib_get_buffer (vm, tail);
1394 
1395  while (len < space_needed)
1396  {
1397  u32 bi;
1398  if (vlib_buffer_alloc (vm, &bi, 1) == 0)
1399  {
1400  vlib_buffer_free_one (vm, tail);
1401  return 0;
1402  }
1403  b->flags = VLIB_BUFFER_NEXT_PRESENT;
1404  b->next_buffer = bi;
1405  b = vlib_get_buffer (vm, bi);
1406  len += data_size;
1407  n_buffers++;
1408  }
1409  sb = vlib_get_buffer (vm, first->next_buffer);
1410  to_free = first->next_buffer;
1411  first->next_buffer = tail;
1412  }
1413  else
1414  sb = vlib_get_buffer (vm, first->next_buffer);
1415 
1416  src_left = sb->current_length;
1417  sp = vlib_buffer_get_current (sb);
1418  dp = vlib_buffer_get_tail (db);
1419 
1420  while (bytes_left)
1421  {
1422  u16 bytes_to_copy;
1423 
1424  if (dst_left == 0)
1425  {
1426  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1427  ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
1428  db = vlib_get_buffer (vm, db->next_buffer);
1429  dst_left = data_size;
1430  if (db->current_data > 0)
1431  {
1432  db->current_data = 0;
1433  }
1434  else
1435  {
1436  dst_left += -db->current_data;
1437  }
1438  dp = vlib_buffer_get_current (db);
1439  }
1440 
1441  while (src_left == 0)
1442  {
1443  ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
1444  sb = vlib_get_buffer (vm, sb->next_buffer);
1445  src_left = sb->current_length;
1446  sp = vlib_buffer_get_current (sb);
1447  }
1448 
1449  bytes_to_copy = clib_min (dst_left, src_left);
1450 
1451  if (dp != sp)
1452  {
1453  if (sb == db)
1454  bytes_to_copy = clib_min (bytes_to_copy, sp - dp);
1455 
1456  clib_memcpy_fast (dp, sp, bytes_to_copy);
1457  }
1458 
1459  src_left -= bytes_to_copy;
1460  dst_left -= bytes_to_copy;
1461  dp += bytes_to_copy;
1462  sp += bytes_to_copy;
1463  bytes_left -= bytes_to_copy;
1464  }
1465  if (db != first)
1466  db->current_data = 0;
1467  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1468 
1469  if (is_cloned && to_free)
1470  vlib_buffer_free_one (vm, to_free);
1471  else
1472  {
1473  if (db->flags & VLIB_BUFFER_NEXT_PRESENT)
1475  db->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1476  b = first;
1477  n_buffers = 1;
1478  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1479  {
1480  b = vlib_get_buffer (vm, b->next_buffer);
1481  ++n_buffers;
1482  }
1483  }
1484 
1485  first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1486 
1487  return n_buffers;
1488 }
1489 
1490 #endif /* included_vlib_buffer_funcs_h */
1491 
1492 /*
1493  * fd.io coding-style-patch-verification: ON
1494  *
1495  * Local Variables:
1496  * eval: (c-set-style "gnu")
1497  * End:
1498  */
u32 opaque2[14]
Definition: buffer.h:170
#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK
Definition: buffer_funcs.h:958
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:427
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:310
u32 flags
Definition: vhost_user.h:141
#define clib_min(x, y)
Definition: clib.h:302
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:102
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:80
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:78
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
#define clib_atomic_add_fetch(a, b)
Definition: atomics.h:30
format_function_t format_vlib_buffer_contents
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:421
#define PREDICT_TRUE(x)
Definition: clib.h:113
static_always_inline u64x2 u32x4_extend_to_u64x2_high(u32x4 v)
Definition: vector_neon.h:125
format_function_t format_vlib_buffer
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
static_always_inline vlib_buffer_pool_t * vlib_get_buffer_pool(vlib_main_t *vm, u8 buffer_pool_index)
Definition: buffer_funcs.h:478
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:476
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
static void vlib_buffer_move(vlib_main_t *vm, vlib_buffer_t *b, i16 offset)
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
int vlib_buffer_add_data(vlib_main_t *vm, u32 *buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:421
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
vl_api_address_t src
Definition: gre.api:51
int i
vlib_buffer_main_t * buffer_main
Definition: main.h:152
foreach_avx2_vec256i static foreach_avx2_vec256u u32x8 u32x8_permute(u32x8 v, u32x8 idx)
Definition: vector_avx2.h:73
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:178
u8 data[128]
Definition: ipsec.api:251
u32 numa_node
Definition: main.h:220
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:462
static u32 vlib_buffer_alloc_to_ring_from_pool(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers into ring from specific buffer pool.
Definition: buffer_funcs.h:657
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:366
static u32 vlib_buffer_alloc_on_numa(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 numa_node)
Allocate buffers from specific numa node into supplied array.
Definition: buffer_funcs.h:595
unsigned char u8
Definition: types.h:56
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:964
u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES]
Definition: buffer.h:459
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, template_end, 64)
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
static_always_inline void vlib_get_buffer_indices(vlib_main_t *vm, vlib_buffer_t **b, u32 *bi, uword count)
Translate array of buffer pointers into buffer indices.
Definition: buffer_funcs.h:337
format_function_t format_vlib_buffer_no_chain
vlib_buffer_pool_thread_t * threads
Definition: buffer.h:436
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
#define static_always_inline
Definition: clib.h:100
#define always_inline
Definition: clib.h:99
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
void di(unformat_input_t *i)
Definition: unformat.c:163
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:378
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:257
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
epu8_epi32 epu16_epi32 u64x2
Definition: vector_sse42.h:674
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:912
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
static u32 vlib_buffer_chain_linearize(vlib_main_t *vm, vlib_buffer_t *b)
#define hash_get(h, key)
Definition: hash.h:249
static vlib_buffer_t * vlib_buffer_copy_no_chain(vlib_main_t *vm, vlib_buffer_t *b, u32 *di)
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
Definition: buffer.c:254
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
Definition: buffer_funcs.h:145
format_function_t format_vlib_buffer_and_data
unsigned short u16
Definition: types.h:57
static void vlib_buffer_validate(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:55
vlib_buffer_t buffer_template
Definition: buffer.h:439
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
static_always_inline uword vlib_buffer_pool_get(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
Definition: buffer_funcs.h:485
static_always_inline u32x4 u32x4_shuffle(u32x4 v, const int a, const int b, const int c, const int d)
Definition: vector_sse42.h:651
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:451
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
#define PREDICT_FALSE(x)
Definition: clib.h:112
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
Definition: buffer_funcs.h:276
vl_api_address_t dst
Definition: gre.api:52
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
Definition: buffer_funcs.h:400
u8 len
Definition: ip_types.api:90
static void vlib_buffer_chain_init(vlib_buffer_t *first)
#define clib_atomic_sub_fetch(a, b)
Definition: atomics.h:31
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:96
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:608
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:458
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:882
static u16 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
static_always_inline void vlib_buffer_pool_put(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
Definition: buffer_funcs.h:680
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define VLIB_BUFFER_MAX_NUMA_NODES
Definition: buffer.h:442
u8 data[]
Packet data.
Definition: buffer.h:181
clib_spinlock_t lock
Definition: buffer.h:433
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
Definition: buffer_funcs.h:350
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:338
static u32 vlib_buffer_space_left_at_end(vlib_main_t *vm, vlib_buffer_t *b)
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:248
vlib_buffer_known_state_t
Definition: buffer_funcs.h:446
static_always_inline u64x2 u32x4_extend_to_u64x2(u32x4 v)
Definition: vector_neon.h:119
signed int i32
Definition: types.h:77
static u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.
Definition: buffer_funcs.h:523
#define uword_to_pointer(u, type)
Definition: types.h:136
vlib buffer structure definition and a few select access methods.
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:784
static u16 vlib_buffer_clone_at_offset(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create multiple clones of buffer and store them in the supplied array.
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
size_t count
Definition: vapi.c:47
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
Definition: buffer_funcs.h:386
static void * vlib_buffer_ptr_from_index(uword buffer_mem_start, u32 buffer_index, uword offset)
Definition: buffer_funcs.h:71
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, int maybe_next)
Definition: buffer_funcs.h:707
static uword pointer_to_uword(const void *p)
Definition: types.h:131
template key/value backing page structure
Definition: bihash_doc.h:44
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
static u32 vlib_buffer_alloc_to_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Allocate buffers into ring.
Definition: buffer_funcs.h:628
static_always_inline void vlib_buffer_copy_indices(u32 *dst, u32 *src, u32 n_indices)
Definition: buffer_funcs.h:102
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *last, u32 next_bi)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:489
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:449
uword * buffer_known_hash
Definition: buffer.h:457
u32 default_data_size
Definition: buffer.h:464
struct clib_bihash_value offset
template key/value backing page structure
u64x4
Definition: vector_avx2.h:121
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:400
static u16 vlib_buffer_clone_256(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create a maximum of 256 clones of buffer and store them in the supplied array.
unsigned long long u32x4
Definition: ixge.c:28
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:898
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
Definition: buffer.c:378
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:612
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
Definition: buffer_funcs.h:163
static void vlib_buffer_free_from_ring_no_next(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring without freeing tail buffers.
Definition: buffer_funcs.h:937
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:153
signed short i16
Definition: types.h:46