FD.io VPP  v18.01.2-1-g9b554f3
Vector Packet Processing
buffer_funcs.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_funcs.h: VLIB buffer related functions/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
42 
43 #include <vppinfra/hash.h>
44 
45 /** \file
46  vlib buffer access methods.
47 */
48 
49 
50 /** \brief Translate buffer index into buffer pointer
51 
52  @param vm - (vlib_main_t *) vlib main data structure pointer
53  @param buffer_index - (u32) buffer index
54  @return - (vlib_buffer_t *) buffer pointer
55 */
57 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
58 {
60  uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
61  ASSERT (offset < bm->buffer_mem_size);
62 
63  return uword_to_pointer (bm->buffer_mem_start + offset, void *);
64 }
65 
66 /** \brief Translate buffer pointer into buffer index
67 
68  @param vm - (vlib_main_t *) vlib main data structure pointer
69  @param p - (void *) buffer pointer
70  @return - (u32) buffer index
71 */
72 
75 {
79  ASSERT (offset < bm->buffer_mem_size);
80  ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
81  return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
82 }
83 
84 /** \brief Get next buffer in buffer linklist, or zero for end of list.
85 
86  @param vm - (vlib_main_t *) vlib main data structure pointer
87  @param b - (void *) buffer pointer
88  @return - (vlib_buffer_t *) next buffer, or NULL
89 */
92 {
93  return (b->flags & VLIB_BUFFER_NEXT_PRESENT
94  ? vlib_get_buffer (vm, b->next_buffer) : 0);
95 }
96 
98  vlib_buffer_t * b_first);
99 
100 /** \brief Get length in bytes of the buffer chain
101 
102  @param vm - (vlib_main_t *) vlib main data structure pointer
103  @param b - (void *) buffer pointer
104  @return - (uword) length of buffer chain
105 */
108 {
109  uword len = b->current_length;
110 
111  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
112  return len;
113 
116 
118 }
119 
120 /** \brief Get length in bytes of the buffer index buffer chain
121 
122  @param vm - (vlib_main_t *) vlib main data structure pointer
123  @param bi - (u32) buffer index
124  @return - (uword) length of buffer chain
125 */
128 {
129  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
130  return vlib_buffer_length_in_chain (vm, b);
131 }
132 
133 /** \brief Copy buffer contents to memory
134 
135  @param vm - (vlib_main_t *) vlib main data structure pointer
136  @param buffer_index - (u32) buffer index
137  @param contents - (u8 *) memory, <strong>must be large enough</strong>
138  @return - (uword) length of buffer chain
139 */
141 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
142 {
143  uword content_len = 0;
144  uword l;
145  vlib_buffer_t *b;
146 
147  while (1)
148  {
149  b = vlib_get_buffer (vm, buffer_index);
150  l = b->current_length;
151  clib_memcpy (contents + content_len, b->data + b->current_data, l);
152  content_len += l;
153  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
154  break;
155  buffer_index = b->next_buffer;
156  }
157 
158  return content_len;
159 }
160 
161 /* Return physical address of buffer->data start. */
164 {
166  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index);
168  b->buffer_pool_index);
169 
171 }
172 
173 /** \brief Prefetch buffer metadata by buffer index
174  The first 64 bytes of buffer contains most header information
175 
176  @param vm - (vlib_main_t *) vlib main data structure pointer
177  @param bi - (u32) buffer index
178  @param type - LOAD, STORE. In most cases, STORE is the right answer
179 */
180 /* Prefetch buffer header given index. */
181 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
182  do { \
183  vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
184  vlib_prefetch_buffer_header (_b, type); \
185  } while (0)
186 
187 #if 0
188 /* Iterate over known allocated vlib bufs. You probably do not want
189  * to do this!
190  @param vm the vlib_main_t
191  @param bi found allocated buffer index
192  @param body operation to perform on buffer index
193  function executes body for each allocated buffer index
194  */
195 #define vlib_buffer_foreach_allocated(vm,bi,body) \
196 do { \
197  vlib_main_t * _vmain = (vm); \
198  vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
199  hash_pair_t * _vbpair; \
200  hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
201  if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
202  (bi) = _vbpair->key; \
203  body; \
204  } \
205  })); \
206 } while (0)
207 #endif
208 
209 typedef enum
210 {
211  /* Index is unknown. */
213 
214  /* Index is known and free/allocated. */
218 
219 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
220  uword n_buffers,
222  expected_state);
223 
225 vlib_buffer_is_known (u32 buffer_index)
226 {
228 
230  uword *p = hash_get (bm->buffer_known_hash, buffer_index);
232  return p ? p[0] : VLIB_BUFFER_UNKNOWN;
233 }
234 
235 always_inline void
238 {
240 
242  hash_set (bm->buffer_known_hash, buffer_index, state);
244 }
245 
246 /* Validates sanity of a single buffer.
247  Returns format'ed vector with error message if any. */
248 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
249  uword follow_chain);
250 
253 {
254  return round_pow2 (size, sizeof (vlib_buffer_t));
255 }
256 
259 {
261 }
262 
263 always_inline void
265 {
266  /* if there is an need for more free lists we should consider
267  storig data in the 2nd cacheline */
270 
273 }
274 
275 /** \brief Allocate buffers from specific freelist into supplied array
276 
277  @param vm - (vlib_main_t *) vlib main data structure pointer
278  @param buffers - (u32 * ) buffer index array
279  @param n_buffers - (u32) number of buffers requested
280  @return - (u32) number of buffers actually allocated, may be
281  less than the number requested or zero
282 */
285  u32 * buffers,
286  u32 n_buffers, u32 free_list_index)
287 {
290  u32 *src;
291  uword len;
292 
294 
295  fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
296 
297  len = vec_len (fl->buffers);
298 
299  if (PREDICT_FALSE (len < n_buffers))
300  {
301  bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
302  len = vec_len (fl->buffers);
303 
304  /* even if fill free list didn't manage to refill free list
305  we should give what we have */
306  n_buffers = clib_min (len, n_buffers);
307 
308  /* following code is intentionaly duplicated to allow compiler
309  to optimize fast path when n_buffers is constant value */
310  src = fl->buffers + len - n_buffers;
311  clib_memcpy (buffers, src, n_buffers * sizeof (u32));
312  _vec_len (fl->buffers) -= n_buffers;
313 
314  /* Verify that buffers are known free. */
315  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
317 
318  return n_buffers;
319  }
320 
321  src = fl->buffers + len - n_buffers;
322  clib_memcpy (buffers, src, n_buffers * sizeof (u32));
323  _vec_len (fl->buffers) -= n_buffers;
324 
325  /* Verify that buffers are known free. */
326  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
328 
329  return n_buffers;
330 }
331 
332 /** \brief Allocate buffers into supplied array
333 
334  @param vm - (vlib_main_t *) vlib main data structure pointer
335  @param buffers - (u32 * ) buffer index array
336  @param n_buffers - (u32) number of buffers requested
337  @return - (u32) number of buffers actually allocated, may be
338  less than the number requested or zero
339 */
341 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
342 {
343  return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
345 }
346 
347 /** \brief Free buffers
348  Frees the entire buffer chain for each buffer
349 
350  @param vm - (vlib_main_t *) vlib main data structure pointer
351  @param buffers - (u32 * ) buffer index array
352  @param n_buffers - (u32) number of buffers to free
353 
354 */
355 always_inline void
357  /* pointer to first buffer */
358  u32 * buffers,
359  /* number of buffers to free */
360  u32 n_buffers)
361 {
363 
365 
366  return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers);
367 }
368 
369 /** \brief Free buffers, does not free the buffer chain for each buffer
370 
371  @param vm - (vlib_main_t *) vlib main data structure pointer
372  @param buffers - (u32 * ) buffer index array
373  @param n_buffers - (u32) number of buffers to free
374 
375 */
376 always_inline void
378  /* pointer to first buffer */
379  u32 * buffers,
380  /* number of buffers to free */
381  u32 n_buffers)
382 {
384 
386 
387  return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers);
388 }
389 
390 /** \brief Free one buffer
391  Shorthand to free a single buffer chain.
392 
393  @param vm - (vlib_main_t *) vlib main data structure pointer
394  @param buffer_index - (u32) buffer index to free
395 */
396 always_inline void
398 {
399  vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1);
400 }
401 
402 /* Add/delete buffer free lists. */
403 u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
404  char *fmt, ...);
405 always_inline void
407 {
409 
411 
412  bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index);
413 }
414 
415 /* Find already existing public free list with given size or create one. */
417  char *fmt, ...);
418 
419 /* Merge two free lists */
422 
423 /* Make sure we have at least given number of unaligned buffers. */
426  free_list,
427  uword n_unaligned_buffers);
428 
431 {
433 
434  size = vlib_buffer_round_size (size);
435  uword *p = hash_get (bm->free_list_by_size, size);
436  return p ? p[0] : ~0;
437 }
438 
441  u32 * index)
442 {
444  u32 i;
445 
446  *index = i = vlib_buffer_get_free_list_index (b);
448 }
449 
452 {
455 
456  f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
457 
458  /* Sanity: indices must match. */
459  ASSERT (f->index == free_list_index);
460 
461  return f;
462 }
463 
466 {
468  vlib_buffer_get_free_list (vm, free_list_index);
469  return f->n_data_bytes;
470 }
471 
472 void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes);
473 
474 /* Reasonably fast buffer copy routine. */
475 always_inline void
476 vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
477 {
478  while (n >= 4)
479  {
480  dst[0] = src[0];
481  dst[1] = src[1];
482  dst[2] = src[2];
483  dst[3] = src[3];
484  dst += 4;
485  src += 4;
486  n -= 4;
487  }
488  while (n > 0)
489  {
490  dst[0] = src[0];
491  dst += 1;
492  src += 1;
493  n -= 1;
494  }
495 }
496 
497 /* Append given data to end of buffer, possibly allocating new buffers. */
499  u32 free_list_index,
500  u32 buffer_index, void *data, u32 n_data_bytes);
501 
502 /* duplicate all buffers in chain */
505 {
506  vlib_buffer_t *s, *d, *fd;
507  uword n_alloc, n_buffers = 1;
509  int i;
510 
511  s = b;
512  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
513  {
514  n_buffers++;
515  s = vlib_get_buffer (vm, s->next_buffer);
516  }
517  u32 new_buffers[n_buffers];
518 
519  n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
520 
521  /* No guarantee that we'll get all the buffers we asked for */
522  if (PREDICT_FALSE (n_alloc < n_buffers))
523  {
524  if (n_alloc > 0)
525  vlib_buffer_free (vm, new_buffers, n_alloc);
526  return 0;
527  }
528 
529  /* 1st segment */
530  s = b;
531  fd = d = vlib_get_buffer (vm, new_buffers[0]);
532  d->current_data = s->current_data;
534  d->flags = s->flags & flag_mask;
537  clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
540 
541  /* next segments */
542  for (i = 1; i < n_buffers; i++)
543  {
544  /* previous */
545  d->next_buffer = new_buffers[i];
546  /* current */
547  s = vlib_get_buffer (vm, s->next_buffer);
548  d = vlib_get_buffer (vm, new_buffers[i]);
549  d->current_data = s->current_data;
553  d->flags = s->flags & flag_mask;
554  }
555 
556  return fd;
557 }
558 
559 /** \brief Create multiple clones of buffer and store them in the supplied array
560 
561  @param vm - (vlib_main_t *) vlib main data structure pointer
562  @param src_buffer - (u32) source buffer index
563  @param buffers - (u32 * ) buffer index array
564  @param n_buffers - (u8) number of buffer clones requested
565  @param head_end_offset - (u16) offset relative to current position
566  where packet head ends
567  @return - (u8) number of buffers actually cloned, may be
568  less than the number requested or zero
569 */
570 
572 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
573  u8 n_buffers, u16 head_end_offset)
574 {
575  u8 i;
576  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
577 
578  ASSERT (s->n_add_refs == 0);
579  ASSERT (n_buffers);
580 
581  if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
582  {
583  buffers[0] = src_buffer;
584  for (i = 1; i < n_buffers; i++)
585  {
586  vlib_buffer_t *d;
587  d = vlib_buffer_copy (vm, s);
588  if (d == 0)
589  return i;
590  buffers[i] = vlib_get_buffer_index (vm, d);
591 
592  }
593  return n_buffers;
594  }
595 
596  n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
598  (s));
599  if (PREDICT_FALSE (n_buffers == 0))
600  {
601  buffers[0] = src_buffer;
602  return 1;
603  }
604 
605  for (i = 0; i < n_buffers; i++)
606  {
607  vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
608  d->current_data = s->current_data;
609  d->current_length = head_end_offset;
614  head_end_offset;
617  clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
619  head_end_offset);
620  d->next_buffer = src_buffer;
621  }
622  vlib_buffer_advance (s, head_end_offset);
623  s->n_add_refs = n_buffers - 1;
624  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
625  {
626  s = vlib_get_buffer (vm, s->next_buffer);
627  s->n_add_refs = n_buffers - 1;
628  }
629 
630  return n_buffers;
631 }
632 
633 /** \brief Attach cloned tail to the buffer
634 
635  @param vm - (vlib_main_t *) vlib main data structure pointer
636  @param head - (vlib_buffer_t *) head buffer
637  @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
638 */
639 
640 always_inline void
642  vlib_buffer_t * tail)
643 {
644  ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
647 
651  head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
652  head->next_buffer = vlib_get_buffer_index (vm, tail);
655 
656 next_segment:
657  __sync_add_and_fetch (&tail->n_add_refs, 1);
658 
659  if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
660  {
661  tail = vlib_get_buffer (vm, tail->next_buffer);
662  goto next_segment;
663  }
664 }
665 
666 /* Initializes the buffer as an empty packet with no chained buffers. */
667 always_inline void
669 {
671  first->current_length = 0;
672  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
674 }
675 
676 /* The provided next_bi buffer index is appended to the end of the packet. */
679  vlib_buffer_t * first,
680  vlib_buffer_t * last, u32 next_bi)
681 {
682  vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
683  last->next_buffer = next_bi;
685  next_buffer->current_length = 0;
686  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
687  return next_buffer;
688 }
689 
690 /* Increases or decreases the packet length.
691  * It does not allocate or deallocate new buffers.
692  * Therefore, the added length must be compatible
693  * with the last buffer. */
694 always_inline void
696  vlib_buffer_t * last, i32 len)
697 {
698  last->current_length += len;
699  if (first != last)
701 }
702 
703 /* Copy data to the end of the packet and increases its length.
704  * It does not allocate new buffers.
705  * Returns the number of copied bytes. */
708  u32 free_list_index,
709  vlib_buffer_t * first,
710  vlib_buffer_t * last, void *data, u16 data_len)
711 {
712  u32 n_buffer_bytes =
713  vlib_buffer_free_list_buffer_size (vm, free_list_index);
714  ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
715  u16 len = clib_min (data_len,
716  n_buffer_bytes - last->current_length -
717  last->current_data);
718  clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
719  len);
720  vlib_buffer_chain_increase_length (first, last, len);
721  return len;
722 }
723 
724 /* Copy data to the end of the packet and increases its length.
725  * Allocates additional buffers from the free list if necessary.
726  * Returns the number of copied bytes.
727  * 'last' value is modified whenever new buffers are allocated and
728  * chained and points to the last buffer in the chain. */
729 u16
731  u32 free_list_index,
732  vlib_buffer_t * first,
733  vlib_buffer_t ** last,
734  void *data, u16 data_len);
736 
739 
740 typedef struct
741 {
742  /* Vector of packet data. */
744 
745  /* Number of buffers to allocate in each call to physmem
746  allocator. */
748 
749  /* Buffer free list for this template. */
751 
754 
757 
760  void *packet_data,
761  uword n_packet_data_bytes,
762  uword min_n_buffers_each_physmem_alloc,
763  char *fmt, ...);
764 
767  u32 * bi_result);
768 
769 always_inline void
771 {
772  vec_free (t->packet_data);
773 }
774 
777 {
778  serialize_stream_t *s = &m->stream;
782  vlib_main_t *vm = sm->vlib_main;
783  u32 n, *f;
784 
786  if (sm->last_buffer != ~0)
787  {
789  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
790  {
791  b = vlib_get_buffer (vm, b->next_buffer);
792  n += b->current_length;
793  }
794  }
795 
796  /* *INDENT-OFF* */
797  clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
798  n += vlib_buffer_index_length_in_chain (vm, f[0]);
799  }));
800 /* *INDENT-ON* */
801 
802  return n;
803 }
804 
805 /* Set a buffer quickly into "uninitialized" state. We want this to
806  be extremely cheap and arrange for all fields that need to be
807  initialized to be in the first 128 bits of the buffer. */
808 always_inline void
811 {
813 
814  /* Make sure vlib_buffer_t is cacheline aligned and sized */
815  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
816  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) ==
818  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) ==
820 
821  /* Make sure buffer template is sane. */
823 
824  clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
825  STRUCT_MARK_PTR (src, template_start),
826  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
827  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
828 
829  /* Not in the first 16 octets. */
830  dst->n_add_refs = src->n_add_refs;
831 
832  /* Make sure it really worked. */
833 #define _(f) ASSERT (dst->f == src->f);
834  _(current_data);
835  _(current_length);
836  _(flags);
837 #undef _
838  /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
839  /* total_length_not_including_first_buffer is not in the template anymore
840  * so it may actually not zeroed for some buffers. One option is to
841  * uncomment the line lower (comes at a cost), the other, is to just not
842  * care */
843  /* dst->total_length_not_including_first_buffer = 0; */
844  ASSERT (dst->n_add_refs == 0);
845 }
846 
847 always_inline void
850  u32 buffer_index, u8 do_init)
851 {
852  vlib_buffer_t *b;
853  b = vlib_get_buffer (vm, buffer_index);
854  if (PREDICT_TRUE (do_init))
856  vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
857 
858  if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE)
859  {
863  /* keep last stored buffers, as they are more likely hot in the cache */
867  f->n_alloc -= VLIB_FRAME_SIZE;
869  }
870 }
871 
872 always_inline void
874  vlib_buffer_t * dst1,
876 {
878 
879  /* Make sure buffer template is sane. */
881 
882  clib_memcpy (STRUCT_MARK_PTR (dst0, template_start),
883  STRUCT_MARK_PTR (src, template_start),
884  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
885  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
886 
887  clib_memcpy (STRUCT_MARK_PTR (dst1, template_start),
888  STRUCT_MARK_PTR (src, template_start),
889  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
890  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
891 
892  /* Not in the first 16 octets. */
893  dst0->n_add_refs = src->n_add_refs;
894  dst1->n_add_refs = src->n_add_refs;
895 
896  /* Make sure it really worked. */
897 #define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f)
898  _(current_data);
899  _(current_length);
900  _(flags);
901 #undef _
902 
905  ASSERT (dst0->n_add_refs == 0);
906  ASSERT (dst1->n_add_refs == 0);
907 }
908 
909 #if CLIB_DEBUG > 0
912 extern void *vlib_buffer_state_heap;
913 #endif
914 
915 static inline void
917 {
918 #if CLIB_DEBUG > 0
919  uword *p;
920  void *oldheap;
921 
923 
924  while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
925  ;
926 
928 
929  /* If we don't know about b, declare it to be in the expected state */
930  if (!p)
931  {
933  goto out;
934  }
935 
936  if (p[0] != expected)
937  {
938  void cj_stop (void);
939  u32 bi;
941 
942  cj_stop ();
943 
944  bi = vlib_get_buffer_index (vm, b);
945 
946  clib_mem_set_heap (oldheap);
947  clib_warning ("%.6f buffer %llx (%d): %s, not %s",
948  vlib_time_now (vm), bi,
949  p[0] ? "busy" : "free", expected ? "busy" : "free");
950  os_panic ();
951  }
952 out:
955  clib_mem_set_heap (oldheap);
956 #endif
957 }
958 
959 static inline void
961 {
962 #if CLIB_DEBUG > 0
963  void *oldheap;
964 
966 
967  while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
968  ;
969 
971 
974  clib_mem_set_heap (oldheap);
975 #endif
976 }
977 
978 #endif /* included_vlib_buffer_funcs_h */
979 
980 /*
981  * fd.io coding-style-patch-verification: ON
982  *
983  * Local Variables:
984  * eval: (c-set-style "gnu")
985  * End:
986  */
vlib_physmem_region_index_t physmem_region
Definition: buffer.h:413
vlib_main_t vlib_global_main
Definition: main.c:1637
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define clib_min(x, y)
Definition: clib.h:340
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:72
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:50
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:438
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:53
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:356
#define STRUCT_MARK_PTR(v, f)
Definition: clib.h:68
format_function_t format_vlib_buffer_contents
Definition: buffer_funcs.h:737
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:132
#define PREDICT_TRUE(x)
Definition: clib.h:106
format_function_t format_vlib_buffer
Definition: buffer_funcs.h:737
vlib_buffer_callbacks_t cb
Definition: buffer.h:452
void vlib_buffer_merge_free_lists(vlib_buffer_free_list_t *dst, vlib_buffer_free_list_t *src)
Definition: buffer.c:336
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
Definition: buffer_funcs.h:641
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, u32 free_list_index, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
Definition: buffer_funcs.h:707
static void vlib_validate_buffer_in_use(vlib_buffer_t *b, u32 expected)
Definition: buffer_funcs.h:916
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:678
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:224
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:695
vlib_buffer_t buffer_init_template
Definition: buffer.h:337
void os_panic(void)
Definition: unix-misc.c:174
struct vlib_main_t * vlib_main
Definition: buffer.h:463
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:49
clib_spinlock_t global_buffers_lock
Definition: buffer.h:359
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:126
vlib_buffer_main_t * buffer_main
Definition: main.h:106
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:62
u8 *( format_function_t)(u8 *s, va_list *args)
Definition: format.h:48
static u8 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u8 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
Definition: buffer_funcs.h:572
uword * vlib_buffer_state_validation_hash
Definition: buffer.c:586
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:107
static u64 vlib_get_buffer_data_physical_address(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:163
vlib_main_t ** vlib_mains
Definition: buffer.c:292
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:504
struct vlib_serialize_buffer_main_t::@27::@30 rx
void cj_stop(void)
Definition: cj.c:58
static void vlib_buffer_delete_free_list(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:406
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, char *fmt,...)
Definition: buffer.c:732
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:528
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:95
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:68
#define always_inline
Definition: clib.h:92
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
Definition: buffer.h:97
int i32
Definition: types.h:81
void vlib_aligned_memcpy(void *_dst, void *_src, int n_bytes)
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:395
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u32 * vlib_buffer_state_validation_lock
Definition: buffer.c:585
unsigned long u64
Definition: types.h:89
static void vlib_copy_buffers(u32 *dst, u32 *src, u32 n)
Definition: buffer_funcs.h:476
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:74
#define fl(x, y)
#define hash_get(h, key)
Definition: hash.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:459
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:72
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
Definition: buffer.c:214
static u32 vlib_buffer_get_free_list_with_size(vlib_main_t *vm, u32 size)
Definition: buffer_funcs.h:430
#define VLIB_BUFFER_FREE_LIST_INDEX_MASK
Definition: buffer.h:91
format_function_t format_vlib_buffer_and_data
Definition: buffer_funcs.h:737
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:195
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:434
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, u32 free_list_index)
Definition: buffer.h:402
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:423
#define PREDICT_FALSE(x)
Definition: clib.h:105
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:465
#define VLIB_FRAME_SIZE
Definition: node.h:328
#define clib_fifo_foreach(v, f, body)
Definition: fifo.h:279
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
Definition: buffer_funcs.h:848
#define uword_to_pointer(u, type)
Definition: types.h:136
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
Definition: buffer_funcs.h:141
static void vlib_buffer_set_known_state(u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:236
static void vlib_buffer_chain_init(vlib_buffer_t *first)
Definition: buffer_funcs.h:668
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:605
static u32 vlib_buffer_get_free_list_index(vlib_buffer_t *b)
Definition: buffer_funcs.h:258
serialize_stream_t stream
Definition: serialize.h:147
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:446
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:393
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:377
u32 current_buffer_index
Definition: serialize.h:62
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:226
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
Definition: buffer_funcs.h:91
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:284
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:298
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:286
vlib_buffer_known_state_t
Definition: buffer_funcs.h:209
uword data_function_opaque
Definition: serialize.h:74
static void vlib_validate_buffer_set_in_use(vlib_buffer_t *b, u32 expected)
Definition: buffer_funcs.h:960
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:781
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:394
void vlib_buffer_free_list_fill_unaligned(vlib_main_t *vm, vlib_buffer_free_list_t *free_list, uword n_unaligned_buffers)
vhost_vring_state_t state
Definition: vhost-user.h:82
u64 size
Definition: vhost-user.h:76
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, u32 *index)
Definition: buffer_funcs.h:440
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, u32 free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:877
u32 vlib_buffer_add_data(vlib_main_t *vm, u32 free_list_index, u32 buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:820
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:208
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
Definition: buffer_funcs.h:127
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:124
void * vlib_buffer_state_heap
Definition: buffer.c:587
u64 uword
Definition: types.h:112
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:142
template key/value backing page structure
Definition: bihash_doc.h:44
unsigned short u16
Definition: types.h:57
static void vlib_buffer_set_free_list_index(vlib_buffer_t *b, u32 index)
Definition: buffer_funcs.h:264
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: buffer.h:433
static u64 vlib_physmem_virtual_to_physical(vlib_main_t *vm, vlib_physmem_region_index_t idx, void *mem)
static void vlib_buffer_init_two_for_free_list(vlib_buffer_t *dst0, vlib_buffer_t *dst1, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:873
uword buffer_mem_start
Definition: buffer.h:421
uword * buffer_known_hash
Definition: buffer.h:445
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:782
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:809
#define VLIB_BUFFER_EXT_HDR_VALID
Definition: buffer.h:101
static vlib_buffer_known_state_t vlib_buffer_is_known(u32 buffer_index)
Definition: buffer_funcs.h:225
u8 data[0]
Packet data.
Definition: buffer.h:159
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:109
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer_funcs.h:770
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:397
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer.c:802
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:451
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:75
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:341
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
uword * free_list_by_size
Definition: buffer.h:439
u32 vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:421
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:252
static u32 unserialize_vlib_buffer_n_bytes(serialize_main_t *m)
Definition: buffer_funcs.h:776