FD.io VPP  v17.07.01-10-g3be13f0
Vector Packet Processing
buffer_funcs.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_funcs.h: VLIB buffer related functions/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
42 
43 #include <vppinfra/hash.h>
44 
45 /** \file
46  vlib buffer access methods.
47 */
48 
49 
50 /** \brief Translate buffer index into buffer pointer
51 
52  @param vm - (vlib_main_t *) vlib main data structure pointer
53  @param buffer_index - (u32) buffer index
54  @return - (vlib_buffer_t *) buffer pointer
55 */
57 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
58 {
59  return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
61 }
62 
63 /** \brief Translate buffer pointer into buffer index
64 
65  @param vm - (vlib_main_t *) vlib main data structure pointer
66  @param p - (void *) buffer pointer
67  @return - (u32) buffer index
68 */
71 {
73  ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
74  return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
75 }
76 
77 /** \brief Get next buffer in buffer linklist, or zero for end of list.
78 
79  @param vm - (vlib_main_t *) vlib main data structure pointer
80  @param b - (void *) buffer pointer
81  @return - (vlib_buffer_t *) next buffer, or NULL
82 */
85 {
86  return (b->flags & VLIB_BUFFER_NEXT_PRESENT
87  ? vlib_get_buffer (vm, b->next_buffer) : 0);
88 }
89 
91  vlib_buffer_t * b_first);
92 
93 /** \brief Get length in bytes of the buffer chain
94 
95  @param vm - (vlib_main_t *) vlib main data structure pointer
96  @param b - (void *) buffer pointer
97  @return - (uword) length of buffer chain
98 */
101 {
107  return l;
108 }
109 
110 /** \brief Get length in bytes of the buffer index buffer chain
111 
112  @param vm - (vlib_main_t *) vlib main data structure pointer
113  @param bi - (u32) buffer index
114  @return - (uword) length of buffer chain
115 */
118 {
119  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
120  return vlib_buffer_length_in_chain (vm, b);
121 }
122 
123 /** \brief Copy buffer contents to memory
124 
125  @param vm - (vlib_main_t *) vlib main data structure pointer
126  @param buffer_index - (u32) buffer index
127  @param contents - (u8 *) memory, <strong>must be large enough</strong>
128  @return - (uword) length of buffer chain
129 */
131 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
132 {
133  uword content_len = 0;
134  uword l;
135  vlib_buffer_t *b;
136 
137  while (1)
138  {
139  b = vlib_get_buffer (vm, buffer_index);
140  l = b->current_length;
141  clib_memcpy (contents + content_len, b->data + b->current_data, l);
142  content_len += l;
143  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
144  break;
145  buffer_index = b->next_buffer;
146  }
147 
148  return content_len;
149 }
150 
151 /* Return physical address of buffer->data start. */
154 {
156  (((uword) buffer_index) <<
159  data));
160 }
161 
162 /** \brief Prefetch buffer metadata by buffer index
163  The first 64 bytes of buffer contains most header information
164 
165  @param vm - (vlib_main_t *) vlib main data structure pointer
166  @param bi - (u32) buffer index
167  @param type - LOAD, STORE. In most cases, STORE is the right answer
168 */
169 /* Prefetch buffer header given index. */
170 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
171  do { \
172  vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
173  vlib_prefetch_buffer_header (_b, type); \
174  } while (0)
175 
176 #if 0
177 /* Iterate over known allocated vlib bufs. You probably do not want
178  * to do this!
179  @param vm the vlib_main_t
180  @param bi found allocated buffer index
181  @param body operation to perform on buffer index
182  function executes body for each allocated buffer index
183  */
184 #define vlib_buffer_foreach_allocated(vm,bi,body) \
185 do { \
186  vlib_main_t * _vmain = (vm); \
187  vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
188  hash_pair_t * _vbpair; \
189  hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
190  if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
191  (bi) = _vbpair->key; \
192  body; \
193  } \
194  })); \
195 } while (0)
196 #endif
197 
198 typedef enum
199 {
200  /* Index is unknown. */
202 
203  /* Index is known and free/allocated. */
207 
210 {
212  ASSERT (vlib_get_thread_index () == 0);
213 
214  uword *p = hash_get (bm->buffer_known_hash, buffer_index);
215  return p ? p[0] : VLIB_BUFFER_UNKNOWN;
216 }
217 
218 always_inline void
220  u32 buffer_index,
222 {
224  ASSERT (vlib_get_thread_index () == 0);
225  hash_set (bm->buffer_known_hash, buffer_index, state);
226 }
227 
228 /* Validates sanity of a single buffer.
229  Returns format'ed vector with error message if any. */
230 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
231  uword follow_chain);
232 
233 clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
234  unsigned socket_id);
235 
236 /** \brief Allocate buffers into supplied array
237 
238  @param vm - (vlib_main_t *) vlib main data structure pointer
239  @param buffers - (u32 * ) buffer index array
240  @param n_buffers - (u32) number of buffers requested
241  @return - (u32) number of buffers actually allocated, may be
242  less than the number requested or zero
243 */
245 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
246 {
248 
250 
251  return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
252 }
253 
256 {
257  return round_pow2 (size, sizeof (vlib_buffer_t));
258 }
259 
260 /** \brief Allocate buffers from specific freelist into supplied array
261 
262  @param vm - (vlib_main_t *) vlib main data structure pointer
263  @param buffers - (u32 * ) buffer index array
264  @param n_buffers - (u32) number of buffers requested
265  @return - (u32) number of buffers actually allocated, may be
266  less than the number requested or zero
267 */
270  u32 * buffers,
271  u32 n_buffers, u32 free_list_index)
272 {
274 
276 
277  return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
278  free_list_index);
279 }
280 
281 /** \brief Free buffers
282  Frees the entire buffer chain for each buffer
283 
284  @param vm - (vlib_main_t *) vlib main data structure pointer
285  @param buffers - (u32 * ) buffer index array
286  @param n_buffers - (u32) number of buffers to free
287 
288 */
289 always_inline void
291  /* pointer to first buffer */
292  u32 * buffers,
293  /* number of buffers to free */
294  u32 n_buffers)
295 {
297 
299 
300  return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers);
301 }
302 
303 /** \brief Free buffers, does not free the buffer chain for each buffer
304 
305  @param vm - (vlib_main_t *) vlib main data structure pointer
306  @param buffers - (u32 * ) buffer index array
307  @param n_buffers - (u32) number of buffers to free
308 
309 */
310 always_inline void
312  /* pointer to first buffer */
313  u32 * buffers,
314  /* number of buffers to free */
315  u32 n_buffers)
316 {
318 
320 
321  return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers);
322 }
323 
324 /** \brief Free one buffer
325  Shorthand to free a single buffer chain.
326 
327  @param vm - (vlib_main_t *) vlib main data structure pointer
328  @param buffer_index - (u32) buffer index to free
329 */
330 always_inline void
332 {
333  vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1);
334 }
335 
336 /* Add/delete buffer free lists. */
337 u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
338  char *fmt, ...);
339 always_inline void
341 {
343 
345 
346  bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index);
347 }
348 
349 /* Find already existing public free list with given size or create one. */
351  char *fmt, ...);
352 
353 /* Merge two free lists */
356 
357 /* Make sure we have at least given number of unaligned buffers. */
360  free_list,
361  uword n_unaligned_buffers);
362 
365 {
367 
368  size = vlib_buffer_round_size (size);
369  uword *p = hash_get (bm->free_list_by_size, size);
370  return p ? p[0] : ~0;
371 }
372 
375  u32 * index)
376 {
378  u32 i;
379 
380  *index = i = b->free_list_index;
382 }
383 
386 {
389 
390  f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
391 
392  /* Sanity: indices must match. */
393  ASSERT (f->index == free_list_index);
394 
395  return f;
396 }
397 
400 {
402  vlib_buffer_get_free_list (vm, free_list_index);
403  return f->n_data_bytes;
404 }
405 
406 void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes);
407 
408 /* Reasonably fast buffer copy routine. */
409 always_inline void
410 vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
411 {
412  while (n >= 4)
413  {
414  dst[0] = src[0];
415  dst[1] = src[1];
416  dst[2] = src[2];
417  dst[3] = src[3];
418  dst += 4;
419  src += 4;
420  n -= 4;
421  }
422  while (n > 0)
423  {
424  dst[0] = src[0];
425  dst += 1;
426  src += 1;
427  n -= 1;
428  }
429 }
430 
431 always_inline void *
433  uword n_bytes, uword alignment)
434 {
435  void *r =
436  vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment);
437  if (!r)
438  *error =
439  clib_error_return (0, "failed to allocate %wd bytes of I/O memory",
440  n_bytes);
441  else
442  *error = 0;
443  return r;
444 }
445 
446 /* By default allocate I/O memory with cache line alignment. */
447 always_inline void *
449 {
450  return vlib_physmem_alloc_aligned (vm, error, n_bytes,
452 }
453 
454 always_inline void
456 {
457  return vm->os_physmem_free (mem);
458 }
459 
462 {
464  uword o = pointer_to_uword (mem) - pm->virtual.start;
465  return vlib_physmem_offset_to_physical (pm, o);
466 }
467 
468 /* Append given data to end of buffer, possibly allocating new buffers. */
470  u32 free_list_index,
471  u32 buffer_index, void *data, u32 n_data_bytes);
472 
473 /* duplicate all buffers in chain */
476 {
477  vlib_buffer_t *s, *d, *fd;
478  uword n_alloc, n_buffers = 1;
480  int i;
481 
482  s = b;
483  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
484  {
485  n_buffers++;
486  s = vlib_get_buffer (vm, s->next_buffer);
487  }
488  u32 new_buffers[n_buffers];
489 
490  n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
491 
492  /* No guarantee that we'll get all the buffers we asked for */
493  if (PREDICT_FALSE (n_alloc < n_buffers))
494  {
495  if (n_alloc > 0)
496  vlib_buffer_free (vm, new_buffers, n_alloc);
497  return 0;
498  }
499 
500  /* 1st segment */
501  s = b;
502  fd = d = vlib_get_buffer (vm, new_buffers[0]);
503  d->current_data = s->current_data;
505  d->flags = s->flags & flag_mask;
508  clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
511 
512  /* next segments */
513  for (i = 1; i < n_buffers; i++)
514  {
515  /* previous */
516  d->next_buffer = new_buffers[i];
517  /* current */
518  s = vlib_get_buffer (vm, s->next_buffer);
519  d = vlib_get_buffer (vm, new_buffers[i]);
520  d->current_data = s->current_data;
524  d->flags = s->flags & flag_mask;
525  }
526 
527  return fd;
528 }
529 
530 /** \brief Create multiple clones of buffer and store them in the supplied array
531 
532  @param vm - (vlib_main_t *) vlib main data structure pointer
533  @param src_buffer - (u32) source buffer index
534  @param buffers - (u32 * ) buffer index array
535  @param n_buffers - (u8) number of buffer clones requested
536  @param head_end_offset - (u16) offset relative to current position
537  where packet head ends
538  @return - (u8) number of buffers actually cloned, may be
539  less than the number requested or zero
540 */
541 
543 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
544  u8 n_buffers, u16 head_end_offset)
545 {
546  u8 i;
547  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
548 
549  ASSERT (s->n_add_refs == 0);
550  ASSERT (n_buffers);
551 
552  if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
553  {
554  buffers[0] = src_buffer;
555  for (i = 1; i < n_buffers; i++)
556  {
557  vlib_buffer_t *d;
558  d = vlib_buffer_copy (vm, s);
559  if (d == 0)
560  return i;
561  buffers[i] = vlib_get_buffer_index (vm, d);
562 
563  }
564  return n_buffers;
565  }
566 
567  n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
568  s->free_list_index);
569  if (PREDICT_FALSE (n_buffers == 0))
570  {
571  buffers[0] = src_buffer;
572  return 1;
573  }
574 
575  for (i = 0; i < n_buffers; i++)
576  {
577  vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
578  d->current_data = s->current_data;
579  d->current_length = head_end_offset;
583  head_end_offset;
586  clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
588  head_end_offset);
589  d->next_buffer = src_buffer;
590  }
591  vlib_buffer_advance (s, head_end_offset);
592  s->n_add_refs = n_buffers - 1;
593  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
594  {
595  s = vlib_get_buffer (vm, s->next_buffer);
596  s->n_add_refs = n_buffers - 1;
597  }
598 
599  return n_buffers;
600 }
601 
602 /** \brief Attach cloned tail to the buffer
603 
604  @param vm - (vlib_main_t *) vlib main data structure pointer
605  @param head - (vlib_buffer_t *) head buffer
606  @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
607 */
608 
609 always_inline void
611  vlib_buffer_t * tail)
612 {
613  ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
614  ASSERT (head->free_list_index == tail->free_list_index);
615 
619  head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
620  head->next_buffer = vlib_get_buffer_index (vm, tail);
623 
624 next_segment:
625  __sync_add_and_fetch (&tail->n_add_refs, 1);
626 
627  if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
628  {
629  tail = vlib_get_buffer (vm, tail->next_buffer);
630  goto next_segment;
631  }
632 }
633 
634 /* Initializes the buffer as an empty packet with no chained buffers. */
635 always_inline void
637 {
639  first->current_length = 0;
640  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
642 }
643 
644 /* The provided next_bi buffer index is appended to the end of the packet. */
647  vlib_buffer_t * first,
648  vlib_buffer_t * last, u32 next_bi)
649 {
650  vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
651  last->next_buffer = next_bi;
653  next_buffer->current_length = 0;
654  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
655  return next_buffer;
656 }
657 
658 /* Increases or decreases the packet length.
659  * It does not allocate or deallocate new buffers.
660  * Therefore, the added length must be compatible
661  * with the last buffer. */
662 always_inline void
664  vlib_buffer_t * last, i32 len)
665 {
666  last->current_length += len;
667  if (first != last)
669 }
670 
671 /* Copy data to the end of the packet and increases its length.
672  * It does not allocate new buffers.
673  * Returns the number of copied bytes. */
676  u32 free_list_index,
677  vlib_buffer_t * first,
678  vlib_buffer_t * last, void *data, u16 data_len)
679 {
680  u32 n_buffer_bytes =
681  vlib_buffer_free_list_buffer_size (vm, free_list_index);
682  ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
683  u16 len = clib_min (data_len,
684  n_buffer_bytes - last->current_length -
685  last->current_data);
686  clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
687  len);
688  vlib_buffer_chain_increase_length (first, last, len);
689  return len;
690 }
691 
692 /* Copy data to the end of the packet and increases its length.
693  * Allocates additional buffers from the free list if necessary.
694  * Returns the number of copied bytes.
695  * 'last' value is modified whenever new buffers are allocated and
696  * chained and points to the last buffer in the chain. */
697 u16
699  u32 free_list_index,
700  vlib_buffer_t * first,
701  vlib_buffer_t ** last,
702  void *data, u16 data_len);
704 
707 
708 typedef struct
709 {
710  /* Vector of packet data. */
712 
713  /* Number of buffers to allocate in each call to physmem
714  allocator. */
716 
717  /* Buffer free list for this template. */
719 
722 
725 
728  void *packet_data,
729  uword n_packet_data_bytes,
730  uword min_n_buffers_each_physmem_alloc,
731  char *fmt, ...);
732 
735  u32 * bi_result);
736 
737 always_inline void
739 {
740  vec_free (t->packet_data);
741 }
742 
745 {
746  serialize_stream_t *s = &m->stream;
750  vlib_main_t *vm = sm->vlib_main;
751  u32 n, *f;
752 
754  if (sm->last_buffer != ~0)
755  {
757  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
758  {
759  b = vlib_get_buffer (vm, b->next_buffer);
760  n += b->current_length;
761  }
762  }
763 
764  /* *INDENT-OFF* */
765  clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
766  n += vlib_buffer_index_length_in_chain (vm, f[0]);
767  }));
768 /* *INDENT-ON* */
769 
770  return n;
771 }
772 
773 /* Set a buffer quickly into "uninitialized" state. We want this to
774  be extremely cheap and arrange for all fields that need to be
775  initialized to be in the first 128 bits of the buffer. */
776 always_inline void
779 {
781 
782  /* Make sure vlib_buffer_t is cacheline aligned and sized */
783  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
784  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) ==
786  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) ==
788 
789  /* Make sure buffer template is sane. */
791 
792  clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
793  STRUCT_MARK_PTR (src, template_start),
794  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
795  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
796 
797  /* Not in the first 16 octets. */
798  dst->n_add_refs = src->n_add_refs;
799 
800  /* Make sure it really worked. */
801 #define _(f) ASSERT (dst->f == src->f);
802  _(current_data);
803  _(current_length);
804  _(flags);
805  _(free_list_index);
806 #undef _
808  ASSERT (dst->n_add_refs == 0);
809 }
810 
811 always_inline void
814  u32 buffer_index, u8 do_init)
815 {
816  vlib_buffer_t *b;
817  b = vlib_get_buffer (vm, buffer_index);
818  if (PREDICT_TRUE (do_init))
820  vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
821 }
822 
823 always_inline void
825  vlib_buffer_t * dst1,
827 {
829 
830  /* Make sure buffer template is sane. */
832 
833  clib_memcpy (STRUCT_MARK_PTR (dst0, template_start),
834  STRUCT_MARK_PTR (src, template_start),
835  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
836  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
837 
838  clib_memcpy (STRUCT_MARK_PTR (dst1, template_start),
839  STRUCT_MARK_PTR (src, template_start),
840  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
841  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
842 
843  /* Not in the first 16 octets. */
844  dst0->n_add_refs = src->n_add_refs;
845  dst1->n_add_refs = src->n_add_refs;
846 
847  /* Make sure it really worked. */
848 #define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f)
849  _(current_data);
850  _(current_length);
851  _(flags);
852  _(free_list_index);
853 #undef _
854 
857  ASSERT (dst0->n_add_refs == 0);
858  ASSERT (dst1->n_add_refs == 0);
859 }
860 
861 #if CLIB_DEBUG > 0
864 extern void *vlib_buffer_state_heap;
865 #endif
866 
867 static inline void
869 {
870 #if CLIB_DEBUG > 0
871  uword *p;
872  void *oldheap;
873 
875 
876  while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
877  ;
878 
880 
881  /* If we don't know about b, declare it to be in the expected state */
882  if (!p)
883  {
885  goto out;
886  }
887 
888  if (p[0] != expected)
889  {
890  void cj_stop (void);
891  u32 bi;
893 
894  cj_stop ();
895 
896  bi = vlib_get_buffer_index (vm, b);
897 
898  clib_mem_set_heap (oldheap);
899  clib_warning ("%.6f buffer %llx (%d): %s, not %s",
900  vlib_time_now (vm), bi,
901  p[0] ? "busy" : "free", expected ? "busy" : "free");
902  os_panic ();
903  }
904 out:
907  clib_mem_set_heap (oldheap);
908 #endif
909 }
910 
911 static inline void
913 {
914 #if CLIB_DEBUG > 0
915  void *oldheap;
916 
918 
919  while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
920  ;
921 
923 
926  clib_mem_set_heap (oldheap);
927 #endif
928 }
929 
930 #endif /* included_vlib_buffer_funcs_h */
931 
932 /*
933  * fd.io coding-style-patch-verification: ON
934  *
935  * Local Variables:
936  * eval: (c-set-style "gnu")
937  * End:
938  */
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define clib_min(x, y)
Definition: clib.h:332
static u64 vlib_physmem_virtual_to_physical(vlib_main_t *vm, void *mem)
Definition: buffer_funcs.h:461
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:290
static void vlib_buffer_set_known_state(vlib_main_t *vm, u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:219
#define STRUCT_MARK_PTR(v, f)
Definition: clib.h:68
format_function_t format_vlib_buffer_contents
Definition: buffer_funcs.h:705
static void * vlib_physmem_alloc(vlib_main_t *vm, clib_error_t **error, uword n_bytes)
Definition: buffer_funcs.h:448
u32 free_list_index
Buffer free list that this buffer was allocated from and will be freed to.
Definition: buffer.h:99
#define PREDICT_TRUE(x)
Definition: clib.h:98
vlib_physmem_main_t physmem_main
Definition: main.h:106
format_function_t format_vlib_buffer
Definition: buffer_funcs.h:705
vlib_buffer_callbacks_t cb
Definition: buffer.h:419
void vlib_buffer_merge_free_lists(vlib_buffer_free_list_t *dst, vlib_buffer_free_list_t *src)
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
Definition: buffer_funcs.h:610
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, u32 free_list_index, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
Definition: buffer_funcs.h:675
static void vlib_validate_buffer_in_use(vlib_buffer_t *b, u32 expected)
Definition: buffer_funcs.h:868
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:646
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:192
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:663
vlib_buffer_t buffer_init_template
Definition: buffer.h:330
void os_panic(void)
Definition: unix-misc.c:174
struct vlib_main_t * vlib_main
Definition: buffer.h:429
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:49
vlib_buffer_main_t * buffer_main
Definition: main.h:104
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:62
u8 *( format_function_t)(u8 *s, va_list *args)
Definition: format.h:48
static u8 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u8 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
Definition: buffer_funcs.h:543
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:209
uword * vlib_buffer_state_validation_hash
Definition: buffer.c:545
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:100
static u64 vlib_get_buffer_data_physical_address(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:153
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:475
clib_error_t * vlib_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
Definition: buffer.c:430
void cj_stop(void)
Definition: cj.c:58
static void vlib_buffer_delete_free_list(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:340
static void vlib_physmem_free(vlib_main_t *vm, void *mem)
Definition: buffer_funcs.h:455
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, char *fmt,...)
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:532
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:87
static void * vlib_physmem_at_offset(vlib_physmem_main_t *pm, uword offset)
Definition: physmem.h:94
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
#define always_inline
Definition: clib.h:84
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
Definition: buffer.h:89
int i32
Definition: types.h:81
void vlib_aligned_memcpy(void *_dst, void *_src, int n_bytes)
#define clib_error_return(e, args...)
Definition: error.h:99
u32 * vlib_buffer_state_validation_lock
Definition: buffer.c:544
unsigned long u64
Definition: types.h:89
static void vlib_copy_buffers(u32 *dst, u32 *src, u32 n)
Definition: buffer_funcs.h:410
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:70
struct vlib_serialize_buffer_main_t::@26::@29 rx
#define hash_get(h, key)
Definition: hash.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:397
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
vlib_main_t vlib_global_main
Definition: main.c:1653
static u32 vlib_buffer_get_free_list_with_size(vlib_main_t *vm, u32 size)
Definition: buffer_funcs.h:364
format_function_t format_vlib_buffer_and_data
Definition: buffer_funcs.h:705
static u64 vlib_physmem_offset_to_physical(vlib_physmem_main_t *pm, uword o)
Definition: physmem.h:64
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, u32 free_list_index)
Definition: buffer.h:387
#define PREDICT_FALSE(x)
Definition: clib.h:97
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:399
#define clib_fifo_foreach(v, f, body)
Definition: fifo.h:279
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
Definition: buffer_funcs.h:812
void(* vlib_buffer_free_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:377
void(* vlib_buffer_free_no_next_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:379
#define uword_to_pointer(u, type)
Definition: types.h:136
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
Definition: buffer_funcs.h:131
static void vlib_buffer_chain_init(vlib_buffer_t *first)
Definition: buffer_funcs.h:636
serialize_stream_t stream
Definition: serialize.h:147
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:311
u32 current_buffer_index
Definition: serialize.h:62
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:185
void *(* os_physmem_alloc_aligned)(vlib_physmem_main_t *pm, uword n_bytes, uword alignment)
Definition: main.h:110
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:340
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:223
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:69
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
Definition: buffer_funcs.h:84
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:269
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:278
vlib_buffer_known_state_t
Definition: buffer_funcs.h:198
uword data_function_opaque
Definition: serialize.h:74
static void vlib_validate_buffer_set_in_use(vlib_buffer_t *b, u32 expected)
Definition: buffer_funcs.h:912
static void * vlib_physmem_alloc_aligned(vlib_main_t *vm, clib_error_t **error, uword n_bytes, uword alignment)
Definition: buffer_funcs.h:432
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
static uword vlib_physmem_offset_of(vlib_physmem_main_t *pm, void *p)
Definition: physmem.h:79
u32(* vlib_buffer_alloc_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:372
u32(* vlib_buffer_alloc_from_free_list_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Definition: buffer.h:374
void vlib_buffer_free_list_fill_unaligned(vlib_main_t *vm, vlib_buffer_free_list_t *free_list, uword n_unaligned_buffers)
vhost_vring_state_t state
Definition: vhost-user.h:81
u64 size
Definition: vhost-user.h:75
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, u32 *index)
Definition: buffer_funcs.h:374
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, u32 free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
u32 vlib_buffer_add_data(vlib_main_t *vm, u32 free_list_index, u32 buffer_index, void *data, u32 n_data_bytes)
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:201
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
Definition: buffer_funcs.h:117
void(* os_physmem_free)(void *x)
Definition: main.h:112
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:124
void * vlib_buffer_state_heap
Definition: buffer.c:546
u64 uword
Definition: types.h:112
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:103
template key/value backing page structure
Definition: bihash_doc.h:44
unsigned short u16
Definition: types.h:57
unsigned char u8
Definition: types.h:56
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: buffer.h:401
static void vlib_buffer_init_two_for_free_list(vlib_buffer_t *dst0, vlib_buffer_t *dst1, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:824
uword * buffer_known_hash
Definition: buffer.h:413
u32 opaque[8]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:131
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:777
#define VLIB_BUFFER_EXT_HDR_VALID
Definition: buffer.h:93
vlib_physmem_region_t virtual
Definition: physmem.h:50
u8 data[0]
Packet data.
Definition: buffer.h:152
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:101
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer_funcs.h:738
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:331
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:385
u32 flags
Definition: vhost-user.h:76
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:245
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
uword * free_list_by_size
Definition: buffer.h:407
u32 vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:255
static u32 unserialize_vlib_buffer_n_bytes(serialize_main_t *m)
Definition: buffer_funcs.h:744