FD.io VPP  v17.07.01-10-g3be13f0
Vector Packet Processing
buffer.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.h: VLIB buffers
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
42 
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vlib/error.h> /* for vlib_error_t */
48 
49 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
50 #define VLIB_BUFFER_DATA_SIZE (2048)
51 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 
53 /** \file
54  vlib buffer structure definition and a few select
55  access methods. This structure and the buffer allocation
56  mechanism should perhaps live in vnet, but it would take a lot
57  of typing to make it so.
58 */
59 
60 /* VLIB buffer representation. */
61 typedef struct
62 {
63  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
64  STRUCT_MARK (template_start);
65  /* Offset within data[] that we are currently processing.
66  If negative current header points into predata area. */
67  i16 current_data; /**< signed offset in data[], pre_data[]
68  that we are currently processing.
69  If negative current header points into predata area.
70  */
71  u16 current_length; /**< Nbytes between current data and
72  the end of this buffer.
73  */
74  u32 flags; /**< buffer flags:
75  <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
76  <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
77  <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
78  <br> VLIB_BUFFER_REPL_FAIL: packet replication failure
79  <br> VLIB_BUFFER_RECYCLE: as it says
80  <br> VLIB_BUFFER_FLOW_REPORT: buffer is a flow report,
81  <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
82  set to avoid adding it to a flow report
83  <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
84  */
85 #define VLIB_BUFFER_IS_TRACED (1 << 0)
86 #define VLIB_BUFFER_LOG2_NEXT_PRESENT (1)
87 #define VLIB_BUFFER_NEXT_PRESENT (1 << VLIB_BUFFER_LOG2_NEXT_PRESENT)
88 #define VLIB_BUFFER_IS_RECYCLED (1 << 2)
89 #define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 3)
90 #define VLIB_BUFFER_REPL_FAIL (1 << 4)
91 #define VLIB_BUFFER_RECYCLE (1 << 5)
92 #define VLIB_BUFFER_FLOW_REPORT (1 << 6)
93 #define VLIB_BUFFER_EXT_HDR_VALID (1 << 7)
94 
95  /* User defined buffer flags. */
96 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
97 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
98 
99  u32 free_list_index; /**< Buffer free list that this buffer was
100  allocated from and will be freed to.
101  */
102 
104  /**< Only valid for first buffer in chain. Current length plus
105  total length given here give total number of bytes in buffer chain.
106  */
107  STRUCT_MARK (template_end);
108 
109  u32 next_buffer; /**< Next buffer for this linked-list of buffers.
110  Only valid if VLIB_BUFFER_NEXT_PRESENT flag is set.
111  */
112 
113  vlib_error_t error; /**< Error code for buffers to be enqueued
114  to error handler.
115  */
116  u32 current_config_index; /**< Used by feature subgraph arcs to
117  visit enabled feature nodes
118  */
119 
120  u8 feature_arc_index; /**< Used to identify feature arcs by intermediate
121  feature node
122  */
123 
124  u8 n_add_refs; /**< Number of additional references to this buffer. */
125 
126  u8 dont_waste_me[2]; /**< Available space in the (precious)
127  first 32 octets of buffer metadata
128  Before allocating any of it, discussion required!
129  */
130 
131  u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes.
132  See .../vnet/vnet/buffer.h
133  */
134  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
135 
136  u32 trace_index; /**< Specifies index into trace buffer
137  if VLIB_PACKET_IS_TRACED flag is set.
138  */
139  u32 recycle_count; /**< Used by L2 path recycle code */
140  u32 opaque2[14]; /**< More opaque data, currently unused */
141 
142  /***** end of second cache line */
143  CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
144  u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data
145  before buffer start.
146  Packet rewrite string will be
147  rewritten backwards and may extend
148  back before buffer->data[0].
149  Must come directly before packet data.
150  */
151 
152  u8 data[0]; /**< Packet data. Hardware DMA here */
153 } vlib_buffer_t; /* Must be a multiple of 64B. */
154 
155 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
156 
157 /** \brief Prefetch buffer metadata.
158  The first 64 bytes of buffer contains most header information
159 
160  @param b - (vlib_buffer_t *) pointer to the buffer
161  @param type - LOAD, STORE. In most cases, STORE is the right answer
162 */
163 
164 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
165 
168 {
169  return (void *) (b + 1) + buffer_bytes;
170 }
171 
172 always_inline void
174 {
175  ASSERT (sizeof (b[0]) % 64 == 0);
176 
177  /* Rewrite data must be before and contiguous with packet data. */
179 }
180 
181 /** \brief Get pointer to current data to process
182 
183  @param b - (vlib_buffer_t *) pointer to the buffer
184  @return - (void *) (b->data + b->current_data)
185 */
186 
187 always_inline void *
189 {
190  /* Check bounds. */
191  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
192  return b->data + b->current_data;
193 }
194 
195 /** \brief Advance current data pointer by the supplied (signed!) amount
196 
197  @param b - (vlib_buffer_t *) pointer to the buffer
198  @param l - (word) signed increment
199 */
200 always_inline void
202 {
203  ASSERT (b->current_length >= l);
204  b->current_data += l;
205  b->current_length -= l;
206 }
207 
208 /** \brief Check if there is enough space in buffer to advance
209 
210  @param b - (vlib_buffer_t *) pointer to the buffer
211  @param l - (word) size to check
212  @return - 0 if there is less space than 'l' in buffer
213 */
216 {
217  return b->current_length >= l;
218 }
219 
220 /** \brief Reset current header & length to state they were in when
221  packet was received.
222 
223  @param b - (vlib_buffer_t *) pointer to the buffer
224 */
225 
226 always_inline void
228 {
229  b->current_length += clib_max (b->current_data, 0);
230  b->current_data = 0;
231 }
232 
233 /** \brief Get pointer to buffer's opaque data array
234 
235  @param b - (vlib_buffer_t *) pointer to the buffer
236  @return - (void *) b->opaque
237 */
238 always_inline void *
240 {
241  return (void *) b->opaque;
242 }
243 
244 /** \brief Get pointer to buffer's opaque2 data array
245 
246  @param b - (vlib_buffer_t *) pointer to the buffer
247  @return - (void *) b->opaque2
248 */
249 always_inline void *
251 {
252  return (void *) b->opaque2;
253 }
254 
255 /** \brief Get pointer to the end of buffer's data
256  * @param b pointer to the buffer
257  * @return pointer to tail of packet's data
258  */
261 {
262  return b->data + b->current_data + b->current_length;
263 }
264 
265 /** \brief Append uninitialized data to buffer
266  * @param b pointer to the buffer
267  * @param size number of uninitialized bytes
268  * @return pointer to beginning of uninitialized data
269  */
270 always_inline void *
272 {
273  void *p = vlib_buffer_get_tail (b);
274  /* XXX make sure there's enough space */
275  b->current_length += size;
276  return p;
277 }
278 
279 /** \brief Prepend uninitialized data to buffer
280  * @param b pointer to the buffer
281  * @param size number of uninitialized bytes
282  * @return pointer to beginning of uninitialized data
283  */
284 always_inline void *
286 {
288  b->current_data -= size;
289  b->current_length += size;
290 
291  return vlib_buffer_get_current (b);
292 }
293 
294 /** \brief Make head room, typically for packet headers
295  * @param b pointer to the buffer
296  * @param size number of head room bytes
297  * @return pointer to start of buffer (current data)
298  */
299 always_inline void *
301 {
303  b->current_data += size;
304  return vlib_buffer_get_current (b);
305 }
306 
307 /** \brief Retrieve bytes from buffer head
308  * @param b pointer to the buffer
309  * @param size number of bytes to pull
310  * @return pointer to start of buffer (current data)
311  */
312 always_inline void *
314 {
316  return 0;
317 
318  void *data = vlib_buffer_get_current (b);
319  vlib_buffer_advance (b, size);
320  return data;
321 }
322 
323 /* Forward declaration. */
324 struct vlib_main_t;
325 
327 {
328  /* Template buffer used to initialize first 16 bytes of buffers
329  allocated on this free list. */
331 
332  /* Our index into vlib_main_t's buffer_free_list_pool. */
334 
335  /* Number of data bytes for buffers in this free list. */
337 
338  /* Number of buffers to allocate when we need to allocate new buffers
339  from physmem heap. */
341 
342  /* Total number of buffers allocated from this free list. */
344 
345  /* Vector of free buffers. Each element is a byte offset into I/O heap. */
347 
348  /* Memory chunks allocated for this free list
349  recorded here so they can be freed when free list
350  is deleted. */
352 
353  /* Free list name. */
355 
356  /* Callback functions to initialize newly allocated buffers.
357  If null buffers are zeroed. */
358  void (*buffer_init_function) (struct vlib_main_t * vm,
359  struct vlib_buffer_free_list_t * fl,
360  u32 * buffers, u32 n_buffers);
361 
362  /* Callback function to announce that buffers have been
363  added to the freelist */
365  (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl);
366 
368 } __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
369 
370 typedef struct
371 {
372  u32 (*vlib_buffer_alloc_cb) (struct vlib_main_t * vm, u32 * buffers,
373  u32 n_buffers);
374  u32 (*vlib_buffer_alloc_from_free_list_cb) (struct vlib_main_t * vm,
375  u32 * buffers, u32 n_buffers,
376  u32 free_list_index);
377  void (*vlib_buffer_free_cb) (struct vlib_main_t * vm, u32 * buffers,
378  u32 n_buffers);
379  void (*vlib_buffer_free_no_next_cb) (struct vlib_main_t * vm, u32 * buffers,
380  u32 n_buffers);
381  void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t,
382  void *packet_data,
383  uword n_packet_data_bytes,
384  uword
386  u8 * name);
387  void (*vlib_buffer_delete_free_list_cb) (struct vlib_main_t * vm,
388  u32 free_list_index);
390 
391 typedef struct
392 {
393  /* Buffer free callback, for subversive activities */
394  u32 (*buffer_free_callback) (struct vlib_main_t * vm,
395  u32 * buffers,
396  u32 n_buffers, u32 follow_buffer_next);
397  /* Pool of buffer free lists.
398  Multiple free lists exist for packet generator which uses
399  separate free lists for each packet stream --- so as to avoid
400  initializing static data for each packet generated. */
402 #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
403 #define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE
404 
405  /* Hash table mapping buffer size (rounded to next unit of
406  sizeof (vlib_buffer_t)) to free list index. */
408 
409  /* Hash table mapping buffer index into number
410  0 => allocated but free, 1 => allocated and not-free.
411  If buffer index is not in hash table then this buffer
412  has never been allocated. */
414 
415  /* List of free-lists needing Blue Light Special announcements */
417 
418  /* Callbacks */
422 
423 void vlib_buffer_cb_init (struct vlib_main_t *vm);
424 int vlib_buffer_cb_register (struct vlib_main_t *vm,
426 
427 typedef struct
428 {
430 
431  u32 first_buffer, last_buffer;
432 
433  union
434  {
435  struct
436  {
437  /* Total accumulated bytes in chain starting with first_buffer. */
439 
440  /* Max number of bytes to accumulate in chain starting with first_buffer.
441  As this limit is reached buffers are enqueued to next node. */
443 
444  /* Next node to enqueue buffers to relative to current process node. */
446 
447  /* Free list to use to allocate new buffers. */
449  } tx;
450 
451  struct
452  {
453  /* CLIB fifo of buffer indices waiting to be unserialized. */
455 
456  /* Event type used to signal that RX buffers have been added to fifo. */
458  } rx;
459  };
461 
465  struct vlib_main_t *vm,
467 
470 void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp);
471 
474 {
475  serialize_stream_t *s = &m->stream;
479  return sm->tx.n_total_data_bytes + s->current_buffer_index +
481 }
482 
483 /*
484  */
485 
486 /** \brief Compile time buffer trajectory tracing option
487  Turn this on if you run into "bad monkey" contexts,
488  and you want to know exactly which nodes they've visited...
489  See vlib/main.c...
490 */
491 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
492 
493 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
494 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) (b)->pre_data[0]=0
495 #else
496 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
497 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
498 
499 #endif /* included_vlib_buffer_h */
500 
501 /*
502  * fd.io coding-style-patch-verification: ON
503  *
504  * Local Variables:
505  * eval: (c-set-style "gnu")
506  * End:
507  */
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:68
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:260
static void vlib_buffer_reset(vlib_buffer_t *b)
Reset current header & length to state they were in when packet was received.
Definition: buffer.h:227
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:116
u32 free_list_index
Buffer free list that this buffer was allocated from and will be freed to.
Definition: buffer.h:99
vlib_buffer_callbacks_t cb
Definition: buffer.h:419
vlib_buffer_t buffer_init_template
Definition: buffer.h:330
struct vlib_main_t * vlib_main
Definition: buffer.h:429
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
#define STRUCT_MARK(mark)
Definition: clib.h:67
void ** buffer_memory_allocated
Definition: buffer.h:351
u32 recycle_count
Used by L2 path recycle code.
Definition: buffer.h:139
void unserialize_open_vlib_buffer(serialize_main_t *m, struct vlib_main_t *vm, vlib_serialize_buffer_main_t *sm)
void unserialize_close_vlib_buffer(serialize_main_t *m)
u32 min_n_buffers_each_physmem_alloc
Definition: buffer.h:340
u32 serialize_close_vlib_buffer(serialize_main_t *m)
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.h:358
#define always_inline
Definition: clib.h:84
void serialize_open_vlib_buffer(serialize_main_t *m, struct vlib_main_t *vm, vlib_serialize_buffer_main_t *sm)
void * vlib_set_buffer_free_callback(struct vlib_main_t *vm, void *fp)
u8 feature_arc_index
Used to identify feature arcs by intermediate feature node.
Definition: buffer.h:120
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:144
static vlib_buffer_t * vlib_buffer_next_contiguous(vlib_buffer_t *b, u32 buffer_bytes)
Definition: buffer.h:167
static u32 serialize_vlib_buffer_n_bytes(serialize_main_t *m)
Definition: buffer.h:473
vlib_buffer_free_list_t ** announce_list
Definition: buffer.h:416
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:300
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u8 size)
Append uninitialized data to buffer.
Definition: buffer.h:271
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
u32 vlib_error_t
Definition: error.h:44
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:113
#define uword_to_pointer(u, type)
Definition: types.h:136
serialize_stream_t stream
Definition: serialize.h:147
u32 current_buffer_index
Definition: serialize.h:62
static void * vlib_get_buffer_opaque2(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque2 data array.
Definition: buffer.h:250
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
Definition: buffer.h:365
uword data_function_opaque
Definition: serialize.h:74
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
int vlib_buffer_cb_register(struct vlib_main_t *vm, vlib_buffer_callbacks_t *cb)
u64 size
Definition: vhost-user.h:75
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:201
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:124
#define clib_max(x, y)
Definition: clib.h:325
u64 uword
Definition: types.h:112
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:103
static void * vlib_buffer_push_uninit(vlib_buffer_t *b, u8 size)
Prepend uninitialized data to buffer.
Definition: buffer.h:285
unsigned short u16
Definition: types.h:57
static u8 vlib_buffer_has_space(vlib_buffer_t *b, word l)
Check if there is enough space in buffer to advance.
Definition: buffer.h:215
i64 word
Definition: types.h:111
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
static void * vlib_get_buffer_opaque(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque data array.
Definition: buffer.h:239
void vlib_buffer_cb_init(struct vlib_main_t *vm)
struct vlib_serialize_buffer_main_t::@26::@28 tx
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: buffer.h:401
uword * buffer_known_hash
Definition: buffer.h:413
int extern_buffer_mgmt
Definition: buffer.h:420
u32 opaque[8]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:131
short i16
Definition: types.h:46
u8 data[0]
Packet data.
Definition: buffer.h:152
static void * vlib_buffer_pull(vlib_buffer_t *b, u8 size)
Retrieve bytes from buffer head.
Definition: buffer.h:313
uword buffer_init_function_opaque
Definition: buffer.h:367
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
static void vlib_buffer_struct_is_sane(vlib_buffer_t *b)
Definition: buffer.h:173
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:136
uword * free_list_by_size
Definition: buffer.h:407
u32 opaque2[14]
More opaque data, currently unused.
Definition: buffer.h:140