FD.io VPP  v17.04.2-2-ga8f93f8
Vector Packet Processing
buffer.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.h: VLIB buffers
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
42 
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vlib/error.h> /* for vlib_error_t */
48 
49 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
50 #define VLIB_BUFFER_DATA_SIZE (2048)
51 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 
53 /** \file
54  vlib buffer structure definition and a few select
55  access methods. This structure and the buffer allocation
56  mechanism should perhaps live in vnet, but it would take a lot
57  of typing to make it so.
58 */
59 
60 /* VLIB buffer representation. */
61 typedef struct
62 {
63  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
64  STRUCT_MARK (template_start);
65  /* Offset within data[] that we are currently processing.
66  If negative current header points into predata area. */
67  i16 current_data; /**< signed offset in data[], pre_data[]
68  that we are currently processing.
69  If negative current header points into predata area.
70  */
71  u16 current_length; /**< Nbytes between current data and
72  the end of this buffer.
73  */
74  u32 flags; /**< buffer flags:
75  <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
76  <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
77  <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
78  <br> VLIB_BUFFER_REPL_FAIL: packet replication failure
79  <br> VLIB_BUFFER_RECYCLE: as it says
80  <br> VLIB_BUFFER_FLOW_REPORT: buffer is a flow report,
81  <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
82  set to avoid adding it to a flow report
83  <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
84  */
85 #define VLIB_BUFFER_IS_TRACED (1 << 0)
86 #define VLIB_BUFFER_LOG2_NEXT_PRESENT (1)
87 #define VLIB_BUFFER_NEXT_PRESENT (1 << VLIB_BUFFER_LOG2_NEXT_PRESENT)
88 #define VLIB_BUFFER_IS_RECYCLED (1 << 2)
89 #define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 3)
90 #define VLIB_BUFFER_REPL_FAIL (1 << 4)
91 #define VLIB_BUFFER_RECYCLE (1 << 5)
92 #define VLIB_BUFFER_FLOW_REPORT (1 << 6)
93 #define VLIB_BUFFER_EXT_HDR_VALID (1 << 7)
94 
95  /* User defined buffer flags. */
96 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
97 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
98 
99  u32 free_list_index; /**< Buffer free list that this buffer was
100  allocated from and will be freed to.
101  */
102 
104  /**< Only valid for first buffer in chain. Current length plus
105  total length given here give total number of bytes in buffer chain.
106  */
107  STRUCT_MARK (template_end);
108 
109  u32 next_buffer; /**< Next buffer for this linked-list of buffers.
110  Only valid if VLIB_BUFFER_NEXT_PRESENT flag is set.
111  */
112 
113  vlib_error_t error; /**< Error code for buffers to be enqueued
114  to error handler.
115  */
116  u32 current_config_index; /**< Used by feature subgraph arcs to
117  visit enabled feature nodes
118  */
119 
120  u8 feature_arc_index; /**< Used to identify feature arcs by intermediate
121  feature node
122  */
123 
124  u8 n_add_refs; /**< Number of additional references to this buffer. */
125 
126  u8 dont_waste_me[2]; /**< Available space in the (precious)
127  first 32 octets of buffer metadata
128  Before allocating any of it, discussion required!
129  */
130 
131  u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes.
132  See .../vnet/vnet/buffer.h
133  */
134  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
135 
136  u32 trace_index; /**< Specifies index into trace buffer
137  if VLIB_PACKET_IS_TRACED flag is set.
138  */
139  u32 recycle_count; /**< Used by L2 path recycle code */
140  u32 opaque2[14]; /**< More opaque data, currently unused */
141 
142  /***** end of second cache line */
143  CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
144  u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data
145  before buffer start.
146  Packet rewrite string will be
147  rewritten backwards and may extend
148  back before buffer->data[0].
149  Must come directly before packet data.
150  */
151 
152  u8 data[0]; /**< Packet data. Hardware DMA here */
153 } vlib_buffer_t; /* Must be a multiple of 64B. */
154 
155 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
156 
157 /** \brief Prefetch buffer metadata.
158  The first 64 bytes of buffer contains most header information
159 
160  @param b - (vlib_buffer_t *) pointer to the buffer
161  @param type - LOAD, STORE. In most cases, STORE is the right answer
162 */
163 
164 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
165 
168 {
169  return (void *) (b + 1) + buffer_bytes;
170 }
171 
172 always_inline void
174 {
175  ASSERT (sizeof (b[0]) % 64 == 0);
176 
177  /* Rewrite data must be before and contiguous with packet data. */
179 }
180 
181 /** \brief Get pointer to current data to process
182 
183  @param b - (vlib_buffer_t *) pointer to the buffer
184  @return - (void *) (b->data + b->current_data)
185 */
186 
187 always_inline void *
189 {
190  /* Check bounds. */
191  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
192  return b->data + b->current_data;
193 }
194 
195 /** \brief Advance current data pointer by the supplied (signed!) amount
196 
197  @param b - (vlib_buffer_t *) pointer to the buffer
198  @param l - (word) signed increment
199 */
200 always_inline void
202 {
203  ASSERT (b->current_length >= l);
204  b->current_data += l;
205  b->current_length -= l;
206 }
207 
208 /** \brief Reset current header & length to state they were in when
209  packet was received.
210 
211  @param b - (vlib_buffer_t *) pointer to the buffer
212 */
213 
214 always_inline void
216 {
217  b->current_length += clib_max (b->current_data, 0);
218  b->current_data = 0;
219 }
220 
221 /** \brief Get pointer to buffer's opaque data array
222 
223  @param b - (vlib_buffer_t *) pointer to the buffer
224  @return - (void *) b->opaque
225 */
226 always_inline void *
228 {
229  return (void *) b->opaque;
230 }
231 
232 /** \brief Get pointer to buffer's opaque2 data array
233 
234  @param b - (vlib_buffer_t *) pointer to the buffer
235  @return - (void *) b->opaque2
236 */
237 always_inline void *
239 {
240  return (void *) b->opaque2;
241 }
242 
243 /** \brief Get pointer to the end of buffer's data
244  * @param b pointer to the buffer
245  * @return pointer to tail of packet's data
246  */
249 {
250  return b->data + b->current_data + b->current_length;
251 }
252 
253 /** \brief Append uninitialized data to buffer
254  * @param b pointer to the buffer
255  * @param size number of uninitialized bytes
256  * @return pointer to beginning of uninitialized data
257  */
258 always_inline void *
260 {
261  void *p = vlib_buffer_get_tail (b);
262  /* XXX make sure there's enough space */
263  b->current_length += size;
264  return p;
265 }
266 
267 /** \brief Prepend uninitialized data to buffer
268  * @param b pointer to the buffer
269  * @param size number of uninitialized bytes
270  * @return pointer to beginning of uninitialized data
271  */
272 always_inline void *
274 {
276  b->current_data -= size;
277  b->current_length += size;
278 
279  return vlib_buffer_get_current (b);
280 }
281 
282 /** \brief Make head room, typically for packet headers
283  * @param b pointer to the buffer
284  * @param size number of head room bytes
285  * @return pointer to start of buffer (current data)
286  */
287 always_inline void *
289 {
291  b->current_data += size;
292  return vlib_buffer_get_current (b);
293 }
294 
295 /** \brief Retrieve bytes from buffer head
296  * @param b pointer to the buffer
297  * @param size number of bytes to pull
298  * @return pointer to start of buffer (current data)
299  */
300 always_inline void *
302 {
304  return 0;
305 
306  void *data = vlib_buffer_get_current (b);
307  vlib_buffer_advance (b, size);
308  return data;
309 }
310 
311 /* Forward declaration. */
312 struct vlib_main_t;
313 
315 {
316  /* Template buffer used to initialize first 16 bytes of buffers
317  allocated on this free list. */
319 
320  /* Our index into vlib_main_t's buffer_free_list_pool. */
322 
323  /* Number of data bytes for buffers in this free list. */
325 
326  /* Number of buffers to allocate when we need to allocate new buffers
327  from physmem heap. */
329 
330  /* Total number of buffers allocated from this free list. */
332 
333  /* Vector of free buffers. Each element is a byte offset into I/O heap. */
335 
336  /* Memory chunks allocated for this free list
337  recorded here so they can be freed when free list
338  is deleted. */
340 
341  /* Free list name. */
343 
344  /* Callback functions to initialize newly allocated buffers.
345  If null buffers are zeroed. */
347  struct vlib_buffer_free_list_t * fl,
348  u32 * buffers, u32 n_buffers);
349 
350  /* Callback function to announce that buffers have been
351  added to the freelist */
353  (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl);
354 
356 } __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
357 
358 typedef struct
359 {
360  u32 (*vlib_buffer_alloc_cb) (struct vlib_main_t * vm, u32 * buffers,
361  u32 n_buffers);
362  u32 (*vlib_buffer_alloc_from_free_list_cb) (struct vlib_main_t * vm,
363  u32 * buffers, u32 n_buffers,
364  u32 free_list_index);
365  void (*vlib_buffer_free_cb) (struct vlib_main_t * vm, u32 * buffers,
366  u32 n_buffers);
367  void (*vlib_buffer_free_no_next_cb) (struct vlib_main_t * vm, u32 * buffers,
368  u32 n_buffers);
369  void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t,
370  void *packet_data,
371  uword n_packet_data_bytes,
372  uword
374  u8 * name);
375  void (*vlib_buffer_delete_free_list_cb) (struct vlib_main_t * vm,
376  u32 free_list_index);
378 
379 typedef struct
380 {
381  /* Buffer free callback, for subversive activities */
382  u32 (*buffer_free_callback) (struct vlib_main_t * vm,
383  u32 * buffers,
384  u32 n_buffers, u32 follow_buffer_next);
385  /* Pool of buffer free lists.
386  Multiple free lists exist for packet generator which uses
387  separate free lists for each packet stream --- so as to avoid
388  initializing static data for each packet generated. */
390 #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
391 #define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE
392 
393  /* Hash table mapping buffer size (rounded to next unit of
394  sizeof (vlib_buffer_t)) to free list index. */
396 
397  /* Hash table mapping buffer index into number
398  0 => allocated but free, 1 => allocated and not-free.
399  If buffer index is not in hash table then this buffer
400  has never been allocated. */
402 
403  /* List of free-lists needing Blue Light Special announcements */
405 
406  /* Callbacks */
410 
411 void vlib_buffer_cb_init (struct vlib_main_t *vm);
414 
415 typedef struct
416 {
418 
419  u32 first_buffer, last_buffer;
420 
421  union
422  {
423  struct
424  {
425  /* Total accumulated bytes in chain starting with first_buffer. */
427 
428  /* Max number of bytes to accumulate in chain starting with first_buffer.
429  As this limit is reached buffers are enqueued to next node. */
431 
432  /* Next node to enqueue buffers to relative to current process node. */
434 
435  /* Free list to use to allocate new buffers. */
437  } tx;
438 
439  struct
440  {
441  /* CLIB fifo of buffer indices waiting to be unserialized. */
443 
444  /* Event type used to signal that RX buffers have been added to fifo. */
446  } rx;
447  };
449 
453  struct vlib_main_t *vm,
455 
458 void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp);
459 
462 {
463  serialize_stream_t *s = &m->stream;
467  return sm->tx.n_total_data_bytes + s->current_buffer_index +
469 }
470 
471 /*
472  */
473 
474 /** \brief Compile time buffer trajectory tracing option
475  Turn this on if you run into "bad monkey" contexts,
476  and you want to know exactly which nodes they've visited...
477  See vlib/main.c...
478 */
479 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
480 
481 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
482 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) (b)->pre_data[0]=0
483 #else
484 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
485 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
486 
487 #endif /* included_vlib_buffer_h */
488 
489 /*
490  * fd.io coding-style-patch-verification: ON
491  *
492  * Local Variables:
493  * eval: (c-set-style "gnu")
494  * End:
495  */
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:68
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:248
static void vlib_buffer_reset(vlib_buffer_t *b)
Reset current header & length to state they were in when packet was received.
Definition: buffer.h:215
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:116
u32 free_list_index
Buffer free list that this buffer was allocated from and will be freed to.
Definition: buffer.h:99
vlib_buffer_callbacks_t cb
Definition: buffer.h:407
vlib_buffer_t buffer_init_template
Definition: buffer.h:318
struct vlib_main_t * vlib_main
Definition: buffer.h:417
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
#define STRUCT_MARK(mark)
Definition: clib.h:67
void ** buffer_memory_allocated
Definition: buffer.h:339
u32 recycle_count
Used by L2 path recycle code.
Definition: buffer.h:139
void unserialize_open_vlib_buffer(serialize_main_t *m, struct vlib_main_t *vm, vlib_serialize_buffer_main_t *sm)
void unserialize_close_vlib_buffer(serialize_main_t *m)
u32 min_n_buffers_each_physmem_alloc
Definition: buffer.h:328
u32 serialize_close_vlib_buffer(serialize_main_t *m)
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.h:346
#define always_inline
Definition: clib.h:84
void serialize_open_vlib_buffer(serialize_main_t *m, struct vlib_main_t *vm, vlib_serialize_buffer_main_t *sm)
void * vlib_set_buffer_free_callback(struct vlib_main_t *vm, void *fp)
Definition: buffer.c:639
u8 feature_arc_index
Used to identify feature arcs by intermediate feature node.
Definition: buffer.h:120
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:144
static vlib_buffer_t * vlib_buffer_next_contiguous(vlib_buffer_t *b, u32 buffer_bytes)
Definition: buffer.h:167
static u32 serialize_vlib_buffer_n_bytes(serialize_main_t *m)
Definition: buffer.h:461
vlib_buffer_free_list_t ** announce_list
Definition: buffer.h:404
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:288
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u8 size)
Append uninitialized data to buffer.
Definition: buffer.h:259
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
u32 vlib_error_t
Definition: error.h:44
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:113
#define uword_to_pointer(u, type)
Definition: types.h:136
serialize_stream_t stream
Definition: serialize.h:147
u32 current_buffer_index
Definition: serialize.h:62
vlib_main_t * vm
Definition: buffer.c:276
static void * vlib_get_buffer_opaque2(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque2 data array.
Definition: buffer.h:238
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
Definition: buffer.h:353
uword data_function_opaque
Definition: serialize.h:74
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
int vlib_buffer_cb_register(struct vlib_main_t *vm, vlib_buffer_callbacks_t *cb)
Definition: buffer.c:1025
u64 size
Definition: vhost-user.h:77
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:201
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:124
#define clib_max(x, y)
Definition: clib.h:325
u64 uword
Definition: types.h:112
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:103
static void * vlib_buffer_push_uninit(vlib_buffer_t *b, u8 size)
Prepend uninitialized data to buffer.
Definition: buffer.h:273
unsigned short u16
Definition: types.h:57
i64 word
Definition: types.h:111
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
static void * vlib_get_buffer_opaque(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque data array.
Definition: buffer.h:227
void vlib_buffer_cb_init(struct vlib_main_t *vm)
Definition: buffer.c:1011
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: buffer.h:389
struct vlib_serialize_buffer_main_t::@23::@25 tx
uword * buffer_known_hash
Definition: buffer.h:401
int extern_buffer_mgmt
Definition: buffer.h:408
u32 opaque[8]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:131
short i16
Definition: types.h:46
u8 data[0]
Packet data.
Definition: buffer.h:152
static void * vlib_buffer_pull(vlib_buffer_t *b, u8 size)
Retrieve bytes from buffer head.
Definition: buffer.h:301
uword buffer_init_function_opaque
Definition: buffer.h:355
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
static void vlib_buffer_struct_is_sane(vlib_buffer_t *b)
Definition: buffer.h:173
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:136
uword * free_list_by_size
Definition: buffer.h:395
u32 opaque2[14]
More opaque data, currently unused.
Definition: buffer.h:140