FD.io VPP  v19.04.4-rc0-5-ge88582fac
Vector Packet Processing
buffer.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.h: VLIB buffers
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
42 
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vppinfra/lock.h>
48 #include <vlib/error.h> /* for vlib_error_t */
49 
50 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 
53 #define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048)
54 
55 /* Minimum buffer chain segment size. Does not apply to last buffer in chain.
56  Dataplane code can safely asume that specified amount of data is not split
57  into 2 chained buffers */
58 #define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE (128)
59 
60 /* Amount of head buffer data copied to each replica head buffer */
61 #define VLIB_BUFFER_CLONE_HEAD_SIZE (256)
62 
63 /** \file
64  vlib buffer structure definition and a few select
65  access methods. This structure and the buffer allocation
66  mechanism should perhaps live in vnet, but it would take a lot
67  of typing to make it so.
68 */
69 
70 /**
71  * Buffer Flags
72  */
73 #define foreach_vlib_buffer_flag \
74  _( 0, IS_TRACED, 0) \
75  _( 1, NEXT_PRESENT, 0) \
76  _( 2, TOTAL_LENGTH_VALID, 0) \
77  _( 3, EXT_HDR_VALID, "ext-hdr-valid")
78 
79 /* NOTE: only buffer generic flags should be defined here, please consider
80  using user flags. i.e. src/vnet/buffer.h */
81 
82 enum
83 {
84 #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)),
86 #undef _
87 };
88 
89 enum
90 {
91 #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit),
93 #undef _
94 };
95 
96  /* User defined buffer flags. */
97 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
98 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
99 #define VLIB_BUFFER_FLAGS_ALL (0x0f)
100 
101 /** VLIB buffer representation. */
102 typedef union
103 {
104  struct
105  {
106  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
107 
108  /** signed offset in data[], pre_data[] that we are currently
109  * processing. If negative current header points into predata area. */
111 
112  /** Nbytes between current data and the end of this buffer. */
114 
115  /** buffer flags:
116  <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
117  <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
118  <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
119  <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
120  <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
121  set to avoid adding it to a flow report
122  <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
123  */
125 
126  /** Generic flow identifier */
128 
129  /** Reference count for this buffer. */
130  volatile u8 ref_count;
131 
132  /** index of buffer pool this buffer belongs. */
134 
135  /** Error code for buffers to be enqueued to error handler. */
137 
138  /** Next buffer for this linked-list of buffers. Only valid if
139  * VLIB_BUFFER_NEXT_PRESENT flag is set. */
141 
142  /** The following fields can be in a union because once a packet enters
143  * the punt path, it is no longer on a feature arc */
144  union
145  {
146  /** Used by feature subgraph arcs to visit enabled feature nodes */
148  /* the reason the packet once punted */
150  };
151 
152  /** Opaque data used by sub-graphs for their own purposes. */
153  u32 opaque[10];
154 
155  /** part of buffer metadata which is initialized on alloc ends here. */
156  STRUCT_MARK (template_end);
157 
158  /** start of 2nd cache line */
159  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
160 
161  /** Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is
162  * set. */
164 
165  /** Only valid for first buffer in chain. Current length plus total length
166  * given here give total number of bytes in buffer chain. */
168 
169  /**< More opaque data, see ../vnet/vnet/buffer.h */
170  u32 opaque2[14];
171 
172  /** start of third cache line */
173  CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
174 
175  /** Space for inserting data before buffer start. Packet rewrite string
176  * will be rewritten backwards and may extend back before
177  * buffer->data[0]. Must come directly before packet data. */
179 
180  /** Packet data */
181  u8 data[0];
182  };
183 #ifdef CLIB_HAVE_VEC128
184  u8x16 as_u8x16[4];
185 #endif
186 #ifdef CLIB_HAVE_VEC256
187  u8x32 as_u8x32[2];
188 #endif
189 #ifdef CLIB_HAVE_VEC512
190  u8x64 as_u8x64[1];
191 #endif
192 } vlib_buffer_t;
193 
194 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
195 
196 /** \brief Prefetch buffer metadata.
197  The first 64 bytes of buffer contains most header information
198 
199  @param b - (vlib_buffer_t *) pointer to the buffer
200  @param type - LOAD, STORE. In most cases, STORE is the right answer
201 */
202 
203 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
204 #define vlib_prefetch_buffer_data(b,type) \
205  CLIB_PREFETCH (vlib_buffer_get_current(b), CLIB_CACHE_LINE_BYTES, type)
206 
207 always_inline void
209 {
210  ASSERT (sizeof (b[0]) % 64 == 0);
211 
212  /* Rewrite data must be before and contiguous with packet data. */
214 }
215 
218 {
219  return pointer_to_uword (b->data);
220 }
221 
222 /** \brief Get pointer to current data to process
223 
224  @param b - (vlib_buffer_t *) pointer to the buffer
225  @return - (void *) (b->data + b->current_data)
226 */
227 
228 always_inline void *
230 {
231  /* Check bounds. */
232  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
233  return b->data + b->current_data;
234 }
235 
238 {
239  return vlib_buffer_get_va (b) + b->current_data;
240 }
241 
242 /** \brief Advance current data pointer by the supplied (signed!) amount
243 
244  @param b - (vlib_buffer_t *) pointer to the buffer
245  @param l - (word) signed increment
246 */
247 always_inline void
249 {
250  ASSERT (b->current_length >= l);
251  b->current_data += l;
252  b->current_length -= l;
253 
254  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 ||
256 }
257 
258 /** \brief Check if there is enough space in buffer to advance
259 
260  @param b - (vlib_buffer_t *) pointer to the buffer
261  @param l - (word) size to check
262  @return - 0 if there is less space than 'l' in buffer
263 */
266 {
267  return b->current_length >= l;
268 }
269 
270 /** \brief Reset current header & length to state they were in when
271  packet was received.
272 
273  @param b - (vlib_buffer_t *) pointer to the buffer
274 */
275 
276 always_inline void
278 {
279  b->current_length += clib_max (b->current_data, 0);
280  b->current_data = 0;
281 }
282 
283 /** \brief Get pointer to buffer's opaque data array
284 
285  @param b - (vlib_buffer_t *) pointer to the buffer
286  @return - (void *) b->opaque
287 */
288 always_inline void *
290 {
291  return (void *) b->opaque;
292 }
293 
294 /** \brief Get pointer to buffer's opaque2 data array
295 
296  @param b - (vlib_buffer_t *) pointer to the buffer
297  @return - (void *) b->opaque2
298 */
299 always_inline void *
301 {
302  return (void *) b->opaque2;
303 }
304 
305 /** \brief Get pointer to the end of buffer's data
306  * @param b pointer to the buffer
307  * @return pointer to tail of packet's data
308  */
311 {
312  return b->data + b->current_data + b->current_length;
313 }
314 
315 /** \brief Append uninitialized data to buffer
316  * @param b pointer to the buffer
317  * @param size number of uninitialized bytes
318  * @return pointer to beginning of uninitialized data
319  */
320 always_inline void *
322 {
323  void *p = vlib_buffer_get_tail (b);
324  /* XXX make sure there's enough space */
325  b->current_length += size;
326  return p;
327 }
328 
329 /** \brief Prepend uninitialized data to buffer
330  * @param b pointer to the buffer
331  * @param size number of uninitialized bytes
332  * @return pointer to beginning of uninitialized data
333  */
334 always_inline void *
336 {
338  b->current_data -= size;
339  b->current_length += size;
340 
341  return vlib_buffer_get_current (b);
342 }
343 
344 /** \brief Make head room, typically for packet headers
345  * @param b pointer to the buffer
346  * @param size number of head room bytes
347  * @return pointer to start of buffer (current data)
348  */
349 always_inline void *
351 {
353  b->current_data += size;
354  return vlib_buffer_get_current (b);
355 }
356 
357 /** \brief Retrieve bytes from buffer head
358  * @param b pointer to the buffer
359  * @param size number of bytes to pull
360  * @return pointer to start of buffer (current data)
361  */
362 always_inline void *
364 {
366  return 0;
367 
368  void *data = vlib_buffer_get_current (b);
369  vlib_buffer_advance (b, size);
370  return data;
371 }
372 
373 /* Forward declaration. */
374 struct vlib_main_t;
375 
376 typedef struct
377 {
378  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
382 typedef struct
383 {
384  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
396 
397  /* per-thread data */
399 
400  /* buffer metadata template */
403 
404 #define VLIB_BUFFER_MAX_NUMA_NODES 32
405 
406 typedef struct
407 {
408  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
409  /* Virtual memory address and size of buffer memory, used for calculating
410  buffer index */
414 
415  /* Hash table mapping buffer index into number
416  0 => allocated but free, 1 => allocated and not-free.
417  If buffer index is not in hash table then this buffer
418  has never been allocated. */
421  u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
422 
423  /* config */
427 
428  /* logging */
431 
433 
434 /*
435  */
436 
437 /** \brief Compile time buffer trajectory tracing option
438  Turn this on if you run into "bad monkey" contexts,
439  and you want to know exactly which nodes they've visited...
440  See vlib/main.c...
441 */
442 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
443 
444 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
445 extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index);
446 extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
447 extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b);
448 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \
449  vlib_buffer_trace_trajectory_init (b);
450 #else
451 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
452 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
453 
454 extern u16 __vlib_buffer_external_hdr_size;
455 #define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
456 static void __clib_constructor \
457 vnet_buffer_set_ext_hdr_size() \
458 { \
459  if (__vlib_buffer_external_hdr_size) \
460  clib_error ("buffer external header space already set"); \
461  __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \
462 }
463 
464 #endif /* included_vlib_buffer_h */
465 
466 /*
467  * fd.io coding-style-patch-verification: ON
468  *
469  * Local Variables:
470  * eval: (c-set-style "gnu")
471  * End:
472  */
u32 opaque2[14]
Definition: buffer.h:170
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:310
static uword vlib_buffer_get_current_va(vlib_buffer_t *b)
Definition: buffer.h:237
static void vlib_buffer_reset(vlib_buffer_t *b)
Reset current header & length to state they were in when packet was received.
Definition: buffer.h:277
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
u32 physmem_map_index
Definition: buffer.h:390
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u8 data[0]
Packet data.
Definition: buffer.h:181
u16 vlib_error_t
Definition: error.h:43
#define STRUCT_MARK(mark)
Definition: clib.h:70
u8 data[128]
Definition: ipsec.api:248
uword log2_page_size
Definition: buffer.h:387
unsigned char u8
Definition: types.h:56
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
vlib_buffer_pool_thread_t * threads
Definition: buffer.h:398
u32 vlib_log_class_t
Definition: vlib.h:50
i64 word
Definition: types.h:111
#define always_inline
Definition: clib.h:98
uword buffer_mem_size
Definition: buffer.h:412
unsigned int u32
Definition: types.h:88
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
uword size
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:350
unsigned short u16
Definition: types.h:57
vlib_buffer_t buffer_template
Definition: buffer.h:401
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:413
vlib_log_class_t log_default
Definition: buffer.h:429
u32 punt_reason
Definition: buffer.h:149
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:420
vlib_main_t * vm
Definition: buffer.c:312
u32 flow_id
Generic flow identifier.
Definition: buffer.h:127
#define VLIB_BUFFER_MAX_NUMA_NODES
Definition: buffer.h:404
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
clib_spinlock_t lock
Definition: buffer.h:395
static void * vlib_get_buffer_opaque2(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque2 data array.
Definition: buffer.h:300
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
u32 buffers_per_numa
Definition: buffer.h:424
#define ASSERT(truth)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:178
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
static uword pointer_to_uword(const void *p)
Definition: types.h:131
#define clib_max(x, y)
Definition: clib.h:288
static void * vlib_buffer_push_uninit(vlib_buffer_t *b, u8 size)
Prepend uninitialized data to buffer.
Definition: buffer.h:335
static u8 vlib_buffer_has_space(vlib_buffer_t *b, word l)
Check if there is enough space in buffer to advance.
Definition: buffer.h:265
static void * vlib_get_buffer_opaque(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque data array.
Definition: buffer.h:289
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:411
uword * buffer_known_hash
Definition: buffer.h:419
u32 default_data_size
Definition: buffer.h:426
#define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE
Definition: buffer.h:58
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:788
static uword vlib_buffer_get_va(vlib_buffer_t *b)
Definition: buffer.h:217
static void * vlib_buffer_pull(vlib_buffer_t *b, u8 size)
Retrieve bytes from buffer head.
Definition: buffer.h:363
#define foreach_vlib_buffer_flag
Buffer Flags.
Definition: buffer.h:73
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u16 size)
Append uninitialized data to buffer.
Definition: buffer.h:321
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
static void vlib_buffer_struct_is_sane(vlib_buffer_t *b)
Definition: buffer.h:208
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:153
signed short i16
Definition: types.h:46