FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
buffer.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.h: VLIB buffers
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
42 
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vppinfra/lock.h>
48 #include <vlib/error.h> /* for vlib_error_t */
49 
50 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 
53 #define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048)
54 
55 /* Minimum buffer chain segment size. Does not apply to last buffer in chain.
56  Dataplane code can safely asume that specified amount of data is not split
57  into 2 chained buffers */
58 #define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE (128)
59 
60 /* Amount of head buffer data copied to each replica head buffer */
61 #define VLIB_BUFFER_CLONE_HEAD_SIZE (256)
62 
63 /** \file
64  vlib buffer structure definition and a few select
65  access methods. This structure and the buffer allocation
66  mechanism should perhaps live in vnet, but it would take a lot
67  of typing to make it so.
68 */
69 
70 /**
71  * Buffer Flags
72  */
73 #define foreach_vlib_buffer_flag \
74  _( 0, IS_TRACED, 0) \
75  _( 1, NEXT_PRESENT, "next-present") \
76  _( 2, TOTAL_LENGTH_VALID, 0) \
77  _( 3, EXT_HDR_VALID, "ext-hdr-valid")
78 
79 /* NOTE: only buffer generic flags should be defined here, please consider
80  using user flags. i.e. src/vnet/buffer.h */
81 
82 enum
83 {
84 #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)),
86 #undef _
87 };
88 
89 enum
90 {
91 #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit),
93 #undef _
94 };
95 
96  /* User defined buffer flags. */
97 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
98 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
99 #define VLIB_BUFFER_FLAGS_ALL (0x0f)
100 
101 /** VLIB buffer representation. */
102 typedef union
103 {
104  struct
105  {
106  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
107 
108  /** signed offset in data[], pre_data[] that we are currently
109  * processing. If negative current header points into predata area. */
111 
112  /** Nbytes between current data and the end of this buffer. */
114 
115  /** buffer flags:
116  <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
117  <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
118  <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
119  <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
120  <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
121  set to avoid adding it to a flow report
122  <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
123  */
125 
126  /** Generic flow identifier */
128 
129  /** Reference count for this buffer. */
130  volatile u8 ref_count;
131 
132  /** index of buffer pool this buffer belongs. */
134 
135  /** Error code for buffers to be enqueued to error handler. */
137 
138  /** Next buffer for this linked-list of buffers. Only valid if
139  * VLIB_BUFFER_NEXT_PRESENT flag is set. */
141 
142  /** The following fields can be in a union because once a packet enters
143  * the punt path, it is no longer on a feature arc */
144  union
145  {
146  /** Used by feature subgraph arcs to visit enabled feature nodes */
148  /* the reason the packet once punted */
150  };
151 
152  /** Opaque data used by sub-graphs for their own purposes. */
153  u32 opaque[10];
154 
155  /** part of buffer metadata which is initialized on alloc ends here. */
156  STRUCT_MARK (template_end);
157 
158  /** start of 2nd cache line */
159  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
160 
161  /** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is
162  * set. */
164 
165  /** Only valid for first buffer in chain. Current length plus total length
166  * given here give total number of bytes in buffer chain. */
168 
169  /**< More opaque data, see ../vnet/vnet/buffer.h */
170  u32 opaque2[14];
171 
172  /** start of third cache line */
173  CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
174 
175  /** Space for inserting data before buffer start. Packet rewrite string
176  * will be rewritten backwards and may extend back before
177  * buffer->data[0]. Must come directly before packet data. */
179 
180  /** Packet data */
181  u8 data[];
182  };
183 #ifdef CLIB_HAVE_VEC128
184  u8x16 as_u8x16[4];
185 #endif
186 #ifdef CLIB_HAVE_VEC256
187  u8x32 as_u8x32[2];
188 #endif
189 #ifdef CLIB_HAVE_VEC512
190  u8x64 as_u8x64[1];
191 #endif
192 } vlib_buffer_t;
193 
194 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
195 
196 /** \brief Prefetch buffer metadata.
197  The first 64 bytes of buffer contains most header information
198 
199  @param b - (vlib_buffer_t *) pointer to the buffer
200  @param type - LOAD, STORE. In most cases, STORE is the right answer
201 */
202 
203 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
204 #define vlib_prefetch_buffer_data(b,type) \
205  CLIB_PREFETCH (vlib_buffer_get_current(b), CLIB_CACHE_LINE_BYTES, type)
206 
207 always_inline void
209 {
210  ASSERT (sizeof (b[0]) % 64 == 0);
211 
212  /* Rewrite data must be before and contiguous with packet data. */
214 }
215 
218 {
219  return pointer_to_uword (b->data);
220 }
221 
222 /** \brief Get pointer to current data to process
223 
224  @param b - (vlib_buffer_t *) pointer to the buffer
225  @return - (void *) (b->data + b->current_data)
226 */
227 
228 always_inline void *
230 {
231  /* Check bounds. */
232  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
233  return b->data + b->current_data;
234 }
235 
238 {
239  return vlib_buffer_get_va (b) + b->current_data;
240 }
241 
242 /** \brief Advance current data pointer by the supplied (signed!) amount
243 
244  @param b - (vlib_buffer_t *) pointer to the buffer
245  @param l - (word) signed increment
246 */
247 always_inline void
249 {
250  ASSERT (b->current_length >= l);
251  b->current_data += l;
252  b->current_length -= l;
253 
254  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 ||
256 }
257 
258 /** \brief Check if there is enough space in buffer to advance
259 
260  @param b - (vlib_buffer_t *) pointer to the buffer
261  @param l - (word) size to check
262  @return - 0 if there is less space than 'l' in buffer
263 */
266 {
267  return b->current_length >= l;
268 }
269 
270 /** \brief Reset current header & length to state they were in when
271  packet was received.
272 
273  @param b - (vlib_buffer_t *) pointer to the buffer
274 */
275 
276 always_inline void
278 {
279  b->current_length += clib_max (b->current_data, 0);
280  b->current_data = 0;
281 }
282 
283 /** \brief Get pointer to buffer's opaque data array
284 
285  @param b - (vlib_buffer_t *) pointer to the buffer
286  @return - (void *) b->opaque
287 */
288 always_inline void *
290 {
291  return (void *) b->opaque;
292 }
293 
294 /** \brief Get pointer to buffer's opaque2 data array
295 
296  @param b - (vlib_buffer_t *) pointer to the buffer
297  @return - (void *) b->opaque2
298 */
299 always_inline void *
301 {
302  return (void *) b->opaque2;
303 }
304 
305 /** \brief Get pointer to the end of buffer's data
306  * @param b pointer to the buffer
307  * @return pointer to tail of packet's data
308  */
311 {
312  return b->data + b->current_data + b->current_length;
313 }
314 
315 /** \brief Append uninitialized data to buffer
316  * @param b pointer to the buffer
317  * @param size number of uninitialized bytes
318  * @return pointer to beginning of uninitialized data
319  */
320 always_inline void *
322 {
323  void *p = vlib_buffer_get_tail (b);
324  /* XXX make sure there's enough space */
325  b->current_length += size;
326  return p;
327 }
328 
329 /** \brief Prepend uninitialized data to buffer
330  * @param b pointer to the buffer
331  * @param size number of uninitialized bytes
332  * @return pointer to beginning of uninitialized data
333  */
334 always_inline void *
336 {
338  b->current_data -= size;
339  b->current_length += size;
340 
341  return vlib_buffer_get_current (b);
342 }
343 
344 /** \brief Make head room, typically for packet headers
345  * @param b pointer to the buffer
346  * @param size number of head room bytes
347  * @return pointer to start of buffer (current data)
348  */
349 always_inline void *
351 {
353  b->current_data += size;
354  return vlib_buffer_get_current (b);
355 }
356 
357 /** \brief Construct a trace handle from thread and pool index
358  * @param thread Thread id
359  * @param pool_index Pool index
360  * @return trace handle
361  */
364 {
365  u32 rv;
366  ASSERT (thread < 0xff);
367  ASSERT (pool_index < 0x00FFFFFF);
368  rv = (thread << 24) | (pool_index & 0x00FFFFFF);
369  return rv;
370 }
371 
372 /** \brief Extract the thread id from a trace handle
373  * @param trace_handle the trace handle
374  * @return the thread id
375  */
378 {
379  u32 trace_handle = b->trace_handle;
380 
381  return trace_handle >> 24;
382 }
383 
384 /** \brief Extract the trace (pool) index from a trace handle
385  * @param trace_handle the trace handle
386  * @return the trace index
387  */
390 {
391  u32 trace_handle = b->trace_handle;
392  return trace_handle & 0x00FFFFFF;
393 }
394 
395 /** \brief Retrieve bytes from buffer head
396  * @param b pointer to the buffer
397  * @param size number of bytes to pull
398  * @return pointer to start of buffer (current data)
399  */
400 always_inline void *
402 {
404  return 0;
405 
406  void *data = vlib_buffer_get_current (b);
407  vlib_buffer_advance (b, size);
408  return data;
409 }
410 
411 /* Forward declaration. */
412 struct vlib_main_t;
413 
414 typedef struct
415 {
416  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
420 typedef struct
421 {
422  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
434 
435  /* per-thread data */
437 
438  /* buffer metadata template */
441 
442 #define VLIB_BUFFER_MAX_NUMA_NODES 32
443 
444 typedef struct
445 {
446  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
447  /* Virtual memory address and size of buffer memory, used for calculating
448  buffer index */
452 
453  /* Hash table mapping buffer index into number
454  0 => allocated but free, 1 => allocated and not-free.
455  If buffer index is not in hash table then this buffer
456  has never been allocated. */
459  u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
460 
461  /* config */
465 
466  /* logging */
469 
471 
472 /*
473  */
474 
475 /** \brief Compile time buffer trajectory tracing option
476  Turn this on if you run into "bad monkey" contexts,
477  and you want to know exactly which nodes they've visited...
478  See vlib/main.c...
479 */
480 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
481 
482 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
483 extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index);
484 extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
485 extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b);
486 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \
487  vlib_buffer_trace_trajectory_init (b);
488 #else
489 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
490 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
491 
492 extern u16 __vlib_buffer_external_hdr_size;
493 #define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
494 static void __clib_constructor \
495 vnet_buffer_set_ext_hdr_size() \
496 { \
497  if (__vlib_buffer_external_hdr_size) \
498  clib_error ("buffer external header space already set"); \
499  __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \
500 }
501 
502 #endif /* included_vlib_buffer_h */
503 
504 /*
505  * fd.io coding-style-patch-verification: ON
506  *
507  * Local Variables:
508  * eval: (c-set-style "gnu")
509  * End:
510  */
u32 opaque2[14]
Definition: buffer.h:170
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:310
static uword vlib_buffer_get_current_va(vlib_buffer_t *b)
Definition: buffer.h:237
static u32 vlib_buffer_make_trace_handle(u32 thread, u32 pool_index)
Construct a trace handle from thread and pool index.
Definition: buffer.h:363
static void vlib_buffer_reset(vlib_buffer_t *b)
Reset current header & length to state they were in when packet was received.
Definition: buffer.h:277
static u32 vlib_buffer_get_trace_index(vlib_buffer_t *b)
Extract the trace (pool) index from a trace handle.
Definition: buffer.h:389
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
u32 physmem_map_index
Definition: buffer.h:428
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u16 vlib_error_t
Definition: error.h:43
#define STRUCT_MARK(mark)
Definition: clib.h:71
u8 data[128]
Definition: ipsec.api:251
uword log2_page_size
Definition: buffer.h:425
unsigned char u8
Definition: types.h:56
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
vlib_buffer_pool_thread_t * threads
Definition: buffer.h:436
u32 vlib_log_class_t
Definition: vlib.h:51
i64 word
Definition: types.h:111
#define always_inline
Definition: clib.h:99
uword buffer_mem_size
Definition: buffer.h:450
pthread_t thread[MAX_CONNS]
Definition: main.c:142
unsigned int u32
Definition: types.h:88
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
uword size
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:350
unsigned short u16
Definition: types.h:57
vlib_buffer_t buffer_template
Definition: buffer.h:439
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:451
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
vlib_log_class_t log_default
Definition: buffer.h:467
u32 punt_reason
Definition: buffer.h:149
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:458
vlib_main_t * vm
Definition: buffer.c:323
u32 flow_id
Generic flow identifier.
Definition: buffer.h:127
#define VLIB_BUFFER_MAX_NUMA_NODES
Definition: buffer.h:442
u8 data[]
Packet data.
Definition: buffer.h:181
static u32 vlib_buffer_get_trace_thread(vlib_buffer_t *b)
Extract the thread id from a trace handle.
Definition: buffer.h:377
clib_spinlock_t lock
Definition: buffer.h:433
static void * vlib_get_buffer_opaque2(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque2 data array.
Definition: buffer.h:300
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
u32 buffers_per_numa
Definition: buffer.h:462
#define ASSERT(truth)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:178
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
static uword pointer_to_uword(const void *p)
Definition: types.h:131
#define clib_max(x, y)
Definition: clib.h:295
static void * vlib_buffer_push_uninit(vlib_buffer_t *b, u8 size)
Prepend uninitialized data to buffer.
Definition: buffer.h:335
static u8 vlib_buffer_has_space(vlib_buffer_t *b, word l)
Check if there is enough space in buffer to advance.
Definition: buffer.h:265
static void * vlib_get_buffer_opaque(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque data array.
Definition: buffer.h:289
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:449
uword * buffer_known_hash
Definition: buffer.h:457
u32 default_data_size
Definition: buffer.h:464
#define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE
Definition: buffer.h:58
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:820
static uword vlib_buffer_get_va(vlib_buffer_t *b)
Definition: buffer.h:217
static void * vlib_buffer_pull(vlib_buffer_t *b, u8 size)
Retrieve bytes from buffer head.
Definition: buffer.h:401
#define foreach_vlib_buffer_flag
Buffer Flags.
Definition: buffer.h:73
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u16 size)
Append uninitialized data to buffer.
Definition: buffer.h:321
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
static void vlib_buffer_struct_is_sane(vlib_buffer_t *b)
Definition: buffer.h:208
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:153
signed short i16
Definition: types.h:46