FD.io VPP  v21.10.1-2-g0a485f517
Vector Packet Processing
buffer_node.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_node.h: VLIB buffer handling node helper macros/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_node_h
41 #define included_vlib_buffer_node_h
42 
43 /** \file
44  vlib buffer/node functions
45 */
46 
47 /** \brief Finish enqueueing two buffers forward in the graph.
48  Standard dual loop boilerplate element. This is a MACRO,
49  with MULTIPLE SIDE EFFECTS. In the ideal case,
50  <code>next_index == next0 == next1</code>,
51  which means that the speculative enqueue at the top of the dual loop
52  has correctly dealt with both packets. In that case, the macro does
53  nothing at all.
54 
55  @param vm vlib_main_t pointer, varies by thread
56  @param node current node vlib_node_runtime_t pointer
57  @param next_index speculated next index used for both packets
58  @param to_next speculated vector pointer used for both packets
59  @param n_left_to_next number of slots left in speculated vector
60  @param bi0 first buffer index
61  @param bi1 second buffer index
62  @param next0 actual next index to be used for the first packet
63  @param next1 actual next index to be used for the second packet
64 
65  @return @c next_index -- speculative next index to be used for future packets
66  @return @c to_next -- speculative frame to be used for future packets
67  @return @c n_left_to_next -- number of slots left in speculative frame
68 */
69 
70 #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
71 do { \
72  ASSERT (bi0 != 0); \
73  ASSERT (bi1 != 0); \
74  int enqueue_code = (next0 != next_index) + 2*(next1 != next_index); \
75  \
76  if (PREDICT_FALSE (enqueue_code != 0)) \
77  { \
78  switch (enqueue_code) \
79  { \
80  case 1: \
81  /* A B A */ \
82  to_next[-2] = bi1; \
83  to_next -= 1; \
84  n_left_to_next += 1; \
85  vlib_set_next_frame_buffer (vm, node, next0, bi0); \
86  break; \
87  \
88  case 2: \
89  /* A A B */ \
90  to_next -= 1; \
91  n_left_to_next += 1; \
92  vlib_set_next_frame_buffer (vm, node, next1, bi1); \
93  break; \
94  \
95  case 3: \
96  /* A B B or A B C */ \
97  to_next -= 2; \
98  n_left_to_next += 2; \
99  vlib_set_next_frame_buffer (vm, node, next0, bi0); \
100  vlib_set_next_frame_buffer (vm, node, next1, bi1); \
101  if (next0 == next1) \
102  { \
103  vlib_put_next_frame (vm, node, next_index, \
104  n_left_to_next); \
105  next_index = next1; \
106  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
107  } \
108  } \
109  } \
110 } while (0)
111 
112 
113 /** \brief Finish enqueueing four buffers forward in the graph.
114  Standard quad loop boilerplate element. This is a MACRO,
115  with MULTIPLE SIDE EFFECTS. In the ideal case,
116  <code>next_index == next0 == next1 == next2 == next3</code>,
117  which means that the speculative enqueue at the top of the quad loop
118  has correctly dealt with all four packets. In that case, the macro does
119  nothing at all.
120 
121  @param vm vlib_main_t pointer, varies by thread
122  @param node current node vlib_node_runtime_t pointer
123  @param next_index speculated next index used for both packets
124  @param to_next speculated vector pointer used for both packets
125  @param n_left_to_next number of slots left in speculated vector
126  @param bi0 first buffer index
127  @param bi1 second buffer index
128  @param bi2 third buffer index
129  @param bi3 fourth buffer index
130  @param next0 actual next index to be used for the first packet
131  @param next1 actual next index to be used for the second packet
132  @param next2 actual next index to be used for the third packet
133  @param next3 actual next index to be used for the fourth packet
134 
135  @return @c next_index -- speculative next index to be used for future packets
136  @return @c to_next -- speculative frame to be used for future packets
137  @return @c n_left_to_next -- number of slots left in speculative frame
138 */
139 
140 #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
141 do { \
142  ASSERT (bi0 != 0); \
143  ASSERT (bi1 != 0); \
144  ASSERT (bi2 != 0); \
145  ASSERT (bi3 != 0); \
146  /* After the fact: check the [speculative] enqueue to "next" */ \
147  u32 fix_speculation = (next_index ^ next0) | (next_index ^ next1) \
148  | (next_index ^ next2) | (next_index ^ next3); \
149  if (PREDICT_FALSE(fix_speculation)) \
150  { \
151  /* rewind... */ \
152  to_next -= 4; \
153  n_left_to_next += 4; \
154  \
155  /* If bi0 belongs to "next", send it there */ \
156  if (next_index == next0) \
157  { \
158  to_next[0] = bi0; \
159  to_next++; \
160  n_left_to_next --; \
161  } \
162  else /* send it where it needs to go */ \
163  vlib_set_next_frame_buffer (vm, node, next0, bi0); \
164  \
165  if (next_index == next1) \
166  { \
167  to_next[0] = bi1; \
168  to_next++; \
169  n_left_to_next --; \
170  } \
171  else \
172  vlib_set_next_frame_buffer (vm, node, next1, bi1); \
173  \
174  if (next_index == next2) \
175  { \
176  to_next[0] = bi2; \
177  to_next++; \
178  n_left_to_next --; \
179  } \
180  else \
181  vlib_set_next_frame_buffer (vm, node, next2, bi2); \
182  \
183  if (next_index == next3) \
184  { \
185  to_next[0] = bi3; \
186  to_next++; \
187  n_left_to_next --; \
188  } \
189  else \
190  { \
191  vlib_set_next_frame_buffer (vm, node, next3, bi3); \
192  \
193  /* Change speculation: last 2 packets went to the same node*/ \
194  if (next2 == next3) \
195  { \
196  vlib_put_next_frame (vm, node, next_index, n_left_to_next); \
197  next_index = next3; \
198  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
199  } \
200  } \
201  } \
202  } while(0);
203 
204 /** \brief Finish enqueueing one buffer forward in the graph.
205  Standard single loop boilerplate element. This is a MACRO,
206  with MULTIPLE SIDE EFFECTS. In the ideal case,
207  <code>next_index == next0</code>,
208  which means that the speculative enqueue at the top of the single loop
209  has correctly dealt with the packet in hand. In that case, the macro does
210  nothing at all.
211 
212  @param vm vlib_main_t pointer, varies by thread
213  @param node current node vlib_node_runtime_t pointer
214  @param next_index speculated next index used for both packets
215  @param to_next speculated vector pointer used for both packets
216  @param n_left_to_next number of slots left in speculated vector
217  @param bi0 first buffer index
218  @param next0 actual next index to be used for the first packet
219 
220  @return @c next_index -- speculative next index to be used for future packets
221  @return @c to_next -- speculative frame to be used for future packets
222  @return @c n_left_to_next -- number of slots left in speculative frame
223 */
224 #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
225 do { \
226  ASSERT (bi0 != 0); \
227  if (PREDICT_FALSE (next0 != next_index)) \
228  { \
229  vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \
230  next_index = next0; \
231  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
232  \
233  to_next[0] = bi0; \
234  to_next += 1; \
235  n_left_to_next -= 1; \
236  } \
237 } while (0)
238 
243  uword sizeof_trace,
244  void *opaque1,
245  uword opaque2,
246  void (*two_buffers) (vlib_main_t * vm,
247  void *opaque1,
248  uword opaque2,
249  vlib_buffer_t * b0,
250  vlib_buffer_t * b1,
251  u32 * next0, u32 * next1),
252  void (*one_buffer) (vlib_main_t * vm,
253  void *opaque1, uword opaque2,
254  vlib_buffer_t * b0,
255  u32 * next0))
256 {
257  u32 n_left_from, *from, *to_next;
258  u32 next_index;
259 
261  n_left_from = frame->n_vectors;
262  next_index = node->cached_next_index;
263 
264  if (node->flags & VLIB_NODE_FLAG_TRACE)
266  /* stride */ 1, sizeof_trace);
267 
268  while (n_left_from > 0)
269  {
270  u32 n_left_to_next;
271 
272  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
273 
274  while (n_left_from >= 4 && n_left_to_next >= 2)
275  {
276  vlib_buffer_t *p0, *p1;
277  u32 pi0, next0;
278  u32 pi1, next1;
279 
280  /* Prefetch next iteration. */
281  {
282  vlib_buffer_t *p2, *p3;
283 
284  p2 = vlib_get_buffer (vm, from[2]);
285  p3 = vlib_get_buffer (vm, from[3]);
286 
287  vlib_prefetch_buffer_header (p2, LOAD);
288  vlib_prefetch_buffer_header (p3, LOAD);
289 
290  clib_prefetch_load (p2->data);
291  clib_prefetch_load (p3->data);
292  }
293 
294  pi0 = to_next[0] = from[0];
295  pi1 = to_next[1] = from[1];
296  from += 2;
297  to_next += 2;
298  n_left_from -= 2;
299  n_left_to_next -= 2;
300 
301  p0 = vlib_get_buffer (vm, pi0);
302  p1 = vlib_get_buffer (vm, pi1);
303 
304  two_buffers (vm, opaque1, opaque2, p0, p1, &next0, &next1);
305 
307  to_next, n_left_to_next,
308  pi0, pi1, next0, next1);
309  }
310 
311  while (n_left_from > 0 && n_left_to_next > 0)
312  {
313  vlib_buffer_t *p0;
314  u32 pi0, next0;
315 
316  pi0 = from[0];
317  to_next[0] = pi0;
318  from += 1;
319  to_next += 1;
320  n_left_from -= 1;
321  n_left_to_next -= 1;
322 
323  p0 = vlib_get_buffer (vm, pi0);
324 
325  one_buffer (vm, opaque1, opaque2, p0, &next0);
326 
328  to_next, n_left_to_next,
329  pi0, next0);
330  }
331 
332  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
333  }
334 
335  return frame->n_vectors;
336 }
337 
338 /* Minimum size for the 'buffers' and 'nexts' arrays to be used when calling
339  * vlib_buffer_enqueue_to_next().
340  * Because of optimizations, vlib_buffer_enqueue_to_next() will access
341  * past 'count' elements in the 'buffers' and 'nexts' arrays, IOW it
342  * will overflow.
343  * Those overflow elements are ignored in the final result so they do not
344  * need to be properly initialized, however if the array is allocated right
345  * before the end of a page and the next page is not mapped, accessing the
346  * overflow elements will trigger a segfault. */
347 #define VLIB_BUFFER_ENQUEUE_MIN_SIZE(n) round_pow2 ((n), 64)
348 
351  u32 * buffers, u16 * nexts, uword count)
352 {
355  (fn) (vm, node, buffers, nexts, count);
356 }
357 
360  u32 **buffers, u16 **nexts, uword count)
361 {
362  const u32 bl = vec_len (*buffers), nl = vec_len (*nexts);
364  ASSERT (bl >= count && nl >= count);
365  vec_validate (*buffers, c);
366  vec_validate (*nexts, c);
368  vec_set_len (*buffers, bl);
369  vec_set_len (*nexts, nl);
370 }
371 
374  vlib_node_runtime_t * node, u32 * buffers,
376 {
379  (fn) (vm, node, buffers, next_index, count);
380 }
381 
384  u32 frame_queue_index, u32 *buffer_indices,
385  u16 *thread_indices, u32 n_packets,
386  int drop_on_congestion)
387 {
390  return (fn) (vm, node, frame_queue_index, buffer_indices, thread_indices,
391  n_packets, drop_on_congestion);
392 }
393 
394 #endif /* included_vlib_buffer_node_h */
395 
396 /*
397  * fd.io coding-style-patch-verification: ON
398  *
399  * Local Variables:
400  * eval: (c-set-style "gnu")
401  * End:
402  */
vlib_buffer_enqueue_to_single_next_fn_t
void() vlib_buffer_enqueue_to_single_next_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index, u32 count)
Definition: buffer_funcs.h:58
vlib_buffer_func_main_t::buffer_enqueue_to_thread_fn
vlib_buffer_enqueue_to_thread_fn_t * buffer_enqueue_to_thread_fn
Definition: buffer_funcs.h:74
vlib_buffer_enqueue_to_thread
static_always_inline u32 vlib_buffer_enqueue_to_thread(vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index, u32 *buffer_indices, u16 *thread_indices, u32 n_packets, int drop_on_congestion)
Definition: buffer_node.h:383
vlib_prefetch_buffer_header
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:231
frame
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: nat44_ei.c:3048
next_index
nat44_ei_hairpin_src_next_t next_index
Definition: nat44_ei_hairpinning.c:412
generic_buffer_node_inline
static uword generic_buffer_node_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, uword sizeof_trace, void *opaque1, uword opaque2, void(*two_buffers)(vlib_main_t *vm, void *opaque1, uword opaque2, vlib_buffer_t *b0, vlib_buffer_t *b1, u32 *next0, u32 *next1), void(*one_buffer)(vlib_main_t *vm, void *opaque1, uword opaque2, vlib_buffer_t *b0, u32 *next0))
Definition: buffer_node.h:240
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
vlib_trace_frame_buffers_only
void vlib_trace_frame_buffers_only(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, uword n_buffers, uword next_buffer_stride, uword n_buffer_data_bytes_in_trace)
Definition: trace.c:48
node
vlib_main_t vlib_node_runtime_t * node
Definition: nat44_ei.c:3047
u16
unsigned short u16
Definition: types.h:57
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
vlib_buffer_enqueue_to_next
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:350
vlib_frame_t
Definition: node.h:372
vlib_buffer_func_main_t::buffer_enqueue_to_next_fn
vlib_buffer_enqueue_to_next_fn_t * buffer_enqueue_to_next_fn
Definition: buffer_funcs.h:72
vlib_buffer_func_main_t::buffer_enqueue_to_single_next_fn
vlib_buffer_enqueue_to_single_next_fn_t * buffer_enqueue_to_single_next_fn
Definition: buffer_funcs.h:73
vlib_buffer_enqueue_to_single_next
static_always_inline void vlib_buffer_enqueue_to_single_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 next_index, u32 count)
Definition: buffer_node.h:373
count
u8 count
Definition: dhcp.api:208
vec_len
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
Definition: vec_bootstrap.h:142
vlib_buffer_func_main
vlib_buffer_func_main_t vlib_buffer_func_main
Definition: buffer_funcs.c:419
VLIB_NODE_FLAG_TRACE
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:291
vlib_frame_vector_args
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:301
vlib_buffer_enqueue_to_next_vec
static_always_inline void vlib_buffer_enqueue_to_next_vec(vlib_main_t *vm, vlib_node_runtime_t *node, u32 **buffers, u16 **nexts, uword count)
Definition: buffer_node.h:359
c
svmdb_client_t * c
Definition: vpp_get_metrics.c:48
static_always_inline
#define static_always_inline
Definition: clib.h:112
uword
u64 uword
Definition: types.h:112
VLIB_BUFFER_ENQUEUE_MIN_SIZE
#define VLIB_BUFFER_ENQUEUE_MIN_SIZE(n)
Definition: buffer_node.h:347
vec_validate
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment)
Definition: vec.h:523
vec_set_len
#define vec_set_len(v, l)
Set vector length to a user-defined value.
Definition: vec_bootstrap.h:179
vlib_validate_buffer_enqueue_x1
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:224
always_inline
#define always_inline
Definition: rdma_mlx5dv.h:23
vlib_buffer_enqueue_to_thread_fn_t
u32() vlib_buffer_enqueue_to_thread_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index, u32 *buffer_indices, u16 *thread_indices, u32 n_packets, int drop_on_congestion)
Definition: buffer_funcs.h:62
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
vlib_put_next_frame
vlib_put_next_frame(vm, node, next_index, 0)
u32
unsigned int u32
Definition: types.h:88
vlib_buffer_enqueue_to_next_fn_t
void() vlib_buffer_enqueue_to_next_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_funcs.h:54
clib_prefetch_load
static_always_inline void clib_prefetch_load(void *p)
Definition: cache.h:92
vlib_main_t
Definition: main.h:102
vlib_buffer_t::data
u8 data[]
Packet data.
Definition: buffer.h:204
nexts
u16 nexts[VLIB_FRAME_SIZE]
Definition: nat44_ei_out2in.c:718
vlib_validate_buffer_enqueue_x2
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
vlib_node_runtime_t
Definition: node.h:454
from
from
Definition: nat44_ei_hairpinning.c:415
vlib_get_next_frame
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:395
n_left_from
n_left_from
Definition: nat44_ei_hairpinning.c:416
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111