37 #error files which #include <vnet/pipeline.h> must define NSTAGES 41 #define STAGE_INLINE inline 72 u32 n_left_from, n_left_to_next, *to_next, next_index, next0;
78 while (n_left_from > 0)
82 pi_limit =
clib_min (n_left_from, n_left_to_next);
84 for (pi = 0; pi <
NSTAGES - 1; pi++)
88 stage0 (vm, node, from[pi]);
91 for (; pi < pi_limit; pi++)
93 stage0 (vm, node, from[pi]);
94 to_next[0] = from[pi - 1];
99 to_next, n_left_to_next,
100 from[pi - 1], next0);
102 if ((
int) n_left_to_next < 0 && n_left_from > 0)
107 for (; pi < (pi_limit + (NSTAGES - 1)); pi++)
109 if (((pi - 1) >= 0) && ((pi - 1) < pi_limit))
111 to_next[0] = from[pi - 1];
116 to_next, n_left_to_next,
117 from[pi - 1], next0);
119 if ((
int) n_left_to_next < 0 && n_left_from > 0)
137 u32 n_left_from, n_left_to_next, *to_next, next_index, next0;
143 while (n_left_from > 0)
147 pi_limit =
clib_min (n_left_from, n_left_to_next);
149 for (pi = 0; pi <
NSTAGES - 1; pi++)
153 stage0 (vm, node, from[pi]);
155 stage1 (vm, node, from[pi - 1]);
158 for (; pi < pi_limit; pi++)
160 stage0 (vm, node, from[pi]);
161 stage1 (vm, node, from[pi - 1]);
162 to_next[0] = from[pi - 2];
167 to_next, n_left_to_next,
168 from[pi - 2], next0);
170 if ((
int) n_left_to_next < 0 && n_left_from > 0)
176 for (; pi < (pi_limit + (NSTAGES - 1)); pi++)
178 if (((pi - 1) >= 0) && ((pi - 1) < pi_limit))
179 stage1 (vm, node, from[pi - 1]);
180 if (((pi - 2) >= 0) && ((pi - 2) < pi_limit))
182 to_next[0] = from[pi - 2];
187 to_next, n_left_to_next,
188 from[pi - 2], next0);
190 if ((
int) n_left_to_next < 0 && n_left_from > 0)
209 u32 n_left_from, n_left_to_next, *to_next, next_index, next0;
215 while (n_left_from > 0)
219 pi_limit =
clib_min (n_left_from, n_left_to_next);
221 for (pi = 0; pi <
NSTAGES - 1; pi++)
225 stage0 (vm, node, from[pi]);
227 stage1 (vm, node, from[pi - 1]);
229 stage2 (vm, node, from[pi - 2]);
232 for (; pi < pi_limit; pi++)
234 stage0 (vm, node, from[pi]);
235 stage1 (vm, node, from[pi - 1]);
236 stage2 (vm, node, from[pi - 2]);
237 to_next[0] = from[pi - 3];
242 to_next, n_left_to_next,
243 from[pi - 3], next0);
245 if ((
int) n_left_to_next < 0 && n_left_from > 0)
251 for (; pi < (pi_limit + (NSTAGES - 1)); pi++)
253 if (((pi - 1) >= 0) && ((pi - 1) < pi_limit))
254 stage1 (vm, node, from[pi - 1]);
255 if (((pi - 2) >= 0) && ((pi - 2) < pi_limit))
256 stage2 (vm, node, from[pi - 2]);
257 if (((pi - 3) >= 0) && ((pi - 3) < pi_limit))
259 to_next[0] = from[pi - 3];
264 to_next, n_left_to_next,
265 from[pi - 3], next0);
267 if ((
int) n_left_to_next < 0 && n_left_from > 0)
287 u32 n_left_from, n_left_to_next, *to_next, next_index, next0;
293 while (n_left_from > 0)
297 pi_limit =
clib_min (n_left_from, n_left_to_next);
299 for (pi = 0; pi <
NSTAGES - 1; pi++)
303 stage0 (vm, node, from[pi]);
305 stage1 (vm, node, from[pi - 1]);
307 stage2 (vm, node, from[pi - 2]);
309 stage3 (vm, node, from[pi - 3]);
312 for (; pi < pi_limit; pi++)
314 stage0 (vm, node, from[pi]);
315 stage1 (vm, node, from[pi - 1]);
316 stage2 (vm, node, from[pi - 2]);
317 stage3 (vm, node, from[pi - 3]);
318 to_next[0] = from[pi - 4];
323 to_next, n_left_to_next,
324 from[pi - 4], next0);
326 if ((
int) n_left_to_next < 0 && n_left_from > 0)
332 for (; pi < (pi_limit + (NSTAGES - 1)); pi++)
334 if (((pi - 1) >= 0) && ((pi - 1) < pi_limit))
335 stage1 (vm, node, from[pi - 1]);
336 if (((pi - 2) >= 0) && ((pi - 2) < pi_limit))
337 stage2 (vm, node, from[pi - 2]);
338 if (((pi - 3) >= 0) && ((pi - 3) < pi_limit))
339 stage3 (vm, node, from[pi - 3]);
340 if (((pi - 4) >= 0) && ((pi - 4) < pi_limit))
342 to_next[0] = from[pi - 4];
347 to_next, n_left_to_next,
348 from[pi - 4], next0);
350 if ((
int) n_left_to_next < 0 && n_left_from > 0)
369 u32 n_left_from, n_left_to_next, *to_next, next_index, next0;
375 while (n_left_from > 0)
379 pi_limit =
clib_min (n_left_from, n_left_to_next);
381 for (pi = 0; pi <
NSTAGES - 1; pi++)
385 stage0 (vm, node, from[pi]);
387 stage1 (vm, node, from[pi - 1]);
389 stage2 (vm, node, from[pi - 2]);
391 stage3 (vm, node, from[pi - 3]);
393 stage4 (vm, node, from[pi - 4]);
396 for (; pi < pi_limit; pi++)
398 stage0 (vm, node, from[pi]);
399 stage1 (vm, node, from[pi - 1]);
400 stage2 (vm, node, from[pi - 2]);
401 stage3 (vm, node, from[pi - 3]);
402 stage4 (vm, node, from[pi - 4]);
403 to_next[0] = from[pi - 5];
408 to_next, n_left_to_next,
409 from[pi - 5], next0);
411 if ((
int) n_left_to_next < 0 && n_left_from > 0)
417 for (; pi < (pi_limit + (NSTAGES - 1)); pi++)
419 if (((pi - 1) >= 0) && ((pi - 1) < pi_limit))
420 stage1 (vm, node, from[pi - 1]);
421 if (((pi - 2) >= 0) && ((pi - 2) < pi_limit))
422 stage2 (vm, node, from[pi - 2]);
423 if (((pi - 3) >= 0) && ((pi - 3) < pi_limit))
424 stage3 (vm, node, from[pi - 3]);
425 if (((pi - 4) >= 0) && ((pi - 4) < pi_limit))
426 stage4 (vm, node, from[pi - 4]);
427 if (((pi - 5) >= 0) && ((pi - 5) < pi_limit))
429 to_next[0] = from[pi - 5];
434 to_next, n_left_to_next,
435 from[pi - 5], next0);
437 if ((
int) n_left_to_next < 0 && n_left_from > 0)
static u32 last_stage(vlib_main_t *vm, vlib_node_runtime_t *node, u32 bi)
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
static void stage1(vlib_main_t *vm, vlib_node_runtime_t *node, u32 bi)
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
#define CLIB_PREFETCH(addr, size, type)
static void stage0(vlib_main_t *vm, vlib_node_runtime_t *node, u32 buffer_index)
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
#define CLIB_CACHE_LINE_BYTES
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static STAGE_INLINE void generic_stage0(vlib_main_t *vm, vlib_node_runtime_t *node, u32 buffer_index)