40 #ifndef included_vlib_buffer_node_h 41 #define included_vlib_buffer_node_h 70 #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \ 74 int enqueue_code = (next0 != next_index) + 2*(next1 != next_index); \ 76 if (PREDICT_FALSE (enqueue_code != 0)) \ 78 switch (enqueue_code) \ 84 n_left_to_next += 1; \ 85 vlib_set_next_frame_buffer (vm, node, next0, bi0); \ 91 n_left_to_next += 1; \ 92 vlib_set_next_frame_buffer (vm, node, next1, bi1); \ 98 n_left_to_next += 2; \ 99 vlib_set_next_frame_buffer (vm, node, next0, bi0); \ 100 vlib_set_next_frame_buffer (vm, node, next1, bi1); \ 101 if (next0 == next1) \ 103 vlib_put_next_frame (vm, node, next_index, \ 105 next_index = next1; \ 106 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \ 140 #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \ 147 u32 fix_speculation = (next_index ^ next0) | (next_index ^ next1) \ 148 | (next_index ^ next2) | (next_index ^ next3); \ 149 if (PREDICT_FALSE(fix_speculation)) \ 153 n_left_to_next += 4; \ 156 if (next_index == next0) \ 163 vlib_set_next_frame_buffer (vm, node, next0, bi0); \ 165 if (next_index == next1) \ 172 vlib_set_next_frame_buffer (vm, node, next1, bi1); \ 174 if (next_index == next2) \ 181 vlib_set_next_frame_buffer (vm, node, next2, bi2); \ 183 if (next_index == next3) \ 191 vlib_set_next_frame_buffer (vm, node, next3, bi3); \ 194 if (next2 == next3) \ 196 vlib_put_next_frame (vm, node, next_index, n_left_to_next); \ 197 next_index = next3; \ 198 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \ 224 #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \ 227 if (PREDICT_FALSE (next0 != next_index)) \ 229 vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \ 230 next_index = next0; \ 231 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \ 235 n_left_to_next -= 1; \ 251 u32 * next0,
u32 * next1),
253 void *opaque1,
uword opaque2,
257 u32 n_left_from, *from, *to_next;
268 while (n_left_from > 0)
274 while (n_left_from >= 4 && n_left_to_next >= 2)
294 pi0 = to_next[0] = from[0];
295 pi1 = to_next[1] = from[1];
304 two_buffers (vm, opaque1, opaque2, p0, p1, &next0, &next1);
307 to_next, n_left_to_next,
308 pi0, pi1, next0, next1);
311 while (n_left_from > 0 && n_left_to_next > 0)
325 one_buffer (vm, opaque1, opaque2, p0, &next0);
328 to_next, n_left_to_next,
342 u32 *to_next, n_left_to_next, max;
345 next_index = nexts[0];
347 max =
clib_min (n_left_to_next, count);
352 if ((nexts[0] != next_index) || n_left_to_next == 0)
355 next_index = nexts[0];
357 max =
clib_min (n_left_to_next, count);
359 #if defined(CLIB_HAVE_VEC512) 361 next32 = (next32 == u16x32_splat (next32[0]));
364 #elif defined(CLIB_HAVE_VEC256) 366 next16 = (next16 == u16x16_splat (next16[0]));
369 #elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK) 371 next8 = (next8 == u16x8_splat (next8[0]));
378 x |= next_index ^ nexts[1];
379 x |= next_index ^ nexts[2];
380 x |= next_index ^ nexts[3];
381 n_enqueued = (x == 0) ? 4 : 1;
390 #ifdef CLIB_HAVE_VEC512 391 if (n_enqueued >= 32)
397 n_left_to_next -= 32;
404 #ifdef CLIB_HAVE_VEC256 405 if (n_enqueued >= 16)
411 n_left_to_next -= 16;
418 #ifdef CLIB_HAVE_VEC128 445 to_next[0] = buffers[0];
463 u32 *to_next, n_left_to_next, n_enq;
470 n_left_to_next -=
count;
475 n_enq = n_left_to_next;
478 n_left_to_next -= n_enq;
487 n_enq =
clib_min (n_left_to_next, count);
495 u32 * buffer_indices,
u16 * thread_indices,
496 u32 n_packets,
int drop_on_congestion)
501 u32 n_left = n_packets;
504 u32 n_left_to_next_thread = 0, *to_next_thread = 0;
505 u32 next_thread_index, current_thread_index = ~0;
513 next_thread_index = thread_indices[0];
515 if (next_thread_index != current_thread_index)
517 if (drop_on_congestion &&
522 dbi[0] = buffer_indices[0];
537 current_thread_index = next_thread_index;
540 to_next_thread[0] = buffer_indices[0];
542 n_left_to_next_thread--;
544 if (n_left_to_next_thread == 0)
549 current_thread_index = ~0;
587 if (drop_on_congestion && n_drop)
590 return n_packets - n_drop;
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
u16x16 u64x4 static_always_inline u32 u8x32_msb_mask(u8x32 v)
u32 buffer_index[VLIB_FRAME_SIZE]
vlib_main_t ** vlib_mains
static vlib_frame_queue_t * is_vlib_frame_queue_congested(u32 frame_queue_index, u32 index, u32 queue_hi_thresh, vlib_frame_queue_t **handoff_queue_by_worker_index)
#define count_trailing_zeros(x)
volatile uword check_frame_queues
#define static_always_inline
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
static_always_inline void vlib_buffer_enqueue_to_single_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 next_index, u32 count)
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
static_always_inline u16 u8x16_msb_mask(u8x16 v)
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
#define CLIB_PREFETCH(addr, size, type)
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
sll srl srl sll sra u16x4 i
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
vlib_frame_queue_per_thread_data_t * per_thread_data
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
vlib_frame_queue_main_t * frame_queue_mains
foreach_avx512_vec512i foreach_avx512_vec512u static_always_inline u32 u16x32_msb_mask(u16x32 v)
static uword generic_buffer_node_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, uword sizeof_trace, void *opaque1, uword opaque2, void(*two_buffers)(vlib_main_t *vm, void *opaque1, uword opaque2, vlib_buffer_t *b0, vlib_buffer_t *b1, u32 *next0, u32 *next1), void(*one_buffer)(vlib_main_t *vm, void *opaque1, uword opaque2, vlib_buffer_t *b0, u32 *next0))
void vlib_trace_frame_buffers_only(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, uword n_buffers, uword next_buffer_stride, uword n_buffer_data_bytes_in_trace)
static_always_inline void vlib_buffer_copy_indices(u32 *dst, u32 *src, u32 n_indices)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
VLIB buffer representation.
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
static_always_inline u32 vlib_buffer_enqueue_to_thread(vlib_main_t *vm, u32 frame_queue_index, u32 *buffer_indices, u16 *thread_indices, u32 n_packets, int drop_on_congestion)
static vlib_thread_main_t * vlib_get_thread_main()
#define CLIB_MEM_OVERFLOW_LOAD(f, src)
u16 flags
Copy of main node flags.
#define VLIB_NODE_FLAG_TRACE
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.