|
FD.io VPP
v21.10.1-2-g0a485f517
Vector Packet Processing
|
Go to the documentation of this file.
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
64 u32 *buffer_indices,
u16 *thread_indices,
u32 n_packets,
65 int drop_on_congestion);
145 u32 n = ring_size - start;
163 u32 n = ring_size - start;
173 #if defined CLIB_HAVE_VEC512
174 b->as_u8x64[0] = bt->as_u8x64[0];
175 #elif defined (CLIB_HAVE_VEC256)
176 b->as_u8x32[0] = bt->as_u8x32[0];
177 b->as_u8x32[1] = bt->as_u8x32[1];
178 #elif defined (CLIB_HAVE_VEC128)
179 b->as_u8x16[0] = bt->as_u8x16[0];
180 b->as_u8x16[1] = bt->as_u8x16[1];
181 b->as_u8x16[2] = bt->as_u8x16[2];
182 b->as_u8x16[3] = bt->as_u8x16[3];
208 #ifdef CLIB_HAVE_VEC512
209 u64x8 of8 = u64x8_splat (buffer_mem_start +
offset);
215 u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
216 u64x8 b1 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 8));
217 u64x8 b2 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 16));
218 u64x8 b3 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 24));
230 u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
237 #elif defined CLIB_HAVE_VEC256
243 u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
244 u64x4 b1 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 4));
245 u64x4 b2 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 8));
246 u64x4 b3 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 12));
247 u64x4 b4 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 16));
248 u64x4 b5 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 20));
249 u64x4 b6 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 24));
250 u64x4 b7 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 28));
267 #ifdef CLIB_HAVE_VEC256
268 u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
271 #elif defined (CLIB_HAVE_VEC128)
273 u32x4 bi4 = u32x4_load_unaligned (bi);
275 #if defined (__aarch64__)
346 #ifdef CLIB_HAVE_VEC256
347 u32x8
mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
353 u64x4 v0 = u64x4_load_unaligned (
b);
354 u64x4 v1 = u64x4_load_unaligned (
b + 4);
369 u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
370 u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
419 return (
b->
flags & VLIB_BUFFER_NEXT_PRESENT
469 uword content_len = 0;
479 if (!(
b->
flags & VLIB_BUFFER_NEXT_PRESENT))
507 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
509 vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
510 vlib_prefetch_buffer_header (_b, type); \
546 uword next_buffer_stride,
549 uword follow_buffer_next);
598 u8 buffer_pool_index)
606 if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
749 u8 buffer_pool_index)
777 u32 n_cached, n_empty;
810 const int queue_size = 128;
812 u8 buffer_pool_index = ~0;
813 u32 n_queue = 0, queue[queue_size + 4];
815 #if defined(CLIB_HAVE_VEC128)
819 .
flags = VLIB_BUFFER_NEXT_PRESENT,
831 #if defined(CLIB_HAVE_VEC128)
854 #if defined(CLIB_HAVE_VEC128)
856 p0 = u8x16_load_unaligned (
b[0]);
857 p1 = u8x16_load_unaligned (
b[1]);
858 p2 = u8x16_load_unaligned (
b[2]);
859 p3 = u8x16_load_unaligned (
b[3]);
861 r = p0 ^ bpi_vec.as_u8x16[0];
862 r |= p1 ^ bpi_vec.as_u8x16[0];
863 r |= p2 ^ bpi_vec.as_u8x16[0];
864 r |= p3 ^ bpi_vec.as_u8x16[0];
865 r &= bpi_mask.as_u8x16[0];
866 r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
868 sum = !u8x16_is_all_zero (
r);
874 sum &= VLIB_BUFFER_NEXT_PRESENT;
905 if (n_queue >= queue_size)
932 #if defined(CLIB_HAVE_VEC128)
946 queue[n_queue++] = bi;
949 if (n_queue == queue_size)
955 if (maybe_next && (
flags & VLIB_BUFFER_NEXT_PRESENT))
1072 #define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \
1073 (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \
1074 VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL)
1086 while (s->
flags & VLIB_BUFFER_NEXT_PRESENT)
1175 if (source +
length <= destination)
1178 memmove (destination, source,
length);
1211 buffers[0] = src_buffer;
1229 buffers[0] = src_buffer;
1255 VLIB_BUFFER_NEXT_PRESENT;
1265 while (s->
flags & VLIB_BUFFER_NEXT_PRESENT)
1301 (buffers + n_cloned),
1302 256, head_end_offset,
offset);
1329 head_end_offset, 0);
1343 ASSERT ((head->
flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1346 head->
flags |= VLIB_BUFFER_NEXT_PRESENT;
1347 head->
flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1348 head->
flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1349 head->
flags |= (tail->
flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
1357 if (tail->
flags & VLIB_BUFFER_NEXT_PRESENT)
1368 first->total_length_not_including_first_buffer = 0;
1369 first->current_length = 0;
1370 first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1371 first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1379 last->next_buffer = next_bi;
1380 last->flags |= VLIB_BUFFER_NEXT_PRESENT;
1382 next_buffer->
flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1396 first->total_length_not_including_first_buffer +=
len;
1408 ASSERT (n_buffer_bytes >=
last->current_length +
last->current_data);
1410 n_buffer_bytes -
last->current_length -
1411 last->current_data);
1447 uword n_packet_data_bytes,
1448 uword min_n_buffers_each_alloc,
1468 #define VLIB_BUFFER_LINEARIZE_MAX 64
1475 u16 rem_len, dst_len, data_size, src_len = 0;
1498 while (0 == src_len)
1527 const int n = (rem_len + data_size - 1) / data_size;
1544 for (
i = 0;
i < n - 1;
i++)
1564 copy_len =
clib_min (src_len, dst_len);
1574 memmove (
dst,
src, copy_len);
1585 dst_len -= copy_len;
1586 src_len -= copy_len;
1587 rem_len -= copy_len;
1597 if (dst_b->
flags & VLIB_BUFFER_NEXT_PRESENT)
1600 dst_b->
flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1605 dst_b->
flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
void() vlib_buffer_enqueue_to_single_next_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index, u32 count)
u32 next_buffer
Next buffer for this linked-list of buffers.
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
u8 buffer_pool_index
index of buffer pool this buffer belongs.
static u16 vlib_buffer_clone_256(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create a maximum of 256 clones of buffer and store them in the supplied array.
vlib_buffer_t * bufs[VLIB_FRAME_SIZE]
foreach_avx2_vec256i static foreach_avx2_vec256u u32x8 u32x8_permute(u32x8 v, u32x8 idx)
vlib_buffer_enqueue_to_thread_fn_t * buffer_enqueue_to_thread_fn
format_function_t format_vlib_buffer
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
static __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring_from_pool(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers into ring from specific buffer pool.
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, template_end, 64)
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
static void vlib_buffer_move(vlib_main_t *vm, vlib_buffer_t *b, i16 offset)
nat44_ei_hairpin_src_next_t next_index
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static uword pointer_to_uword(const void *p)
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
static __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Allocate buffers into ring.
#define clib_atomic_add_fetch(a, b)
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
vlib_main_t vlib_node_runtime_t * node
static void vlib_buffer_copy_indices_to_ring(u32 *ring, u32 *src, u32 start, u32 ring_size, u32 n_buffers)
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
static heap_elt_t * first(heap_header_t *h)
epu8_epi32 epu16_epi32 u64x2
vlib_buffer_main_t * buffer_main
static_always_inline __clib_warn_unused_result uword vlib_buffer_pool_get(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
static __clib_warn_unused_result u32 vlib_buffer_alloc_on_numa(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 numa_node)
Allocate buffers from specific numa node into supplied array.
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
format_function_t format_vlib_buffer_contents
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
static_always_inline void clib_memcpy_u32(u32 *dst, u32 *src, u32 n_left)
vnet_hw_if_output_node_runtime_t * r
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
static_always_inline vlib_buffer_pool_t * vlib_get_buffer_pool(vlib_main_t *vm, u8 buffer_pool_index)
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
static_always_inline u32x4 u32x4_shuffle(u32x4 v, const int a, const int b, const int c, const int d)
int vlib_buffer_add_data(vlib_main_t *vm, u32 *buffer_index, void *data, u32 n_data_bytes)
vlib_buffer_enqueue_to_next_fn_t * buffer_enqueue_to_next_fn
static_always_inline void vlib_get_buffer_indices(vlib_main_t *vm, vlib_buffer_t **b, u32 *bi, uword count)
Translate array of buffer pointers into buffer indices.
format_function_t format_vlib_buffer_no_chain
vlib_buffer_enqueue_to_single_next_fn_t * buffer_enqueue_to_single_next_fn
static uword round_pow2(uword x, uword pow2)
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
vlib_buffer_pool_thread_t * threads
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
static_always_inline u64x2 u64x2_from_u32x4(u32x4 v)
u32 min_n_buffers_each_alloc
static __clib_warn_unused_result u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
static void vlib_buffer_validate(vlib_main_t *vm, vlib_buffer_t *b)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
struct clib_bihash_value offset
template key/value backing page structure
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
vlib_buffer_t buffer_template
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
#define VLIB_BUFFER_MAX_NUMA_NODES
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
#define static_always_inline
static heap_elt_t * last(heap_header_t *h)
u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES]
format_function_t format_vlib_buffer_and_data
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
static u16 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
volatile u8 ref_count
Reference count for this buffer.
clib_spinlock_t buffer_known_hash_lockp
static u16 vlib_buffer_clone_at_offset(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create multiple clones of buffer and store them in the supplied array.
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
static u32 vlib_buffer_chain_linearize(vlib_main_t *vm, vlib_buffer_t *b)
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
#define CLIB_CACHE_LINE_BYTES
static vlib_buffer_t * vlib_buffer_copy_no_chain(vlib_main_t *vm, vlib_buffer_t *b, u32 *di)
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
u16 current_length
Nbytes between current data and the end of this buffer.
static u32 vlib_buffer_space_left_at_end(vlib_main_t *vm, vlib_buffer_t *b)
#define clib_atomic_sub_fetch(a, b)
static void vlib_buffer_chain_init(vlib_buffer_t *first)
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, int maybe_next)
u32 cached_buffers[VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ]
static_always_inline void vlib_buffer_pool_put(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
#define vec_free(V)
Free vector's memory (no header).
template key/value backing page structure
vlib_buffer_pool_t * buffer_pools
u32() vlib_buffer_enqueue_to_thread_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index, u32 *buffer_indices, u16 *thread_indices, u32 n_packets, int drop_on_congestion)
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer's data.
#define VLIB_BUFFER_PRE_DATA_SIZE
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
vlib_buffer_alloc_free_callback_t * alloc_callback_fn
void() vlib_buffer_enqueue_to_next_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
uword * buffer_known_hash
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
#define CLIB_LOG2_CACHE_LINE_BYTES
static_always_inline void vlib_buffer_copy_indices(u32 *dst, u32 *src, u32 n_indices)
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *last, u32 next_bi)
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
static void vlib_buffer_copy_indices_from_ring(u32 *dst, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
vlib_buffer_func_main_t vlib_buffer_func_main
vlib_buffer_known_state_t
#define VLIB_BUFFER_LINEARIZE_MAX
static void * vlib_buffer_ptr_from_index(uword buffer_mem_start, u32 buffer_index, uword offset)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
#define uword_to_pointer(u, type)
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK
static_always_inline u64x2 u64x2_from_u32x4_high(u32x4 v)
u16 nexts[VLIB_FRAME_SIZE]
@ VLIB_BUFFER_KNOWN_ALLOCATED
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
u32() vlib_frame_queue_dequeue_fn_t(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
vlib_frame_queue_dequeue_fn_t * frame_queue_dequeue_fn
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
static __clib_warn_unused_result u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.
vlib_buffer_alloc_free_callback_t * free_callback_fn
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
#define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,...
VLIB buffer representation.
static void vlib_buffer_free_from_ring_no_next(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring without freeing tail buffers.
vl_api_wireguard_peer_flags_t flags