46 #include <rte_config.h> 48 #include <rte_common.h> 50 #include <rte_memory.h> 51 #include <rte_memzone.h> 52 #include <rte_tailq.h> 54 #include <rte_per_lcore.h> 55 #include <rte_launch.h> 56 #include <rte_atomic.h> 57 #include <rte_cycles.h> 58 #include <rte_prefetch.h> 59 #include <rte_lcore.h> 60 #include <rte_per_lcore.h> 61 #include <rte_branch_prediction.h> 62 #include <rte_interrupts.h> 64 #include <rte_random.h> 65 #include <rte_debug.h> 66 #include <rte_ether.h> 67 #include <rte_ethdev.h> 69 #include <rte_mempool.h> 71 #include <rte_version.h> 80 "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
120 mb = rte_pktmbuf_prefree_seg (mb);
142 #ifndef CLIB_MULTIARCH_VARIANT 174 if (merge_index != ~0 && merge_index != free_list_index)
184 memset (f, 0xab,
sizeof (f[0]));
192 memset (f, 0xab,
sizeof (f[0]));
202 uword min_free_buffers)
207 u32 bi0, bi1, bi2, bi3;
208 unsigned socket_id = rte_socket_id ();
211 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
219 n = min_free_buffers -
vec_len (
fl->buffers);
221 return min_free_buffers;
227 n =
clib_max (n,
fl->min_n_buffers_each_physmem_alloc);
270 fl->buffers[f++] = bi0;
271 fl->buffers[f++] = bi1;
272 fl->buffers[f++] = bi2;
273 fl->buffers[f++] = bi3;
277 if (
fl->buffer_init_function)
279 fl->buffer_init_function (vm,
fl, &bi0, 1);
280 fl->buffer_init_function (vm,
fl, &bi1, 1);
281 fl->buffer_init_function (vm,
fl, &bi2, 1);
282 fl->buffer_init_function (vm,
fl, &bi3, 1);
294 fl->buffers[f++] = bi0;
297 if (
fl->buffer_init_function)
298 fl->buffer_init_function (vm,
fl, &bi0, 1);
338 goto already_announced;
353 u32 * buffers,
u32 n_buffers,
u32 follow_buffer_next)
362 u32 follow_buffer_next);
367 n_buffers = (*cb) (
vm, buffers, n_buffers, follow_buffer_next);
372 while (i + 7 < n_buffers)
396 while (i < n_buffers)
442 #ifndef CLIB_MULTIARCH_VARIANT 447 uword n_packet_data_bytes,
448 uword min_n_buffers_each_physmem_alloc,
u8 * name)
453 memset (t, 0,
sizeof (t[0]));
462 u32 num_elts,
u32 pool_priv_size,
u16 cache_size,
u8 numa,
465 struct rte_mempool *mp;
471 obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0);
472 #if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0) 473 size = rte_mempool_xmem_size (num_elts, obj_size, 21);
475 size = rte_mempool_xmem_size (num_elts, obj_size, 21, 0);
486 rte_mempool_create_empty ((
i8 *) pool_name, num_elts, elt_size,
487 512, pool_priv_size, numa, 0);
491 rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS,
NULL);
493 #if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0) 502 if (ret != (
i32) mp->size)
504 rte_mempool_free (mp);
518 struct rte_mempool *rmp;
531 pool_name =
format (0,
"dpdk_mbuf_pool_socket%u%c", socket_id, 0);
533 elt_size =
sizeof (
struct rte_mbuf) +
534 VLIB_BUFFER_HDR_SIZE +
535 VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE;
546 priv.
mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE +
551 rte_pktmbuf_pool_init (rmp, &priv);
554 rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0);
571 clib_warning (
"WARNING: Failed to allocate mempool for CPU socket " 572 "%u. Threads running on socket %u will use socket %u " 573 "mempool.", socket_id, socket_id, i);
609 struct dpdk_validate_buf_result
615 #define DPDK_TRAJECTORY_POISON 31 618 dpdk_buffer_validate_trajectory (
struct rte_mempool *mp,
void *opaque,
619 void *obj,
unsigned obj_idx)
622 struct dpdk_validate_buf_result *counter = opaque;
626 if (b->
pre_data[0] == DPDK_TRAJECTORY_POISON)
627 counter->uninitialized++;
634 dpdk_buffer_validate_trajectory_all (
u32 * uninitialized)
637 struct dpdk_validate_buf_result counter = { 0 };
642 dpdk_buffer_validate_trajectory, &counter);
644 *uninitialized = counter.uninitialized;
645 return counter.invalid;
649 dpdk_buffer_poison_trajectory (
struct rte_mempool *mp,
void *opaque,
650 void *obj,
unsigned obj_idx)
654 b->
pre_data[0] = DPDK_TRAJECTORY_POISON;
658 dpdk_buffer_poison_trajectory_all (
void)
664 rte_mempool_obj_iter (dm->
pktmbuf_pools[i], dpdk_buffer_poison_trajectory,
699 static void __clib_constructor
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define vlib_buffer_from_rte_mbuf(x)
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
sll srl srl sll sra u16x4 i
void vlib_buffer_merge_free_lists(vlib_buffer_free_list_t *dst, vlib_buffer_free_list_t *src)
dpdk_buffer_main_t dpdk_buffer_main
static clib_error_t * dpdk_buffer_init(vlib_main_t *vm)
#define VLIB_BUFFER_RECYCLE
void * mheap_alloc(void *memory, uword size)
u32 * vlib_buffer_state_validation_lock
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512
#define VLIB_BUFFER_PRE_DATA_SIZE
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
u8 buffer_pool_index
index of buffer pool this buffer belongs.
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
vlib_buffer_main_t * buffer_main
struct rte_mempool * last_pool
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
void( vlib_buffer_free_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
vlib_main_t ** vlib_mains
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define vlib_worker_thread_barrier_sync(X)
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2
#define VLIB_BUFFER_NEXT_PRESENT
#define static_always_inline
uword * vlib_buffer_state_validation_hash
#define VLIB_INIT_FUNCTION(x)
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
static void __clib_constructor dpdk_input_multiarch_select(void)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
vlib_buffer_free_list_t ** announce_list
#define rte_mbuf_from_vlib_buffer(x)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
static clib_error_t * buffer_state_validation_init(vlib_main_t *vm)
static u32 vlib_buffer_get_free_list_with_size(vlib_main_t *vm, u32 size)
#define pool_put(P, E)
Free an object E in pool P.
uword CLIB_MULTIARCH_FN() dpdk_buffer_fill_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b)
u8 last_buffer_pool_index
clib_error_t * dpdk_pool_create(vlib_main_t *vm, u8 *pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, struct rte_mempool **_mp, vlib_physmem_region_index_t *pri)
struct rte_pktmbuf_pool_private mbp_priv
STATIC_ASSERT(VLIB_BUFFER_PRE_DATA_SIZE==RTE_PKTMBUF_HEADROOM,"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM")
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
static_always_inline uword vlib_get_thread_index(void)
u8 vlib_buffer_add_physmem_region(vlib_main_t *vm, vlib_physmem_region_index_t pri)
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
#define CLIB_PREFETCH(addr, size, type)
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512
#define vec_free(V)
Free vector's memory (no header).
static_always_inline void clib_memcpy64_x4(void *d0, void *d1, void *d2, void *d3, void *s)
static void * clib_mem_set_heap(void *heap)
#define clib_warning(format, args...)
#define clib_memcpy(a, b, c)
void CLIB_MULTIARCH_FN() dpdk_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
static uword round_pow2(uword x, uword pow2)
void * vlib_buffer_state_heap
static void dpdk_buffer_delete_free_list(vlib_main_t *vm, u32 free_list_index)
#define VLIB_BUFFER_DATA_SIZE
#define VLIB_BUFFER_HDR_SIZE
void CLIB_MULTIARCH_FN() dpdk_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
void( vlib_buffer_free_no_next_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
#define hash_create(elts, value_bytes)
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2
vlib_buffer_free_cb_t * vlib_buffer_free_cb
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, u32 *index)
u32 next_buffer
Next buffer for this linked-list of buffers.
dpdk_buffer_per_thread_data * ptd
struct rte_mempool ** pktmbuf_pools
#define clib_error_report(e)
struct rte_mbuf *** mbuf_pending_free_list
VLIB_BUFFER_REGISTER_CALLBACKS(dpdk, static)
u8 n_add_refs
Number of additional references to this buffer.
static_always_inline void dpdk_prefetch_buffer_by_index(vlib_main_t *vm, u32 bi)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
vlib_buffer_free_list_t * buffer_free_list_pool
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
static vlib_thread_main_t * vlib_get_thread_main()
static void dpdk_packet_template_init(vlib_main_t *vm, void *vt, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, u8 *name)
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx2
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
uword( vlib_buffer_fill_free_list_cb_t)(struct vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
#define CLIB_CACHE_LINE_BYTES
static_always_inline void dpdk_rte_pktmbuf_free(vlib_main_t *vm, u32 thread_index, vlib_buffer_t *b)
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
u8 vlib_physmem_region_index_t
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
#define CLIB_MULTIARCH_FN(fn)
#define vec_resize_aligned(V, N, A)
Resize a vector (no header, alignment specified).