48 #include <rte_config.h> 50 #include <rte_common.h> 52 #include <rte_memory.h> 53 #include <rte_memzone.h> 54 #include <rte_tailq.h> 56 #include <rte_per_lcore.h> 57 #include <rte_launch.h> 58 #include <rte_atomic.h> 59 #include <rte_cycles.h> 60 #include <rte_prefetch.h> 61 #include <rte_lcore.h> 62 #include <rte_per_lcore.h> 63 #include <rte_branch_prediction.h> 64 #include <rte_interrupts.h> 66 #include <rte_random.h> 67 #include <rte_debug.h> 68 #include <rte_ether.h> 69 #include <rte_ethdev.h> 71 #include <rte_mempool.h> 73 #include <rte_version.h> 82 "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
116 if ((mb = rte_pktmbuf_prefree_seg (mb)))
117 rte_mempool_put (mb->pool, mb);
119 if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
126 #ifndef CLIB_MARCH_VARIANT 176 uword min_free_buffers)
180 struct rte_mbuf **mb;
183 unsigned socket_id = rte_socket_id ();
196 n_alloc = min_free_buffers -
vec_len (
fl->buffers);
198 return min_free_buffers;
204 n_alloc =
clib_max (n_alloc,
fl->min_n_buffers_each_alloc);
209 if (rte_mempool_get_bulk (rmp, (
void *) d->
mbuf_alloc_list, n_alloc) < 0)
243 sizeof (
struct rte_mbuf));
258 if (
fl->buffer_init_function)
259 fl->buffer_init_function (vm,
fl,
fl->buffers + first, n_alloc);
261 fl->n_alloc += n_alloc;
286 u32 * buffers,
u32 n_buffers,
u32 follow_buffer_next)
292 u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST |
293 VLIB_BUFFER_NEXT_PRESENT);
296 u32 follow_buffer_next);
301 n_buffers = (*cb) (
vm, buffers, n_buffers, follow_buffer_next);
326 for (i = 0; i < 4; i++)
331 if (or_flags & simple_mask)
375 #ifndef CLIB_MARCH_VARIANT 378 u32 num_elts,
u32 pool_priv_size,
u16 cache_size,
u8 numa,
379 struct rte_mempool **_mp,
u32 * map_index)
381 struct rte_mempool *mp;
382 enum rte_iova_mode iova_mode;
386 size_t min_chunk_size, align;
392 mp = rte_mempool_create_empty ((
char *) pool_name, num_elts, elt_size,
393 512, pool_priv_size, numa, 0);
397 rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS,
NULL);
399 size = rte_mempool_op_calc_mem_size_default (mp, num_elts, 21,
400 &min_chunk_size, &align);
403 0, numa, map_index)))
405 rte_mempool_free (mp);
414 rte_pktmbuf_pool_init (mp, &priv);
416 if (rte_eth_dev_count_avail () == 0)
419 iova_mode = rte_eal_iova_mode ();
420 for (i = 0; i < pm->
n_pages; i++)
423 char *va = ((
char *) pm->
base) + i * page_sz;
424 uword pa = iova_mode == RTE_IOVA_VA ?
426 ret = rte_mempool_populate_iova (mp, va, pa, page_sz, 0, 0);
429 rte_mempool_free (mp);
434 if (map_dma && rte_vfio_dma_map (pointer_to_uword (va), pa, page_sz))
448 struct rte_mempool *rmp;
460 pool_name =
format (0,
"dpdk_mbuf_pool_socket%u%c", socket_id, 0);
462 elt_size =
sizeof (
struct rte_mbuf) +
463 VLIB_BUFFER_HDR_SIZE +
464 VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE;
475 rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0);
493 clib_warning (
"WARNING: Failed to allocate mempool for CPU socket " 494 "%u. Threads running on socket %u will use socket %u " 495 "mempool.", socket_id, socket_id, i);
531 struct dpdk_validate_buf_result
537 #define DPDK_TRAJECTORY_POISON 31 540 dpdk_buffer_validate_trajectory (
struct rte_mempool *mp,
void *opaque,
541 void *obj,
unsigned obj_idx)
544 struct dpdk_validate_buf_result *counter = opaque;
548 if (b->
pre_data[0] == DPDK_TRAJECTORY_POISON)
549 counter->uninitialized++;
556 dpdk_buffer_validate_trajectory_all (
u32 * uninitialized)
559 struct dpdk_validate_buf_result counter = { 0 };
564 dpdk_buffer_validate_trajectory, &counter);
566 *uninitialized = counter.uninitialized;
567 return counter.invalid;
571 dpdk_buffer_poison_trajectory (
struct rte_mempool *mp,
void *opaque,
572 void *obj,
unsigned obj_idx)
576 b->
pre_data[0] = DPDK_TRAJECTORY_POISON;
580 dpdk_buffer_poison_trajectory_all (
void)
586 rte_mempool_obj_iter (dm->
pktmbuf_pools[i], dpdk_buffer_poison_trajectory,
624 static void __clib_constructor
#define vlib_buffer_from_rte_mbuf(x)
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
dpdk_buffer_main_t dpdk_buffer_main
static clib_error_t * dpdk_buffer_init(vlib_main_t *vm)
uword() vlib_buffer_fill_free_list_cb_t(struct vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
static_always_inline void dpdk_prefetch_buffer(vlib_buffer_t *b)
u8 vlib_buffer_register_physmem_map(vlib_main_t *vm, u32 physmem_map_index)
u32 * vlib_buffer_state_validation_lock
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512
#define VLIB_BUFFER_PRE_DATA_SIZE
u8 buffer_pool_index
index of buffer pool this buffer belongs.
static void dpdk_buffer_delete_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
vlib_main_t ** vlib_mains
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2
#define static_always_inline
uword * vlib_buffer_state_validation_hash
#define VLIB_INIT_FUNCTION(x)
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
static void __clib_constructor dpdk_input_multiarch_select(void)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static heap_elt_t * first(heap_header_t *h)
#define rte_mbuf_from_vlib_buffer(x)
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
clib_error_t * vlib_physmem_shared_map_create(vlib_main_t *vm, char *name, uword size, u32 log2_page_sz, u32 numa_node, u32 *map_index)
static clib_error_t * buffer_state_validation_init(vlib_main_t *vm)
void() vlib_buffer_free_cb_t(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static_always_inline void dpdk_rte_pktmbuf_free(vlib_main_t *vm, u32 thread_index, vlib_buffer_t *b, int maybe_next)
#define pool_put(P, E)
Free an object E in pool P.
uword CLIB_MULTIARCH_FN() dpdk_buffer_fill_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
vlib_buffer_free_list_t * buffer_free_list_pool
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
STATIC_ASSERT(VLIB_BUFFER_PRE_DATA_SIZE==RTE_PKTMBUF_HEADROOM, "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM")
struct rte_pktmbuf_pool_private mbp_priv
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
static_always_inline uword vlib_get_thread_index(void)
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
#define CLIB_PREFETCH(addr, size, type)
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512
#define vec_free(V)
Free vector's memory (no header).
static_always_inline void clib_memcpy64_x4(void *d0, void *d1, void *d2, void *d3, void *s)
static void * clib_mem_set_heap(void *heap)
#define clib_warning(format, args...)
void CLIB_MULTIARCH_FN() dpdk_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
static uword round_pow2(uword x, uword pow2)
void * vlib_buffer_state_heap
#define VLIB_BUFFER_DATA_SIZE
#define VLIB_BUFFER_HDR_SIZE
void CLIB_MULTIARCH_FN() dpdk_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
#define hash_create(elts, value_bytes)
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2
vlib_buffer_free_cb_t * vlib_buffer_free_cb
void * mheap_alloc_with_lock(void *memory, uword size, int locked)
u32 next_buffer
Next buffer for this linked-list of buffers.
dpdk_buffer_per_thread_data * ptd
struct rte_mempool ** pktmbuf_pools
#define clib_error_report(e)
clib_error_t * dpdk_pool_create(vlib_main_t *vm, u8 *pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, struct rte_mempool **_mp, u32 *map_index)
VLIB_BUFFER_REGISTER_CALLBACKS(dpdk, static)
static uword pointer_to_uword(const void *p)
u8 n_add_refs
Number of additional references to this buffer.
void() vlib_buffer_free_no_next_cb_t(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b, u32 follow_buffer_next)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
static vlib_thread_main_t * vlib_get_thread_main()
vlib_buffer_main_t buffer_main
u8 vlib_buffer_free_list_index_t
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx2
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
struct rte_mbuf ** mbuf_alloc_list
#define CLIB_MULTIARCH_FN(fn)
#define vec_resize_aligned(V, N, A)
Resize a vector (no header, alignment specified).