48 #include <sys/ioctl.h> 50 #include <rte_config.h> 52 #include <rte_common.h> 54 #include <rte_memory.h> 55 #include <rte_memzone.h> 56 #include <rte_tailq.h> 58 #include <rte_per_lcore.h> 59 #include <rte_launch.h> 60 #include <rte_atomic.h> 61 #include <rte_cycles.h> 62 #include <rte_prefetch.h> 63 #include <rte_lcore.h> 64 #include <rte_per_lcore.h> 65 #include <rte_branch_prediction.h> 66 #include <rte_interrupts.h> 68 #include <rte_random.h> 69 #include <rte_debug.h> 70 #include <rte_ether.h> 71 #include <rte_ethdev.h> 73 #include <rte_mempool.h> 75 #include <rte_version.h> 86 "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
120 if ((mb = rte_pktmbuf_prefree_seg (mb)))
121 rte_mempool_put (mb->pool, mb);
123 if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
130 #ifndef CLIB_MARCH_VARIANT 147 memset (f, 0xab,
sizeof (f[0]));
180 uword min_free_buffers)
184 struct rte_mbuf **mb;
187 unsigned socket_id = rte_socket_id ();
200 n_alloc = min_free_buffers -
vec_len (
fl->buffers);
202 return min_free_buffers;
208 n_alloc =
clib_max (n_alloc,
fl->min_n_buffers_each_alloc);
213 if (rte_mempool_get_bulk (rmp, (
void *) d->
mbuf_alloc_list, n_alloc) < 0)
247 sizeof (
struct rte_mbuf));
262 if (
fl->buffer_init_function)
263 fl->buffer_init_function (vm,
fl,
fl->buffers + first, n_alloc);
265 fl->n_alloc += n_alloc;
290 u32 * buffers,
u32 n_buffers,
u32 follow_buffer_next)
296 u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST |
297 VLIB_BUFFER_NEXT_PRESENT);
300 u32 follow_buffer_next);
305 n_buffers = (*cb) (
vm, buffers, n_buffers, follow_buffer_next);
330 for (i = 0; i < 4; i++)
335 if (or_flags & simple_mask)
379 #ifndef CLIB_MARCH_VARIANT 384 uword n_packet_data_bytes,
390 memset (t, 0,
sizeof (t[0]));
402 const char fn[] =
"/dev/vfio/vfio";
403 char buff[
sizeof (fn)] = { 0 };
405 u8 *path =
format (0,
"%v%c", path_name, 0);
407 if (readlink ((
char *) path, buff,
sizeof (fn)) + 1 !=
sizeof (fn))
410 if (strncmp (fn, buff,
sizeof (fn)))
413 fd = atoi ((
char *) file_name);
424 u32 num_elts,
u32 pool_priv_size,
u16 cache_size,
u8 numa,
425 struct rte_mempool ** _mp,
429 struct rte_mempool *mp;
433 size_t min_chunk_size, align;
439 mp = rte_mempool_create_empty ((
char *) pool_name, num_elts, elt_size,
440 512, pool_priv_size, numa, 0);
444 rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS,
NULL);
446 size = rte_mempool_op_calc_mem_size_default (mp, num_elts, 21,
447 &min_chunk_size, &align);
454 rte_mempool_free (mp);
464 rte_pktmbuf_pool_init (mp, &priv);
466 for (i = 0; i < pr->
n_pages; i++)
469 ret = rte_mempool_populate_iova (mp, ((
char *) pr->
mem) + i * page_size,
473 rte_mempool_free (mp);
488 struct vfio_iommu_type1_dma_map dm = { 0 };
490 dm.argsz =
sizeof (
struct vfio_iommu_type1_dma_map);
491 dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
503 if (rv != 0 && errno != EINVAL)
515 struct rte_mempool *rmp;
527 pool_name =
format (0,
"dpdk_mbuf_pool_socket%u%c", socket_id, 0);
529 elt_size =
sizeof (
struct rte_mbuf) +
530 VLIB_BUFFER_HDR_SIZE +
531 VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE;
543 rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0);
560 clib_warning (
"WARNING: Failed to allocate mempool for CPU socket " 561 "%u. Threads running on socket %u will use socket %u " 562 "mempool.", socket_id, socket_id, i);
598 struct dpdk_validate_buf_result
604 #define DPDK_TRAJECTORY_POISON 31 607 dpdk_buffer_validate_trajectory (
struct rte_mempool *mp,
void *opaque,
608 void *obj,
unsigned obj_idx)
611 struct dpdk_validate_buf_result *counter = opaque;
615 if (b->
pre_data[0] == DPDK_TRAJECTORY_POISON)
616 counter->uninitialized++;
623 dpdk_buffer_validate_trajectory_all (
u32 * uninitialized)
626 struct dpdk_validate_buf_result counter = { 0 };
631 dpdk_buffer_validate_trajectory, &counter);
633 *uninitialized = counter.uninitialized;
634 return counter.invalid;
638 dpdk_buffer_poison_trajectory (
struct rte_mempool *mp,
void *opaque,
639 void *obj,
unsigned obj_idx)
643 b->
pre_data[0] = DPDK_TRAJECTORY_POISON;
647 dpdk_buffer_poison_trajectory_all (
void)
653 rte_mempool_obj_iter (dm->
pktmbuf_pools[i], dpdk_buffer_poison_trajectory,
692 static void __clib_constructor
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define vlib_buffer_from_rte_mbuf(x)
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
dpdk_buffer_main_t dpdk_buffer_main
static clib_error_t * dpdk_buffer_init(vlib_main_t *vm)
static void dpdk_packet_template_init(vlib_main_t *vm, void *vt, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, u8 *name)
#define VLIB_PHYSMEM_F_HUGETLB
static_always_inline void dpdk_prefetch_buffer(vlib_buffer_t *b)
u32 * vlib_buffer_state_validation_lock
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512
#define VLIB_BUFFER_PRE_DATA_SIZE
u8 buffer_pool_index
index of buffer pool this buffer belongs.
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
static void dpdk_buffer_delete_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
void( vlib_buffer_free_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
vlib_main_t ** vlib_mains
#define vlib_worker_thread_barrier_sync(X)
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2
#define VLIB_PHYSMEM_F_SHARED
memset(h->entries, 0, sizeof(h->entries[0])*entries)
#define static_always_inline
uword * vlib_buffer_state_validation_hash
#define VLIB_INIT_FUNCTION(x)
u8 vlib_buffer_pool_create(vlib_main_t *vm, vlib_physmem_region_index_t pri, u16 buffer_size)
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
static void __clib_constructor dpdk_input_multiarch_select(void)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
static heap_elt_t * first(heap_header_t *h)
#define rte_mbuf_from_vlib_buffer(x)
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
static clib_error_t * buffer_state_validation_init(vlib_main_t *vm)
static_always_inline void dpdk_rte_pktmbuf_free(vlib_main_t *vm, u32 thread_index, vlib_buffer_t *b, int maybe_next)
#define pool_put(P, E)
Free an object E in pool P.
static clib_error_t * scan_vfio_fd(void *arg, u8 *path_name, u8 *file_name)
uword CLIB_MULTIARCH_FN() dpdk_buffer_fill_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
vlib_buffer_free_list_t * buffer_free_list_pool
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
clib_error_t * dpdk_pool_create(vlib_main_t *vm, u8 *pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, struct rte_mempool **_mp, vlib_physmem_region_index_t *pri)
struct rte_pktmbuf_pool_private mbp_priv
STATIC_ASSERT(VLIB_BUFFER_PRE_DATA_SIZE==RTE_PKTMBUF_HEADROOM,"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM")
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
static_always_inline uword vlib_get_thread_index(void)
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
#define CLIB_PREFETCH(addr, size, type)
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512
#define vec_free(V)
Free vector's memory (no header).
static_always_inline void clib_memcpy64_x4(void *d0, void *d1, void *d2, void *d3, void *s)
static void * clib_mem_set_heap(void *heap)
#define clib_warning(format, args...)
void CLIB_MULTIARCH_FN() dpdk_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
static uword round_pow2(uword x, uword pow2)
void * vlib_buffer_state_heap
#define VLIB_BUFFER_DATA_SIZE
#define VLIB_BUFFER_HDR_SIZE
void CLIB_MULTIARCH_FN() dpdk_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
void( vlib_buffer_free_no_next_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
#define hash_create(elts, value_bytes)
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2
vlib_buffer_free_cb_t * vlib_buffer_free_cb
void * mheap_alloc_with_lock(void *memory, uword size, int locked)
u32 next_buffer
Next buffer for this linked-list of buffers.
dpdk_buffer_per_thread_data * ptd
struct rte_mempool ** pktmbuf_pools
#define clib_error_report(e)
VLIB_BUFFER_REGISTER_CALLBACKS(dpdk, static)
static uword pointer_to_uword(const void *p)
u8 n_add_refs
Number of additional references to this buffer.
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b, u32 follow_buffer_next)
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
#define clib_unix_warning(format, args...)
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
static vlib_thread_main_t * vlib_get_thread_main()
vlib_buffer_main_t buffer_main
u8 vlib_buffer_free_list_index_t
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx2
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
uword( vlib_buffer_fill_free_list_cb_t)(struct vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
u8 vlib_physmem_region_index_t
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
linux_vfio_main_t vfio_main
struct rte_mbuf ** mbuf_alloc_list
#define CLIB_MULTIARCH_FN(fn)
#define vec_resize_aligned(V, N, A)
Resize a vector (no header, alignment specified).