46 #include <rte_config.h> 48 #include <rte_common.h> 50 #include <rte_memory.h> 51 #include <rte_memzone.h> 52 #include <rte_tailq.h> 54 #include <rte_per_lcore.h> 55 #include <rte_launch.h> 56 #include <rte_atomic.h> 57 #include <rte_cycles.h> 58 #include <rte_prefetch.h> 59 #include <rte_lcore.h> 60 #include <rte_per_lcore.h> 61 #include <rte_branch_prediction.h> 62 #include <rte_interrupts.h> 64 #include <rte_random.h> 65 #include <rte_debug.h> 66 #include <rte_ether.h> 67 #include <rte_ethdev.h> 69 #include <rte_mempool.h> 71 #include <rte_version.h> 80 "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
101 rte_pktmbuf_free_seg (mb);
140 if (merge_index != ~0 && merge_index != free_list_index)
150 memset (f, 0xab,
sizeof (f[0]));
158 memset (f, 0xab,
sizeof (f[0]));
171 u32 bi0, bi1, bi2, bi3;
172 unsigned socket_id = rte_socket_id ();
174 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
183 return min_free_buffers;
216 ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
217 ASSERT (rte_mbuf_refcnt_read (mb1) == 0);
218 ASSERT (rte_mbuf_refcnt_read (mb2) == 0);
219 ASSERT (rte_mbuf_refcnt_read (mb3) == 0);
221 rte_mbuf_refcnt_set (mb0, 1);
222 rte_mbuf_refcnt_set (mb1, 1);
223 rte_mbuf_refcnt_set (mb2, 1);
224 rte_mbuf_refcnt_set (mb3, 1);
260 ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
261 rte_mbuf_refcnt_set (mb0, 1);
283 u32 * alloc_buffers,
u32 n_alloc_buffers)
295 ASSERT (len >= n_alloc_buffers);
297 src = free_list->
buffers + len - n_alloc_buffers;
300 _vec_len (free_list->
buffers) -= n_alloc_buffers;
302 return n_alloc_buffers;
324 u32 n_buffers,
u32 free_list_index)
334 u32 * buffers,
u32 n_buffers,
u32 follow_buffer_next)
341 u32 follow_buffer_next);
346 n_buffers = (*cb) (
vm, buffers, n_buffers, follow_buffer_next);
351 for (i = 0; i < n_buffers; i++)
370 goto already_announced;
412 uword n_packet_data_bytes,
413 uword min_n_buffers_each_physmem_alloc,
u8 * name)
418 memset (t, 0,
sizeof (t[0]));
431 struct rte_mempool *rmp;
440 u8 *pool_name =
format (0,
"mbuf_pool_socket%u%c", socket_id, 0);
442 rmp = rte_pktmbuf_pool_create ((
char *) pool_name,
453 uword this_pool_start;
454 uword this_pool_size;
455 uword save_vpm_start, save_vpm_end, save_vpm_size;
456 struct rte_mempool_memhdr *memhdr;
458 this_pool_start = ~0ULL;
461 STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
463 if (((
uword) (memhdr->addr + memhdr->len)) > this_pool_end)
464 this_pool_end = (
uword) (memhdr->addr + memhdr->len);
465 if (((
uword) memhdr->addr) < this_pool_start)
466 this_pool_start = (
uword) (memhdr->addr);
468 ASSERT (this_pool_start < ~0ULL && this_pool_end > 0);
469 this_pool_size = this_pool_end - this_pool_start;
473 clib_warning (
"%s: pool start %llx pool end %llx pool size %lld",
474 pool_name, this_pool_start, this_pool_end,
477 (
"before: virtual.start %llx virtual.end %llx virtual.size %lld",
485 if ((this_pool_start < vpm->
virtual.start) || vpm->
virtual.
start == 0)
495 (
"after: virtual.start %llx virtual.end %llx virtual.size %lld",
526 (
"WARNING: Failed to allocate mempool for CPU socket %u. " 527 "Threads running on socket %u will use socket %u mempool.",
528 socket_id, socket_id, i);
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
#define vlib_buffer_from_rte_mbuf(x)
sll srl srl sll sra u16x4 i
void vlib_buffer_merge_free_lists(vlib_buffer_free_list_t *dst, vlib_buffer_free_list_t *src)
static clib_error_t * dpdk_buffer_init(vlib_main_t *vm)
#define VLIB_BUFFER_RECYCLE
int vlib_buffer_cb_register(struct vlib_main_t *vm, vlib_buffer_callbacks_t *cb)
vlib_physmem_main_t physmem_main
void * mheap_alloc(void *memory, uword size)
u32 * vlib_buffer_state_validation_lock
#define VLIB_BUFFER_PRE_DATA_SIZE
#define CLIB_LOG2_CACHE_LINE_BYTES
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
vlib_buffer_main_t * buffer_main
u32 min_n_buffers_each_physmem_alloc
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
vlib_main_t ** vlib_mains
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
u32 dpdk_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
static uword fill_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
#define VLIB_BUFFER_NEXT_PRESENT
#define static_always_inline
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
uword * vlib_buffer_state_validation_hash
#define VLIB_INIT_FUNCTION(x)
static_always_inline void dpdk_rte_pktmbuf_free(vlib_main_t *vm, vlib_buffer_t *b)
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
#define clib_error_return(e, args...)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
vlib_buffer_free_list_t ** announce_list
#define rte_mbuf_from_vlib_buffer(x)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static clib_error_t * buffer_state_validation_init(vlib_main_t *vm)
static u32 vlib_buffer_get_free_list_with_size(vlib_main_t *vm, u32 size)
uword os_get_cpu_number(void)
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
#define pool_put(P, E)
Free an object E in pool P.
clib_error_t * vlib_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
static u32 alloc_from_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *free_list, u32 *alloc_buffers, u32 n_alloc_buffers)
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
u32 dpdk_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
STATIC_ASSERT(VLIB_BUFFER_PRE_DATA_SIZE==RTE_PKTMBUF_HEADROOM,"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM")
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
#define vec_free(V)
Free vector's memory (no header).
static void * clib_mem_set_heap(void *heap)
#define clib_warning(format, args...)
#define clib_memcpy(a, b, c)
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_sync(vlib_main_t *vm)
static uword round_pow2(uword x, uword pow2)
void * vlib_buffer_state_heap
static void dpdk_buffer_delete_free_list(vlib_main_t *vm, u32 free_list_index)
#define VLIB_BUFFER_DATA_SIZE
#define VLIB_BUFFER_HDR_SIZE
#define hash_create(elts, value_bytes)
u32(* vlib_buffer_alloc_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, u32 *index)
u32 next_buffer
Next buffer for this linked-list of buffers.
struct rte_mempool ** pktmbuf_pools
u8 n_add_refs
Number of additional references to this buffer.
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
vlib_buffer_free_list_t * buffer_free_list_pool
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
vlib_physmem_region_t virtual
static void dpdk_packet_template_init(vlib_main_t *vm, void *vt, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, u8 *name)
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
static void dpdk_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static void dpdk_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)