75 s =
format (s,
"current data %d, length %d, free-list %d, clone-count %u",
80 s =
format (s,
", totlen-nifb %d",
93 format (s,
"\n%Unext-buffer 0x%x, segment length %d, clone-count %u",
126 t =
"known-allocated";
138 return format (s,
"%s", t);
161 uword follow_buffer_next,
uword ** unique_hash)
169 return format (0,
"unknown free list 0x%x",
180 return format (0,
"%d-%d beyond end of buffer %d",
190 return format (0,
"next 0x%x: %U",
223 uword next_buffer_stride,
226 uword follow_buffer_next)
229 u32 bi, *b = buffers;
231 u8 *msg = 0, *result = 0;
234 for (i = 0; i < n_buffers; i++)
237 b += next_buffer_stride;
242 msg =
format (0,
"not unique");
247 if (k != known_state)
249 msg =
format (0,
"is %U; expected %U",
265 result =
format (0,
"0x%x: %v", bi, msg);
284 } __attribute__ ((packed)) __bootstrap_vlib_main_vector
309 if (vlib_buffer_callbacks)
314 for (i = 0; i < n_buffers; i++)
321 if (known != expected_state)
325 (vm,
"%s %U buffer 0x%x",
326 is_free ?
"freeing" :
"allocating",
355 u32 is_public,
u32 is_default,
u8 * name)
365 u32 default_free_free_list_index;
368 default_free_free_list_index =
376 ASSERT (default_free_free_list_index ==
380 return default_free_free_list_index;
385 memset (f, 0,
sizeof (f[0]));
404 for (i = 1; i <
vec_len (vlib_mains); i++)
489 if (merge_index != ~0 && merge_index != free_list_index)
499 memset (f, 0xab,
sizeof (f[0]));
503 for (i = 1; i <
vec_len (vlib_mains); i++)
507 memset (f, 0xab,
sizeof (f[0]));
516 uword min_free_buffers)
522 u32 n_remaining, n_alloc, n_this_chunk;
527 return min_free_buffers;
542 return min_free_buffers;
553 while (n_remaining > 0)
555 n_this_chunk =
clib_min (n_remaining, 16);
557 n_bytes = n_this_chunk * (
sizeof (b[0]) + fl->
n_data_bytes);
563 buffer_pools[0].physmem_region, n_bytes,
572 n_alloc += n_this_chunk;
573 n_remaining -= n_this_chunk;
577 for (i = 0; i < n_this_chunk; i++)
586 memset (buffers, 0, n_bytes);
590 for (i = 0; i < n_this_chunk; i++)
614 u32 * buffers,
u32 n_buffers,
u32 follow_buffer_next)
621 u32 follow_buffer_next);
626 n_buffers = (*cb) (
vm, buffers, n_buffers, follow_buffer_next);
631 for (i = 0; i < n_buffers; i++)
651 goto already_announced;
678 while (follow_buffer_next
712 static void __attribute__ ((unused))
715 u32 * buffers,
u32 n_buffers)
722 for (i = 0; i < n_buffers; i++)
735 uword n_packet_data_bytes,
736 uword min_n_buffers_each_physmem_alloc,
751 min_n_buffers_each_physmem_alloc,
756 memset (t, 0,
sizeof (t[0]));
762 (vm, n_packet_data_bytes,
822 u32 buffer_index,
void *data,
u32 n_data_bytes)
824 u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
834 n_left = n_data_bytes;
851 n =
clib_min (n_left_this_buffer, n_left);
881 void *data,
u16 data_len)
901 u16 len = (data_len > max) ? max : data_len;
926 else if (start < bm->buffer_mem_start)
943 clib_panic (
"buffer memory size out of range!");
957 u32 threadnum = va_arg (*va,
u32);
958 uword bytes_alloc, bytes_free, n_free,
size;
961 return format (s,
"%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
962 "Thread",
"Name",
"Index",
"Size",
"Alloc",
"Free",
967 bytes_alloc = size * f->
n_alloc;
968 bytes_free = size * n_free;
970 s =
format (s,
"%7d%30v%12d%12d%=12U%=12U%=12d%=12d", threadnum,
996 vlib_cli_output (vm,
"%U", format_vlib_buffer_free_list, f, vm_index);
1009 .path =
"show buffers",
1010 .short_help =
"Show packet buffer allocation",
1025 if (vlib_buffer_callbacks)
1070 if (
unformat (input,
"memory-size-in-mb %d", &size_in_mb))
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
vlib_physmem_region_index_t physmem_region
vlib_main_t vlib_global_main
#define hash_set(h, key, value)
sll srl srl sll sra u16x4 i
void vlib_buffer_merge_free_lists(vlib_buffer_free_list_t *dst, vlib_buffer_free_list_t *src)
#define VLIB_PHYSMEM_F_INIT_MHEAP
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
#define VLIB_BUFFER_RECYCLE
static clib_error_t * vlib_buffers_configure(vlib_main_t *vm, unformat_input_t *input)
#define clib_error(format, args...)
vlib_buffer_callbacks_t cb
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, u32 next_bi)
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
vlib_buffer_t buffer_init_template
#define VLIB_BUFFER_PRE_DATA_SIZE
#define CLIB_LOG2_CACHE_LINE_BYTES
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
clib_spinlock_t global_buffers_lock
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
static heap_elt_t * last(heap_header_t *h)
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
void ** buffer_memory_allocated
vlib_buffer_main_t * buffer_main
static void vlib_buffer_free_no_next_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
u32 min_n_buffers_each_physmem_alloc
static clib_error_t * show_buffers(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
vlib_main_t ** vlib_mains
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
#define vlib_worker_thread_barrier_sync(X)
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
void vlib_buffer_delete_free_list_internal(vlib_main_t *vm, u32 free_list_index)
static u8 * format_vlib_buffer_free_list(u8 *s, va_list *va)
#define VLIB_BUFFER_NEXT_PRESENT
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
#define static_always_inline
static u32 vlib_buffer_physmem_sz
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
void(* os_physmem_free)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx, void *x)
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
void *(* os_physmem_alloc_aligned)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx, uword n_bytes, uword alignment)
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
static vlib_buffer_t * vlib_buffer_next_contiguous(vlib_buffer_t *b, u32 buffer_bytes)
u8 * format_vlib_buffer(u8 *s, va_list *args)
static uword pointer_to_uword(const void *p)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
static void clib_spinlock_init(clib_spinlock_t *p)
vlib_buffer_free_list_t ** announce_list
static heap_elt_t * first(heap_header_t *h)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
void * vlib_set_buffer_free_callback(vlib_main_t *vm, void *fp)
u16 current_length
Nbytes between current data and the end of this buffer.
static u32 vlib_buffer_get_free_list_with_size(vlib_main_t *vm, u32 size)
static u32 vlib_buffer_create_free_list_helper(vlib_main_t *vm, u32 n_data_bytes, u32 is_public, u32 is_default, u8 *name)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
#define pool_put(P, E)
Free an object E in pool P.
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, u32 free_list_index)
vlib_buffer_pool_t * buffer_pools
static u8 * vlib_validate_buffer_helper(vlib_main_t *vm, u32 bi, uword follow_buffer_next, uword **unique_hash)
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, u32 free_list_index)
static void vlib_packet_template_buffer_init(vlib_main_t *vm, vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
#define uword_to_pointer(u, type)
static void vlib_buffer_set_known_state(u32 buffer_index, vlib_buffer_known_state_t state)
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, u32 free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P (general version).
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
static u32 vlib_buffer_get_free_list_index(vlib_buffer_t *b)
clib_spinlock_t buffer_known_hash_lockp
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
u32 vlib_buffer_add_data(vlib_main_t *vm, u32 free_list_index, u32 buffer_index, void *data, u32 n_data_bytes)
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
static_always_inline uword vlib_get_thread_index(void)
u8 vlib_buffer_add_physmem_region(vlib_main_t *vm, vlib_physmem_region_index_t pri)
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
u8 * format_vlib_buffer_contents(u8 *s, va_list *va)
#define vec_free(V)
Free vector's memory (no header).
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 bi, uword follow_buffer_next)
#define VLIB_BUFFER_IS_TRACED
#define clib_memcpy(a, b, c)
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Allocate buffers from specific freelist into supplied array.
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
static uword round_pow2(uword x, uword pow2)
vlib_buffer_known_state_t
#define VLIB_CLI_COMMAND(x,...)
#define hash_set1(h, key)
#define hash_create(elts, value_bytes)
vlib_buffer_free_cb_t * vlib_buffer_free_cb
static u8 * format_vlib_buffer_known_state(u8 *s, va_list *args)
void(* vlib_packet_template_init_cb)(struct vlib_main_t *vm, void *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, u8 *name)
vhost_vring_state_t state
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, u32 *index)
u32 next_buffer
Next buffer for this linked-list of buffers.
u8 * format_vlib_buffer_and_data(u8 *s, va_list *args)
u32 vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
u8 n_add_refs
Number of additional references to this buffer.
static vlib_main_t * vlib_get_main(void)
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
static void vlib_buffer_set_free_list_index(vlib_buffer_t *b, u32 index)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
vlib_buffer_free_list_t * buffer_free_list_pool
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, char *fmt,...)
#define VLIB_PHYSMEM_F_FAKE
#define clib_error_free(e)
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
static vlib_buffer_known_state_t vlib_buffer_is_known(u32 buffer_index)
static uword clib_mem_is_vec(void *v)
Predicate function, says whether the supplied vector is a clib heap object.
uword buffer_init_function_opaque
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
u8 vlib_physmem_region_index_t
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
u32 min_n_buffers_each_physmem_alloc
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
uword * free_list_by_size
#define clib_panic(format, args...)
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
#define vlib_panic_with_msg(vm, args...)
static uword vlib_buffer_fill_free_list_internal(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
static void vlib_buffer_free_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
static u32 vlib_buffer_round_size(u32 size)
static uword pool_elts(void *v)
Number of active elements in a pool.