51 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384 52 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192 54 #ifdef CLIB_HAVE_VEC128 64 u16 __vlib_buffer_external_hdr_size = 0;
84 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
90 b_first->
flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
101 #define _(bit, name, v) \ 102 if (v && (b->flags & VLIB_BUFFER_##name)) \ 103 a = format (a, "%s ", v); 106 s =
format (s,
"current data %d, length %d, buffer-pool %d, " 110 if (b->
flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
111 s =
format (s,
", totlen-nifb %d",
114 if (b->
flags & VLIB_BUFFER_IS_TRACED)
133 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
139 format (s,
"\n%Unext-buffer 0x%x, segment length %d, ref-count %u",
172 t =
"known-allocated";
184 return format (s,
"%s", t);
196 if (!(b->
flags & VLIB_BUFFER_NEXT_PRESENT))
207 uword follow_buffer_next,
uword ** unique_hash)
223 if (follow_buffer_next && (b->
flags & VLIB_BUFFER_NEXT_PRESENT))
230 return format (0,
"next 0x%x: %U",
263 uword next_buffer_stride,
266 uword follow_buffer_next)
269 u32 bi, *b = buffers;
271 u8 *msg = 0, *result = 0;
274 for (i = 0; i < n_buffers; i++)
277 b += next_buffer_stride;
282 msg =
format (0,
"not unique");
287 if (k != known_state)
289 msg =
format (0,
"is %U; expected %U",
305 result =
format (0,
"0x%x: %v", bi, msg);
324 } __attribute__ ((packed)) __bootstrap_vlib_main_vector
352 for (i = 0; i < n_buffers; i++)
364 if (known != expected_state)
366 clib_panic (
"%s %U buffer 0x%x", is_free ?
"freeing" :
"allocating",
381 uword n_packet_data_bytes,
382 uword min_n_buffers_each_alloc,
char *fmt, ...)
424 u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
433 n_left = n_data_bytes;
437 b->
flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
440 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
450 n =
clib_min (n_left_this_buffer, n_left);
462 b->
flags |= VLIB_BUFFER_NEXT_PRESENT;
497 u16 len = (data_len > max) ? max : data_len;
509 u32 physmem_map_index)
517 u32 alloc_size, n_alloc_per_page;
529 else if (start < bm->buffer_mem_start)
546 clib_panic (
"buffer memory size out of range!");
572 for (j = 0; j < m->
n_pages; j++)
573 for (i = 0; i < n_alloc_per_page; i++)
602 return format (s,
"%-20s%=6s%=6s%=6s%=11s%=6s%=8s%=8s%=8s",
603 "Pool Name",
"Index",
"NUMA",
"Size",
"Data Size",
604 "Total",
"Avail",
"Cached",
"Used");
611 s =
format (s,
"%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u",
639 .path =
"show buffers",
640 .short_help =
"Show packet buffer allocation",
672 u32 physmem_map_index;
673 uword n_pages, pagesize;
674 u32 buffers_per_numa;
682 name =
format (0,
"buffers-numa-%d%c", numa_node, 0);
688 n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1;
699 "backed buffer pool");
710 name =
format (name,
"default-numa-%d%c", numa_node, 0);
716 if (*index == (
u8) ~ 0)
811 u8 *
name = 0, first_valid_buffer_pool_index = ~0;
822 unformat_bitmap_list, &bmp)))
825 if ((err =
clib_sysfs_read (
"/sys/devices/system/node/has_memory",
"%U",
826 unformat_bitmap_list, &bmp_has_memory)))
829 if (bmp && bmp_has_memory)
837 clib_panic (
"system have more than %u NUMA nodes",
852 if (first_valid_buffer_pool_index == 0xff)
853 first_valid_buffer_pool_index = index[0];
857 if (first_valid_buffer_pool_index == (
u8) ~ 0)
868 first_valid_buffer_pool_index;
878 name =
format (name,
"/buffer-pools/%s/cached%c", bp->
name, 0);
883 name =
format (name,
"/buffer-pools/%s/used%c", bp->
name, 0);
888 name =
format (name,
"/buffer-pools/%s/available%c", bp->
name, 0);
913 else if (
unformat (input,
"default data-size %u",
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
vlib_main_t vlib_global_main
u8 * format_clib_error(u8 *s, va_list *va)
#define hash_set(h, key, value)
#define vlib_log_warn(...)
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
#define CLIB_CACHE_LINE_ROUND(x)
static clib_error_t * vlib_buffers_configure(vlib_main_t *vm, unformat_input_t *input)
void vlib_buffer_main_alloc(vlib_main_t *vm)
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, template_end, 64)
static void buffer_gauges_update_used_fn(stat_segment_directory_entry_t *e, u32 index)
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
int vlib_buffer_add_data(vlib_main_t *vm, u32 *buffer_index, void *data, u32 n_data_bytes)
#define VLIB_BUFFER_PRE_DATA_SIZE
u16 current_length
Nbytes between current data and the end of this buffer.
#define CLIB_LOG2_CACHE_LINE_BYTES
static heap_elt_t * last(heap_header_t *h)
vlib_buffer_main_t * buffer_main
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
static u8 * format_vlib_buffer_pool(u8 *s, va_list *va)
static clib_error_t * show_buffers(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES]
static uword min_log2(uword x)
u8 buffer_pool_index
index of buffer pool this buffer belongs.
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define vlib_worker_thread_barrier_sync(X)
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
vlib_buffer_pool_thread_t * threads
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
static void buffer_gauges_update_available_fn(stat_segment_directory_entry_t *e, u32 index)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
u8 * format_vlib_buffer_no_chain(u8 *s, va_list *args)
u8 * format_vlib_buffer(u8 *s, va_list *args)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
static void clib_spinlock_init(clib_spinlock_t *p)
static heap_elt_t * first(heap_header_t *h)
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
clib_error_t * vlib_physmem_shared_map_create(vlib_main_t *vm, char *name, uword size, u32 log2_page_sz, u32 numa_node, u32 *map_index)
u32 min_n_buffers_each_alloc
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
static uword clib_bitmap_last_set(uword *ai)
Return the higest numbered set bit in a bitmap.
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
uword clib_mem_get_page_size(void)
vlib_buffer_t buffer_template
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
vlib_buffer_pool_t * buffer_pools
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
static u8 * vlib_validate_buffer_helper(vlib_main_t *vm, u32 bi, uword follow_buffer_next, uword **unique_hash)
static u32 buffer_get_cached(vlib_buffer_pool_t *bp)
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
vlib_log_class_t log_default
#define STATIC_ASSERT_FITS_IN(s, e, o)
static vlib_buffer_pool_t * buffer_get_by_index(vlib_buffer_main_t *bm, u32 index)
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
clib_spinlock_t buffer_known_hash_lockp
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
clib_error_t * stat_segment_register_gauge(u8 *name, stat_segment_update_fn update_fn, u32 caller_index)
u8 * format_vlib_buffer_contents(u8 *s, va_list *va)
#define vec_free(V)
Free vector's memory (no header).
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 bi, uword follow_buffer_next)
#define VLIB_BUFFER_MAX_NUMA_NODES
#define clib_warning(format, args...)
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA
vlib_buffer_known_state_t
#define VLIB_CLI_COMMAND(x,...)
static void buffer_gauges_update_cached_fn(stat_segment_directory_entry_t *e, u32 index)
#define hash_set1(h, key)
#define hash_create(elts, value_bytes)
static u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.
static u8 * format_vlib_buffer_known_state(u8 *s, va_list *args)
u8 * format_vlib_buffer_and_data(u8 *s, va_list *args)
#define clib_error_report(e)
uword clib_mem_get_default_hugepage_size(void)
static void * clib_mem_alloc(uword size)
static uword pointer_to_uword(const void *p)
static vlib_main_t * vlib_get_main(void)
VLIB_WORKER_INIT_FUNCTION(vlib_buffer_worker_init)
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *last, u32 next_bi)
#define VLIB_BUFFER_DEFAULT_DATA_SIZE
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
VLIB buffer representation.
uword * buffer_known_hash
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
#define clib_error_free(e)
clib_error_t * vlib_buffer_worker_init(vlib_main_t *vm)
static void * clib_mem_alloc_aligned(uword size, uword align)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
vl_api_dhcp_client_state_t state
#define vec_foreach(var, vec)
Vector iterator.
u8 vlib_buffer_pool_create(vlib_main_t *vm, char *name, u32 data_size, u32 physmem_map_index)
#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV
#define CLIB_CACHE_LINE_BYTES
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
volatile u8 ref_count
Reference count for this buffer.
static uword * clib_bitmap_and(uword *ai, uword *bi)
Logical operator across two bitmaps.
static clib_error_t * vlib_buffer_main_init_numa_node(struct vlib_main_t *vm, u32 numa_node, u8 *index)
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
#define clib_panic(format, args...)
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)