34 return dlminfo.fordblks;
76 return &fsh->
slices[slice_index];
94 u32 max_chunk_sz, max_chunks;
197 _vec_len (fs) =
vec_len (fs) - 1;
237 u32 timeout_in_seconds)
268 u32 fl_index,
u32 data_bytes)
284 memset (f, 0,
sizeof (*f));
298 u32 fl_index, fl_size, n_alloc = 0;
309 memset (f, 0,
sizeof (*f));
351 data_bytes += fl_size;
367 fl_size = fl_size >> 1;
371 f->start_chunk =
first;
381 u32 fl_index,
u32 batch_size)
383 u32 size, hdrs, rounded_data_size;
391 hdrs =
sizeof (*f) +
sizeof (*c);
392 size = (hdrs + rounded_data_size) * batch_size;
405 for (i = 0; i < batch_size; i++)
408 memset (f, 0,
sizeof (*f));
413 c->
length = rounded_data_size;
416 fmem += hdrs + rounded_data_size;
438 u32 fifo_sz, fl_index;
444 fifo_sz += 1 <<
max_log2 (data_bytes);
464 if (fifo_sz <= n_free_bytes)
475 if (data_bytes <= fss->n_fl_chunk_bytes)
494 ASSERT (slice_index < fs->n_slices);
501 f->slice_index = slice_index;
507 if (f->start_chunk->next != f->start_chunk)
522 fss->
fifos->prev = f;
523 f->next = fss->
fifos;
557 f->prev->next = f->next;
559 fss->
fifos = f->next;
561 f->next->prev = f->prev;
571 cur = f->start_chunk;
582 while (cur != f->start_chunk);
584 f->start_chunk = f->end_chunk = f->new_chunks = 0;
585 f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0;
596 f->master_session_index = ~0;
597 f->master_thread_index = ~0;
616 size = (
sizeof (*f)) * batch_size;
629 for (i = 0; i < batch_size; i++)
632 memset (f, 0,
sizeof (*f));
645 u32 chunk_size,
u32 batch_size)
647 u32 size, rounded_data_size, fl_index;
657 clib_warning (
"chunk size out of range %d", chunk_size);
663 size = (
sizeof (*c) + rounded_data_size) * batch_size;
678 for (i = 0; i < batch_size; i++)
682 c->
length = rounded_data_size;
685 cmem +=
sizeof (*c) + rounded_data_size;
699 u32 rx_fifo_size,
u32 tx_fifo_size,
702 u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc;
703 u32 hdrs, pairs_per_slice, alloc_now;
705 int rx_fl_index, tx_fl_index,
i;
707 uword space_available;
710 if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
715 clib_warning (
"rx fifo_size out of range %d", rx_fifo_size);
721 clib_warning (
"tx fifo_size out of range %d", tx_fifo_size);
725 rx_rounded_data_size = (1 << (
max_log2 (rx_fifo_size)));
727 tx_rounded_data_size = (1 << (
max_log2 (tx_fifo_size)));
733 pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size;
735 pairs_to_alloc = space_available / pair_size;
736 pairs_to_alloc =
clib_min (pairs_to_alloc, *n_fifo_pairs);
737 pairs_per_slice = pairs_to_alloc / fs->
n_slices;
738 pairs_per_slice += pairs_to_alloc % fs->
n_slices ? 1 : 0;
740 if (!pairs_per_slice)
746 alloc_now =
clib_min (pairs_per_slice, *n_fifo_pairs);
748 clib_warning (
"rx prealloc failed: pairs %u", alloc_now);
750 clib_warning (
"tx prealloc failed: pairs %u", alloc_now);
753 *n_fifo_pairs -= alloc_now;
866 for (slice_index = 0; slice_index < fs->
n_slices; slice_index++)
877 u32 count = 0, rounded_size, fl_index;
899 rounded_size = (1 << (
max_log2 (size)));
925 for (slice_index = 0; slice_index < fs->
n_slices; slice_index++)
953 for (slice_index = 0; slice_index < fs->
n_slices; slice_index++)
969 for (slice_index = 0; slice_index < fs->
n_slices; slice_index++)
996 s =
format (s,
"%s",
"private-heap");
998 s =
format (s,
"%s",
"memfd");
1000 s =
format (s,
"%s",
"shm");
1002 s =
format (s,
"%s",
"unknown");
1012 u32 count, indent, active_fifos, free_fifos, fifo_hdr = 0;
1014 int verbose __attribute__ ((unused)) = va_arg (*args,
int);
1015 uword est_chunk_bytes, est_free_seg_bytes, free_chunks;
1016 uword chunk_bytes = 0, free_seg_bytes, chunk_size;
1026 #if USE_DLMALLOC == 0 1029 s =
format (s,
"%U segment has %u active fifos\n",
1035 s =
format (s,
"%-15s%15s%15s%15s%15s%15s",
"Name",
"Type",
1036 "HeapSize (M)",
"ActiveFifos",
"FreeFifos",
"Address");
1046 free_fifos, address);
1060 for (slice_index = 0; slice_index < fs->
n_slices; slice_index++)
1077 chunk_size >> 10, count);
1079 chunk_bytes += count * chunk_size;
1089 s =
format (s,
"\n%Useg free bytes: %U (%lu) estimated: %U (%lu)\n",
1092 est_free_seg_bytes, est_free_seg_bytes);
1093 s =
format (s,
"%Uchunk free bytes: %U (%lu) estimated: %U (%lu)\n",
1097 s =
format (s,
"%Ufifo hdr free bytes: %U (%u) reserved %U (%lu)\n",
u32 length
length of chunk in bytes
void fifo_segment_info(fifo_segment_t *seg, char **address, size_t *size)
#define FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE
4kB min fifo size
fifo_segment_header_t * h
fifo segment data
svm_fifo_chunk_t * svm_fifo_collect_chunks(svm_fifo_t *f)
Removes chunks that are after fifo end byte.
static void fsh_check_mem(fifo_segment_header_t *fsh)
static uword fsh_n_free_bytes(fifo_segment_header_t *fsh)
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
svm_fifo_t * fifo_segment_get_slice_fifo_list(fifo_segment_t *fs, u32 slice_index)
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
void svm_fifo_free_chunk_lookup(svm_fifo_t *f)
Cleanup fifo chunk lookup rb tree.
void fifo_segment_delete(fifo_segment_main_t *sm, fifo_segment_t *s)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
svm_fifo_t * fifo_segment_alloc_fifo_w_slice(fifo_segment_t *fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype)
Allocate fifo in fifo segment.
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static heap_elt_t * last(heap_header_t *h)
void svm_fifo_init(svm_fifo_t *f, u32 size)
Initialize fifo.
ssvm_shared_header_t * sh
DLMALLOC_EXPORT struct dlmallinfo mspace_mallinfo(mspace msp)
static u32 fs_slice_num_free_fifos(fifo_segment_slice_t *fss)
static u32 fs_freelist_index_to_size(u32 fl_index)
static int fs_try_alloc_fifo_batch(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index, u32 batch_size)
u32 fifo_segment_num_free_fifos(fifo_segment_t *fs)
void ssvm_delete(ssvm_private_t *ssvm)
static void fsh_update_free_bytes(fifo_segment_header_t *fsh)
struct _svm_fifo svm_fifo_t
uword next_baseva
Where to put the next one.
void svm_fifo_init_chunks(svm_fifo_t *f)
Initialize fifo chunks and rbtree.
u32 fifo_segment_num_free_chunks(fifo_segment_t *fs, u32 size)
Find number of free chunks of given size.
fifo_segment_t * fifo_segment_get_segment(fifo_segment_main_t *sm, u32 segment_index)
static fifo_segment_slice_t * fsh_slice_get(fifo_segment_header_t *fsh, u32 slice_index)
u8 * ssvm_name(const ssvm_private_t *ssvm)
enum ssvm_segment_type_ ssvm_segment_type_t
svm_fifo_chunk_t * svm_fifo_chunk_alloc(u32 size)
Creates a fifo chunk in the current heap.
static u32 fs_freelist_for_size(u32 size)
static int fs_chunk_size_is_valid(fifo_segment_header_t *fsh, u32 size)
void svm_fifo_free_ooo_data(svm_fifo_t *f)
Cleanup fifo ooo data.
static svm_fifo_t * fs_try_alloc_fifo_freelist_multi_chunk(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 data_bytes)
uword n_fl_chunk_bytes
Chunk bytes on freelist.
static void * ssvm_push_heap(ssvm_shared_header_t *sh)
int attach_timeout
shm segments attach timeout (sec)
int ssvm_master_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
static void ssvm_pop_heap(void *oldheap)
char * segment_name
segment name
static heap_elt_t * first(heap_header_t *h)
int fifo_segment_attach(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Attach as slave to a fifo segment.
void fifo_segment_preallocate_fifo_pairs(fifo_segment_t *fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 *n_fifo_pairs)
Pre-allocates fifo pairs in fifo segment.
int fifo_segment_prealloc_fifo_chunks(fifo_segment_t *fs, u32 slice_index, u32 chunk_size, u32 batch_size)
Try to preallocate fifo chunks on segment.
u8 * format_mheap(u8 *s, va_list *va)
u8 * format_fifo_segment_type(u8 *s, va_list *args)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
struct svm_fifo_chunk_ * next
pointer to next chunk in linked-lists
static uword fsh_free_space(fifo_segment_header_t *fsh)
Fifo segment free space.
svm_fifo_t * free_fifos
Freelists by fifo size.
u8 * format_fifo_segment(u8 *s, va_list *args)
Segment format function.
#define pool_put(P, E)
Free an object E in pool P.
int fifo_segment_grow_fifo(fifo_segment_t *fs, svm_fifo_t *f, u32 chunk_size)
Grow fifo size by adding an additional chunk of memory.
#define clib_atomic_fetch_add_rel(a, b)
int fifo_segment_create(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Create a fifo segment and initialize as master.
#define FIFO_SEGMENT_MIN_FIFO_SIZE
4kB min fifo size
svm_fifo_t * fifos
Linked list of active RX fifos.
u32 segment_size
size of the segment
ssvm_private_t ssvm
ssvm segment data
void fifo_segment_main_init(fifo_segment_main_t *sm, u64 baseva, u32 timeout_in_seconds)
#define clib_warning(format, args...)
void svm_fifo_add_chunk(svm_fifo_t *f, svm_fifo_chunk_t *c)
Grow fifo size by adding chunk to chunk list.
uword fifo_segment_fl_chunk_bytes(fifo_segment_t *fs)
Number of bytes on chunk free lists.
int ssvm_slave_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
u32 start_byte
chunk start byte
int memfd_fd
fd for memfd segments
u8 n_slices
number of fifo segment slices
#define clib_atomic_load_relax_n(a)
u32 fifo_segment_index(fifo_segment_main_t *sm, fifo_segment_t *s)
manual_print typedef address
void fifo_segment_free_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated in fifo segment.
fifo_segment_t * segments
pool of fifo segments
#define FIFO_SEGMENT_MAX_FIFO_SIZE
2GB max fifo size
static u32 fs_slice_num_free_chunks(fifo_segment_slice_t *fss, u32 size)
static svm_fifo_t * fs_try_alloc_fifo(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 data_bytes)
Try to allocate new fifo.
static void * clib_mem_alloc(uword size)
#define clib_atomic_store_rel_n(a, b)
#define clib_atomic_fetch_sub_rel(a, b)
ssvm_segment_type_t segment_type
type of segment requested
u32 fifo_segment_num_fifos(fifo_segment_t *fs)
Get number of active fifos.
static void fsh_free_bytes_sub(fifo_segment_header_t *fsh, int size)
static svm_fifo_t * fs_try_alloc_fifo_freelist(fifo_segment_slice_t *fss, u32 fl_index, u32 data_bytes)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static uword max_log2(uword x)
int fifo_segment_prealloc_fifo_hdrs(fifo_segment_t *fs, u32 slice_index, u32 batch_size)
Try to preallocate fifo headers.
void fifo_segment_update_free_bytes(fifo_segment_t *fs)
Update fifo segment free bytes estimate.
u8 fifo_segment_has_fifos(fifo_segment_t *fs)
static void * clib_mem_alloc_aligned(uword size, uword align)
int fifo_segment_collect_fifo_chunks(fifo_segment_t *fs, svm_fifo_t *f)
Collect unused chunks for fifo.
#define FIFO_SEGMENT_ALLOC_BATCH_SIZE
u32 timeout_in_seconds
Time to wait during attach.
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
#define CLIB_CACHE_LINE_BYTES
int fifo_segment_init(fifo_segment_t *fs)
Initialize fifo segment shared header.
static void fsh_active_fifos_update(fifo_segment_header_t *fsh, int inc)
uword fifo_segment_free_bytes(fifo_segment_t *fs)
Fifo segment estimate of number of free bytes.
svm_fifo_chunk_t ** free_chunks
Freelists by chunk size.
u32 * new_segment_indices
return vec of new seg indices
ssvm_segment_type_t ssvm_type(const ssvm_private_t *ssvm)
svm_fifo_t * svm_fifo_create(u32 data_size_in_bytes)
Creates a fifo in the current heap.