36 #define segment_manager_foreach_segment_w_lock(VAR, SM, BODY) \ 38 clib_rwlock_reader_lock (&(SM)->segments_rwlock); \ 39 pool_foreach((VAR), ((SM)->segments), (BODY)); \ 40 clib_rwlock_reader_unlock (&(SM)->segments_rwlock); \ 76 return (seg - sm->segments);
88 uword baseva = (
uword) ~ 0ULL, alloc_size, page_size;
89 u32 rnd_margin = 128 << 10, fs_index = ~0;
99 if (!props->add_segment && !segment_size)
102 return VNET_API_ERROR_INVALID_VALUE;
116 segment_size = segment_size ? segment_size : props->add_segment_size;
119 segment_size =
clib_max (segment_size + page_size - 1, segment_size);
120 segment_size = segment_size & ~(page_size - 1);
125 alloc_size = (
uword) segment_size + rnd_margin;
135 seg_name =
format (0,
"%s%c",
"process-private", 0);
143 clib_warning (
"svm_master_init ('%v', %u) failed", seg_name,
161 fs_index = fs - sm->segments;
243 *sm_index = segment_handle >> 32;
244 *segment_index = segment_handle & 0xFFFFFFFF;
251 return (((
u64) segment_manager_index << 32) | segment_index);
257 u32 sm_index, segment_index;
311 u32 prealloc_fifo_pairs)
313 u32 rx_fifo_size, tx_fifo_size, pair_size;
314 u32 rx_rounded_data_size, tx_rounded_data_size;
315 u64 approx_total_size, max_seg_size = ((
u64) 1 << 32) - (128 << 10);
318 u32 approx_segment_count;
324 if (prealloc_fifo_pairs)
327 rx_rounded_data_size = (1 << (
max_log2 (props->rx_fifo_size)));
328 tx_rounded_data_size = (1 << (
max_log2 (props->tx_fifo_size)));
330 rx_fifo_size =
sizeof (
svm_fifo_t) + rx_rounded_data_size;
331 tx_fifo_size =
sizeof (
svm_fifo_t) + tx_rounded_data_size;
332 pair_size = rx_fifo_size + tx_fifo_size;
334 approx_total_size = (
u64) prealloc_fifo_pairs *pair_size;
335 if (first_seg_size > approx_total_size)
336 max_seg_size = first_seg_size;
337 approx_segment_count = (approx_total_size + (max_seg_size - 1))
341 for (i = 0; i < approx_segment_count + 1; i++)
357 &prealloc_fifo_pairs);
359 if (prealloc_fifo_pairs == 0)
398 segment_manager_del_segment (sm, fifo_segment);
486 for (slice_index = 0; slice_index < fs->
n_slices; slice_index++)
498 f->master_thread_index);
518 u32 rx_fifo_size,
u32 tx_fifo_size,
560 int alloc_fail = 1, rv = 0, new_fs_index;
563 u32 sm_index, fs_index;
564 u8 added_a_segment = 0;
593 ASSERT (rx_fifo && tx_fifo);
596 (*tx_fifo)->segment_manager = sm_index;
597 (*rx_fifo)->segment_manager = sm_index;
598 (*tx_fifo)->segment_index = fs_index;
599 (*rx_fifo)->segment_index = fs_index;
616 if (props->add_segment)
620 clib_warning (
"Added a segment, still can't allocate a fifo");
622 return SESSION_ERROR_NEW_SEG_NO_SPACE;
627 return SESSION_ERROR_SEG_CREATE;
639 clib_warning (
"Can't add new seg and no space to allocate fifos!");
640 return SESSION_ERROR_NO_SPACE;
651 if (!rx_fifo || !tx_fifo)
659 segment_index = rx_fifo->segment_index;
675 if (segment_index != 0 || !sm->first_is_protected)
736 u32 fifo_evt_size, notif_q_size, q_hdrs;
737 u32 msg_q_sz, fifo_evt_ring_sz, session_ntf_ring_sz;
739 fifo_evt_size = 1 <<
max_log2 (
sizeof (session_event_t));
740 notif_q_size =
clib_max (16, q_len >> 4);
743 fifo_evt_ring_sz = q_len * fifo_evt_size;
744 session_ntf_ring_sz = notif_q_size * 256;
747 return (msg_q_sz + fifo_evt_ring_sz + session_ntf_ring_sz + q_hdrs);
759 u32 fifo_evt_size, session_evt_size = 256, notif_q_size;
764 fifo_evt_size =
sizeof (session_event_t);
765 notif_q_size =
clib_max (16, props->evt_q_size >> 4);
768 {props->evt_q_size, fifo_evt_size, 0},
769 {notif_q_size, session_evt_size, 0}
782 if (props->use_mq_eventfd)
793 return sm->event_queue;
841 u8 show_segments = 0, verbose = 0;
849 else if (
unformat (input,
"verbose"))
864 vlib_cli_output (vm,
"%-10d%=15d%=12d", segment_manager_index (sm),
865 sm->app_wrk_index, pool_elts (sm->segments));
876 segment_manager_foreach_segment_w_lock (seg, sm, ({
877 vlib_cli_output (vm,
"%U", format_fifo_segment, seg, verbose);
889 .path =
"show segment-manager",
890 .short_help =
"show segment-manager [segments][verbose]",
910 "API Client",
"SegManager");
923 for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
925 f = fifo_segment_get_slice_fifo_list (fs, slice_index);
928 u32 session_index, thread_index;
931 session_index = f->master_session_index;
932 thread_index = f->master_thread_index;
934 session = session_get (session_index, thread_index);
935 str = format (0,
"%U", format_session, session, verbose);
938 s = format (s,
"%-40s%-20s%-15u%-10u", str, app_name,
939 app_wrk->api_client_index, app_wrk->connects_seg_manager);
941 s = format (s,
"%-40s%-20s", str, app_name);
943 vlib_cli_output (vm,
"%v", s);
944 vec_reset_length (s);
u32 segment_manager_index(segment_manager_t *sm)
static void clib_rwlock_reader_lock(clib_rwlock_t *p)
void segment_manager_segment_reader_unlock(segment_manager_t *sm)
u64 segment_manager_segment_handle(segment_manager_t *sm, fifo_segment_t *segment)
svm_msg_q_t * segment_manager_alloc_queue(fifo_segment_t *segment, segment_manager_props_t *props)
Allocates shm queue in the first segment.
static void clib_rwlock_writer_lock(clib_rwlock_t *p)
int segment_manager_add_segment(segment_manager_t *sm, uword segment_size)
Adds segment to segment manager's pool.
svm_fifo_t * fifo_segment_get_slice_fifo_list(fifo_segment_t *fs, u32 slice_index)
fifo_segment_t * segment_manager_get_segment(segment_manager_t *sm, u32 segment_index)
Reads a segment from the segment manager's pool without lock.
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
struct segment_manager_main_ segment_manager_main_t
svm_fifo_t * fifo_segment_alloc_fifo_w_slice(fifo_segment_t *fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype)
Allocate fifo in fifo segment.
static void clib_rwlock_free(clib_rwlock_t *p)
static session_t * session_get_if_valid(u64 si, u32 thread_index)
fifo_segment_t * segment_manager_get_segment_w_handle(u64 segment_handle)
void segment_manager_format_sessions(segment_manager_t *sm, int verbose)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
clib_valloc_main_t va_allocator
Virtual address allocator.
static segment_manager_main_t sm_main
void segment_manager_free(segment_manager_t *sm)
Cleanup segment manager.
ssvm_shared_header_t * sh
int app_worker_add_segment_notify(app_worker_t *app_wrk, u64 segment_handle)
Send an API message to the external app, to map new segment.
segment_manager_t * segment_managers
Pool of segment managers.
int segment_manager_init(segment_manager_t *sm, uword first_seg_size, u32 prealloc_fifo_pairs)
Initializes segment manager based on options provided.
u32 segment_manager_evt_q_expected_size(u32 q_len)
static void segment_manager_parse_segment_handle(u64 segment_handle, u32 *sm_index, u32 *segment_index)
static u32 segment_manager_segment_index(segment_manager_t *sm, fifo_segment_t *seg)
void ssvm_delete(ssvm_private_t *ssvm)
static session_handle_t session_handle(session_t *s)
struct _svm_fifo svm_fifo_t
#define fifo_segment_flags(_fs)
u64 segment_manager_make_segment_handle(u32 segment_manager_index, u32 segment_index)
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
svm_msg_q_t * svm_msg_q_alloc(svm_msg_q_cfg_t *cfg)
Allocate message queue.
static u8 segment_manager_app_detached(segment_manager_t *sm)
void segment_manager_dealloc_fifos(svm_fifo_t *rx_fifo, svm_fifo_t *tx_fifo)
static void * ssvm_push_heap(ssvm_shared_header_t *sh)
#define clib_error_return(e, args...)
u32 default_fifo_size
default rx/tx fifo size
static void segment_manager_lock_and_del_segment(segment_manager_t *sm, u32 fs_index)
Removes segment after acquiring writer lock.
int ssvm_master_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
static void ssvm_pop_heap(void *oldheap)
static heap_elt_t * first(heap_header_t *h)
void fifo_segment_preallocate_fifo_pairs(fifo_segment_t *fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 *n_fifo_pairs)
Pre-allocates fifo pairs in fifo segment.
u32 seg_name_counter
Counter for segment names.
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static void clib_rwlock_init(clib_rwlock_t *p)
static void clib_rwlock_reader_unlock(clib_rwlock_t *p)
void segment_manager_app_detach(segment_manager_t *sm)
u32 default_app_mq_size
default app msg q size
static session_t * session_get_from_handle(session_handle_t handle)
void segment_manager_del_segment(segment_manager_t *sm, fifo_segment_t *fs)
Remove segment without lock.
void segment_manager_dealloc_queue(segment_manager_t *sm, svm_queue_t *q)
Frees shm queue allocated in the first segment.
uword clib_mem_get_page_size(void)
const u8 * application_name_from_index(u32 app_index)
Returns app name for app-index.
#define pool_put(P, E)
Free an object E in pool P.
void segment_manager_init_free(segment_manager_t *sm)
Initiate segment manager cleanup.
struct _segment_manager_props segment_manager_props_t
int fifo_segment_grow_fifo(fifo_segment_t *fs, svm_fifo_t *f, u32 chunk_size)
Grow fifo size by adding an additional chunk of memory.
u8 segment_manager_has_fifos(segment_manager_t *sm)
uword size
size in bytes of this chunk
int segment_manager_grow_fifo(segment_manager_t *sm, svm_fifo_t *f, u32 size)
Grows fifo owned by segment manager.
int segment_manager_collect_fifo_chunks(segment_manager_t *sm, svm_fifo_t *f)
Collect fifo chunks that are no longer used.
uword baseva
base VA for this chunk
#define SEGMENT_MANAGER_INVALID_APP_INDEX
static void clib_rwlock_writer_unlock(clib_rwlock_t *p)
u32 n_rings
number of msg rings
svm_msg_q_t * segment_manager_event_queue(segment_manager_t *sm)
ssvm_private_t ssvm
ssvm segment data
segment_manager_t * segment_manager_alloc(void)
static segment_manager_props_t * segment_manager_properties_get(segment_manager_t *sm)
#define clib_warning(format, args...)
int svm_fifo_reduce_size(svm_fifo_t *f, u32 len, u8 try_shrink)
Request to reduce fifo size by amount of bytes.
#define segment_manager_foreach_segment_w_lock(VAR, SM, BODY)
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
int segment_manager_alloc_session_fifos(segment_manager_t *sm, u32 thread_index, svm_fifo_t **rx_fifo, svm_fifo_t **tx_fifo)
void svm_queue_free(svm_queue_t *q)
#define VLIB_CLI_COMMAND(x,...)
u8 n_slices
number of fifo segment slices
segment_manager_props_t * application_get_segment_manager_properties(u32 app_index)
uword clib_valloc_alloc(clib_valloc_main_t *vam, uword size, int os_out_of_memory_on_failure)
Allocate virtual space.
segment_manager_props_t * segment_manager_props_init(segment_manager_props_t *props)
void fifo_segment_free_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated in fifo segment.
fifo_segment_t * segment_manager_get_segment_w_lock(segment_manager_t *sm, u32 segment_index)
Reads a segment from the segment manager's pool and acquires reader lock.
void segment_manager_main_init(segment_manager_main_init_args_t *a)
svm_msg_q_ring_cfg_t * ring_cfgs
array of ring cfgs
static vlib_main_t * vlib_get_main(void)
int app_worker_del_segment_notify(app_worker_t *app_wrk, u64 segment_handle)
void segment_manager_segment_writer_unlock(segment_manager_t *sm)
app_worker_t * app_worker_get(u32 wrk_index)
u32 q_nitems
msg queue size (not rings)
static uword max_log2(uword x)
void session_close(session_t *s)
Initialize session closing procedure.
void segment_manager_del_sessions(segment_manager_t *sm)
Initiate disconnects for all sessions 'owned' by a segment manager.
uword clib_valloc_free(clib_valloc_main_t *vam, uword baseva)
Free virtual space.
struct _segment_manager segment_manager_t
u32 default_segment_size
default fifo segment size
segment_manager_t * segment_manager_get(u32 index)
struct _svm_queue svm_queue_t
void fifo_segment_update_free_bytes(fifo_segment_t *fs)
Update fifo segment free bytes estimate.
u8 fifo_segment_has_fifos(fifo_segment_t *fs)
u32 app_index
Index of owning app.
int fifo_segment_collect_fifo_chunks(fifo_segment_t *fs, svm_fifo_t *f)
Collect unused chunks for fifo.
void clib_valloc_init(clib_valloc_main_t *vam, clib_valloc_chunk_t *template, int need_lock)
Initialize a virtual memory allocation arena.
int segment_manager_try_alloc_fifos(fifo_segment_t *fifo_segment, u32 thread_index, u32 rx_fifo_size, u32 tx_fifo_size, svm_fifo_t **rx_fifo, svm_fifo_t **tx_fifo)
static u32 vlib_num_workers()
#define vec_foreach(var, vec)
Vector iterator.
static clib_error_t * segment_manager_show_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
int consumer_pid
pid of msg consumer
int svm_msg_q_alloc_producer_eventfd(svm_msg_q_t *mq)
Allocate event fd for queue consumer.
format_function_t format_fifo_segment
int fifo_segment_init(fifo_segment_t *fs)
Initialize fifo segment shared header.
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
segment_manager_t * segment_manager_get_if_valid(u32 index)
int segment_manager_shrink_fifo(segment_manager_t *sm, svm_fifo_t *f, u32 size, u8 is_producer)
Request to shrink fifo owned by segment manager.
ssvm_segment_type_t ssvm_type(const ssvm_private_t *ssvm)
static uword pool_elts(void *v)
Number of active elements in a pool.