19 #ifndef __included_ssvm_fifo_h__ 20 #define __included_ssvm_fifo_h__ 37 #define SVM_FIFO_TRACE (0) 38 #define OOO_SEGMENT_INVALID_INDEX ((u32)~0) 39 #define SVM_FIFO_INVALID_SESSION_INDEX ((u32)~0) 40 #define SVM_FIFO_INVALID_INDEX ((u32)~0) 41 #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7 76 typedef struct _svm_fifo
89 volatile u32 has_event;
90 u32 master_session_index;
91 u32 client_session_index;
92 u8 master_thread_index;
93 u8 client_thread_index;
97 struct _svm_fifo *
next;
98 struct _svm_fifo *prev;
105 volatile u32 want_deq_ntf;
106 volatile u32 has_deq_ntf;
115 volatile u8 n_subscribers;
137 #define svm_fifo_trace_add(_f, _s, _l, _t) \ 139 svm_fifo_trace_elem_t *trace_elt; \ 140 vec_add2(_f->trace, trace_elt, 1); \ 141 trace_elt->offset = _s; \ 142 trace_elt->len = _l; \ 143 trace_elt->action = _t; \ 146 #define svm_fifo_trace_add(_f, _s, _l, _t) 201 return ((f->size + a - b) % f->size);
212 return ((f->size + b - a) % f->size);
223 return (head <= tail ? tail - head : f->
size + tail - head);
234 return (f->nitems -
f_cursize (f, head, tail));
652 return tail >= head ? (tail - head) : (f->size - head);
663 return tail >= head ? f->size - tail :
f_free_count (f, head, tail);
670 return (f->head_chunk->data + (f->head - f->head_chunk->start_byte));
677 return (f->tail_chunk->data + (f->tail - f->tail_chunk->start_byte));
683 return f->n_subscribers;
778 f->want_deq_ntf |= ntf_type;
792 f->want_deq_ntf &= ~ntf_type;
842 u8 want_ntf = f->want_deq_ntf;
851 u32 nitems = f->nitems;
852 if (!f->has_deq_ntf && max_deq < nitems
853 && max_deq + n_last_deq >= nitems)
void svm_fifo_add_subscriber(svm_fifo_t *f, u8 sub)
Add io events subscriber to list.
u32 length
length of chunk in bytes
static void f_load_head_tail_all_acq(svm_fifo_t *f, u32 *head, u32 *tail)
Load head and tail independent of producer/consumer role.
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
struct svm_fifo_chunk_ svm_fifo_chunk_t
int svm_fifo_enqueue(svm_fifo_t *f, u32 len, const u8 *src)
Enqueue data to fifo.
static u32 svm_fifo_max_enqueue_prod(svm_fifo_t *f)
Maximum number of bytes that can be enqueued into fifo.
void svm_fifo_clone(svm_fifo_t *df, svm_fifo_t *sf)
Clone fifo.
void svm_fifo_try_shrink(svm_fifo_t *f, u32 head, u32 tail)
Try to shrink fifo size.
static int svm_fifo_is_full(svm_fifo_t *f)
u8 * svm_fifo_replay(u8 *s, svm_fifo_t *f, u8 no_read, u8 verbose)
static u8 svm_fifo_has_ooo_data(svm_fifo_t *f)
Check if fifo has out-of-order data.
format_function_t format_svm_fifo
void svm_fifo_free_ooo_data(svm_fifo_t *f)
Cleanup fifo ooo data.
static u32 f_free_count(svm_fifo_t *f, u32 head, u32 tail)
Fifo free bytes, i.e., number of free bytes.
Fixed length block allocator.
static int svm_fifo_is_empty_prod(svm_fifo_t *f)
Check if fifo is empty optimized for producer.
u32 prev
Previous linked-list element pool index.
static void f_load_head_tail_cons(svm_fifo_t *f, u32 *head, u32 *tail)
Load head and tail optimized for consumer.
static int svm_fifo_is_empty_cons(svm_fifo_t *f)
Check if fifo is empty optimized for consumer.
static void svm_fifo_reset_has_deq_ntf(svm_fifo_t *f)
Clear has notification flag.
static u32 svm_fifo_max_enqueue(svm_fifo_t *f)
int svm_fifo_reduce_size(svm_fifo_t *f, u32 len, u8 try_shrink)
Request to reduce fifo size by amount of bytes.
void svm_fifo_add_chunk(svm_fifo_t *f, svm_fifo_chunk_t *c)
Grow fifo size by adding chunk to chunk list.
static u8 * svm_fifo_tail(svm_fifo_t *f)
struct _svm_fifo svm_fifo_t
static int svm_fifo_is_empty(svm_fifo_t *f)
Check if fifo is empty.
int svm_fifo_segments(svm_fifo_t *f, svm_fifo_seg_t *fs)
static u32 svm_fifo_max_dequeue(svm_fifo_t *f)
Fifo max bytes to dequeue.
static u32 svm_fifo_max_dequeue_cons(svm_fifo_t *f)
Fifo max bytes to dequeue optimized for consumer.
static u32 svm_fifo_max_write_chunk(svm_fifo_t *f)
Max contiguous chunk of data that can be written.
static u32 ooo_segment_length(svm_fifo_t *f, ooo_segment_t *s)
u8 * svm_fifo_dump_trace(u8 *s, svm_fifo_t *f)
struct svm_fifo_seg_ svm_fifo_seg_t
static u8 * svm_fifo_head(svm_fifo_t *f)
static u32 svm_fifo_max_read_chunk(svm_fifo_t *f)
Max contiguous chunk of data that can be read.
static int svm_fifo_is_full_prod(svm_fifo_t *f)
Check if fifo is full optimized for producer.
static void svm_fifo_newest_ooo_segment_reset(svm_fifo_t *f)
int svm_fifo_peek(svm_fifo_t *f, u32 offset, u32 len, u8 *dst)
Peek data from fifo.
u8 svm_fifo_is_sane(svm_fifo_t *f)
Check if fifo is sane.
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
void svm_fifo_dequeue_drop_all(svm_fifo_t *f)
Dequeue and drop all bytes from fifo.
u32 svm_fifo_n_ooo_segments(svm_fifo_t *f)
Number of out-of-order segments for fifo.
struct svm_fifo_chunk_ * next
pointer to next chunk in linked-lists
u8 svm_fifo_set_single_thread_owned(svm_fifo_t *f)
Declare this fifo is used by only a single thread.
static void svm_fifo_unset_event(svm_fifo_t *f)
Unset fifo event flag.
void svm_fifo_init_chunks(svm_fifo_t *f)
Initialize fifo chunks and rbtree.
svm_fifo_t * svm_fifo_create(u32 size)
Create fifo of requested size.
static u32 svm_fifo_max_dequeue_prod(svm_fifo_t *f)
Fifo max bytes to dequeue optimized for producer.
static ooo_segment_t * svm_fifo_newest_ooo_segment(svm_fifo_t *f)
static u8 svm_fifo_set_event(svm_fifo_t *f)
Set fifo event flag.
svm_fifo_chunk_t * svm_fifo_chunk_alloc(u32 size)
Allocate a fifo chunk on heap.
#define SVM_FIFO_MAX_EVT_SUBSCRIBERS
static u8 svm_fifo_needs_deq_ntf(svm_fifo_t *f, u32 n_last_deq)
Check if fifo needs dequeue notification.
static u32 f_cursize(svm_fifo_t *f, u32 head, u32 tail)
Fifo current size, i.e., number of bytes enqueued.
int svm_fifo_dequeue(svm_fifo_t *f, u32 len, u8 *dst)
Dequeue data from fifo.
void svm_fifo_overwrite_head(svm_fifo_t *f, u8 *src, u32 len)
Overwrite fifo head with new data.
static void f_load_head_tail_prod(svm_fifo_t *f, u32 *head, u32 *tail)
Load head and tail optimized for producer.
u32 start_byte
chunk start byte
svm_fifo_chunk_t * svm_fifo_collect_chunks(svm_fifo_t *f)
Removes chunks that are after fifo end byte.
#define OOO_SEGMENT_INVALID_INDEX
void svm_fifo_free_chunk_lookup(svm_fifo_t *f)
Cleanup fifo chunk lookup rb tree.
#define clib_atomic_swap_acq_n(a, b)
int svm_fifo_dequeue_drop(svm_fifo_t *f, u32 len)
Dequeue and drop bytes from fifo.
Notify on transition to empty.
u8 data[0]
start of chunk data
void svm_fifo_free(svm_fifo_t *f)
Free fifo and associated state.
static void svm_fifo_clear_deq_ntf(svm_fifo_t *f)
Clear the want notification flag and set has notification.
static void svm_fifo_add_want_deq_ntf(svm_fifo_t *f, u8 ntf_type)
Set specific want notification flag.
static u8 svm_fifo_is_wrapped(svm_fifo_t *f)
Check if fifo is wrapped.
Notify on transition from full.
static u32 f_distance_to(svm_fifo_t *f, u32 a, u32 b)
Distance to a from b, i.e., a - b in the fifo.
u32 length
Length of segment.
void svm_fifo_enqueue_nocopy(svm_fifo_t *f, u32 len)
Advance tail pointer.
void svm_fifo_init(svm_fifo_t *f, u32 size)
Initialize fifo.
u32 next
Next linked-list element pool index.
template key/value backing page structure
#define clib_atomic_swap_rel_n(a, b)
enum svm_fifo_deq_ntf_ svm_fifo_deq_ntf_t
No notification requested.
enum svm_fifo_flag_ svm_fifo_flag_t
static void svm_fifo_del_want_deq_ntf(svm_fifo_t *f, u8 ntf_type)
Clear specific want notification flag.
static u32 f_distance_from(svm_fifo_t *f, u32 a, u32 b)
Distance from a to b, i.e., b - a in the fifo.
static u32 ooo_segment_offset_prod(svm_fifo_t *f, ooo_segment_t *s)
static u8 svm_fifo_n_subscribers(svm_fifo_t *f)
void svm_fifo_init_pointers(svm_fifo_t *f, u32 head, u32 tail)
Init fifo head and tail.
void svm_fifo_segments_free(svm_fifo_t *f, svm_fifo_seg_t *fs)
static int svm_fifo_has_event(svm_fifo_t *f)
Check if fifo has io event.
#define clib_atomic_load_acq_n(a)
void svm_fifo_del_subscriber(svm_fifo_t *f, u8 subscriber)
Remove io events subscriber form list.
int svm_fifo_enqueue_with_offset(svm_fifo_t *f, u32 offset, u32 len, u8 *src)
Enqueue data to fifo with offset.
u32 start
Start of segment, normalized.
ooo_segment_t * svm_fifo_first_ooo_segment(svm_fifo_t *f)
First out-of-order segment for fifo.
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".