FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
svm_fifo.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __included_ssvm_fifo_h__
16 #define __included_ssvm_fifo_h__
17 
18 #include <vppinfra/clib.h>
19 #include <vppinfra/vec.h>
20 #include <vppinfra/mheap.h>
21 #include <vppinfra/heap.h>
22 #include <vppinfra/pool.h>
23 #include <vppinfra/format.h>
24 #include <pthread.h>
25 
26 /** Out-of-order segment */
27 typedef struct
28 {
29  u32 next; /**< Next linked-list element pool index */
30  u32 prev; /**< Previous linked-list element pool index */
31 
32  u32 start; /**< Start of segment, normalized*/
33  u32 length; /**< Length of segment */
35 
38 
39 #define SVM_FIFO_TRACE (0)
40 #define OOO_SEGMENT_INVALID_INDEX ((u32)~0)
41 #define SVM_FIFO_INVALID_SESSION_INDEX ((u32)~0)
42 
43 typedef struct
44 {
49 
50 typedef struct _svm_fifo
51 {
52  volatile u32 cursize; /**< current fifo size */
53  u32 nitems;
54  CLIB_CACHE_LINE_ALIGN_MARK (end_cursize);
55 
56  volatile u32 has_event; /**< non-zero if deq event exists */
57 
58  /* Backpointers */
59  u32 master_session_index;
60  u32 client_session_index;
61  u8 master_thread_index;
62  u8 client_thread_index;
63  u32 segment_manager;
64  u32 ct_session_index; /**< Local session index for vpp */
65  CLIB_CACHE_LINE_ALIGN_MARK (end_shared);
66  u32 head;
67  volatile u32 want_tx_evt; /**< producer wants nudge */
68  CLIB_CACHE_LINE_ALIGN_MARK (end_consumer);
69 
70  /* producer */
71  u32 tail;
72 
73  ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
74  u32 ooos_list_head; /**< Head of out-of-order linked-list */
75  u32 ooos_newest; /**< Last segment to have been updated */
76  struct _svm_fifo *next; /**< next in freelist/active chain */
77  struct _svm_fifo *prev; /**< prev in active chain */
78 #if SVM_FIFO_TRACE
80 #endif
81  u32 freelist_index; /**< aka log2(allocated_size) - const. */
82  i8 refcnt; /**< reference count */
84 } svm_fifo_t;
85 
86 typedef enum
87 {
90 
91 typedef struct svm_fifo_segment_
92 {
93  u8 *data;
96 
97 #if SVM_FIFO_TRACE
98 #define svm_fifo_trace_add(_f, _s, _l, _t) \
99 { \
100  svm_fifo_trace_elem_t *trace_elt; \
101  vec_add2(_f->trace, trace_elt, 1); \
102  trace_elt->offset = _s; \
103  trace_elt->len = _l; \
104  trace_elt->action = _t; \
105 }
106 #else
107 #define svm_fifo_trace_add(_f, _s, _l, _t)
108 #endif
109 
110 u8 *svm_fifo_dump_trace (u8 * s, svm_fifo_t * f);
111 u8 *svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose);
112 
113 static inline u32
115 {
116  return f->cursize;
117 }
118 
119 static inline int
121 {
122  return (f->cursize == f->nitems);
123 }
124 
125 static inline int
127 {
128  return (f->cursize == 0);
129 }
130 
131 static inline u32
133 {
134  return f->nitems - svm_fifo_max_dequeue (f);
135 }
136 
137 static inline int
139 {
140  return f->has_event;
141 }
142 
143 static inline u8
145 {
146  return f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX;
147 }
148 
149 /**
150  * Sets fifo event flag.
151  *
152  * Also acts as a release barrier.
153  *
154  * @return 1 if flag was not set.
155  */
158 {
159  /* return __sync_lock_test_and_set (&f->has_event, 1) == 0;
160  return __sync_bool_compare_and_swap (&f->has_event, 0, 1); */
161  return !__atomic_exchange_n (&f->has_event, 1, __ATOMIC_RELEASE);
162 }
163 
164 /**
165  * Unsets fifo event flag.
166  *
167  * Also acts as a release barrier.
168  */
169 always_inline void
171 {
172  __sync_lock_release (&f->has_event);
173 }
174 
175 static inline void
177 {
178  f->want_tx_evt = want_evt;
179 }
180 
181 static inline u8
183 {
184  return f->want_tx_evt;
185 }
186 
187 svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes);
188 void svm_fifo_free (svm_fifo_t * f);
189 
190 int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
191  const u8 * copy_from_here);
193  u32 required_bytes, u8 * copy_from_here);
194 int svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here);
195 
196 int svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 max_bytes, u8 * copy_here);
197 int svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes);
203 void svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer);
204 void svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len);
205 
207 
210 {
211  if (f->ooos_newest == OOO_SEGMENT_INVALID_INDEX)
212  return 0;
213  return pool_elt_at_index (f->ooo_segments, f->ooos_newest);
214 }
215 
216 always_inline void
218 {
219  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
220 }
221 
222 /**
223  * Max contiguous chunk of data that can be read
224  */
227 {
228  return ((f->tail > f->head) ? (f->tail - f->head) : (f->nitems - f->head));
229 }
230 
231 /**
232  * Max contiguous chunk of data that can be written
233  */
236 {
237  return ((f->tail >= f->head) ? (f->nitems - f->tail) : (f->head - f->tail));
238 }
239 
240 /**
241  * Advance tail pointer
242  *
243  * Useful for moving tail pointer after external enqueue.
244  */
245 always_inline void
247 {
248  ASSERT (bytes <= svm_fifo_max_enqueue (f));
249  f->tail = (f->tail + bytes) % f->nitems;
250  f->cursize += bytes;
251 }
252 
255 {
256  return (f->data + f->head);
257 }
258 
261 {
262  return (f->data + f->tail);
263 }
264 
267 {
268  /* Ambiguous. Assumption is that ooo segments don't touch tail */
269  if (PREDICT_FALSE (pos == f->tail && f->tail == f->head))
270  return f->nitems;
271 
272  return (((f->nitems + pos) - f->tail) % f->nitems);
273 }
274 
277 {
278  return (((f->nitems + f->tail) - pos) % f->nitems);
279 }
280 
283 {
284  return ooo_segment_distance_from_tail (f, s->start);
285 }
286 
289 {
290  return ooo_segment_distance_from_tail (f, s->start) + s->length;
291 }
292 
295 {
296  return s->length;
297 }
298 
301 {
303  return 0;
304  return pool_elt_at_index (f->ooo_segments, s->prev);
305 }
306 
309 {
311  return 0;
312  return pool_elt_at_index (f->ooo_segments, s->next);
313 }
314 
315 #endif /* __included_ssvm_fifo_h__ */
316 
317 /*
318  * fd.io coding-style-patch-verification: ON
319  *
320  * Local Variables:
321  * eval: (c-set-style "gnu")
322  * End:
323  */
int svm_fifo_dequeue_drop(svm_fifo_t *f, u32 max_bytes)
Definition: svm_fifo.c:726
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:862
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
format_function_t format_ooo_segment
Definition: svm_fifo.h:36
static int svm_fifo_is_full(svm_fifo_t *f)
Definition: svm_fifo.h:120
u8 * svm_fifo_replay(u8 *s, svm_fifo_t *f, u8 no_read, u8 verbose)
Definition: svm_fifo.c:92
static u8 svm_fifo_has_ooo_data(svm_fifo_t *f)
Definition: svm_fifo.h:144
format_function_t format_svm_fifo
Definition: svm_fifo.h:206
Fixed length block allocator.
u32 prev
Previous linked-list element pool index.
Definition: svm_fifo.h:30
void svm_fifo_segments_free(svm_fifo_t *f, svm_fifo_segment_t *fs)
Definition: svm_fifo.c:801
static u32 svm_fifo_max_enqueue(svm_fifo_t *f)
Definition: svm_fifo.h:132
u8 *( format_function_t)(u8 *s, va_list *args)
Definition: format.h:48
svm_fifo_t * svm_fifo_create(u32 data_size_in_bytes)
create an svm fifo, in the current heap.
Definition: svm_fifo.c:200
static u8 * svm_fifo_tail(svm_fifo_t *f)
Definition: svm_fifo.h:260
unsigned char u8
Definition: types.h:56
static u32 ooo_segment_distance_from_tail(svm_fifo_t *f, u32 pos)
Definition: svm_fifo.h:266
struct _svm_fifo svm_fifo_t
static void svm_fifo_enqueue_nocopy(svm_fifo_t *f, u32 bytes)
Advance tail pointer.
Definition: svm_fifo.h:246
static int svm_fifo_is_empty(svm_fifo_t *f)
Definition: svm_fifo.h:126
static u32 ooo_segment_end_offset(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:288
#define always_inline
Definition: clib.h:94
static u8 svm_fifo_want_tx_evt(svm_fifo_t *f)
Definition: svm_fifo.h:182
static u32 svm_fifo_max_dequeue(svm_fifo_t *f)
Definition: svm_fifo.h:114
static u32 svm_fifo_max_write_chunk(svm_fifo_t *f)
Max contiguous chunk of data that can be written.
Definition: svm_fifo.h:235
static u32 ooo_segment_length(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:294
u8 * svm_fifo_dump_trace(u8 *s, svm_fifo_t *f)
Definition: svm_fifo.c:68
int svm_fifo_dequeue_nowait(svm_fifo_t *f, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:670
unsigned int u32
Definition: types.h:88
static u8 * svm_fifo_head(svm_fifo_t *f)
Definition: svm_fifo.h:254
static u32 svm_fifo_max_read_chunk(svm_fifo_t *f)
Max contiguous chunk of data that can be read.
Definition: svm_fifo.h:226
static void svm_fifo_newest_ooo_segment_reset(svm_fifo_t *f)
Definition: svm_fifo.h:217
void svm_fifo_overwrite_head(svm_fifo_t *f, u8 *data, u32 len)
Definition: svm_fifo.c:599
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
void svm_fifo_dequeue_drop_all(svm_fifo_t *f)
Definition: svm_fifo.c:766
static u32 ooo_segment_offset(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:282
#define PREDICT_FALSE(x)
Definition: clib.h:107
static ooo_segment_t * ooo_segment_next(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:308
static void svm_fifo_unset_event(svm_fifo_t *f)
Unsets fifo event flag.
Definition: svm_fifo.h:170
signed char i8
Definition: types.h:45
static ooo_segment_t * svm_fifo_newest_ooo_segment(svm_fifo_t *f)
Definition: svm_fifo.h:209
static u8 svm_fifo_set_event(svm_fifo_t *f)
Sets fifo event flag.
Definition: svm_fifo.h:157
struct svm_fifo_segment_ svm_fifo_segment_t
format_function_t format_ooo_list
Definition: svm_fifo.h:37
int svm_fifo_enqueue_nowait(svm_fifo_t *f, u32 max_bytes, const u8 *copy_from_here)
Definition: svm_fifo.c:523
static void svm_fifo_set_want_tx_evt(svm_fifo_t *f, u8 want_evt)
Definition: svm_fifo.h:176
static ooo_segment_t * ooo_segment_get_prev(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:300
#define OOO_SEGMENT_INVALID_INDEX
Definition: svm_fifo.h:40
void svm_fifo_init_pointers(svm_fifo_t *f, u32 pointer)
Set fifo pointers to requested offset.
Definition: svm_fifo.c:835
#define ASSERT(truth)
int svm_fifo_segments(svm_fifo_t *f, svm_fifo_segment_t *fs)
Definition: svm_fifo.c:773
void svm_fifo_free(svm_fifo_t *f)
Definition: svm_fifo.c:221
Out-of-order segment.
Definition: svm_fifo.h:27
u32 length
Length of segment.
Definition: svm_fifo.h:33
u32 next
Next linked-list element pool index.
Definition: svm_fifo.h:29
template key/value backing page structure
Definition: bihash_doc.h:44
int svm_fifo_peek(svm_fifo_t *f, u32 offset, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:718
u32 svm_fifo_number_ooo_segments(svm_fifo_t *f)
Definition: svm_fifo.c:820
svm_fifo_err_t
Definition: svm_fifo.h:86
static int svm_fifo_has_event(svm_fifo_t *f)
Definition: svm_fifo.h:138
u32 start
Start of segment, normalized.
Definition: svm_fifo.h:32
ooo_segment_t * svm_fifo_first_ooo_segment(svm_fifo_t *f)
Definition: svm_fifo.c:826
int svm_fifo_enqueue_with_offset(svm_fifo_t *f, u32 offset, u32 required_bytes, u8 *copy_from_here)
Definition: svm_fifo.c:590
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
static u32 ooo_segment_distance_to_tail(svm_fifo_t *f, u32 pos)
Definition: svm_fifo.h:276