FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
svm_fifo.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
3  * Copyright (c) 2019 Arm Limited
4  * Copyright (c) 2010-2017 Intel Corporation and/or its affiliates.
5  * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6  * Inspired from DPDK rte_ring.h (SPSC only) (derived from freebsd bufring.h).
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at:
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 #ifndef __included_ssvm_fifo_h__
20 #define __included_ssvm_fifo_h__
21 
22 #include <vppinfra/clib.h>
23 #include <vppinfra/vec.h>
24 #include <vppinfra/pool.h>
25 #include <vppinfra/format.h>
26 #include <vppinfra/rbtree.h>
27 
28 /** Out-of-order segment */
29 typedef struct
30 {
31  u32 next; /**< Next linked-list element pool index */
32  u32 prev; /**< Previous linked-list element pool index */
33  u32 start; /**< Start of segment, normalized*/
34  u32 length; /**< Length of segment */
36 
37 #define SVM_FIFO_TRACE (0)
38 #define OOO_SEGMENT_INVALID_INDEX ((u32)~0)
39 #define SVM_FIFO_INVALID_SESSION_INDEX ((u32)~0)
40 #define SVM_FIFO_INVALID_INDEX ((u32)~0)
41 #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7
42 
43 typedef enum svm_fifo_deq_ntf_
44 {
45  SVM_FIFO_NO_DEQ_NOTIF = 0, /**< No notification requested */
46  SVM_FIFO_WANT_DEQ_NOTIF = 1, /**< Notify on dequeue */
47  SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL = 2, /**< Notify on transition from full */
48  SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY = 4, /**< Notify on transition to empty */
50 
51 typedef struct
52 {
57 
58 typedef struct svm_fifo_chunk_
59 {
60  u32 start_byte; /**< chunk start byte */
61  u32 length; /**< length of chunk in bytes */
62  struct svm_fifo_chunk_ *next; /**< pointer to next chunk in linked-lists */
63  u8 data[0]; /**< start of chunk data */
65 
66 typedef enum svm_fifo_flag_
67 {
69  SVM_FIFO_F_GROW = 1 << 1,
75 
76 typedef struct _svm_fifo
77 {
78  CLIB_CACHE_LINE_ALIGN_MARK (shared_first);
79  u32 size; /**< size of the fifo in bytes */
80  u32 nitems; /**< usable size (size-1) */
81  u8 flags; /**< fifo flags */
82  svm_fifo_chunk_t *start_chunk;/**< first chunk in fifo chunk list */
83  svm_fifo_chunk_t *end_chunk; /**< end chunk in fifo chunk list */
84  svm_fifo_chunk_t *new_chunks; /**< chunks yet to be added to list */
85  rb_tree_t chunk_lookup;
86 
87  CLIB_CACHE_LINE_ALIGN_MARK (shared_second);
88  volatile u32 has_event; /**< non-zero if deq event exists */
89  u32 master_session_index; /**< session layer session index */
90  u32 client_session_index; /**< app session index */
91  u8 master_thread_index; /**< session layer thread index */
92  u8 client_thread_index; /**< app worker index */
93  i8 refcnt; /**< reference count */
94  u32 segment_manager; /**< session layer segment manager index */
95  u32 segment_index; /**< segment index in segment manager */
96  struct _svm_fifo *next; /**< next in freelist/active chain */
97  struct _svm_fifo *prev; /**< prev in active chain */
98  u32 size_decrement; /**< bytes to remove from fifo */
99 
100  CLIB_CACHE_LINE_ALIGN_MARK (consumer);
101  u32 head; /**< fifo head position/byte */
102  svm_fifo_chunk_t *head_chunk; /**< tracks chunk where head lands */
103  svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */
104  volatile u32 want_deq_ntf; /**< producer wants nudge */
105  volatile u32 has_deq_ntf;
106 
107  CLIB_CACHE_LINE_ALIGN_MARK (producer);
108  u32 tail; /**< fifo tail position/byte */
109  u32 ooos_list_head; /**< Head of out-of-order linked-list */
110  svm_fifo_chunk_t *tail_chunk; /**< tracks chunk where tail lands */
111  svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */
112  ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
113  u32 ooos_newest; /**< Last segment to have been updated */
114  volatile u8 n_subscribers; /**< Number of subscribers for io events */
115  u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
116 
117 #if SVM_FIFO_TRACE
119 #endif
120 
121 } svm_fifo_t;
122 
123 typedef enum
124 {
128 
129 typedef struct svm_fifo_seg_
130 {
134 
135 #if SVM_FIFO_TRACE
136 #define svm_fifo_trace_add(_f, _s, _l, _t) \
137 { \
138  svm_fifo_trace_elem_t *trace_elt; \
139  vec_add2(_f->trace, trace_elt, 1); \
140  trace_elt->offset = _s; \
141  trace_elt->len = _l; \
142  trace_elt->action = _t; \
143 }
144 #else
145 #define svm_fifo_trace_add(_f, _s, _l, _t)
146 #endif
147 
148 u8 *svm_fifo_dump_trace (u8 * s, svm_fifo_t * f);
149 u8 *svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose);
150 
151 /**
152  * Load head and tail optimized for consumer
153  *
154  * Internal function.
155  */
156 static inline void
158 {
159  /* load-relaxed: consumer owned index */
160  *head = f->head;
161  /* load-acq: consumer foreign index (paired with store-rel in producer) */
162  *tail = clib_atomic_load_acq_n (&f->tail);
163 }
164 
165 /** Load head and tail optimized for producer
166  *
167  * Internal function
168  */
169 static inline void
171 {
172  /* load relaxed: producer owned index */
173  *tail = f->tail;
174  /* load-acq: producer foreign index (paired with store-rel in consumer) */
175  *head = clib_atomic_load_acq_n (&f->head);
176 }
177 
178 /**
179  * Load head and tail independent of producer/consumer role
180  *
181  * Internal function.
182  */
183 static inline void
185 {
186  /* load-acq : consumer foreign index (paired with store-rel) */
187  *tail = clib_atomic_load_acq_n (&f->tail);
188  /* load-acq : producer foriegn index (paired with store-rel) */
189  *head = clib_atomic_load_acq_n (&f->head);
190 }
191 
192 /**
193  * Distance to a from b, i.e., a - b in the fifo
194  *
195  * Internal function.
196  */
197 static inline u32
199 {
200  return ((f->size + a - b) % f->size);
201 }
202 
203 /**
204  * Distance from a to b, i.e., b - a in the fifo
205  *
206  * Internal function.
207  */
208 static inline u32
210 {
211  return ((f->size + b - a) % f->size);
212 }
213 
214 /**
215  * Fifo current size, i.e., number of bytes enqueued
216  *
217  * Internal function.
218  */
219 static inline u32
220 f_cursize (svm_fifo_t * f, u32 head, u32 tail)
221 {
222  return (head <= tail ? tail - head : f->size + tail - head);
223 }
224 
225 /**
226  * Fifo free bytes, i.e., number of free bytes
227  *
228  * Internal function
229  */
230 static inline u32
231 f_free_count (svm_fifo_t * f, u32 head, u32 tail)
232 {
233  return (f->nitems - f_cursize (f, head, tail));
234 }
235 
236 /**
237  * Try to shrink fifo size.
238  *
239  * Internal function.
240  */
241 void svm_fifo_try_shrink (svm_fifo_t * f, u32 head, u32 tail);
242 
243 /**
244  * Create fifo of requested size
245  *
246  * Allocates fifo on current heap.
247  *
248  * @param size data size in bytes for fifo to be allocated. Will be
249  * rounded to the next highest power-of-two value.
250  * @return pointer to new fifo
251  */
253 /**
254  * Initialize fifo
255  *
256  * @param f fifo
257  * @param size size for fifo
258  */
259 void svm_fifo_init (svm_fifo_t * f, u32 size);
260 /**
261  * Initialize fifo chunks and rbtree
262  *
263  * @param f fifo
264  */
266 /**
267  * Allocate a fifo chunk on heap
268  *
269  * If the chunk is allocated on a fifo segment, this should be called
270  * with the segment's heap pushed.
271  *
272  * @param size chunk size in bytes. Will be rounded to the next highest
273  * power-of-two
274  * @return new chunk or 0 if alloc failed
275  */
277 /**
278  * Grow fifo size by adding chunk to chunk list
279  *
280  * If fifos are allocated on a segment, this should be called with
281  * the segment's heap pushed.
282  *
283  * @param f fifo to be extended
284  * @param c chunk or linked list of chunks to be added
285  */
287 /**
288  * Request to reduce fifo size by amount of bytes
289  *
290  * Because the producer might be enqueuing data when this is called, the
291  * actual size update is only applied when producer tries to enqueue new
292  * data, unless @param try_shrink is set.
293  *
294  * @param f fifo
295  * @param len number of bytes to remove from fifo. The actual number
296  * of bytes to be removed will be less or equal to this
297  * value.
298  * @param try_shrink flg to indicate if it's safe to try to shrink fifo
299  * size. It should be set only if this is called by the
300  * producer of if the producer is not using the fifo
301  * @return actual length fifo size will be reduced by
302  */
303 int svm_fifo_reduce_size (svm_fifo_t * f, u32 len, u8 try_shrink);
304 /**
305  * Removes chunks that are after fifo end byte
306  *
307  * Needs to be called with segment heap pushed.
308  *
309  * @param f fifo
310  */
312 /**
313  * Free fifo and associated state
314  *
315  * @param f fifo
316  */
317 void svm_fifo_free (svm_fifo_t * f);
318 /**
319  * Cleanup fifo chunk lookup rb tree
320  *
321  * The rb tree is allocated in segment heap so this should be called
322  * with it pushed.
323  *
324  * @param f fifo to cleanup
325  */
327 /**
328  * Cleanup fifo ooo data
329  *
330  * The ooo data is allocated in producer process memory. The fifo
331  * segment heap should not be pushed.
332  *
333  * @param f fifo to cleanup
334  */
336 /**
337  * Init fifo head and tail
338  *
339  * @param f fifo
340  * @param head head value that will be matched to a chunk
341  * @param tail tail value that will be matched to a chunk
342  */
343 void svm_fifo_init_pointers (svm_fifo_t * f, u32 head, u32 tail);
344 /**
345  * Clone fifo
346  *
347  * Clones single/default chunk fifo. It does not work for fifos with
348  * multiple chunks.
349  */
350 void svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf);
351 /**
352  * Enqueue data to fifo
353  *
354  * Data is enqueued and tail pointer is updated atomically. If the new data
355  * enqueued partly overlaps or "touches" an out-of-order segment, said segment
356  * is "consumed" and the number of bytes returned is appropriately updated.
357  *
358  * @param f fifo
359  * @param len length of data to copy
360  * @param src buffer from where to copy the data
361  * @return number of contiguous bytes that can be consumed or error
362  */
363 int svm_fifo_enqueue (svm_fifo_t * f, u32 len, const u8 * src);
364 /**
365  * Enqueue data to fifo with offset
366  *
367  * Data is enqueued without updating tail pointer. Instead, an out-of-order
368  * list of segments is generated and maintained. Fifo takes care of coalescing
369  * contiguous or overlapping segments.
370  *
371  * @param f fifo
372  * @param offset offset at which to copy the data
373  * @param len len of data to copy
374  * @param src buffer from where to copy the data
375  * @return 0 if enqueue was successful, error otherwise
376  */
378  u8 * src);
379 
380 /**
381  * Advance tail pointer
382  *
383  * Useful for moving tail pointer after external enqueue.
384  *
385  * @param f fifo
386  * @param len number of bytes to add to tail
387  */
389 /**
390  * Overwrite fifo head with new data
391  *
392  * This should be typically used by dgram transport protocols that need
393  * to update the dgram header after dequeueing a chunk of data. It assumes
394  * that the dgram header is at most spread over two chunks.
395  *
396  * @param f fifo
397  * @param src src of new data
398  * @param len length of new data
399  */
401 /**
402  * Dequeue data from fifo
403  *
404  * Data is dequeued to consumer provided buffer and head is atomically
405  * updated.
406  *
407  * @param f fifo
408  * @param len length of data to dequeue
409  * @param dst buffer to where to dequeue the data
410  * @return number of bytes dequeued or error
411  */
412 int svm_fifo_dequeue (svm_fifo_t * f, u32 len, u8 * dst);
413 /**
414  * Peek data from fifo
415  *
416  * Data is copied from requested offset into provided dst buffer. Head is
417  * not updated.
418  *
419  * @param f fifo
420  * @param offset offset from which to copy the data
421  * @param len length of data to copy
422  * @param dst buffer to where to dequeue the data
423  * @return number of bytes peeked
424  */
425 int svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst);
426 /**
427  * Dequeue and drop bytes from fifo
428  *
429  * Advances fifo head by requested amount of bytes.
430  *
431  * @param f fifo
432  * @param len number of bytes to drop
433  * @return number of bytes dropped
434  */
436 /**
437  * Dequeue and drop all bytes from fifo
438  *
439  * Advances head to tail position.
440  *
441  * @param f fifo
442  */
446 /**
447  * Add io events subscriber to list
448  *
449  * @param f fifo
450  * @param sub subscriber opaque index (typically app worker index)
451  */
452 void svm_fifo_add_subscriber (svm_fifo_t * f, u8 sub);
453 /**
454  * Remove io events subscriber form list
455  *
456  * @param f fifo
457  * @param sub subscriber index to be removed
458  */
459 void svm_fifo_del_subscriber (svm_fifo_t * f, u8 subscriber);
460 /**
461  * Number of out-of-order segments for fifo
462  *
463  * @param f fifo
464  * @return number of out of order segments
465  */
467 /**
468  * First out-of-order segment for fifo
469  *
470  * @param f fifo
471  * @return first out-of-order segment for fifo
472  */
474 /**
475  * Check if fifo is sane. Debug only.
476  *
477  * @param f fifo
478  * @return 1 if sane, 0 otherwise
479  */
481 /**
482  * Declare this fifo is used by only a single thread.
483  * In this special case, fifo-growth can be done in an efficient way without delay.
484  *
485  * @param f fifo
486  * @return 1 if the fifo is already owned by another thread, 0 otherwise
487  */
490 
491 /**
492  * Fifo max bytes to dequeue optimized for consumer
493  *
494  * @param f fifo
495  * @return max number of bytes that can be dequeued
496  */
497 static inline u32
499 {
500  u32 tail, head;
501  f_load_head_tail_cons (f, &head, &tail);
502  return f_cursize (f, head, tail);
503 }
504 
505 /**
506  * Fifo max bytes to dequeue optimized for producer
507  *
508  * @param f fifo
509  * @return max number of bytes that can be dequeued
510  */
511 static inline u32
513 {
514  u32 tail, head;
515  f_load_head_tail_prod (f, &head, &tail);
516  return f_cursize (f, head, tail);
517 }
518 
519 /**
520  * Fifo max bytes to dequeue
521  *
522  * Note: use producer or consumer specific functions for performance:
523  * @ref svm_fifo_max_dequeue_cons (svm_fifo_t *f)
524  * @ref svm_fifo_max_dequeue_prod (svm_fifo_t *f)
525  */
526 static inline u32
528 {
529  u32 tail, head;
530  f_load_head_tail_all_acq (f, &head, &tail);
531  return f_cursize (f, head, tail);
532 }
533 
534 /**
535  * Check if fifo is full optimized for producer
536  *
537  * @param f fifo
538  * @return 1 if fifo is full 0 otherwise
539  */
540 static inline int
542 {
543  return (svm_fifo_max_dequeue_prod (f) == f->nitems);
544 }
545 
546 /* Check if fifo is full.
547  *
548  * Note: use producer or consumer specific functions for performance.
549  * @ref svm_fifo_is_full_prod (svm_fifo_t * f)
550  * add cons version if needed
551  */
552 static inline int
554 {
555  return (svm_fifo_max_dequeue (f) == f->nitems);
556 }
557 
558 /**
559  * Check if fifo is empty optimized for consumer
560  *
561  * @param f fifo
562  * @return 1 if fifo is empty 0 otherwise
563  */
564 static inline int
566 {
567  return (svm_fifo_max_dequeue_cons (f) == 0);
568 }
569 
570 /**
571  * Check if fifo is empty optimized for producer
572  *
573  * @param f fifo
574  * @return 1 if fifo is empty 0 otherwise
575  */
576 static inline int
578 {
579  return (svm_fifo_max_dequeue_prod (f) == 0);
580 }
581 
582 /**
583  * Check if fifo is empty
584  *
585  * Note: use producer or consumer specific functions for perfomance.
586  * @ref svm_fifo_is_empty_cons (svm_fifo_t * f)
587  * @ref svm_fifo_is_empty_prod (svm_fifo_t * f)
588  */
589 static inline int
591 {
592  return (svm_fifo_max_dequeue (f) == 0);
593 }
594 
595 /**
596  * Check if fifo is wrapped
597  *
598  * @param f fifo
599  * @return 1 if 'normalized' head is ahead of tail
600  */
601 static inline u8
603 {
604  u32 head, tail;
605  f_load_head_tail_all_acq (f, &head, &tail);
606  return head > tail;
607 }
608 
609 /**
610  * Maximum number of bytes that can be enqueued into fifo
611  *
612  * Optimized for producer
613  *
614  * @param f fifo
615  * @return max number of bytes that can be enqueued into fifo
616  */
617 static inline u32
619 {
620  u32 head, tail;
621  f_load_head_tail_prod (f, &head, &tail);
622  if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK))
623  svm_fifo_try_shrink (f, head, tail);
624  return f_free_count (f, head, tail);
625 }
626 
627 /* Maximum number of bytes that can be enqueued into fifo
628  *
629  * Note: use producer or consumer specific functions for performance.
630  * @ref svm_fifo_max_enqueue_prod (svm_fifo_t *f)
631  * add consumer specific version if needed.
632  */
633 static inline u32
635 {
636  u32 head, tail;
637  f_load_head_tail_all_acq (f, &head, &tail);
638  if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK))
639  svm_fifo_try_shrink (f, head, tail);
640  return f_free_count (f, head, tail);
641 }
642 
643 /**
644  * Max contiguous chunk of data that can be read
645  */
646 static inline u32
648 {
649  u32 head, tail;
650  f_load_head_tail_cons (f, &head, &tail);
651  return tail >= head ? (tail - head) : (f->size - head);
652 }
653 
654 /**
655  * Max contiguous chunk of data that can be written
656  */
657 static inline u32
659 {
660  u32 head, tail;
661  f_load_head_tail_prod (f, &head, &tail);
662  return tail >= head ? f->size - tail : f_free_count (f, head, tail);
663 }
664 
665 static inline u8 *
667 {
668  /* load-relaxed: consumer owned index */
669  return (f->head_chunk->data + (f->head - f->head_chunk->start_byte));
670 }
671 
672 static inline u8 *
674 {
675  /* load-relaxed: producer owned index */
676  return (f->tail_chunk->data + (f->tail - f->tail_chunk->start_byte));
677 }
678 
679 static inline u8
681 {
682  return f->n_subscribers;
683 }
684 
685 /**
686  * Check if fifo has out-of-order data
687  *
688  * @param f fifo
689  * @return 1 if fifo has ooo data, 0 otherwise
690  */
691 static inline u8
693 {
694  return f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX;
695 }
696 
697 static inline ooo_segment_t *
699 {
700  if (f->ooos_newest == OOO_SEGMENT_INVALID_INDEX)
701  return 0;
702  return pool_elt_at_index (f->ooo_segments, f->ooos_newest);
703 }
704 
705 static inline void
707 {
708  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
709 }
710 
711 static inline u32
713 {
714  u32 tail;
715  /* load-relaxed: producer owned index */
716  tail = f->tail;
717 
718  return f_distance_to (f, s->start, tail);
719 }
720 
721 static inline u32
723 {
724  return s->length;
725 }
726 
727 /**
728  * Check if fifo has io event
729  *
730  * @param f fifo
731  * @return 1 if fifo has event, 0 otherwise
732  */
733 static inline int
735 {
736  return f->has_event;
737 }
738 
739 /**
740  * Set fifo event flag.
741  *
742  * Forces release semantics.
743  *
744  * @param f fifo
745  * @return 1 if flag was not set, 0 otherwise
746  */
749 {
750  return !clib_atomic_swap_rel_n (&f->has_event, 1);
751 }
752 
753 /**
754  * Unset fifo event flag.
755  *
756  * Forces acquire semantics
757  *
758  * @param f fifo
759  */
760 always_inline void
762 {
763  clib_atomic_swap_acq_n (&f->has_event, 0);
764 }
765 
766 /**
767  * Set specific want notification flag
768  *
769  * For list of flags see @ref svm_fifo_deq_ntf_t
770  *
771  * @param f fifo
772  * @param ntf_type type of notification requested
773  */
774 static inline void
776 {
777  f->want_deq_ntf |= ntf_type;
778 }
779 
780 /**
781  * Clear specific want notification flag
782  *
783  * For list of flags see @ref svm_fifo_ntf_t
784  *
785  * @param f fifo
786  * @param ntf_type type of notification to be cleared
787  */
788 static inline void
790 {
791  f->want_deq_ntf &= ~ntf_type;
792 }
793 
794 /**
795  * Clear the want notification flag and set has notification
796  *
797  * Should be used after enqueuing an event. This clears the
798  * SVM_FIFO_WANT_NOTIF flag but it does not clear
799  * SVM_FIFO_WANT_NOTIF_IF_FULL. If the latter was set, has_ntf is
800  * set to avoid enqueueing events for for all dequeue operations until
801  * it is manually cleared.
802  *
803  * @param f fifo
804  */
805 static inline void
807 {
808  /* Set the flag if want_notif_if_full was the only ntf requested */
809  f->has_deq_ntf = f->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL;
811 }
812 
813 /**
814  * Clear has notification flag
815  *
816  * The fifo generates only one event per SVM_FIFO_WANT_NOTIF_IF_FULL
817  * request and sets has_ntf. To received new events the flag must be
818  * cleared using this function.
819  *
820  * @param f fifo
821  */
822 static inline void
824 {
825  f->has_deq_ntf = 0;
826 }
827 
828 /**
829  * Check if fifo needs dequeue notification
830  *
831  * Determines based on notification request flags and state of the fifo if
832  * an event should be generated.
833  *
834  * @param f fifo
835  * @param n_last_deq number of bytes last dequeued
836  * @return 1 if event should be generated, 0 otherwise
837  */
838 static inline u8
840 {
841  u8 want_ntf = f->want_deq_ntf;
842 
843  if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF))
844  return 0;
845  else if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF)
846  return 1;
847  if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL)
848  {
849  u32 max_deq = svm_fifo_max_dequeue_cons (f);
850  u32 nitems = f->nitems;
851  if (!f->has_deq_ntf && max_deq < nitems
852  && max_deq + n_last_deq >= nitems)
853  return 1;
854  }
855  if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY)
856  {
857  if (!f->has_deq_ntf && svm_fifo_is_empty (f))
858  return 1;
859  }
860  return 0;
861 }
862 
863 #endif /* __included_ssvm_fifo_h__ */
864 
865 /*
866  * fd.io coding-style-patch-verification: ON
867  *
868  * Local Variables:
869  * eval: (c-set-style "gnu")
870  * End:
871  */
void svm_fifo_add_subscriber(svm_fifo_t *f, u8 sub)
Add io events subscriber to list.
Definition: svm_fifo.c:1175
u32 length
length of chunk in bytes
Definition: svm_fifo.h:61
static void f_load_head_tail_all_acq(svm_fifo_t *f, u32 *head, u32 *tail)
Load head and tail independent of producer/consumer role.
Definition: svm_fifo.h:184
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:889
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
u32 flags
Definition: vhost_user.h:141
struct svm_fifo_chunk_ svm_fifo_chunk_t
int svm_fifo_enqueue(svm_fifo_t *f, u32 len, const u8 *src)
Enqueue data to fifo.
Definition: svm_fifo.c:888
static u32 svm_fifo_max_enqueue_prod(svm_fifo_t *f)
Maximum number of bytes that can be enqueued into fifo.
Definition: svm_fifo.h:618
a
Definition: bitmap.h:538
void svm_fifo_clone(svm_fifo_t *df, svm_fifo_t *sf)
Clone fifo.
Definition: svm_fifo.c:1130
void svm_fifo_try_shrink(svm_fifo_t *f, u32 head, u32 tail)
Try to shrink fifo size.
Definition: svm_fifo.c:734
static int svm_fifo_is_full(svm_fifo_t *f)
Definition: svm_fifo.h:553
Notify on dequeue.
Definition: svm_fifo.h:46
u8 * svm_fifo_replay(u8 *s, svm_fifo_t *f, u8 no_read, u8 verbose)
Definition: svm_fifo.c:1292
static u8 svm_fifo_has_ooo_data(svm_fifo_t *f)
Check if fifo has out-of-order data.
Definition: svm_fifo.h:692
format_function_t format_svm_fifo
Definition: svm_fifo.h:489
void svm_fifo_free_ooo_data(svm_fifo_t *f)
Cleanup fifo ooo data.
Definition: svm_fifo.c:132
#define PREDICT_TRUE(x)
Definition: clib.h:113
svm_fifo_flag_
Definition: svm_fifo.h:66
static u32 f_free_count(svm_fifo_t *f, u32 head, u32 tail)
Fifo free bytes, i.e., number of free bytes.
Definition: svm_fifo.h:231
Fixed length block allocator.
static int svm_fifo_is_empty_prod(svm_fifo_t *f)
Check if fifo is empty optimized for producer.
Definition: svm_fifo.h:577
u32 prev
Previous linked-list element pool index.
Definition: svm_fifo.h:32
static void f_load_head_tail_cons(svm_fifo_t *f, u32 *head, u32 *tail)
Load head and tail optimized for consumer.
Definition: svm_fifo.h:157
vl_api_address_t src
Definition: gre.api:51
static int svm_fifo_is_empty_cons(svm_fifo_t *f)
Check if fifo is empty optimized for consumer.
Definition: svm_fifo.h:565
static void svm_fifo_reset_has_deq_ntf(svm_fifo_t *f)
Clear has notification flag.
Definition: svm_fifo.h:823
static u32 svm_fifo_max_enqueue(svm_fifo_t *f)
Definition: svm_fifo.h:634
int svm_fifo_reduce_size(svm_fifo_t *f, u32 len, u8 try_shrink)
Request to reduce fifo size by amount of bytes.
Definition: svm_fifo.c:807
void svm_fifo_add_chunk(svm_fifo_t *f, svm_fifo_chunk_t *c)
Grow fifo size by adding chunk to chunk list.
Definition: svm_fifo.c:579
static u8 * svm_fifo_tail(svm_fifo_t *f)
Definition: svm_fifo.h:673
unsigned char u8
Definition: types.h:56
struct _svm_fifo svm_fifo_t
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
static int svm_fifo_is_empty(svm_fifo_t *f)
Check if fifo is empty.
Definition: svm_fifo.h:590
int svm_fifo_segments(svm_fifo_t *f, svm_fifo_seg_t *fs)
Definition: svm_fifo.c:1077
#define always_inline
Definition: clib.h:99
static u32 svm_fifo_max_dequeue(svm_fifo_t *f)
Fifo max bytes to dequeue.
Definition: svm_fifo.h:527
static u32 svm_fifo_max_dequeue_cons(svm_fifo_t *f)
Fifo max bytes to dequeue optimized for consumer.
Definition: svm_fifo.h:498
static u32 svm_fifo_max_write_chunk(svm_fifo_t *f)
Max contiguous chunk of data that can be written.
Definition: svm_fifo.h:658
static u32 ooo_segment_length(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:722
u8 * svm_fifo_dump_trace(u8 *s, svm_fifo_t *f)
Definition: svm_fifo.c:1268
unsigned int u32
Definition: types.h:88
struct svm_fifo_seg_ svm_fifo_seg_t
svm_fifo_deq_ntf_
Definition: svm_fifo.h:43
static u8 * svm_fifo_head(svm_fifo_t *f)
Definition: svm_fifo.h:666
static u32 svm_fifo_max_read_chunk(svm_fifo_t *f)
Max contiguous chunk of data that can be read.
Definition: svm_fifo.h:647
static int svm_fifo_is_full_prod(svm_fifo_t *f)
Check if fifo is full optimized for producer.
Definition: svm_fifo.h:541
static void svm_fifo_newest_ooo_segment_reset(svm_fifo_t *f)
Definition: svm_fifo.h:706
int svm_fifo_peek(svm_fifo_t *f, u32 offset, u32 len, u8 *dst)
Peek data from fifo.
Definition: svm_fifo.c:1007
u8 svm_fifo_is_sane(svm_fifo_t *f)
Check if fifo is sane.
Definition: svm_fifo.c:1198
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
uword size
void svm_fifo_dequeue_drop_all(svm_fifo_t *f)
Dequeue and drop all bytes from fifo.
Definition: svm_fifo.c:1061
u32 svm_fifo_n_ooo_segments(svm_fifo_t *f)
Number of out-of-order segments for fifo.
Definition: svm_fifo.c:1141
struct svm_fifo_chunk_ * next
pointer to next chunk in linked-lists
Definition: svm_fifo.h:62
#define PREDICT_FALSE(x)
Definition: clib.h:112
u8 svm_fifo_set_single_thread_owned(svm_fifo_t *f)
Declare this fifo is used by only a single thread.
Definition: svm_fifo.c:1237
static void svm_fifo_unset_event(svm_fifo_t *f)
Unset fifo event flag.
Definition: svm_fifo.h:761
void svm_fifo_init_chunks(svm_fifo_t *f)
Initialize fifo chunks and rbtree.
Definition: svm_fifo.c:408
signed char i8
Definition: types.h:45
svm_fifo_t * svm_fifo_create(u32 size)
Create fifo of requested size.
Definition: svm_fifo.c:436
static u32 svm_fifo_max_dequeue_prod(svm_fifo_t *f)
Fifo max bytes to dequeue optimized for producer.
Definition: svm_fifo.h:512
vl_api_address_t dst
Definition: gre.api:52
static ooo_segment_t * svm_fifo_newest_ooo_segment(svm_fifo_t *f)
Definition: svm_fifo.h:698
u8 len
Definition: ip_types.api:90
static u8 svm_fifo_set_event(svm_fifo_t *f)
Set fifo event flag.
Definition: svm_fifo.h:748
svm_fifo_chunk_t * svm_fifo_chunk_alloc(u32 size)
Allocate a fifo chunk on heap.
Definition: svm_fifo.c:471
#define SVM_FIFO_MAX_EVT_SUBSCRIBERS
Definition: svm_fifo.h:41
static u8 svm_fifo_needs_deq_ntf(svm_fifo_t *f, u32 n_last_deq)
Check if fifo needs dequeue notification.
Definition: svm_fifo.h:839
static u32 f_cursize(svm_fifo_t *f, u32 head, u32 tail)
Fifo current size, i.e., number of bytes enqueued.
Definition: svm_fifo.h:220
int svm_fifo_dequeue(svm_fifo_t *f, u32 len, u8 *dst)
Dequeue data from fifo.
Definition: svm_fifo.c:981
void svm_fifo_overwrite_head(svm_fifo_t *f, u8 *src, u32 len)
Overwrite fifo head with new data.
Definition: svm_fifo.c:866
static void f_load_head_tail_prod(svm_fifo_t *f, u32 *head, u32 *tail)
Load head and tail optimized for producer.
Definition: svm_fifo.h:170
u32 start_byte
chunk start byte
Definition: svm_fifo.h:60
svm_fifo_chunk_t * svm_fifo_collect_chunks(svm_fifo_t *f)
Removes chunks that are after fifo end byte.
Definition: svm_fifo.c:715
#define OOO_SEGMENT_INVALID_INDEX
Definition: svm_fifo.h:38
void svm_fifo_free_chunk_lookup(svm_fifo_t *f)
Cleanup fifo chunk lookup rb tree.
Definition: svm_fifo.c:847
#define clib_atomic_swap_acq_n(a, b)
Definition: atomics.h:51
int svm_fifo_dequeue_drop(svm_fifo_t *f, u32 len)
Dequeue and drop bytes from fifo.
Definition: svm_fifo.c:1029
Notify on transition to empty.
Definition: svm_fifo.h:48
u8 data[0]
start of chunk data
Definition: svm_fifo.h:63
void svm_fifo_free(svm_fifo_t *f)
Free fifo and associated state.
Definition: svm_fifo.c:853
static void svm_fifo_clear_deq_ntf(svm_fifo_t *f)
Clear the want notification flag and set has notification.
Definition: svm_fifo.h:806
static void svm_fifo_add_want_deq_ntf(svm_fifo_t *f, u8 ntf_type)
Set specific want notification flag.
Definition: svm_fifo.h:775
static u8 svm_fifo_is_wrapped(svm_fifo_t *f)
Check if fifo is wrapped.
Definition: svm_fifo.h:602
Out-of-order segment.
Definition: svm_fifo.h:29
Notify on transition from full.
Definition: svm_fifo.h:47
static u32 f_distance_to(svm_fifo_t *f, u32 a, u32 b)
Distance to a from b, i.e., a - b in the fifo.
Definition: svm_fifo.h:198
u32 length
Length of segment.
Definition: svm_fifo.h:34
void svm_fifo_enqueue_nocopy(svm_fifo_t *f, u32 len)
Advance tail pointer.
Definition: svm_fifo.c:964
void svm_fifo_init(svm_fifo_t *f, u32 size)
Initialize fifo.
Definition: svm_fifo.c:392
u32 next
Next linked-list element pool index.
Definition: svm_fifo.h:31
template key/value backing page structure
Definition: bihash_doc.h:44
#define clib_atomic_swap_rel_n(a, b)
Definition: atomics.h:52
enum svm_fifo_deq_ntf_ svm_fifo_deq_ntf_t
No notification requested.
Definition: svm_fifo.h:45
enum svm_fifo_flag_ svm_fifo_flag_t
static void svm_fifo_del_want_deq_ntf(svm_fifo_t *f, u8 ntf_type)
Clear specific want notification flag.
Definition: svm_fifo.h:789
static u32 f_distance_from(svm_fifo_t *f, u32 a, u32 b)
Distance from a to b, i.e., b - a in the fifo.
Definition: svm_fifo.h:209
static u32 ooo_segment_offset_prod(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:712
static u8 svm_fifo_n_subscribers(svm_fifo_t *f)
Definition: svm_fifo.h:680
void svm_fifo_init_pointers(svm_fifo_t *f, u32 head, u32 tail)
Init fifo head and tail.
Definition: svm_fifo.c:1156
void svm_fifo_segments_free(svm_fifo_t *f, svm_fifo_seg_t *fs)
Definition: svm_fifo.c:1109
svm_fifo_err_t
Definition: svm_fifo.h:123
static int svm_fifo_has_event(svm_fifo_t *f)
Check if fifo has io event.
Definition: svm_fifo.h:734
#define clib_atomic_load_acq_n(a)
Definition: atomics.h:48
void svm_fifo_del_subscriber(svm_fifo_t *f, u8 subscriber)
Remove io events subscriber form list.
Definition: svm_fifo.c:1183
int svm_fifo_enqueue_with_offset(svm_fifo_t *f, u32 offset, u32 len, u8 *src)
Enqueue data to fifo with offset.
Definition: svm_fifo.c:931
u32 start
Start of segment, normalized.
Definition: svm_fifo.h:33
ooo_segment_t * svm_fifo_first_ooo_segment(svm_fifo_t *f)
First out-of-order segment for fifo.
Definition: svm_fifo.c:1147
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".