FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
fifo_types.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef SRC_SVM_FIFO_TYPES_H_
17 #define SRC_SVM_FIFO_TYPES_H_
18 
19 #include <svm/ssvm.h>
20 #include <vppinfra/clib.h>
21 #include <vppinfra/rbtree.h>
22 #include <vppinfra/lock.h>
23 
24 #define FS_MIN_LOG2_CHUNK_SZ 12 /**< also min fifo size */
25 #define FS_MAX_LOG2_CHUNK_SZ 22 /**< 4MB max chunk size */
26 #define FS_CHUNK_VEC_LEN 11 /**< number of chunk sizes */
27 
29  FS_CHUNK_VEC_LEN - 1,
30  "update chunk sizes");
31 
32 #define SVM_FIFO_TRACE (0)
33 #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7
34 
36 typedef uword fs_sptr_t;
37 
38 typedef struct svm_fifo_chunk_
39 {
40  u32 start_byte; /**< chunk start byte */
41  u32 length; /**< length of chunk in bytes */
42  fs_sptr_t next; /**< pointer to next chunk in linked-lists */
43  rb_node_index_t enq_rb_index; /**< enq node index if chunk in rbtree */
44  rb_node_index_t deq_rb_index; /**< deq node index if chunk in rbtree */
45  u8 data[0]; /**< start of chunk data */
47 
48 typedef struct
49 {
50  u32 next; /**< Next linked-list element pool index */
51  u32 prev; /**< Previous linked-list element pool index */
52  u32 start; /**< Start of segment, normalized*/
53  u32 length; /**< Length of segment */
55 
56 typedef struct
57 {
62 
63 typedef struct svm_fifo_shr_
64 {
66  fs_sptr_t start_chunk; /**< first chunk in fifo chunk list */
67  fs_sptr_t end_chunk; /**< end chunk in fifo chunk list */
68  volatile u32 has_event; /**< non-zero if deq event exists */
69  u32 min_alloc; /**< min chunk alloc if space available */
70  u32 size; /**< size of the fifo in bytes */
71  u32 master_session_index; /**< session layer session index */
72  u32 client_session_index; /**< app session index */
73  u8 slice_index; /**< segment slice for fifo */
74  fs_sptr_t next; /**< next in freelist */
75 
76  CLIB_CACHE_LINE_ALIGN_MARK (consumer);
77  fs_sptr_t head_chunk; /**< tracks chunk where head lands */
78  u32 head; /**< fifo head position/byte */
79  volatile u32 want_deq_ntf; /**< producer wants nudge */
80  volatile u32 has_deq_ntf;
81 
82  CLIB_CACHE_LINE_ALIGN_MARK (producer);
83  u32 tail; /**< fifo tail position/byte */
84  fs_sptr_t tail_chunk; /**< tracks chunk where tail lands */
85  volatile u8 n_subscribers; /**< Number of subscribers for io events */
88 
89 typedef struct _svm_fifo
90 {
91  CLIB_CACHE_LINE_ALIGN_MARK (cacheline);
92  svm_fifo_shared_t *shr; /**< shared fifo in fifo segment memory */
93  fifo_segment_header_t *fs_hdr; /**< fifo segment header for fifo */
94  rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */
95  rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */
96  svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */
97  svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */
98  ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
99  u32 ooos_list_head; /**< Head of out-of-order linked-list */
100  u32 ooos_newest; /**< Last segment to have been updated */
101 
102  u8 flags; /**< fifo flags */
103  u8 master_thread_index; /**< session layer thread index */
104  u8 client_thread_index; /**< app worker index */
105  i8 refcnt; /**< reference count */
106  u32 segment_manager; /**< session layer segment manager index */
107  u32 segment_index; /**< segment index in segment manager */
108 
109  struct _svm_fifo *next; /**< prev in active chain */
110  struct _svm_fifo *prev; /**< prev in active chain */
111 
112  svm_fifo_chunk_t *chunks_at_attach; /**< chunks to be accounted at detach */
113  svm_fifo_shared_t *hdr_at_attach; /**< hdr to be freed at detach */
114 
115 #if SVM_FIFO_TRACE
117 #endif
118 } svm_fifo_t;
119 
120 typedef struct fifo_segment_slice_
121 {
122  CLIB_CACHE_LINE_ALIGN_MARK (cacheline);
123  fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
124  fs_sptr_t free_fifos; /**< Freelists of fifo shared hdrs */
125  uword n_fl_chunk_bytes; /**< Chunk bytes on freelist */
126  uword virtual_mem; /**< Slice sum of all fifo sizes */
127  u32 num_chunks[FS_CHUNK_VEC_LEN]; /**< Allocated chunks by chunk size */
129 
130 typedef struct fifo_slice_private_
131 {
132  clib_mem_bulk_handle_t fifos; /**< Bulk fifo allocator */
133  uword virtual_mem; /**< Slice sum of all fifo sizes */
134  svm_fifo_t *active_fifos; /**< Linked list of active RX fifos */
136 
138 {
139  uword n_cached_bytes; /**< Cached bytes */
140  u32 n_active_fifos; /**< Number of active fifos */
141  u32 n_reserved_bytes; /**< Bytes not to be allocated */
142  u32 max_log2_fifo_size; /**< Max log2(chunk size) for fs */
143  u8 flags; /**< Segment flags */
144  u8 n_slices; /**< Number of slices */
145  u8 high_watermark; /**< Memory pressure watermark high */
146  u8 low_watermark; /**< Memory pressure watermark low */
147  u8 pct_first_alloc; /**< Pct of fifo size to alloc */
148  u8 n_mqs; /**< Num mqs for mqs segment */
149  CLIB_CACHE_LINE_ALIGN_MARK (allocator);
154  fifo_segment_slice_t slices[0]; /** Fixed array of slices */
155 };
156 
157 void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
158  int n_bytes);
159 
160 always_inline void *
162 {
163  return sp ? (void *) ((u8 *) fsh + sp) : 0;
164 }
165 
168 {
169  return p ? (fs_sptr_t) ((u8 *) p - (u8 *) fsh) : 0;
170 }
171 
174 {
175  return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + cp) : 0;
176 }
177 
180 {
181  return c ? (fs_sptr_t) ((u8 *) c - (u8 *) fsh) : 0;
182 }
183 
184 #endif /* SRC_SVM_FIFO_TYPES_H_ */
185 
186 /*
187  * fd.io coding-style-patch-verification: ON
188  *
189  * Local Variables:
190  * eval: (c-set-style "gnu")
191  * End:
192  */
trace
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:899
fifo_segment_header_::max_byte_index
uword max_byte_index
Definition: fifo_types.h:151
svm_fifo_shr_::master_session_index
u32 master_session_index
session layer session index
Definition: fifo_types.h:71
svm_fifo_shr_::want_deq_ntf
volatile u32 want_deq_ntf
producer wants nudge
Definition: fifo_types.h:79
fs_sptr
static fs_sptr_t fs_sptr(fifo_segment_header_t *fsh, void *p)
Definition: fifo_types.h:167
svm_fifo_shr_::head
u32 head
fifo head position/byte
Definition: fifo_types.h:78
svm_fifo_shr_::size
u32 size
size of the fifo in bytes
Definition: fifo_types.h:70
fsh_virtual_mem_update
void fsh_virtual_mem_update(fifo_segment_header_t *fsh, u32 slice_index, int n_bytes)
Definition: fifo_segment.c:123
clib_mem_bulk_handle_t
void * clib_mem_bulk_handle_t
Definition: mem.h:560
fifo_segment_header_::slices
fifo_segment_slice_t slices[0]
Definition: fifo_types.h:154
svm_fifo_shr_::has_event
volatile u32 has_event
non-zero if deq event exists
Definition: fifo_types.h:68
fs_ptr
static void * fs_ptr(fifo_segment_header_t *fsh, fs_sptr_t sp)
Definition: fifo_types.h:161
fs_chunk_sptr
static fs_sptr_t fs_chunk_sptr(fifo_segment_header_t *fsh, svm_fifo_chunk_t *c)
Definition: fifo_types.h:179
CLIB_CACHE_LINE_ALIGN_MARK
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
clib.h
fifo_segment_slice_::virtual_mem
uword virtual_mem
Slice sum of all fifo sizes.
Definition: fifo_types.h:126
fifo_segment_header_::byte_index
uword byte_index
Definition: fifo_types.h:150
next
u16 * next
Definition: nat44_ei_out2in.c:718
svm_fifo_shr_::slice_index
u8 slice_index
segment slice for fifo
Definition: fifo_types.h:73
fifo_segment_header_::n_mqs
u8 n_mqs
Num mqs for mqs segment.
Definition: fifo_types.h:148
fifo_segment_header_::n_active_fifos
u32 n_active_fifos
Number of active fifos.
Definition: fifo_types.h:140
svm_fifo_shr_::tail_chunk
fs_sptr_t tail_chunk
tracks chunk where tail lands
Definition: fifo_types.h:84
svm_fifo_shr_::CLIB_CACHE_LINE_ALIGN_MARK
CLIB_CACHE_LINE_ALIGN_MARK(shared)
svm_fifo_chunk_
Definition: fifo_types.h:38
fifo_slice_private_::virtual_mem
uword virtual_mem
Slice sum of all fifo sizes.
Definition: fifo_types.h:133
svm_fifo_chunk_::start_byte
u32 start_byte
chunk start byte
Definition: fifo_types.h:40
fifo_segment_header_::n_cached_bytes
uword n_cached_bytes
Cached bytes.
Definition: fifo_types.h:139
ooo_segment_t::next
u32 next
Next linked-list element pool index.
Definition: fifo_types.h:50
svm_fifo_shr_
Definition: fifo_types.h:63
FS_MIN_LOG2_CHUNK_SZ
#define FS_MIN_LOG2_CHUNK_SZ
also min fifo size
Definition: fifo_types.h:24
svm_fifo_shr_::head_chunk
fs_sptr_t head_chunk
tracks chunk where head lands
Definition: fifo_types.h:77
svm_fifo_chunk_::enq_rb_index
rb_node_index_t enq_rb_index
enq node index if chunk in rbtree
Definition: fifo_types.h:43
fifo_segment_slice_::n_fl_chunk_bytes
uword n_fl_chunk_bytes
Chunk bytes on freelist.
Definition: fifo_types.h:125
svm_fifo_shr_::min_alloc
u32 min_alloc
min chunk alloc if space available
Definition: fifo_types.h:69
fifo_segment_header_::n_reserved_bytes
u32 n_reserved_bytes
Bytes not to be allocated.
Definition: fifo_types.h:141
svm_fifo_shr_::next
fs_sptr_t next
next in freelist
Definition: fifo_types.h:74
ooo_segment_t::length
u32 length
Length of segment.
Definition: fifo_types.h:53
svm_fifo_t
struct _svm_fifo svm_fifo_t
fifo_segment_header_::low_watermark
u8 low_watermark
Memory pressure watermark low.
Definition: fifo_types.h:146
fifo_segment_header_::max_log2_fifo_size
u32 max_log2_fifo_size
Max log2(chunk size) for fs.
Definition: fifo_types.h:142
svm_fifo_shr_::end_chunk
fs_sptr_t end_chunk
end chunk in fifo chunk list
Definition: fifo_types.h:67
SVM_FIFO_MAX_EVT_SUBSCRIBERS
#define SVM_FIFO_MAX_EVT_SUBSCRIBERS
Definition: fifo_types.h:33
svm_fifo_trace_elem_t
Definition: fifo_types.h:56
svm_fifo_shr_::client_session_index
u32 client_session_index
app session index
Definition: fifo_types.h:72
svm_fifo_trace_elem_t::offset
u32 offset
Definition: fifo_types.h:58
svm_fifo_shr_::tail
u32 tail
fifo tail position/byte
Definition: fifo_types.h:83
lock.h
ooo_segment_t::start
u32 start
Start of segment, normalized.
Definition: fifo_types.h:52
fifo_segment_header_
Definition: fifo_types.h:137
FS_MAX_LOG2_CHUNK_SZ
#define FS_MAX_LOG2_CHUNK_SZ
4MB max chunk size
Definition: fifo_types.h:25
c
svmdb_client_t * c
Definition: vpp_get_metrics.c:48
fs_sptr_t
uword fs_sptr_t
Definition: fifo_types.h:36
uword
u64 uword
Definition: types.h:112
svm_fifo_chunk_::deq_rb_index
rb_node_index_t deq_rb_index
deq node index if chunk in rbtree
Definition: fifo_types.h:44
rb_node_index_t
u32 rb_node_index_t
Definition: rbtree.h:24
svm_fifo_shared_t
struct svm_fifo_shr_ svm_fifo_shared_t
rbtree.h
svm_fifo_chunk_::next
fs_sptr_t next
pointer to next chunk in linked-lists
Definition: fifo_types.h:42
svm_fifo_chunk_::length
u32 length
length of chunk in bytes
Definition: fifo_types.h:41
svm_fifo_shr_::start_chunk
fs_sptr_t start_chunk
first chunk in fifo chunk list
Definition: fifo_types.h:66
fifo_segment_slice_::num_chunks
u32 num_chunks[FS_CHUNK_VEC_LEN]
Allocated chunks by chunk size.
Definition: fifo_types.h:127
fifo_segment_slice_::free_fifos
fs_sptr_t free_fifos
Freelists of fifo shared hdrs
Definition: fifo_types.h:124
fifo_slice_private_t
struct fifo_slice_private_ fifo_slice_private_t
always_inline
#define always_inline
Definition: rdma_mlx5dv.h:23
fifo_segment_slice_t
struct fifo_segment_slice_ fifo_segment_slice_t
fifo_segment_slice_::CLIB_CACHE_LINE_ALIGN_MARK
CLIB_CACHE_LINE_ALIGN_MARK(cacheline)
fifo_slice_private_
Definition: fifo_types.h:130
svm_fifo_shr_::subscribers
u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS]
Definition: fifo_types.h:86
svm_fifo_chunk_::data
u8 data[0]
start of chunk data
Definition: fifo_types.h:45
FS_CHUNK_VEC_LEN
#define FS_CHUNK_VEC_LEN
number of chunk sizes
Definition: fifo_types.h:26
i8
signed char i8
Definition: types.h:45
ooo_segment_t::prev
u32 prev
Previous linked-list element pool index.
Definition: fifo_types.h:51
u32
unsigned int u32
Definition: types.h:88
svm_fifo_trace_elem_t::action
u32 action
Definition: fifo_types.h:60
n_bytes
u32 n_bytes
Definition: interface_output.c:401
ooo_segment_t
Definition: fifo_types.h:48
rb_tree_
Definition: rbtree.h:42
STATIC_ASSERT
STATIC_ASSERT((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ)==FS_CHUNK_VEC_LEN - 1, "update chunk sizes")
svm_fifo_trace_elem_t::len
u32 len
Definition: fifo_types.h:59
fifo_segment_slice_
Definition: fifo_types.h:120
fifo_segment_header_::CLIB_CACHE_LINE_ALIGN_MARK
CLIB_CACHE_LINE_ALIGN_MARK(allocator)
fs_chunk_ptr
static svm_fifo_chunk_t * fs_chunk_ptr(fifo_segment_header_t *fsh, fs_sptr_t cp)
Definition: fifo_types.h:173
fifo_segment_header_::start_byte_index
uword start_byte_index
Definition: fifo_types.h:152
u8
unsigned char u8
Definition: types.h:56
fifo_segment_header_::n_slices
u8 n_slices
Number of slices.
Definition: fifo_types.h:144
fifo_segment_slice_::free_chunks
fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]
Free chunks by size.
Definition: fifo_types.h:123
fifo_slice_private_::active_fifos
svm_fifo_t * active_fifos
Linked list of active RX fifos.
Definition: fifo_types.h:134
ssvm.h
svm_fifo_shr_::has_deq_ntf
volatile u32 has_deq_ntf
Definition: fifo_types.h:80
fifo_segment_header_::flags
u8 flags
Segment flags.
Definition: fifo_types.h:143
fifo_slice_private_::fifos
clib_mem_bulk_handle_t fifos
Bulk fifo allocator.
Definition: fifo_types.h:132
svm_fifo_chunk_t
struct svm_fifo_chunk_ svm_fifo_chunk_t
svm_fifo_shr_::n_subscribers
volatile u8 n_subscribers
Number of subscribers for io events.
Definition: fifo_types.h:85
fifo_segment_header_::pct_first_alloc
u8 pct_first_alloc
Pct of fifo size to alloc.
Definition: fifo_types.h:147
fifo_segment_header_::high_watermark
u8 high_watermark
Memory pressure watermark high.
Definition: fifo_types.h:145
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105