FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
buffer_funcs.c
Go to the documentation of this file.
1 /* SPDX-License-Identifier: Apache-2.0
2  * Copyright(c) 2021 Cisco Systems, Inc.
3  */
4 
5 #include <vppinfra/clib.h>
6 #include <vlib/vlib.h>
8 
11  u16 next_index, u32 *buffers, u16 *nexts, u32 n_buffers,
12  u32 n_left, u32 *tmp)
13 {
14  u64 match_bmp[VLIB_FRAME_SIZE / 64];
15  vlib_frame_t *f;
16  u32 n_extracted, n_free;
17  u32 *to;
18 
20 
22 
23  /* if frame contains enough space for worst case scenario, we can avoid
24  * use of tmp */
25  if (n_free >= n_left)
27  else
28  to = tmp;
29 
31 
32  n_extracted = clib_compress_u32 (to, buffers, match_bmp, n_buffers);
33 
34  for (int i = 0; i < ARRAY_LEN (match_bmp); i++)
35  used_elt_bmp[i] |= match_bmp[i];
36 
37  if (to != tmp)
38  {
39  /* indices already written to frame, just close it */
40  vlib_put_next_frame (vm, node, next_index, n_free - n_extracted);
41  }
42  else if (n_free >= n_extracted)
43  {
44  /* enough space in the existing frame */
46  vlib_buffer_copy_indices (to, tmp, n_extracted);
47  vlib_put_next_frame (vm, node, next_index, n_free - n_extracted);
48  }
49  else
50  {
51  /* full frame */
55 
56  /* second frame */
57  u32 n_2nd_frame = n_extracted - n_free;
60  vlib_buffer_copy_indices (to, tmp + n_free, n_2nd_frame);
62  VLIB_FRAME_SIZE - n_2nd_frame);
63  }
64 
65  return n_left - n_extracted;
66 }
67 
68 void __clib_section (".vlib_buffer_enqueue_to_next_fn")
69 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_next_fn)
71  uword count)
72 {
74  u32 n_left;
76 
77  while (count >= VLIB_FRAME_SIZE)
78  {
79  u64 used_elt_bmp[VLIB_FRAME_SIZE / 64] = {};
81  u32 off = 0;
82 
83  next_index = nexts[0];
84  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers, nexts,
86 
87  while (n_left)
88  {
89  while (PREDICT_FALSE (used_elt_bmp[off] == ~0))
90  off++;
91 
92  next_index =
93  nexts[off * 64 + count_trailing_zeros (~used_elt_bmp[off])];
94  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers,
96  }
97 
98  buffers += VLIB_FRAME_SIZE;
101  }
102 
103  if (count)
104  {
105  u64 used_elt_bmp[VLIB_FRAME_SIZE / 64] = {};
106  next_index = nexts[0];
107  n_left = count;
108  u32 off = 0;
109 
110  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers, nexts,
111  count, n_left, tmp);
112 
113  while (n_left)
114  {
115  while (PREDICT_FALSE (used_elt_bmp[off] == ~0))
116  off++;
117 
118  next_index =
119  nexts[off * 64 + count_trailing_zeros (~used_elt_bmp[off])];
120  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers,
121  nexts, count, n_left, tmp);
122  }
123  }
124 }
125 
126 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_next_fn);
127 
128 void __clib_section (".vlib_buffer_enqueue_to_single_next_fn")
129 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_single_next_fn)
131  u32 count)
132 {
133  u32 *to_next, n_left_to_next, n_enq;
134 
135  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
136 
137  if (PREDICT_TRUE (n_left_to_next >= count))
138  {
139  vlib_buffer_copy_indices (to_next, buffers, count);
140  n_left_to_next -= count;
141  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
142  return;
143  }
144 
145  n_enq = n_left_to_next;
146 next:
147  vlib_buffer_copy_indices (to_next, buffers, n_enq);
148  n_left_to_next -= n_enq;
149 
150  if (PREDICT_FALSE (count > n_enq))
151  {
152  count -= n_enq;
153  buffers += n_enq;
154 
155  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
156  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
157  n_enq = clib_min (n_left_to_next, count);
158  goto next;
159  }
160  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
161 }
162 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_single_next_fn);
163 
164 u32 __clib_section (".vlib_buffer_enqueue_to_thread_fn")
165 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_thread_fn)
166 (vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index,
167  u32 *buffer_indices, u16 *thread_indices, u32 n_packets,
168  int drop_on_congestion)
169 {
173  u32 n_left = n_packets;
174  u32 drop_list[VLIB_FRAME_SIZE], *dbi = drop_list, n_drop = 0;
175  vlib_frame_queue_elt_t *hf = 0;
176  u32 n_left_to_next_thread = 0, *to_next_thread = 0;
177  u32 next_thread_index, current_thread_index = ~0;
178  int i;
179 
180  fqm = vec_elt_at_index (tm->frame_queue_mains, frame_queue_index);
182 
183  while (n_left)
184  {
185  next_thread_index = thread_indices[0];
186 
187  if (next_thread_index != current_thread_index)
188  {
189  if (drop_on_congestion &&
191  frame_queue_index, next_thread_index, fqm->queue_hi_thresh,
193  {
194  dbi[0] = buffer_indices[0];
195  dbi++;
196  n_drop++;
197  goto next;
198  }
199 
200  if (hf)
201  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_thread;
202 
204  frame_queue_index, next_thread_index,
206 
207  n_left_to_next_thread = VLIB_FRAME_SIZE - hf->n_vectors;
208  to_next_thread = &hf->buffer_index[hf->n_vectors];
209  current_thread_index = next_thread_index;
210  }
211 
212  to_next_thread[0] = buffer_indices[0];
213  to_next_thread++;
214  n_left_to_next_thread--;
215 
216  if (n_left_to_next_thread == 0)
217  {
220  vlib_get_main_by_index (current_thread_index)->check_frame_queues =
221  1;
222  current_thread_index = ~0;
224  hf = 0;
225  }
226 
227  /* next */
228  next:
229  thread_indices += 1;
230  buffer_indices += 1;
231  n_left -= 1;
232  }
233 
234  if (hf)
235  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_thread;
236 
237  /* Ship frames to the thread nodes */
238  for (i = 0; i < vec_len (ptd->handoff_queue_elt_by_thread_index); i++)
239  {
241  {
243  /*
244  * It works better to let the handoff node
245  * rate-adapt, always ship the handoff queue element.
246  */
247  if (1 || hf->n_vectors == hf->last_n_vectors)
248  {
252  }
253  else
254  hf->last_n_vectors = hf->n_vectors;
255  }
257  (vlib_frame_queue_t *) (~0);
258  }
259 
260  if (drop_on_congestion && n_drop)
261  vlib_buffer_free (vm, drop_list, n_drop);
262 
263  return n_packets - n_drop;
264 }
265 
266 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_thread_fn);
267 
268 /*
269  * Check the frame queue to see if any frames are available.
270  * If so, pull the packets off the frames and put them to
271  * the handoff node.
272  */
273 u32 __clib_section (".vlib_frame_queue_dequeue_fn")
274 CLIB_MULTIARCH_FN (vlib_frame_queue_dequeue_fn)
276 {
277  u32 thread_id = vm->thread_index;
278  vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id];
280  u32 *from, *to;
281  vlib_frame_t *f;
282  int msg_type;
283  int processed = 0;
284  u32 vectors = 0;
285 
286  ASSERT (fq);
287  ASSERT (vm == vlib_global_main.vlib_mains[thread_id]);
288 
289  if (PREDICT_FALSE (fqm->node_index == ~0))
290  return 0;
291  /*
292  * Gather trace data for frame queues
293  */
294  if (PREDICT_FALSE (fq->trace))
295  {
296  frame_queue_trace_t *fqt;
298  u32 elix;
299 
300  fqt = &fqm->frame_queue_traces[thread_id];
301 
302  fqt->nelts = fq->nelts;
303  fqt->head = fq->head;
304  fqt->head_hint = fq->head_hint;
305  fqt->tail = fq->tail;
306  fqt->threshold = fq->vector_threshold;
307  fqt->n_in_use = fqt->tail - fqt->head;
308  if (fqt->n_in_use >= fqt->nelts)
309  {
310  // if beyond max then use max
311  fqt->n_in_use = fqt->nelts - 1;
312  }
313 
314  /* Record the number of elements in use in the histogram */
315  fqh = &fqm->frame_queue_histogram[thread_id];
316  fqh->count[fqt->n_in_use]++;
317 
318  /* Record a snapshot of the elements in use */
319  for (elix = 0; elix < fqt->nelts; elix++)
320  {
321  elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1));
322  if (1 || elt->valid)
323  {
324  fqt->n_vectors[elix] = elt->n_vectors;
325  }
326  }
327  fqt->written = 1;
328  }
329 
330  while (1)
331  {
332  vlib_buffer_t *b;
333  if (fq->head == fq->tail)
334  {
335  fq->head_hint = fq->head;
336  return processed;
337  }
338 
339  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
340 
341  if (!elt->valid)
342  {
343  fq->head_hint = fq->head;
344  return processed;
345  }
346 
347  from = elt->buffer_index;
348  msg_type = elt->msg_type;
349 
351  ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE);
352 
353  f = vlib_get_frame_to_node (vm, fqm->node_index);
354 
355  /* If the first vector is traced, set the frame trace flag */
356  b = vlib_get_buffer (vm, from[0]);
357  if (b->flags & VLIB_BUFFER_IS_TRACED)
359 
361 
362  vlib_buffer_copy_indices (to, from, elt->n_vectors);
363 
364  vectors += elt->n_vectors;
365  f->n_vectors = elt->n_vectors;
366  vlib_put_frame_to_node (vm, fqm->node_index, f);
367 
368  elt->valid = 0;
369  elt->n_vectors = 0;
370  elt->msg_type = 0xfefefefe;
372  fq->head++;
373  processed++;
374 
375  /*
376  * Limit the number of packets pushed into the graph
377  */
378  if (vectors >= fq->vector_threshold)
379  {
380  fq->head_hint = fq->head;
381  return processed;
382  }
383  }
384  ASSERT (0);
385  return processed;
386 }
387 CLIB_MARCH_FN_REGISTRATION (vlib_frame_queue_dequeue_fn);
388 
389 #ifndef CLIB_MARCH_VARIANT
391 
392 static clib_error_t *
394 {
397  CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_next_fn);
399  CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_single_next_fn);
401  CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_thread_fn);
403  CLIB_MARCH_FN_POINTER (vlib_frame_queue_dequeue_fn);
404  return 0;
405 }
406 
408 #endif
vlib.h
to
u32 * to
Definition: interface_output.c:1078
tmp
u32 * tmp
Definition: interface_output.c:1078
vlib_frame_t::n_vectors
u16 n_vectors
Definition: node.h:387
vlib_buffer_free
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:982
count_trailing_zeros
#define count_trailing_zeros(x)
Definition: clib.h:161
vlib_buffer_func_main_t::buffer_enqueue_to_thread_fn
vlib_buffer_enqueue_to_thread_fn_t * buffer_enqueue_to_thread_fn
Definition: buffer_funcs.h:74
n_buffers
u32 n_buffers
Definition: interface_output.c:401
next_index
nat44_ei_hairpin_src_next_t next_index
Definition: nat44_ei_hairpinning.c:412
vlib_frame_queue_t::tail
volatile u64 tail
Definition: threads.h:122
vlib_buffer_copy_indices
vlib_buffer_copy_indices(to, tmp, n_free)
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
f
vlib_frame_t * f
Definition: interface_output.c:1080
vlib_frame_queue_t::vector_threshold
u64 vector_threshold
Definition: threads.h:129
frame_queue_trace_t::n_in_use
u32 n_in_use
Definition: node.h:769
clib.h
VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME
@ VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME
Definition: threads.h:69
vlib_frame_queue_per_thread_data_t
Definition: threads.h:142
next
u16 * next
Definition: nat44_ei_out2in.c:718
VLIB_FRAME_SIZE
#define VLIB_FRAME_SIZE
Definition: node.h:368
node
vlib_main_t vlib_node_runtime_t * node
Definition: nat44_ei.c:3047
is_vlib_frame_queue_congested
static vlib_frame_queue_t * is_vlib_frame_queue_congested(u32 frame_queue_index, u32 index, u32 queue_hi_thresh, vlib_frame_queue_t **handoff_queue_by_worker_index)
Definition: threads.h:552
u16
unsigned short u16
Definition: types.h:57
CLIB_MULTIARCH_FN
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:53
frame_queue_trace_t::written
u32 written
Definition: node.h:771
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
clib_mask_compare_u16
static_always_inline void clib_mask_compare_u16(u16 v, u16 *a, u64 *mask, u32 n_elts)
Compare 16-bit elemments with provied value and return bitmap.
Definition: vector_funcs.h:67
vlib_frame_queue_t::trace
u64 trace
Definition: threads.h:128
vlib_frame_t::frame_flags
u16 frame_flags
Definition: node.h:375
vlib_frame_t
Definition: node.h:372
vlib_get_frame_to_node
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:184
vlib_main_t::check_frame_queues
volatile uword check_frame_queues
Definition: main.h:257
vlib_buffer_func_main_t::buffer_enqueue_to_next_fn
vlib_buffer_enqueue_to_next_fn_t * buffer_enqueue_to_next_fn
Definition: buffer_funcs.h:72
vlib_get_worker_handoff_queue_elt
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
Definition: threads.h:584
vlib_buffer_func_main_t::buffer_enqueue_to_single_next_fn
vlib_buffer_enqueue_to_single_next_fn_t * buffer_enqueue_to_single_next_fn
Definition: buffer_funcs.h:73
vlib_frame_queue_elt_t
Definition: threads.h:72
vlib_put_frame_to_node
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:218
enqueue_one
static_always_inline u32 enqueue_one(vlib_main_t *vm, vlib_node_runtime_t *node, u64 *used_elt_bmp, u16 next_index, u32 *buffers, u16 *nexts, u32 n_buffers, u32 n_left, u32 *tmp)
Definition: buffer_funcs.c:10
vlib_frame_queue_per_thread_data_t::handoff_queue_elt_by_thread_index
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
Definition: threads.h:144
count
u8 count
Definition: dhcp.api:208
vec_len
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
Definition: vec_bootstrap.h:142
vlib_buffer_func_main
vlib_buffer_func_main_t vlib_buffer_func_main
Definition: buffer_funcs.c:390
vlib_frame_queue_elt_t::buffer_index
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:81
vlib_frame_queue_elt_t::last_n_vectors
u32 last_n_vectors
Definition: threads.h:78
vlib_frame_queue_t::head_hint
volatile u64 head_hint
Definition: threads.h:133
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
VLIB_NODE_FLAG_TRACE
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:291
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
vector_funcs.h
ARRAY_LEN
#define ARRAY_LEN(x)
Definition: clib.h:70
vlib_frame_vector_args
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:301
vlib_frame_queue_elt_t::n_vectors
u32 n_vectors
Definition: threads.h:77
vlib_global_main
vlib_global_main_t vlib_global_main
Definition: main.c:1786
static_always_inline
#define static_always_inline
Definition: clib.h:112
frame_queue_trace_t::threshold
u32 threshold
Definition: node.h:772
uword
u64 uword
Definition: types.h:112
frame_queue_trace_t::nelts
u32 nelts
Definition: node.h:770
CLIB_MARCH_FN_REGISTRATION
CLIB_MARCH_FN_REGISTRATION(vlib_buffer_enqueue_to_next_fn)
vlib_main_t::thread_index
u32 thread_index
Definition: main.h:213
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
frame_queue_trace_t::head
u64 head
Definition: node.h:766
next_thread_index
static u32 next_thread_index(vnet_main_t *vnm, u32 thread_index)
Definition: rx_queue.c:30
vlib_buffer_func_main_t
Definition: buffer_funcs.h:70
CLIB_MEMORY_BARRIER
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:137
clib_min
#define clib_min(x, y)
Definition: clib.h:342
vlib_frame_queue_per_thread_data_t::congested_handoff_queue_by_thread_index
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
Definition: threads.h:145
frame_queue_trace_t::n_vectors
i32 n_vectors[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:773
frame_queue_trace_t::head_hint
u64 head_hint
Definition: node.h:767
vlib_buffer_funcs_init
static clib_error_t * vlib_buffer_funcs_init(vlib_main_t *vm)
Definition: buffer_funcs.c:393
u64
unsigned long u64
Definition: types.h:89
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
n_free
u32 n_free
Definition: interface_output.c:1078
off
u32 off
Definition: interface_output.c:1078
vlib_put_next_frame
vlib_put_next_frame(vm, node, next_index, 0)
u32
unsigned int u32
Definition: types.h:88
VLIB_INIT_FUNCTION
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:172
vlib_thread_main_t
Definition: threads.h:264
n_left
u32 n_left
Definition: interface_output.c:1078
vlib_get_main_by_index
static vlib_main_t * vlib_get_main_by_index(u32 thread_index)
Definition: global_funcs.h:29
elt
app_rx_mq_elt_t * elt
Definition: application.c:488
frame_queue_nelt_counter_t::count
u64 count[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:778
vlib_main_t
Definition: main.h:102
vlib_frame_queue_t::nelts
u32 nelts
Definition: threads.h:138
vlib_frame_queue_main_t::queue_hi_thresh
u32 queue_hi_thresh
Definition: threads.h:152
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
vlib_frame_queue_t::elts
vlib_frame_queue_elt_t * elts
Definition: threads.h:137
clib_error_t
Definition: clib_error.h:21
frame_queue_trace_t
Definition: node.h:763
vlib_init_function_t
clib_error_t *() vlib_init_function_t(struct vlib_main_t *vm)
Definition: init.h:51
nexts
u16 nexts[VLIB_FRAME_SIZE]
Definition: nat44_ei_out2in.c:718
vlib_frame_queue_main_t::per_thread_data
vlib_frame_queue_per_thread_data_t * per_thread_data
Definition: threads.h:155
vlib_buffer_func_main_t::frame_queue_dequeue_fn
vlib_frame_queue_dequeue_fn_t * frame_queue_dequeue_fn
Definition: buffer_funcs.h:75
vlib_frame_queue_t
Definition: threads.h:118
vlib_node_runtime_t
Definition: node.h:454
vlib_put_frame_queue_elt
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
Definition: threads.h:514
CLIB_MARCH_FN_POINTER
#define CLIB_MARCH_FN_POINTER(fn)
Definition: cpu.h:84
from
from
Definition: nat44_ei_hairpinning.c:415
PREDICT_TRUE
#define PREDICT_TRUE(x)
Definition: clib.h:125
vlib_get_thread_main
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:56
vlib_global_main_t::vlib_mains
vlib_main_t ** vlib_mains
Definition: main.h:279
vlib_get_next_frame
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:395
vlib_thread_main_t::frame_queue_mains
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:310
vlib_frame_queue_main_t
Definition: threads.h:148
vlib_get_next_frame_internal
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:384
vlib_frame_queue_t::head
volatile u64 head
Definition: threads.h:127
clib_compress_u32
static_always_inline u32 clib_compress_u32(u32 *dst, u32 *src, u64 *mask, u32 n_elts)
Compare array of 32-bit elemments into destination array based on mask.
Definition: vector_funcs.h:210
vlib_buffer_t::flags
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,...
Definition: buffer.h:133
frame_queue_trace_t::tail
u64 tail
Definition: node.h:768
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
frame_queue_nelt_counter_t
Definition: node.h:776