FD.io VPP  v21.10.1-2-g0a485f517
Vector Packet Processing
input.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
23 
24 #include <avf/avf.h>
25 
26 #define foreach_avf_input_error \
27  _(BUFFER_ALLOC, "buffer alloc error")
28 
29 typedef enum
30 {
31 #define _(f,s) AVF_INPUT_ERROR_##f,
33 #undef _
36 
37 static __clib_unused char *avf_input_error_strings[] = {
38 #define _(n,s) s,
40 #undef _
41 };
42 
43 #define AVF_INPUT_REFILL_TRESHOLD 32
44 
47 {
48 #ifdef CLIB_HAVE_VEC256
49  u64x4 v = { addr, 0, 0, 0 };
50  u64x4_store_unaligned (v, (void *) d);
51 #else
52  d->qword[0] = addr;
53  d->qword[1] = 0;
54 #endif
55 }
56 
59  int use_va_dma)
60 {
61  u16 n_refill, mask, n_alloc, slot, size;
62  vlib_buffer_t *b[8];
63  avf_rx_desc_t *d, *first_d;
64  void *p[8];
65 
66  size = rxq->size;
67  mask = size - 1;
68  n_refill = mask - rxq->n_enqueued;
69  if (PREDICT_TRUE (n_refill <= AVF_INPUT_REFILL_TRESHOLD))
70  return;
71 
72  slot = (rxq->next - n_refill - 1) & mask;
73 
74  n_refill &= ~7; /* round to 8 */
75  n_alloc =
77  rxq->buffer_pool_index);
78 
79  if (PREDICT_FALSE (n_alloc != n_refill))
80  {
81  vlib_error_count (vm, node->node_index,
82  AVF_INPUT_ERROR_BUFFER_ALLOC, 1);
83  if (n_alloc)
84  vlib_buffer_free_from_ring (vm, rxq->bufs, slot, size, n_alloc);
85  return;
86  }
87 
88  rxq->n_enqueued += n_alloc;
89  first_d = rxq->descs;
90 
91  ASSERT (slot % 8 == 0);
92 
93  while (n_alloc >= 8)
94  {
95  d = first_d + slot;
96 
97  if (use_va_dma)
98  {
100  sizeof (vlib_buffer_t));
101  avf_rx_desc_write (d + 0, pointer_to_uword (p[0]));
102  avf_rx_desc_write (d + 1, pointer_to_uword (p[1]));
103  avf_rx_desc_write (d + 2, pointer_to_uword (p[2]));
104  avf_rx_desc_write (d + 3, pointer_to_uword (p[3]));
105  avf_rx_desc_write (d + 4, pointer_to_uword (p[4]));
106  avf_rx_desc_write (d + 5, pointer_to_uword (p[5]));
107  avf_rx_desc_write (d + 6, pointer_to_uword (p[6]));
108  avf_rx_desc_write (d + 7, pointer_to_uword (p[7]));
109  }
110  else
111  {
112  vlib_get_buffers (vm, rxq->bufs + slot, b, 8);
113  avf_rx_desc_write (d + 0, vlib_buffer_get_pa (vm, b[0]));
114  avf_rx_desc_write (d + 1, vlib_buffer_get_pa (vm, b[1]));
115  avf_rx_desc_write (d + 2, vlib_buffer_get_pa (vm, b[2]));
116  avf_rx_desc_write (d + 3, vlib_buffer_get_pa (vm, b[3]));
117  avf_rx_desc_write (d + 4, vlib_buffer_get_pa (vm, b[4]));
118  avf_rx_desc_write (d + 5, vlib_buffer_get_pa (vm, b[5]));
119  avf_rx_desc_write (d + 6, vlib_buffer_get_pa (vm, b[6]));
120  avf_rx_desc_write (d + 7, vlib_buffer_get_pa (vm, b[7]));
121  }
122 
123  /* next */
124  slot = (slot + 8) & mask;
125  n_alloc -= 8;
126  }
127 
128  avf_tail_write (rxq->qrx_tail, slot);
129 }
130 
131 
134  u64 qw1, avf_rx_tail_t * t)
135 {
136  vlib_buffer_t *hb = b;
137  u32 tlnifb = 0, i = 0;
138 
139  if (qw1 & AVF_RXD_STATUS_EOP)
140  return 0;
141 
142  while ((qw1 & AVF_RXD_STATUS_EOP) == 0)
143  {
145  ASSERT (qw1 & AVF_RXD_STATUS_DD);
146  qw1 = t->qw1s[i];
147  b->next_buffer = t->buffers[i];
148  b->flags |= VLIB_BUFFER_NEXT_PRESENT;
151  tlnifb += b->current_length = qw1 >> AVF_RXD_LEN_SHIFT;
152  i++;
153  }
154 
156  hb->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
157  return tlnifb;
158 }
159 
162  uword n_rx_packets)
163 {
164  uword n;
166 
167  for (n = 0; n < n_rx_packets; n++)
168  {
169  if ((ptd->qw1s[n] & AVF_RXD_STATUS_FLM) == 0)
170  continue;
171 
172  fle = pool_elt_at_index (ad->flow_lookup_entries, ptd->flow_ids[n]);
173 
174  if (fle->next_index != (u16) ~0)
175  {
176  ptd->next[n] = fle->next_index;
177  }
178 
179  if (fle->flow_id != ~0)
180  {
181  ptd->bufs[n]->flow_id = fle->flow_id;
182  }
183 
184  if (fle->buffer_advance != ~0)
185  {
186  vlib_buffer_advance (ptd->bufs[n], fle->buffer_advance);
187  }
188  }
189 }
190 
194  int maybe_multiseg)
195 {
196  vlib_buffer_t bt;
197  vlib_buffer_t **b = ptd->bufs;
198  u64 *qw1 = ptd->qw1s;
199  avf_rx_tail_t *tail = ptd->tails;
200  uword n_rx_bytes = 0;
201 
202  /* copy template into local variable - will save per packet load */
204 
205  while (n_left >= 4)
206  {
207  if (n_left >= 12)
208  {
209  vlib_prefetch_buffer_header (b[8], LOAD);
210  vlib_prefetch_buffer_header (b[9], LOAD);
211  vlib_prefetch_buffer_header (b[10], LOAD);
212  vlib_prefetch_buffer_header (b[11], LOAD);
213  }
214 
215  vlib_buffer_copy_template (b[0], &bt);
216  vlib_buffer_copy_template (b[1], &bt);
217  vlib_buffer_copy_template (b[2], &bt);
218  vlib_buffer_copy_template (b[3], &bt);
219 
220  n_rx_bytes += b[0]->current_length = qw1[0] >> AVF_RXD_LEN_SHIFT;
221  n_rx_bytes += b[1]->current_length = qw1[1] >> AVF_RXD_LEN_SHIFT;
222  n_rx_bytes += b[2]->current_length = qw1[2] >> AVF_RXD_LEN_SHIFT;
223  n_rx_bytes += b[3]->current_length = qw1[3] >> AVF_RXD_LEN_SHIFT;
224 
225  if (maybe_multiseg)
226  {
227  n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[0], qw1[0], tail + 0);
228  n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[1], qw1[1], tail + 1);
229  n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[2], qw1[2], tail + 2);
230  n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[3], qw1[3], tail + 3);
231  }
232 
233  /* next */
234  qw1 += 4;
235  tail += 4;
236  b += 4;
237  n_left -= 4;
238  }
239 
240  while (n_left)
241  {
242  vlib_buffer_copy_template (b[0], &bt);
243 
244  n_rx_bytes += b[0]->current_length = qw1[0] >> AVF_RXD_LEN_SHIFT;
245 
246  if (maybe_multiseg)
247  n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[0], qw1[0], tail + 0);
248 
249  /* next */
250  qw1 += 1;
251  tail += 1;
252  b += 1;
253  n_left -= 1;
254  }
255  return n_rx_bytes;
256 }
257 
260  vlib_frame_t *frame, avf_device_t *ad, u16 qid,
261  int with_flows)
262 {
263  avf_main_t *am = &avf_main;
264  vnet_main_t *vnm = vnet_get_main ();
265  u32 thr_idx = vlib_get_thread_index ();
266  avf_per_thread_data_t *ptd =
267  vec_elt_at_index (am->per_thread_data, thr_idx);
268  avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
269  u32 n_trace, n_rx_packets = 0, n_rx_bytes = 0;
270  u16 n_tail_desc = 0;
271  u64 or_qw1 = 0;
272  u32 *bi, *to_next, n_left_to_next;
273  vlib_buffer_t *bt = &ptd->buffer_template;
275  u16 next = rxq->next;
276  u16 size = rxq->size;
277  u16 mask = size - 1;
278  avf_rx_desc_t *d, *fd = rxq->descs;
279 #ifdef CLIB_HAVE_VEC256
280  u64x4 q1x4, or_q1x4 = { 0 };
281  u32x4 fdidx4;
282  u64x4 dd_eop_mask4 = u64x4_splat (AVF_RXD_STATUS_DD | AVF_RXD_STATUS_EOP);
283 #elif defined(CLIB_HAVE_VEC128)
284  u32x4 q1x4_lo, q1x4_hi, or_q1x4 = { 0 };
285  u32x4 fdidx4;
286  u32x4 dd_eop_mask4 = u32x4_splat (AVF_RXD_STATUS_DD | AVF_RXD_STATUS_EOP);
287 #endif
288  int single_next = 1;
289 
290  /* is there anything on the ring */
291  d = fd + next;
292  if ((d->qword[1] & AVF_RXD_STATUS_DD) == 0)
293  goto done;
294 
295  if (PREDICT_FALSE (ad->per_interface_next_index != ~0))
297 
300 
301  vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next);
302 
303  /* fetch up to AVF_RX_VECTOR_SZ from the rx ring, unflatten them and
304  copy needed data from descriptor to rx vector */
305  bi = to_next;
306 
307  while (n_rx_packets < AVF_RX_VECTOR_SZ)
308  {
309  if (next + 11 < size)
310  {
311  int stride = 8;
312  clib_prefetch_load ((void *) (fd + (next + stride)));
313  clib_prefetch_load ((void *) (fd + (next + stride + 1)));
314  clib_prefetch_load ((void *) (fd + (next + stride + 2)));
315  clib_prefetch_load ((void *) (fd + (next + stride + 3)));
316  }
317 
318 #ifdef CLIB_HAVE_VEC256
319  if (n_rx_packets >= AVF_RX_VECTOR_SZ - 4 || next >= size - 4)
320  goto one_by_one;
321 
322  q1x4 = u64x4_gather ((void *) &d[0].qword[1], (void *) &d[1].qword[1],
323  (void *) &d[2].qword[1], (void *) &d[3].qword[1]);
324 
325  /* not all packets are ready or at least one of them is chained */
326  if (!u64x4_is_equal (q1x4 & dd_eop_mask4, dd_eop_mask4))
327  goto one_by_one;
328 
329  or_q1x4 |= q1x4;
330 
331  u64x4_store_unaligned (q1x4, ptd->qw1s + n_rx_packets);
332 #elif defined(CLIB_HAVE_VEC128)
333  if (n_rx_packets >= AVF_RX_VECTOR_SZ - 4 || next >= size - 4)
334  goto one_by_one;
335 
336  q1x4_lo =
337  u32x4_gather ((void *) &d[0].qword[1], (void *) &d[1].qword[1],
338  (void *) &d[2].qword[1], (void *) &d[3].qword[1]);
339 
340  /* not all packets are ready or at least one of them is chained */
341  if (!u32x4_is_equal (q1x4_lo & dd_eop_mask4, dd_eop_mask4))
342  goto one_by_one;
343 
344  q1x4_hi = u32x4_gather (
345  (void *) &d[0].qword[1] + 4, (void *) &d[1].qword[1] + 4,
346  (void *) &d[2].qword[1] + 4, (void *) &d[3].qword[1] + 4);
347 
348  or_q1x4 |= q1x4_lo;
349  ptd->qw1s[n_rx_packets + 0] = (u64) q1x4_hi[0] << 32 | (u64) q1x4_lo[0];
350  ptd->qw1s[n_rx_packets + 1] = (u64) q1x4_hi[1] << 32 | (u64) q1x4_lo[1];
351  ptd->qw1s[n_rx_packets + 2] = (u64) q1x4_hi[2] << 32 | (u64) q1x4_lo[2];
352  ptd->qw1s[n_rx_packets + 3] = (u64) q1x4_hi[3] << 32 | (u64) q1x4_lo[3];
353 #endif
354 #if defined(CLIB_HAVE_VEC256) || defined(CLIB_HAVE_VEC128)
355 
356  if (with_flows)
357  {
358  fdidx4 = u32x4_gather (
359  (void *) &d[0].fdid_flex_hi, (void *) &d[1].fdid_flex_hi,
360  (void *) &d[2].fdid_flex_hi, (void *) &d[3].fdid_flex_hi);
361  u32x4_store_unaligned (fdidx4, ptd->flow_ids + n_rx_packets);
362  }
363 
364  vlib_buffer_copy_indices (bi, rxq->bufs + next, 4);
365 
366  /* next */
367  next = (next + 4) & mask;
368  d = fd + next;
369  n_rx_packets += 4;
370  bi += 4;
371  continue;
372  one_by_one:
373 #endif
374  clib_prefetch_load ((void *) (fd + ((next + 8) & mask)));
375 
376  if (avf_rxd_is_not_dd (d))
377  break;
378 
379  bi[0] = rxq->bufs[next];
380 
381  /* deal with chained buffers */
383  {
384  u16 tail_desc = 0;
385  u16 tail_next = next;
386  avf_rx_tail_t *tail = ptd->tails + n_rx_packets;
387  avf_rx_desc_t *td;
388  do
389  {
390  tail_next = (tail_next + 1) & mask;
391  td = fd + tail_next;
392 
393  /* bail out in case of incomplete transaction */
394  if (avf_rxd_is_not_dd (td))
395  goto no_more_desc;
396 
397  or_qw1 |= tail->qw1s[tail_desc] = td[0].qword[1];
398  tail->buffers[tail_desc] = rxq->bufs[tail_next];
399  tail_desc++;
400  }
401  while (avf_rxd_is_not_eop (td));
402  next = tail_next;
403  n_tail_desc += tail_desc;
404  }
405 
406  or_qw1 |= ptd->qw1s[n_rx_packets] = d[0].qword[1];
407  if (PREDICT_FALSE (with_flows))
408  {
409  ptd->flow_ids[n_rx_packets] = d[0].fdid_flex_hi;
410  }
411 
412  /* next */
413  next = (next + 1) & mask;
414  d = fd + next;
415  n_rx_packets++;
416  bi++;
417  }
418 no_more_desc:
419 
420  if (n_rx_packets == 0)
421  goto done;
422 
423  rxq->next = next;
424  rxq->n_enqueued -= n_rx_packets + n_tail_desc;
425 
426  /* avoid eating our own tail */
427  rxq->descs[(next + rxq->n_enqueued) & mask].qword[1] = 0;
428 
429 #if defined(CLIB_HAVE_VEC256) || defined(CLIB_HAVE_VEC128)
430  or_qw1 |= or_q1x4[0] | or_q1x4[1] | or_q1x4[2] | or_q1x4[3];
431 #endif
432 
433  vlib_get_buffers (vm, to_next, ptd->bufs, n_rx_packets);
434 
435  vnet_buffer (bt)->sw_if_index[VLIB_RX] = ad->sw_if_index;
436  vnet_buffer (bt)->sw_if_index[VLIB_TX] = ~0;
438  bt->ref_count = 1;
439 
440  if (n_tail_desc)
441  n_rx_bytes = avf_process_rx_burst (vm, node, ptd, n_rx_packets, 1);
442  else
443  n_rx_bytes = avf_process_rx_burst (vm, node, ptd, n_rx_packets, 0);
444 
445  /* the MARKed packets may have different next nodes */
446  if (PREDICT_FALSE (with_flows && (or_qw1 & AVF_RXD_STATUS_FLM)))
447  {
448  u32 n;
449  single_next = 0;
450  for (n = 0; n < n_rx_packets; n++)
451  ptd->next[n] = next_index;
452 
453  avf_process_flow_offload (ad, ptd, n_rx_packets);
454  }
455 
456  /* packet trace if enabled */
457  if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
458  {
459  u32 n_left = n_rx_packets;
460  u32 i, j;
461  u16 *next_indices = ptd->next;
462 
463  i = 0;
464  while (n_trace && n_left)
465  {
466  vlib_buffer_t *b = ptd->bufs[i];
467  if (PREDICT_FALSE (single_next == 0))
468  next_index = next_indices[0];
469 
470  if (PREDICT_TRUE
472  (vm, node, next_index, b, /* follow_chain */ 0)))
473  {
474  avf_input_trace_t *tr =
475  vlib_add_trace (vm, node, b, sizeof (*tr));
476  tr->next_index = next_index;
477  tr->qid = qid;
478  tr->hw_if_index = ad->hw_if_index;
479  tr->qw1s[0] = ptd->qw1s[i];
480  tr->flow_id =
481  (tr->qw1s[0] & AVF_RXD_STATUS_FLM) ? ptd->flow_ids[i] : 0;
482  for (j = 1; j < AVF_RX_MAX_DESC_IN_CHAIN; j++)
483  tr->qw1s[j] = ptd->tails[i].qw1s[j - 1];
484 
485  n_trace--;
486  }
487 
488  /* next */
489  n_left--;
490  i++;
491  next_indices++;
492  }
493  vlib_set_trace_count (vm, node, n_trace);
494  }
495 
496  /* enqueu the packets to the next nodes */
497  if (PREDICT_FALSE (with_flows && (or_qw1 & AVF_RXD_STATUS_FLM)))
498  {
499  /* release next node's frame vector, in this case we use
500  vlib_buffer_enqueue_to_next to place the packets
501  */
502  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
503 
504  /* enqueue buffers to the next node */
505  vlib_buffer_enqueue_to_next (vm, node, to_next, ptd->next, n_rx_packets);
506  }
507  else
508  {
510  {
511  vlib_next_frame_t *nf;
512  vlib_frame_t *f;
515  f = vlib_get_frame (vm, nf->frame);
517 
518  ef = vlib_frame_scalar_args (f);
519  ef->sw_if_index = ad->sw_if_index;
520  ef->hw_if_index = ad->hw_if_index;
521 
522  if ((or_qw1 & AVF_RXD_ERROR_IPE) == 0)
525  }
526 
527  n_left_to_next -= n_rx_packets;
528  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
529  }
530 
532  + VNET_INTERFACE_COUNTER_RX, thr_idx,
533  ad->hw_if_index, n_rx_packets, n_rx_bytes);
534 
535 done:
536  /* refill rx ring */
537  if (ad->flags & AVF_DEVICE_F_VA_DMA)
538  avf_rxq_refill (vm, node, rxq, 1 /* use_va_dma */ );
539  else
540  avf_rxq_refill (vm, node, rxq, 0 /* use_va_dma */ );
541 
542  return n_rx_packets;
543 }
544 
547 {
548  u32 n_rx = 0;
550 
552 
553  for (int i = 0; i < vec_len (pv); i++)
554  {
555  avf_device_t *ad = avf_get_device (pv[i].dev_instance);
556  if ((ad->flags & AVF_DEVICE_F_ADMIN_UP) == 0)
557  continue;
558  if (PREDICT_FALSE (ad->flags & AVF_DEVICE_F_RX_FLOW_OFFLOAD))
559  n_rx +=
560  avf_device_input_inline (vm, node, frame, ad, pv[i].queue_id, 1);
561  else
562  n_rx +=
563  avf_device_input_inline (vm, node, frame, ad, pv[i].queue_id, 0);
564  }
565 
566  return n_rx;
567 }
568 
569 /* *INDENT-OFF* */
571  .name = "avf-input",
572  .sibling_of = "device-input",
573  .format_trace = format_avf_input_trace,
574  .type = VLIB_NODE_TYPE_INPUT,
575  .state = VLIB_NODE_STATE_DISABLED,
576  .n_errors = AVF_INPUT_N_ERROR,
577  .error_strings = avf_input_error_strings,
579 };
580 
581 /* *INDENT-ON* */
582 
583 
584 /*
585  * fd.io coding-style-patch-verification: ON
586  *
587  * Local Variables:
588  * eval: (c-set-style "gnu")
589  * End:
590  */
avf_input_trace_t::hw_if_index
u32 hw_if_index
Definition: avf.h:473
vlib.h
vlib_get_frame
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:273
vlib_buffer_t::next_buffer
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:149
format_avf_input_trace
format_function_t format_avf_input_trace
Definition: avf.h:359
avf_per_thread_data_t::tails
avf_rx_tail_t tails[AVF_RX_VECTOR_SZ]
Definition: avf.h:317
vlib_buffer_t::buffer_pool_index
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:142
avf_rx_desc_t::fdid_flex_hi
u32 fdid_flex_hi
Definition: avf.h:145
avf_rx_tail_t::qw1s
u64 qw1s[AVF_RX_MAX_DESC_IN_CHAIN - 1]
Definition: avf.h:306
avf_rxq_t::descs
avf_rx_desc_t * descs
Definition: avf.h:175
frame
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: nat44_ei.c:3048
vlib_prefetch_buffer_header
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:231
avf_input_trace_t::next_index
u16 next_index
Definition: avf.h:472
vlib_buffer_alloc_to_ring_from_pool
static __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring_from_pool(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers into ring from specific buffer pool.
Definition: buffer_funcs.h:747
next_index
nat44_ei_hairpin_src_next_t next_index
Definition: nat44_ei_hairpinning.c:412
avf_rxq_t::next
u16 next
Definition: avf.h:173
AVF_RXD_ERROR_IPE
#define AVF_RXD_ERROR_IPE
Definition: avf.h:56
vlib_buffer_copy_indices
vlib_buffer_copy_indices(to, tmp, n_free)
avf_per_thread_data_t
Definition: avf.h:310
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
f
vlib_frame_t * f
Definition: interface_output.c:1098
pointer_to_uword
static uword pointer_to_uword(const void *p)
Definition: types.h:131
avf_main
avf_main_t avf_main
Definition: device.c:48
pool_elt_at_index
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:549
avf_rx_desc_write
static_always_inline void avf_rx_desc_write(avf_rx_desc_t *d, u64 addr)
Definition: input.c:46
AVF_INPUT_REFILL_TRESHOLD
#define AVF_INPUT_REFILL_TRESHOLD
Definition: input.c:43
vlib_get_buffers_with_offset
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:204
foreach_avf_input_error
#define foreach_avf_input_error
Definition: input.c:26
vlib_get_buffers
vlib_get_buffers(vm, from, b, n_left_from)
next
u16 * next
Definition: nat44_ei_out2in.c:718
avf_device_t::hw_if_index
u32 hw_if_index
Definition: avf.h:223
node
vlib_main_t vlib_node_runtime_t * node
Definition: nat44_ei.c:3047
avf_rxq_t
Definition: avf.h:169
avf_input_error_t
avf_input_error_t
Definition: input.c:29
vnet_device_input_have_features
static_always_inline int vnet_device_input_have_features(u32 sw_if_index)
Definition: feature.h:336
avf_main_t
Definition: avf.h:321
VLIB_NODE_TYPE_INPUT
@ VLIB_NODE_TYPE_INPUT
Definition: node.h:76
avf_device_input_inline
static_always_inline uword avf_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, avf_device_t *ad, u16 qid, int with_flows)
Definition: input.c:259
u16
unsigned short u16
Definition: types.h:57
am
app_main_t * am
Definition: application.c:489
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
VLIB_RX
@ VLIB_RX
Definition: defs.h:46
avf_rxq_t::bufs
u32 * bufs
Definition: avf.h:176
vlib_get_trace_count
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:212
avf_per_thread_data_t::next
u16 next[AVF_RX_VECTOR_SZ]
Definition: avf.h:314
vlib_buffer_enqueue_to_next
vlib_buffer_enqueue_to_next(vm, node, from,(u16 *) nexts, frame->n_vectors)
avf_rx_tail_t::buffers
u32 buffers[AVF_RX_MAX_DESC_IN_CHAIN - 1]
Definition: avf.h:307
vlib_buffer_get_pa
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:488
addr
vhost_vring_addr_t addr
Definition: vhost_user.h:130
VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT
@ VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT
Definition: devices.h:28
vlib_error_count
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
vlib_frame_t
Definition: node.h:372
avf_device_t
Definition: avf.h:215
ethernet.h
avf_device_t::rxqs
avf_rxq_t * rxqs
Definition: avf.h:230
AVF_INPUT_N_ERROR
@ AVF_INPUT_N_ERROR
Definition: input.c:34
avf_rx_attach_tail
static_always_inline uword avf_rx_attach_tail(vlib_main_t *vm, vlib_buffer_t *bt, vlib_buffer_t *b, u64 qw1, avf_rx_tail_t *t)
Definition: input.c:133
avf_process_rx_burst
static_always_inline uword avf_process_rx_burst(vlib_main_t *vm, vlib_node_runtime_t *node, avf_per_thread_data_t *ptd, u32 n_left, int maybe_multiseg)
Definition: input.c:192
vlib_frame_no_append
static void vlib_frame_no_append(vlib_frame_t *f)
Definition: node_funcs.h:281
vlib_buffer_free_from_ring
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:1026
vlib_buffer_advance
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:276
vec_len
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
Definition: vec_bootstrap.h:142
avf_per_thread_data_t::flow_ids
u32 flow_ids[AVF_RX_VECTOR_SZ]
Definition: avf.h:316
slot
u8 slot
Definition: pci_types.api:22
VLIB_NODE_FN
#define VLIB_NODE_FN(node)
Definition: node.h:202
vnet_buffer
#define vnet_buffer(b)
Definition: buffer.h:441
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
vnet_get_main
vnet_main_t * vnet_get_main(void)
Definition: pnat_test_stubs.h:56
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
vlib_get_thread_index
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:187
rx_queue_funcs.h
avf_input_error_strings
static __clib_unused char * avf_input_error_strings[]
Definition: input.c:37
avf_flow_lookup_entry_t::buffer_advance
i16 buffer_advance
Definition: avf.h:212
AVF_RX_MAX_DESC_IN_CHAIN
#define AVF_RX_MAX_DESC_IN_CHAIN
Definition: avf.h:54
static_always_inline
#define static_always_inline
Definition: clib.h:112
avf_input_trace_t::qw1s
u64 qw1s[AVF_RX_MAX_DESC_IN_CHAIN]
Definition: avf.h:475
avf_rx_tail_t
Definition: avf.h:304
uword
u64 uword
Definition: types.h:112
vlib_buffer_copy_template
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
Definition: buffer_funcs.h:171
VLIB_NODE_FLAG_TRACE_SUPPORTED
#define VLIB_NODE_FLAG_TRACE_SUPPORTED
Definition: node.h:295
mask
vl_api_pnat_mask_t mask
Definition: pnat.api:45
AVF_RXD_STATUS_DD
#define AVF_RXD_STATUS_DD
Definition: avf.h:48
vlib_buffer_t::ref_count
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:139
avf_flow_lookup_entry_t
Definition: avf.h:208
avf_process_flow_offload
static_always_inline void avf_process_flow_offload(avf_device_t *ad, avf_per_thread_data_t *ptd, uword n_rx_packets)
Definition: input.c:161
AVF_RXD_LEN_SHIFT
#define AVF_RXD_LEN_SHIFT
Definition: avf.h:53
avf_flow_lookup_entry_t::next_index
u16 next_index
Definition: avf.h:211
vlib_buffer_t::flow_id
u32 flow_id
Generic flow identifier.
Definition: buffer.h:136
VNET_INTERFACE_COUNTER_RX
@ VNET_INTERFACE_COUNTER_RX
Definition: interface.h:915
avf_input_trace_t::flow_id
u32 flow_id
Definition: avf.h:474
avf.h
avf_per_thread_data_t::qw1s
u64 qw1s[AVF_RX_VECTOR_SZ]
Definition: avf.h:315
vlib_set_trace_count
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:226
vlib_node_registration_t
struct _vlib_node_registration vlib_node_registration_t
vnet_interface_main_t::combined_sw_if_counters
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:1024
avf_rxq_t::size
u16 size
Definition: avf.h:174
avf_rxq_t::n_enqueued
u16 n_enqueued
Definition: avf.h:177
u64x4_gather
static_always_inline u64x4 u64x4_gather(void *p0, void *p1, void *p2, void *p3)
Definition: vector_avx2.h:256
vlib_buffer_t::current_length
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:122
vlib_frame_scalar_args
static void * vlib_frame_scalar_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:315
avf_rx_desc_t::qword
u64 qword[4]
Definition: avf.h:147
ethernet_input_frame_t::sw_if_index
u32 sw_if_index
Definition: ethernet.h:58
vnet_main_t
Definition: vnet.h:76
avf_device_t::sw_if_index
u32 sw_if_index
Definition: avf.h:222
size
u32 size
Definition: vhost_user.h:125
avf_tail_write
static void avf_tail_write(volatile u32 *addr, u32 val)
Definition: avf.h:447
avf_flow_lookup_entry_t::flow_id
u32 flow_id
Definition: avf.h:210
AVF_RXD_STATUS_FLM
#define AVF_RXD_STATUS_FLM
Definition: avf.h:50
u64
unsigned long u64
Definition: types.h:89
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
avf_per_thread_data_t::buffer_template
vlib_buffer_t buffer_template
Definition: avf.h:318
vlib_put_next_frame
vlib_put_next_frame(vm, node, next_index, 0)
vlib_frame_t::flags
u16 flags
Definition: node.h:378
avf_rxq_refill
static_always_inline void avf_rxq_refill(vlib_main_t *vm, vlib_node_runtime_t *node, avf_rxq_t *rxq, int use_va_dma)
Definition: input.c:58
u32
unsigned int u32
Definition: types.h:88
avf_rxd_is_not_eop
static_always_inline int avf_rxd_is_not_eop(avf_rx_desc_t *d)
Definition: avf.h:458
clib_prefetch_load
static_always_inline void clib_prefetch_load(void *p)
Definition: cache.h:92
AVF_RXD_STATUS_EOP
#define AVF_RXD_STATUS_EOP
Definition: avf.h:49
avf_rxq_t::buffer_pool_index
u8 buffer_pool_index
Definition: avf.h:179
pci.h
for
for(i=1;i<=collision_buckets;i++)
Definition: flowhash_template.h:378
n_left
u32 n_left
Definition: interface_output.c:1096
ETH_INPUT_FRAME_F_IP4_CKSUM_OK
#define ETH_INPUT_FRAME_F_IP4_CKSUM_OK
Definition: ethernet.h:54
u32x4
unsigned long long u32x4
Definition: ixge.c:28
vnet_hw_if_get_rxq_poll_vector
static_always_inline vnet_hw_if_rxq_poll_vector_t * vnet_hw_if_get_rxq_poll_vector(vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: rx_queue_funcs.h:70
ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX
#define ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX
Definition: ethernet.h:51
vlib_get_new_next_frame
#define vlib_get_new_next_frame(vm, node, next_index, vectors, n_vectors_left)
Definition: node_funcs.h:400
avf_device_t::per_interface_next_index
u32 per_interface_next_index
Definition: avf.h:219
ethernet_input_frame_t::hw_if_index
u32 hw_if_index
Definition: ethernet.h:59
AVF_RX_VECTOR_SZ
#define AVF_RX_VECTOR_SZ
Definition: avf.h:270
avf_device_t::flow_lookup_entries
avf_flow_lookup_entry_t * flow_lookup_entries
Definition: avf.h:260
vlib_main_t
Definition: main.h:102
vlib_add_trace
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:628
avf_device_t::flags
u32 flags
Definition: avf.h:218
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
unix.h
avf_rx_desc_t
Definition: avf.h:127
vnet_feature_start_device_input_x1
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:343
ethernet_input_frame_t
Definition: ethernet.h:56
i
int i
Definition: flowhash_template.h:376
avf_input_trace_t::qid
u16 qid
Definition: avf.h:471
u32x4_gather
static_always_inline u32x4 u32x4_gather(void *p0, void *p1, void *p2, void *p3)
Definition: vector_neon.h:180
avf_rxq_t::qrx_tail
volatile u32 * qrx_tail
Definition: avf.h:172
avf_input_trace_t
Definition: avf.h:469
avf_rxd_is_not_dd
static_always_inline int avf_rxd_is_not_dd(avf_rx_desc_t *d)
Definition: avf.h:464
vlib_node_runtime_t
Definition: node.h:454
vlib_next_frame_t
Definition: node.h:393
vlib_trace_buffer
static __clib_warn_unused_result int vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:153
PREDICT_TRUE
#define PREDICT_TRUE(x)
Definition: clib.h:125
vlib_buffer_t::total_length_not_including_first_buffer
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:176
avf_per_thread_data_t::bufs
vlib_buffer_t * bufs[AVF_RX_VECTOR_SZ]
Definition: avf.h:313
VLIB_TX
@ VLIB_TX
Definition: defs.h:47
vlib_node_runtime_get_next_frame
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:321
vnet_main_t::interface_main
vnet_interface_main_t interface_main
Definition: vnet.h:81
vlib_increment_combined_counter
vlib_increment_combined_counter(ccm, ti, sw_if_index, n_buffers, n_bytes)
vlib_next_frame_t::frame
vlib_frame_t * frame
Definition: node.h:396
avf_get_device
static_always_inline avf_device_t * avf_get_device(u32 dev_instance)
Definition: avf.h:368
u64x4
u64x4
Definition: vector_avx2.h:142
vlib_buffer_t::flags
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,...
Definition: buffer.h:133
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
vnet_hw_if_rxq_poll_vector_t
Definition: interface.h:746
VLIB_REGISTER_NODE
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
avf_input_node
vlib_node_registration_t avf_input_node
(constructor) VLIB_REGISTER_NODE (avf_input_node)
Definition: input.c:570