FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2016 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <fcntl.h>
21 #include <net/if.h>
22 #include <linux/if_tun.h>
23 #include <sys/ioctl.h>
24 #include <sys/eventfd.h>
25 
26 #include <vlib/vlib.h>
27 #include <vlib/unix/unix.h>
28 #include <vnet/ethernet/ethernet.h>
29 #include <vnet/feature/feature.h>
30 #include <vnet/gso/gro_func.h>
32 #include <vnet/ip/ip4_packet.h>
33 #include <vnet/ip/ip6_packet.h>
34 #include <vnet/udp/udp_packet.h>
37 
38 static char *virtio_input_error_strings[] = {
39 #define _(n, s) s,
41 #undef _
42 };
43 
44 typedef struct
45 {
50  virtio_net_hdr_v1_t hdr;
52 
53 static u8 *
54 format_virtio_input_trace (u8 * s, va_list * args)
55 {
56  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
57  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
58  virtio_input_trace_t *t = va_arg (*args, virtio_input_trace_t *);
59  u32 indent = format_get_indent (s);
60 
61  s = format (s, "virtio: hw_if_index %d next-index %d vring %u len %u",
62  t->hw_if_index, t->next_index, t->ring, t->len);
63  s = format (s, "\n%Uhdr: flags 0x%02x gso_type 0x%02x hdr_len %u "
64  "gso_size %u csum_start %u csum_offset %u num_buffers %u",
65  format_white_space, indent + 2,
66  t->hdr.flags, t->hdr.gso_type, t->hdr.hdr_len, t->hdr.gso_size,
67  t->hdr.csum_start, t->hdr.csum_offset, t->hdr.num_buffers);
68  return s;
69 }
70 
72 virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
73  u8 * l4_proto, u8 * l4_hdr_sz, virtio_if_type_t type)
74 {
75  if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
76  {
77  u16 ethertype = 0, l2hdr_sz = 0;
78  vnet_buffer_oflags_t oflags = 0;
79 
80  if (type == VIRTIO_IF_TYPE_TUN)
81  {
82  switch (b0->data[0] & 0xf0)
83  {
84  case 0x40:
85  ethertype = ETHERNET_TYPE_IP4;
86  break;
87  case 0x60:
88  ethertype = ETHERNET_TYPE_IP6;
89  break;
90  }
91  }
92  else
93  {
94  ethernet_header_t *eh =
96  ethertype = clib_net_to_host_u16 (eh->type);
97  l2hdr_sz = sizeof (ethernet_header_t);
98 
99  if (ethernet_frame_is_tagged (ethertype))
100  {
101  ethernet_vlan_header_t *vlan =
102  (ethernet_vlan_header_t *) (eh + 1);
103 
104  ethertype = clib_net_to_host_u16 (vlan->type);
105  l2hdr_sz += sizeof (*vlan);
106  if (ethertype == ETHERNET_TYPE_VLAN)
107  {
108  vlan++;
109  ethertype = clib_net_to_host_u16 (vlan->type);
110  l2hdr_sz += sizeof (*vlan);
111  }
112  }
113  }
114 
115  vnet_buffer (b0)->l2_hdr_offset = 0;
116  vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz;
117 
118  if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4))
119  {
120  ip4_header_t *ip4 =
121  (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
122  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
123  *l4_proto = ip4->protocol;
124  oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
125  b0->flags |=
126  (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
127  VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
128  VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
129  }
130  else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
131  {
132  ip6_header_t *ip6 =
133  (ip6_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
134  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t);
135  /* FIXME IPv6 EH traversal */
136  *l4_proto = ip6->protocol;
137  b0->flags |= (VNET_BUFFER_F_IS_IP6 |
138  VNET_BUFFER_F_L2_HDR_OFFSET_VALID
139  | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
140  VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
141  }
142  if (*l4_proto == IP_PROTOCOL_TCP)
143  {
144  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
147  (b0)->l4_hdr_offset);
148  *l4_hdr_sz = tcp_header_bytes (tcp);
149  }
150  else if (*l4_proto == IP_PROTOCOL_UDP)
151  {
152  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
155  (b0)->l4_hdr_offset);
156  *l4_hdr_sz = sizeof (*udp);
157  }
158  if (oflags)
159  vnet_buffer_offload_flags_set (b0, oflags);
160  }
161 }
162 
164 fill_gso_buffer_flags (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
165  u8 l4_proto, u8 l4_hdr_sz)
166 {
167  if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4)
168  {
169  ASSERT (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM);
170  vnet_buffer2 (b0)->gso_size = hdr->gso_size;
171  vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
172  b0->flags |= VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4;
173  }
174  if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV6)
175  {
176  ASSERT (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM);
177  vnet_buffer2 (b0)->gso_size = hdr->gso_size;
178  vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
179  b0->flags |= VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6;
180  }
181 }
182 
184 virtio_n_left_to_process (virtio_vring_t * vring, const int packed)
185 {
186  if (packed)
187  return vring->desc_in_use;
188  else
189  return vring->used->idx - vring->last_used_idx;
190 }
191 
193 virtio_get_slot_id (virtio_vring_t * vring, const int packed, u16 last,
194  u16 mask)
195 {
196  if (packed)
197  return vring->packed_desc[last].id;
198  else
199  return vring->used->ring[last & mask].id;
200 }
201 
203 virtio_get_len (virtio_vring_t * vring, const int packed, const int hdr_sz,
204  u16 last, u16 mask)
205 {
206  if (packed)
207  return vring->packed_desc[last].len - hdr_sz;
208  else
209  return vring->used->ring[last & mask].len - hdr_sz;
210 }
211 
212 #define increment_last(last, packed, vring) \
213  do { \
214  last++; \
215  if (packed && last >= vring->size) \
216  { \
217  last = 0; \
218  vring->used_wrap_counter ^= 1; \
219  } \
220  } while (0)
221 
224  vlib_frame_t * frame, virtio_if_t * vif,
226  int gso_enabled, int checksum_offload_enabled,
227  int packed)
228 {
229  vnet_main_t *vnm = vnet_get_main ();
231  uword n_trace = vlib_get_trace_count (vm, node);
232  u32 next_index;
233  const int hdr_sz = vif->virtio_net_hdr_sz;
234  u32 *to_next = 0;
235  u32 n_rx_packets = 0;
236  u32 n_rx_bytes = 0;
237  u16 mask = vring->size - 1;
238  u16 last = vring->last_used_idx;
239  u16 n_left = virtio_n_left_to_process (vring, packed);
240  vlib_buffer_t bt;
241 
242  if (n_left == 0)
243  return 0;
244 
245  if (type == VIRTIO_IF_TYPE_TUN)
246  {
248  }
249  else
250  {
252  if (PREDICT_FALSE (vif->per_interface_next_index != ~0))
254 
255  /* only for l2, redirect if feature path enabled */
257  }
258 
259  while (n_left)
260  {
261  u32 n_left_to_next;
262  u32 next0 = next_index;
263 
264  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
265 
266  while (n_left && n_left_to_next)
267  {
268  if (packed)
269  {
270  vring_packed_desc_t *d = &vring->packed_desc[last];
271  u16 flags = d->flags;
272  if ((flags & VRING_DESC_F_AVAIL) !=
273  (vring->used_wrap_counter << 7)
274  || (flags & VRING_DESC_F_USED) !=
275  (vring->used_wrap_counter << 15))
276  {
277  n_left = 0;
278  break;
279  }
280  }
281  u8 l4_proto = 0, l4_hdr_sz = 0;
282  u16 num_buffers = 1;
283  virtio_net_hdr_v1_t *hdr;
284  u16 slot = virtio_get_slot_id (vring, packed, last, mask);
285  u16 len = virtio_get_len (vring, packed, hdr_sz, last, mask);
286  u32 bi0 = vring->buffers[slot];
287  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
288  hdr = vlib_buffer_get_current (b0);
289  if (hdr_sz == sizeof (virtio_net_hdr_v1_t))
290  num_buffers = hdr->num_buffers;
291 
292  b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
293  b0->current_data = 0;
294  b0->current_length = len;
295 
296  if (checksum_offload_enabled)
297  virtio_needs_csum (b0, hdr, &l4_proto, &l4_hdr_sz, type);
298 
299  if (gso_enabled)
300  fill_gso_buffer_flags (b0, hdr, l4_proto, l4_hdr_sz);
301 
302  vnet_buffer (b0)->sw_if_index[VLIB_RX] = vif->sw_if_index;
303  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
304 
305  /* if multisegment packet */
306  if (PREDICT_FALSE (num_buffers > 1))
307  {
308  vlib_buffer_t *pb, *cb;
309  pb = b0;
311  while (num_buffers > 1)
312  {
313  increment_last (last, packed, vring);
314  u16 cslot = virtio_get_slot_id (vring, packed, last, mask);
315  /* hdr size is 0 after 1st packet in chain buffers */
316  u16 clen = virtio_get_len (vring, packed, 0, last, mask);
317  u32 cbi = vring->buffers[cslot];
318  cb = vlib_get_buffer (vm, cbi);
319 
320  /* current buffer */
321  cb->current_length = clen;
322 
323  /* previous buffer */
324  pb->next_buffer = cbi;
325  pb->flags |= VLIB_BUFFER_NEXT_PRESENT;
326 
327  /* first buffer */
329 
330  pb = cb;
331  vring->desc_in_use--;
332  num_buffers--;
333  n_left--;
334  }
336  }
337 
338  if (type == VIRTIO_IF_TYPE_TUN)
339  {
340  switch (b0->data[0] & 0xf0)
341  {
342  case 0x40:
344  break;
345  case 0x60:
347  break;
348  default:
350  break;
351  }
352 
353  if (PREDICT_FALSE (vif->per_interface_next_index != ~0))
354  next0 = vif->per_interface_next_index;
355  }
356  else
357  {
358  /* copy feature arc data from template */
360  vnet_buffer (b0)->feature_arc_index =
361  vnet_buffer (&bt)->feature_arc_index;
362  }
363 
364  /* trace */
365  if (PREDICT_FALSE (n_trace > 0 && vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */
366  1)))
367  {
369  vlib_set_trace_count (vm, node, --n_trace);
370  tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
371  tr->next_index = next0;
372  tr->hw_if_index = vif->hw_if_index;
373  tr->len = len;
374  clib_memcpy_fast (&tr->hdr, hdr, hdr_sz);
375  }
376 
377  /* enqueue buffer */
378  to_next[0] = bi0;
379  vring->desc_in_use--;
380  to_next += 1;
381  n_left_to_next--;
382  n_left--;
383  increment_last (last, packed, vring);
384 
385  /* only tun interfaces may have different next index */
386  if (type == VIRTIO_IF_TYPE_TUN)
388  n_left_to_next, bi0, next0);
389 
390  /* next packet */
391  n_rx_packets++;
392  n_rx_bytes += len;
393  }
394  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
395  }
396  vring->last_used_idx = last;
397 
400  vif->sw_if_index, n_rx_packets,
401  n_rx_bytes);
402 
403  return n_rx_packets;
404 }
405 
408  vlib_frame_t * frame, virtio_if_t * vif, u16 qid,
410 {
411  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
412  const int hdr_sz = vif->virtio_net_hdr_sz;
413  u16 txq_id = vm->thread_index % vif->num_txqs;
414  virtio_vring_t *txq_vring = vec_elt_at_index (vif->txq_vrings, txq_id);
415  uword rv;
416 
417  if (clib_spinlock_trylock_if_init (&txq_vring->lockp))
418  {
419  if (vif->packet_coalesce)
421  (vm, txq_vring->flow_table);
422  else if (vif->packet_buffering)
424  (vm, txq_vring->buffering);
425  clib_spinlock_unlock_if_init (&txq_vring->lockp);
426  }
427 
428  if (vif->is_packed)
429  {
430  if (vif->gso_enabled)
431  rv =
433  1, 1, 1);
434  else if (vif->csum_offload_enabled)
435  rv =
437  0, 1, 1);
438  else
439  rv =
441  0, 0, 1);
442 
443  virtio_refill_vring_packed (vm, vif, type, vring, hdr_sz,
444  node->node_index);
445  }
446  else
447  {
448  if (vif->gso_enabled)
449  rv =
451  1, 1, 0);
452  else if (vif->csum_offload_enabled)
453  rv =
455  0, 1, 0);
456  else
457  rv =
459  0, 0, 0);
460 
461  virtio_refill_vring_split (vm, vif, type, vring, hdr_sz,
462  node->node_index);
463  }
464  return rv;
465 }
466 
470 {
471  u32 n_rx = 0;
472  virtio_main_t *vim = &virtio_main;
475 
476  vec_foreach (p, pv)
477  {
478  virtio_if_t *vif;
479  vif = vec_elt_at_index (vim->interfaces, p->dev_instance);
480  if (vif->flags & VIRTIO_IF_FLAG_ADMIN_UP)
481  {
482  if (vif->type == VIRTIO_IF_TYPE_TAP)
484  vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_TAP);
485  else if (vif->type == VIRTIO_IF_TYPE_PCI)
487  vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_PCI);
488  else if (vif->type == VIRTIO_IF_TYPE_TUN)
490  vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_TUN);
491  }
492  }
493 
494  return n_rx;
495 }
496 
497 /* *INDENT-OFF* */
499  .name = "virtio-input",
500  .sibling_of = "device-input",
501  .format_trace = format_virtio_input_trace,
503  .type = VLIB_NODE_TYPE_INPUT,
504  .state = VLIB_NODE_STATE_INTERRUPT,
505  .n_errors = VIRTIO_INPUT_N_ERROR,
506  .error_strings = virtio_input_error_strings,
507 };
508 /* *INDENT-ON* */
509 
510 /*
511  * fd.io coding-style-patch-verification: ON
512  *
513  * Local Variables:
514  * eval: (c-set-style "gnu")
515  * End:
516  */
vlib.h
virtio_device_input_inline
static_always_inline uword virtio_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, virtio_if_t *vif, u16 qid, virtio_if_type_t type)
Definition: node.c:407
vlib_buffer_t::next_buffer
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:149
tcp_header_bytes
static int tcp_header_bytes(tcp_header_t *t)
Definition: tcp_packet.h:93
virtio_input_error_strings
static char * virtio_input_error_strings[]
Definition: node.c:38
thread_index
u32 thread_index
Definition: nat44_ei_hairpinning.c:492
virtio_vring_t::flow_table
gro_flow_table_t * flow_table
Definition: virtio.h:110
frame
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: nat44_ei.c:3048
ip4
vl_api_ip4_address_t ip4
Definition: one.api:376
next_index
nat44_ei_hairpin_src_next_t next_index
Definition: nat44_ei_hairpinning.c:412
virtio_vring_t
Definition: virtio.h:63
virtio.h
vnet_hw_if_rxq_poll_vector_t::queue_id
u32 queue_id
Definition: interface.h:749
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
ethernet_vlan_header_t
Definition: packet.h:128
ethernet_frame_is_tagged
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:78
virtio_if_t::flags
u32 flags
Definition: virtio.h:132
tcp_header_t
struct _tcp_header tcp_header_t
virtio_input_trace_t::hdr
virtio_net_hdr_v1_t hdr
Definition: node.c:50
VNET_DEVICE_INPUT_NEXT_DROP
@ VNET_DEVICE_INPUT_NEXT_DROP
Definition: devices.h:29
node
vlib_main_t vlib_node_runtime_t * node
Definition: nat44_ei.c:3047
vnet_buffer_offload_flags_set
static_always_inline void vnet_buffer_offload_flags_set(vlib_buffer_t *b, vnet_buffer_oflags_t oflags)
Definition: buffer.h:522
vnet_buffer_oflags_t
vnet_buffer_oflags_t
Definition: buffer.h:118
VLIB_NODE_TYPE_INPUT
@ VLIB_NODE_TYPE_INPUT
Definition: node.h:76
virtio_needs_csum
static_always_inline void virtio_needs_csum(vlib_buffer_t *b0, virtio_net_hdr_v1_t *hdr, u8 *l4_proto, u8 *l4_hdr_sz, virtio_if_type_t type)
Definition: node.c:72
virtio_if_t::per_interface_next_index
u32 per_interface_next_index
Definition: virtio.h:133
u16
unsigned short u16
Definition: types.h:57
virtio_input_trace_t
Definition: node.c:44
ethernet_header_t::type
u16 type
Definition: packet.h:59
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
VLIB_RX
@ VLIB_RX
Definition: defs.h:46
VRING_DESC_F_USED
#define VRING_DESC_F_USED
Definition: virtio_std.h:76
vlib_get_trace_count
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:212
virtio_if_t::packet_coalesce
int packet_coalesce
Definition: virtio.h:156
vnet_buffer2
#define vnet_buffer2(b)
Definition: buffer.h:499
vring_used_elem_t::id
u32 id
Definition: virtio_std.h:111
VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT
@ VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT
Definition: devices.h:28
vlib_frame_t
Definition: node.h:372
clib_memcpy_fast
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
Definition: string.h:92
virtio_vring_t::size
u16 size
Definition: virtio.h:83
udp_header_t
Definition: udp_packet.h:45
ip4_header_t
Definition: ip4_packet.h:87
ethernet.h
format_virtio_input_trace
static u8 * format_virtio_input_trace(u8 *s, va_list *args)
Definition: node.c:54
ip6_packet.h
virtio_if_t::packet_buffering
int packet_buffering
Definition: virtio.h:157
virtio_main
virtio_main_t virtio_main
Definition: virtio.c:37
vlib_buffer_t::current_data
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:119
virtio_refill_vring_packed
static_always_inline void virtio_refill_vring_packed(vlib_main_t *vm, virtio_if_t *vif, virtio_if_type_t type, virtio_vring_t *vring, const int hdr_sz, u32 node_index)
Definition: virtio_inline.h:96
VIRTIO_NET_HDR_GSO_TCPV4
#define VIRTIO_NET_HDR_GSO_TCPV4
Definition: virtio_std.h:144
fill_gso_buffer_flags
static_always_inline void fill_gso_buffer_flags(vlib_buffer_t *b, u32 gso_size, u8 l4_hdr_sz)
Definition: node.c:112
virtio_if_t::txq_vrings
virtio_vring_t * txq_vrings
Definition: virtio.h:137
len
u8 len
Definition: ip_types.api:103
virtio_vring_t::used
vring_used_t * used
Definition: virtio.h:72
slot
u8 slot
Definition: pci_types.api:22
virtio_get_len
static_always_inline u16 virtio_get_len(virtio_vring_t *vring, const int packed, const int hdr_sz, u16 last, u16 mask)
Definition: node.c:203
VLIB_NODE_FN
#define VLIB_NODE_FN(node)
Definition: node.h:202
feature.h
virtio_device_input_gso_inline
static_always_inline uword virtio_device_input_gso_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, virtio_if_t *vif, virtio_vring_t *vring, virtio_if_type_t type, int gso_enabled, int checksum_offload_enabled, int packed)
Definition: node.c:223
virtio_vring_t::lockp
clib_spinlock_t lockp
Definition: virtio.h:66
VRING_DESC_F_AVAIL
#define VRING_DESC_F_AVAIL
Definition: virtio_std.h:75
CLIB_UNUSED
#define CLIB_UNUSED(x)
Definition: clib.h:90
vnet_buffer
#define vnet_buffer(b)
Definition: buffer.h:437
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
vnet_get_main
vnet_main_t * vnet_get_main(void)
Definition: pnat_test_stubs.h:56
virtio_if_type_t
virtio_if_type_t
Definition: virtio.h:50
VNET_DEVICE_INPUT_NEXT_IP6_INPUT
@ VNET_DEVICE_INPUT_NEXT_IP6_INPUT
Definition: devices.h:26
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
VNET_DEVICE_INPUT_NEXT_IP4_INPUT
@ VNET_DEVICE_INPUT_NEXT_IP4_INPUT
Definition: devices.h:25
rx_queue_funcs.h
static_always_inline
#define static_always_inline
Definition: clib.h:112
virtio_input_trace_t::next_index
u32 next_index
Definition: node.c:46
virtio_input_trace_t::hw_if_index
u32 hw_if_index
Definition: node.c:47
virtio_if_t::csum_offload_enabled
int csum_offload_enabled
Definition: virtio.h:139
udp_packet.h
uword
u64 uword
Definition: types.h:112
last
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
ethernet_header_t
Definition: packet.h:52
vlib_main_t::thread_index
u32 thread_index
Definition: main.h:213
virtio_if_t::hw_if_index
u32 hw_if_index
Definition: virtio.h:152
VLIB_NODE_FLAG_TRACE_SUPPORTED
#define VLIB_NODE_FLAG_TRACE_SUPPORTED
Definition: node.h:295
mask
vl_api_pnat_mask_t mask
Definition: pnat.api:45
VNET_INTERFACE_COUNTER_RX
@ VNET_INTERFACE_COUNTER_RX
Definition: interface.h:914
vlib_set_trace_count
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:226
vlib_node_registration_t
struct _vlib_node_registration vlib_node_registration_t
vnet_interface_main_t::combined_sw_if_counters
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:1023
virtio_if_t::sw_if_index
u32 sw_if_index
Definition: virtio.h:153
vlib_buffer_t::current_length
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:122
virtio_vring_t::last_used_idx
u16 last_used_idx
Definition: virtio.h:88
ip4_packet.h
vlib_buffer_t::current_config_index
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:156
virtio_main_t
Definition: virtio.h:214
virtio_input_trace_t::len
u16 len
Definition: node.c:49
vnet_main_t
Definition: vnet.h:76
vlib_validate_buffer_enqueue_x1
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:224
vnet_gro_flow_table_schedule_node_on_dispatcher
static_always_inline void vnet_gro_flow_table_schedule_node_on_dispatcher(vlib_main_t *vm, gro_flow_table_t *flow_table)
Definition: gro_func.h:409
virtio_input_trace_t::ring
u16 ring
Definition: node.c:48
virtio_if_t::num_txqs
u16 num_txqs
Definition: virtio.h:135
format
description fragment has unexpected format
Definition: map.api:433
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
format_get_indent
static u32 format_get_indent(u8 *s)
Definition: format.h:72
vlib_put_next_frame
vlib_put_next_frame(vm, node, next_index, 0)
u32
unsigned int u32
Definition: types.h:88
virtio_if_t::virtio_net_hdr_sz
u16 virtio_net_hdr_sz
Definition: virtio.h:149
virtio_input_node
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
Definition: node.c:498
ip6
vl_api_ip6_address_t ip6
Definition: one.api:424
VIRTIO_NET_HDR_GSO_TCPV6
#define VIRTIO_NET_HDR_GSO_TCPV6
Definition: virtio_std.h:146
virtio_n_left_to_process
static_always_inline u16 virtio_n_left_to_process(virtio_vring_t *vring, const int packed)
Definition: node.c:184
virtio_vring_t::packed_desc
vring_packed_desc_t * packed_desc
Definition: virtio.h:77
increment_last
#define increment_last(last, packed, vring)
Definition: node.c:212
vec_foreach
#define vec_foreach(var, vec)
Vector iterator.
Definition: vec_bootstrap.h:213
n_left
u32 n_left
Definition: interface_output.c:1078
virtio_if_t::type
virtio_if_type_t type
Definition: virtio.h:150
vring_used_t::idx
u16 idx
Definition: virtio_std.h:118
virtio_if_t
Definition: virtio.h:128
VIRTIO_NET_HDR_F_NEEDS_CSUM
#define VIRTIO_NET_HDR_F_NEEDS_CSUM
Definition: virtio_std.h:140
vnet_hw_if_get_rxq_poll_vector
static_always_inline vnet_hw_if_rxq_poll_vector_t * vnet_hw_if_get_rxq_poll_vector(vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: rx_queue_funcs.h:69
ip6_header_t
Definition: ip6_packet.h:294
virtio_if_t::rxq_vrings
virtio_vring_t * rxq_vrings
Definition: virtio.h:136
clib_spinlock_trylock_if_init
static_always_inline int clib_spinlock_trylock_if_init(clib_spinlock_t *p)
Definition: lock.h:113
virtio_main_t::interfaces
virtio_if_t * interfaces
Definition: virtio.h:220
VIRTIO_INPUT_N_ERROR
@ VIRTIO_INPUT_N_ERROR
Definition: virtio_inline.h:27
vlib_main_t
Definition: main.h:102
vlib_node_t
Definition: node.h:247
virtio_vring_t::buffers
u32 * buffers
Definition: virtio.h:82
vlib_add_trace
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:628
u8
unsigned char u8
Definition: types.h:56
unix.h
vlib_buffer_get_current
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:257
vring_used_t::ring
vring_used_elem_t ring[0]
Definition: virtio_std.h:119
virtio_vring_t::buffering
virtio_vring_buffering_t * buffering
Definition: virtio.h:109
virtio_vring_t::desc_in_use
u16 desc_in_use
Definition: virtio.h:86
vlib_buffer_t::data
u8 data[]
Packet data.
Definition: buffer.h:204
vnet_feature_start_device_input_x1
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:343
virtio_refill_vring_split
static_always_inline void virtio_refill_vring_split(vlib_main_t *vm, virtio_if_t *vif, virtio_if_type_t type, virtio_vring_t *vring, const int hdr_sz, u32 node_index)
Definition: virtio_inline.h:31
ethernet_vlan_header_t::type
u16 type
Definition: packet.h:136
virtio_inline.h
ip4_header_bytes
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:190
rv
int __clib_unused rv
Definition: application.c:491
foreach_virtio_input_error
#define foreach_virtio_input_error
Definition: virtio_inline.h:18
vlib_node_runtime_t
Definition: node.h:454
clib_spinlock_unlock_if_init
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:129
vring_used_elem_t::len
u32 len
Definition: virtio_std.h:112
vlib_trace_buffer
static __clib_warn_unused_result int vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:153
virtio_vring_buffering_schedule_node_on_dispatcher
static_always_inline void virtio_vring_buffering_schedule_node_on_dispatcher(vlib_main_t *vm, virtio_vring_buffering_t *buffering)
Definition: virtio_buffering.h:208
PREDICT_TRUE
#define PREDICT_TRUE(x)
Definition: clib.h:125
vlib_buffer_t::total_length_not_including_first_buffer
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:176
vlib_get_next_frame
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:395
virtio_if_t::is_packed
int is_packed
Definition: virtio.h:211
VLIB_TX
@ VLIB_TX
Definition: defs.h:47
virtio_vring_t::used_wrap_counter
u16 used_wrap_counter
Definition: virtio.h:101
vnet_hw_if_rxq_poll_vector_t::dev_instance
u32 dev_instance
Definition: interface.h:748
type
vl_api_fib_path_type_t type
Definition: fib_types.api:123
vnet_main_t::interface_main
vnet_interface_main_t interface_main
Definition: vnet.h:81
vlib_increment_combined_counter
vlib_increment_combined_counter(ccm, ti, sw_if_index, n_buffers, n_bytes)
format_white_space
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
vlib_buffer_t::flags
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,...
Definition: buffer.h:133
virtio_if_t::gso_enabled
int gso_enabled
Definition: virtio.h:138
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
vnet_hw_if_rxq_poll_vector_t
Definition: interface.h:746
virtio_get_slot_id
static_always_inline u16 virtio_get_slot_id(virtio_vring_t *vring, const int packed, u16 last, u16 mask)
Definition: node.c:193
VLIB_REGISTER_NODE
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105
gro_func.h