FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
vhost_user_output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * vhost-user-output
4  *
5  * Copyright (c) 2014-2018 Cisco and/or its affiliates.
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at:
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *------------------------------------------------------------------
18  */
19 
20 #include <stddef.h>
21 #include <fcntl.h> /* for open */
22 #include <sys/ioctl.h>
23 #include <sys/socket.h>
24 #include <sys/un.h>
25 #include <sys/stat.h>
26 #include <sys/types.h>
27 #include <sys/uio.h> /* for iovec */
28 #include <netinet/in.h>
29 #include <sys/vfs.h>
30 
31 #include <linux/if_arp.h>
32 #include <linux/if_tun.h>
33 
34 #include <vlib/vlib.h>
35 #include <vlib/unix/unix.h>
36 
37 #include <vnet/ip/ip.h>
38 
39 #include <vnet/ethernet/ethernet.h>
40 #include <vnet/devices/devices.h>
41 #include <vnet/feature/feature.h>
42 
45 
47 /*
48  * On the transmit side, we keep processing the buffers from vlib in the while
49  * loop and prepare the copy order to be executed later. However, the static
50  * array which we keep the copy order is limited to VHOST_USER_COPY_ARRAY_N
51  * entries. In order to not corrupt memory, we have to do the copy when the
52  * static array reaches the copy threshold. We subtract 40 in case the code
53  * goes into the inner loop for a maximum of 64k frames which may require
54  * more array entries. We subtract 200 because our default buffer size is
55  * 2048 and the default desc len is likely 1536. While it takes less than 40
56  * vlib buffers for the jumbo frame, it may take twice as much descriptors
57  * for the same jumbo frame. Use 200 for the extra head room.
58  */
59 #define VHOST_USER_TX_COPY_THRESHOLD (VHOST_USER_COPY_ARRAY_N - 200)
60 
62 
63 #define foreach_vhost_user_tx_func_error \
64  _(NONE, "no error") \
65  _(NOT_READY, "vhost vring not ready") \
66  _(DOWN, "vhost interface is down") \
67  _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \
68  _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \
69  _(MMAP_FAIL, "mmap failure") \
70  _(INDIRECT_OVERFLOW, "indirect descriptor table overflow")
71 
72 typedef enum
73 {
74 #define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f,
76 #undef _
79 
80 static __clib_unused char *vhost_user_tx_func_error_strings[] = {
81 #define _(n,s) s,
83 #undef _
84 };
85 
86 static __clib_unused u8 *
87 format_vhost_user_interface_name (u8 * s, va_list * args)
88 {
89  u32 i = va_arg (*args, u32);
90  u32 show_dev_instance = ~0;
92 
94  show_dev_instance = vum->show_dev_instance_by_real_dev_instance[i];
95 
96  if (show_dev_instance != ~0)
97  i = show_dev_instance;
98 
99  s = format (s, "VirtualEthernet0/0/%d", i);
100  return s;
101 }
102 
103 static __clib_unused int
105 {
106  // FIXME: check if the new dev instance is already used
109  hi->dev_instance);
110 
112  hi->dev_instance, ~0);
113 
115  new_dev_instance;
116 
117  vu_log_debug (vui, "renumbered vhost-user interface dev_instance %d to %d",
118  hi->dev_instance, new_dev_instance);
119 
120  return 0;
121 }
122 
123 /**
124  * @brief Try once to lock the vring
125  * @return 0 on success, non-zero on failure.
126  */
129 {
130  return clib_atomic_test_and_set (vui->vring_locks[qid]);
131 }
132 
133 /**
134  * @brief Spin until the vring is successfully locked
135  */
138 {
139  while (vhost_user_vring_try_lock (vui, qid))
140  ;
141 }
142 
143 /**
144  * @brief Unlock the vring lock
145  */
148 {
149  clib_atomic_release (vui->vring_locks[qid]);
150 }
151 
154  vhost_user_intf_t * vui, u16 qid,
155  vlib_buffer_t * b, vhost_user_vring_t * rxvq)
156 {
158  u32 last_avail_idx = rxvq->last_avail_idx;
159  u32 desc_current = rxvq->avail->ring[last_avail_idx & rxvq->qsz_mask];
160  vring_desc_t *hdr_desc = 0;
161  u32 hint = 0;
162 
163  clib_memset (t, 0, sizeof (*t));
164  t->device_index = vui - vum->vhost_user_interfaces;
165  t->qid = qid;
166 
167  hdr_desc = &rxvq->desc[desc_current];
168  if (rxvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT)
169  {
170  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
171  /* Header is the first here */
172  hdr_desc = map_guest_mem (vui, rxvq->desc[desc_current].addr, &hint);
173  }
174  if (rxvq->desc[desc_current].flags & VRING_DESC_F_NEXT)
175  {
176  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
177  }
178  if (!(rxvq->desc[desc_current].flags & VRING_DESC_F_NEXT) &&
179  !(rxvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT))
180  {
181  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
182  }
183 
184  t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
185 }
186 
189  u16 copy_len, u32 * map_hint)
190 {
191  void *dst0, *dst1, *dst2, *dst3;
192  if (PREDICT_TRUE (copy_len >= 4))
193  {
194  if (PREDICT_FALSE (!(dst2 = map_guest_mem (vui, cpy[0].dst, map_hint))))
195  return 1;
196  if (PREDICT_FALSE (!(dst3 = map_guest_mem (vui, cpy[1].dst, map_hint))))
197  return 1;
198  while (PREDICT_TRUE (copy_len >= 4))
199  {
200  dst0 = dst2;
201  dst1 = dst3;
202 
203  if (PREDICT_FALSE
204  (!(dst2 = map_guest_mem (vui, cpy[2].dst, map_hint))))
205  return 1;
206  if (PREDICT_FALSE
207  (!(dst3 = map_guest_mem (vui, cpy[3].dst, map_hint))))
208  return 1;
209 
210  CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD);
211  CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD);
212 
213  clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len);
214  clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len);
215 
216  vhost_user_log_dirty_pages_2 (vui, cpy[0].dst, cpy[0].len, 1);
217  vhost_user_log_dirty_pages_2 (vui, cpy[1].dst, cpy[1].len, 1);
218  copy_len -= 2;
219  cpy += 2;
220  }
221  }
222  while (copy_len)
223  {
224  if (PREDICT_FALSE (!(dst0 = map_guest_mem (vui, cpy->dst, map_hint))))
225  return 1;
226  clib_memcpy_fast (dst0, (void *) cpy->src, cpy->len);
227  vhost_user_log_dirty_pages_2 (vui, cpy->dst, cpy->len, 1);
228  copy_len -= 1;
229  cpy += 1;
230  }
231  return 0;
232 }
233 
236  virtio_net_hdr_t * hdr)
237 {
238  generic_header_offset_t gho = { 0 };
239  int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
240  int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
241 
242  ASSERT (!(is_ip4 && is_ip6));
243  vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
244  if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
245  {
246  ip4_header_t *ip4;
247 
248  ip4 =
250  ip4->checksum = ip4_header_checksum (ip4);
251  }
252 
253  /* checksum offload */
254  if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
255  {
256  hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
257  hdr->csum_start = gho.l4_hdr_offset;
258  hdr->csum_offset = offsetof (udp_header_t, checksum);
259  }
260  else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
261  {
262  hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
263  hdr->csum_start = gho.l4_hdr_offset;
264  hdr->csum_offset = offsetof (tcp_header_t, checksum);
265  }
266 
267  /* GSO offload */
268  if (b->flags & VNET_BUFFER_F_GSO)
269  {
270  if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
271  {
272  if (is_ip4 &&
273  (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
274  {
275  hdr->gso_size = vnet_buffer2 (b)->gso_size;
276  hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
277  }
278  else if (is_ip6 &&
279  (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)))
280  {
281  hdr->gso_size = vnet_buffer2 (b)->gso_size;
282  hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
283  }
284  }
285  else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
286  (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
287  {
288  hdr->gso_size = vnet_buffer2 (b)->gso_size;
289  hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
290  }
291  }
292 }
293 
296  u16 * n_descs_processed, u8 chained,
297  vlib_frame_t * frame, u32 n_left)
298 {
299  u16 desc_idx, flags;
300  vring_packed_desc_t *desc_table = rxvq->packed_desc;
301  u16 last_used_idx = rxvq->last_used_idx;
302 
303  if (PREDICT_FALSE (*n_descs_processed == 0))
304  return;
305 
306  if (rxvq->used_wrap_counter)
307  flags = desc_table[last_used_idx & rxvq->qsz_mask].flags |
309  else
310  flags = desc_table[last_used_idx & rxvq->qsz_mask].flags &
312 
314 
315  for (desc_idx = 1; desc_idx < *n_descs_processed; desc_idx++)
316  {
317  if (rxvq->used_wrap_counter)
318  desc_table[rxvq->last_used_idx & rxvq->qsz_mask].flags |=
320  else
321  desc_table[rxvq->last_used_idx & rxvq->qsz_mask].flags &=
324  }
325 
326  desc_table[last_used_idx & rxvq->qsz_mask].flags = flags;
327 
328  *n_descs_processed = 0;
329 
330  if (chained)
331  {
332  vring_packed_desc_t *desc_table = rxvq->packed_desc;
333 
334  while (desc_table[rxvq->last_used_idx & rxvq->qsz_mask].flags &
337 
338  /* Advance past the current chained table entries */
340  }
341 
342  /* interrupt (call) handling */
343  if ((rxvq->callfd_idx != ~0) &&
344  (rxvq->avail_event->flags != VRING_EVENT_F_DISABLE))
345  {
347 
348  rxvq->n_since_last_int += frame->n_vectors - n_left;
349  if (rxvq->n_since_last_int > vum->coalesce_frames)
350  vhost_user_send_call (vm, rxvq);
351  }
352 }
353 
356  u16 qid, vlib_buffer_t * b,
357  vhost_user_vring_t * rxvq)
358 {
360  u32 last_avail_idx = rxvq->last_avail_idx;
361  u32 desc_current = last_avail_idx & rxvq->qsz_mask;
362  vring_packed_desc_t *hdr_desc = 0;
363  u32 hint = 0;
364 
365  clib_memset (t, 0, sizeof (*t));
366  t->device_index = vui - vum->vhost_user_interfaces;
367  t->qid = qid;
368 
369  hdr_desc = &rxvq->packed_desc[desc_current];
370  if (rxvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT)
371  {
372  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
373  /* Header is the first here */
374  hdr_desc = map_guest_mem (vui, rxvq->packed_desc[desc_current].addr,
375  &hint);
376  }
377  if (rxvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT)
378  {
379  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
380  }
381  if (!(rxvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT) &&
382  !(rxvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT))
383  {
384  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
385  }
386 
387  t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
388 }
389 
393 {
394  u32 *buffers = vlib_frame_vector_args (frame);
395  u32 n_left = frame->n_vectors;
397  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
398  vhost_user_intf_t *vui =
400  u32 qid;
401  vhost_user_vring_t *rxvq;
402  u8 error;
403  u32 thread_index = vm->thread_index;
404  vhost_cpu_t *cpu = &vum->cpus[thread_index];
405  u32 map_hint = 0;
406  u8 retry = 8;
407  u16 copy_len;
408  u16 tx_headers_len;
409  vring_packed_desc_t *desc_table;
410  u32 or_flags;
411  u16 desc_head, desc_index, desc_len;
412  u16 n_descs_processed;
413  u8 indirect, chained;
414 
415  qid = VHOST_VRING_IDX_RX (*vec_elt_at_index (vui->per_cpu_tx_qid,
416  thread_index));
417  rxvq = &vui->vrings[qid];
418 
419 retry:
420  error = VHOST_USER_TX_FUNC_ERROR_NONE;
421  tx_headers_len = 0;
422  copy_len = 0;
423  n_descs_processed = 0;
424 
425  while (n_left > 0)
426  {
427  vlib_buffer_t *b0, *current_b0;
428  uword buffer_map_addr;
429  u32 buffer_len;
430  u16 bytes_left;
431  u32 total_desc_len = 0;
432  u16 n_entries = 0;
433 
434  indirect = 0;
435  chained = 0;
436  if (PREDICT_TRUE (n_left > 1))
437  vlib_prefetch_buffer_with_index (vm, buffers[1], LOAD);
438 
439  b0 = vlib_get_buffer (vm, buffers[0]);
440  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
441  {
442  cpu->current_trace = vlib_add_trace (vm, node, b0,
443  sizeof (*cpu->current_trace));
444  vhost_user_tx_trace_packed (cpu->current_trace, vui, qid / 2, b0,
445  rxvq);
446  }
447 
448  desc_table = rxvq->packed_desc;
449  desc_head = desc_index = rxvq->last_avail_idx & rxvq->qsz_mask;
450  if (PREDICT_FALSE (!vhost_user_packed_desc_available (rxvq, desc_head)))
451  {
452  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
453  goto done;
454  }
455  /*
456  * Go deeper in case of indirect descriptor.
457  * To test it, turn off mrg_rxbuf.
458  */
459  if (desc_table[desc_head].flags & VRING_DESC_F_INDIRECT)
460  {
461  indirect = 1;
462  if (PREDICT_FALSE (desc_table[desc_head].len <
463  sizeof (vring_packed_desc_t)))
464  {
465  error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
466  goto done;
467  }
468  n_entries = desc_table[desc_head].len >> 4;
469  desc_table = map_guest_mem (vui, desc_table[desc_index].addr,
470  &map_hint);
471  if (PREDICT_FALSE (desc_table == 0))
472  {
473  error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
474  goto done;
475  }
476  desc_index = 0;
477  }
478  else if (rxvq->packed_desc[desc_head].flags & VRING_DESC_F_NEXT)
479  chained = 1;
480 
481  desc_len = vui->virtio_net_hdr_sz;
482  buffer_map_addr = desc_table[desc_index].addr;
483  buffer_len = desc_table[desc_index].len;
484 
485  /* Get a header from the header array */
486  virtio_net_hdr_mrg_rxbuf_t *hdr = &cpu->tx_headers[tx_headers_len];
487  tx_headers_len++;
488  hdr->hdr.flags = 0;
489  hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
490  hdr->num_buffers = 1;
491 
492  or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
493  (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
494  (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
495 
496  /* Guest supports csum offload and buffer requires checksum offload? */
497  if (or_flags &&
498  (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)))
499  vhost_user_handle_tx_offload (vui, b0, &hdr->hdr);
500 
501  /* Prepare a copy order executed later for the header */
502  ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N);
503  vhost_copy_t *cpy = &cpu->copy[copy_len];
504  copy_len++;
505  cpy->len = vui->virtio_net_hdr_sz;
506  cpy->dst = buffer_map_addr;
507  cpy->src = (uword) hdr;
508 
509  buffer_map_addr += vui->virtio_net_hdr_sz;
510  buffer_len -= vui->virtio_net_hdr_sz;
511  bytes_left = b0->current_length;
512  current_b0 = b0;
513  while (1)
514  {
515  if (buffer_len == 0)
516  {
517  /* Get new output */
518  if (chained)
519  {
520  /*
521  * Next one is chained
522  * Test it with both indirect and mrg_rxbuf off
523  */
524  if (PREDICT_FALSE (!(desc_table[desc_index].flags &
525  VRING_DESC_F_NEXT)))
526  {
527  /*
528  * Last descriptor in chain.
529  * Dequeue queued descriptors for this packet
530  */
532  &n_descs_processed);
533  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
534  goto done;
535  }
537  desc_index = rxvq->last_avail_idx & rxvq->qsz_mask;
538  n_descs_processed++;
539  buffer_map_addr = desc_table[desc_index].addr;
540  buffer_len = desc_table[desc_index].len;
541  total_desc_len += desc_len;
542  desc_len = 0;
543  }
544  else if (indirect)
545  {
546  /*
547  * Indirect table
548  * Test it with mrg_rxnuf off
549  */
550  if (PREDICT_TRUE (n_entries > 0))
551  n_entries--;
552  else
553  {
554  /* Dequeue queued descriptors for this packet */
556  &n_descs_processed);
557  error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
558  goto done;
559  }
560  total_desc_len += desc_len;
561  desc_index = (desc_index + 1) & rxvq->qsz_mask;
562  buffer_map_addr = desc_table[desc_index].addr;
563  buffer_len = desc_table[desc_index].len;
564  desc_len = 0;
565  }
566  else if (vui->virtio_net_hdr_sz == 12)
567  {
568  /*
569  * MRG is available
570  * This is the default setting for the guest VM
571  */
572  virtio_net_hdr_mrg_rxbuf_t *hdr =
573  &cpu->tx_headers[tx_headers_len - 1];
574 
575  desc_table[desc_index].len = desc_len;
577  desc_head = desc_index =
578  rxvq->last_avail_idx & rxvq->qsz_mask;
579  hdr->num_buffers++;
580  n_descs_processed++;
581  desc_len = 0;
582 
584  (rxvq, desc_index)))
585  {
586  /* Dequeue queued descriptors for this packet */
587  vhost_user_dequeue_descs (rxvq, hdr,
588  &n_descs_processed);
589  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
590  goto done;
591  }
592 
593  buffer_map_addr = desc_table[desc_index].addr;
594  buffer_len = desc_table[desc_index].len;
595  }
596  else
597  {
598  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
599  goto done;
600  }
601  }
602 
603  ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N);
604  vhost_copy_t *cpy = &cpu->copy[copy_len];
605  copy_len++;
606  cpy->len = bytes_left;
607  cpy->len = (cpy->len > buffer_len) ? buffer_len : cpy->len;
608  cpy->dst = buffer_map_addr;
609  cpy->src = (uword) vlib_buffer_get_current (current_b0) +
610  current_b0->current_length - bytes_left;
611 
612  bytes_left -= cpy->len;
613  buffer_len -= cpy->len;
614  buffer_map_addr += cpy->len;
615  desc_len += cpy->len;
616 
618 
619  /* Check if vlib buffer has more data. If not, get more or break */
620  if (PREDICT_TRUE (!bytes_left))
621  {
622  if (PREDICT_FALSE
623  (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT))
624  {
625  current_b0 = vlib_get_buffer (vm, current_b0->next_buffer);
626  bytes_left = current_b0->current_length;
627  }
628  else
629  {
630  /* End of packet */
631  break;
632  }
633  }
634  }
635 
636  /* Move from available to used ring */
637  total_desc_len += desc_len;
638  rxvq->packed_desc[desc_head].len = total_desc_len;
639 
640  vhost_user_advance_last_avail_table_idx (vui, rxvq, chained);
641  n_descs_processed++;
642 
643  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
644  cpu->current_trace->hdr = cpu->tx_headers[tx_headers_len - 1];
645 
646  n_left--;
647 
648  /*
649  * Do the copy periodically to prevent
650  * cpu->copy array overflow and corrupt memory
651  */
652  if (PREDICT_FALSE (copy_len >= VHOST_USER_TX_COPY_THRESHOLD) || chained)
653  {
654  if (PREDICT_FALSE (vhost_user_tx_copy (vui, cpu->copy, copy_len,
655  &map_hint)))
656  vlib_error_count (vm, node->node_index,
657  VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
658  copy_len = 0;
659 
660  /* give buffers back to driver */
661  vhost_user_mark_desc_available (vm, rxvq, &n_descs_processed,
662  chained, frame, n_left);
663  }
664 
665  buffers++;
666  }
667 
668 done:
669  if (PREDICT_TRUE (copy_len))
670  {
671  if (PREDICT_FALSE (vhost_user_tx_copy (vui, cpu->copy, copy_len,
672  &map_hint)))
673  vlib_error_count (vm, node->node_index,
674  VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
675 
676  vhost_user_mark_desc_available (vm, rxvq, &n_descs_processed, chained,
677  frame, n_left);
678  }
679 
680  /*
681  * When n_left is set, error is always set to something too.
682  * In case error is due to lack of remaining buffers, we go back up and
683  * retry.
684  * The idea is that it is better to waste some time on packets
685  * that have been processed already than dropping them and get
686  * more fresh packets with a good likelyhood that they will be dropped too.
687  * This technique also gives more time to VM driver to pick-up packets.
688  * In case the traffic flows from physical to virtual interfaces, this
689  * technique will end-up leveraging the physical NIC buffer in order to
690  * absorb the VM's CPU jitter.
691  */
692  if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry)
693  {
694  retry--;
695  goto retry;
696  }
697 
698  vhost_user_vring_unlock (vui, qid);
699 
700  if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
701  {
702  vlib_error_count (vm, node->node_index, error, n_left);
705  VNET_INTERFACE_COUNTER_DROP, thread_index, vui->sw_if_index, n_left);
706  }
707 
708  vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors);
709  return frame->n_vectors;
710 }
711 
715 {
716  u32 *buffers = vlib_frame_vector_args (frame);
717  u32 n_left = frame->n_vectors;
719  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
720  vhost_user_intf_t *vui =
722  u32 qid = ~0;
723  vhost_user_vring_t *rxvq;
724  u8 error;
725  u32 thread_index = vm->thread_index;
726  vhost_cpu_t *cpu = &vum->cpus[thread_index];
727  u32 map_hint = 0;
728  u8 retry = 8;
729  u16 copy_len;
730  u16 tx_headers_len;
731  u32 or_flags;
732 
733  if (PREDICT_FALSE (!vui->admin_up))
734  {
735  error = VHOST_USER_TX_FUNC_ERROR_DOWN;
736  goto done3;
737  }
738 
739  if (PREDICT_FALSE (!vui->is_ready))
740  {
741  error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
742  goto done3;
743  }
744 
745  qid = VHOST_VRING_IDX_RX (*vec_elt_at_index (vui->per_cpu_tx_qid,
746  thread_index));
747  rxvq = &vui->vrings[qid];
748  if (PREDICT_FALSE (rxvq->avail == 0))
749  {
750  error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
751  goto done3;
752  }
753 
754  if (PREDICT_FALSE (vui->use_tx_spinlock))
755  vhost_user_vring_lock (vui, qid);
756 
758  return (vhost_user_device_class_packed (vm, node, frame));
759 
760 retry:
761  error = VHOST_USER_TX_FUNC_ERROR_NONE;
762  tx_headers_len = 0;
763  copy_len = 0;
764  while (n_left > 0)
765  {
766  vlib_buffer_t *b0, *current_b0;
767  u16 desc_head, desc_index, desc_len;
768  vring_desc_t *desc_table;
769  uword buffer_map_addr;
770  u32 buffer_len;
771  u16 bytes_left;
772 
773  if (PREDICT_TRUE (n_left > 1))
774  vlib_prefetch_buffer_with_index (vm, buffers[1], LOAD);
775 
776  b0 = vlib_get_buffer (vm, buffers[0]);
777 
778  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
779  {
780  cpu->current_trace = vlib_add_trace (vm, node, b0,
781  sizeof (*cpu->current_trace));
782  vhost_user_tx_trace (cpu->current_trace, vui, qid / 2, b0, rxvq);
783  }
784 
785  if (PREDICT_FALSE (rxvq->last_avail_idx == rxvq->avail->idx))
786  {
787  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
788  goto done;
789  }
790 
791  desc_table = rxvq->desc;
792  desc_head = desc_index =
793  rxvq->avail->ring[rxvq->last_avail_idx & rxvq->qsz_mask];
794 
795  /* Go deeper in case of indirect descriptor
796  * I don't know of any driver providing indirect for RX. */
797  if (PREDICT_FALSE (rxvq->desc[desc_head].flags & VRING_DESC_F_INDIRECT))
798  {
799  if (PREDICT_FALSE
800  (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
801  {
802  error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
803  goto done;
804  }
805  if (PREDICT_FALSE
806  (!(desc_table =
807  map_guest_mem (vui, rxvq->desc[desc_index].addr,
808  &map_hint))))
809  {
810  error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
811  goto done;
812  }
813  desc_index = 0;
814  }
815 
816  desc_len = vui->virtio_net_hdr_sz;
817  buffer_map_addr = desc_table[desc_index].addr;
818  buffer_len = desc_table[desc_index].len;
819 
820  {
821  // Get a header from the header array
822  virtio_net_hdr_mrg_rxbuf_t *hdr = &cpu->tx_headers[tx_headers_len];
823  tx_headers_len++;
824  hdr->hdr.flags = 0;
825  hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
826  hdr->num_buffers = 1; //This is local, no need to check
827 
828  or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
829  (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
830  (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
831 
832  /* Guest supports csum offload and buffer requires checksum offload? */
833  if (or_flags
834  && (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)))
835  vhost_user_handle_tx_offload (vui, b0, &hdr->hdr);
836 
837  // Prepare a copy order executed later for the header
838  ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N);
839  vhost_copy_t *cpy = &cpu->copy[copy_len];
840  copy_len++;
841  cpy->len = vui->virtio_net_hdr_sz;
842  cpy->dst = buffer_map_addr;
843  cpy->src = (uword) hdr;
844  }
845 
846  buffer_map_addr += vui->virtio_net_hdr_sz;
847  buffer_len -= vui->virtio_net_hdr_sz;
848  bytes_left = b0->current_length;
849  current_b0 = b0;
850  while (1)
851  {
852  if (buffer_len == 0)
853  { //Get new output
854  if (desc_table[desc_index].flags & VRING_DESC_F_NEXT)
855  {
856  //Next one is chained
857  desc_index = desc_table[desc_index].next;
858  buffer_map_addr = desc_table[desc_index].addr;
859  buffer_len = desc_table[desc_index].len;
860  }
861  else if (vui->virtio_net_hdr_sz == 12) //MRG is available
862  {
863  virtio_net_hdr_mrg_rxbuf_t *hdr =
864  &cpu->tx_headers[tx_headers_len - 1];
865 
866  //Move from available to used buffer
867  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].id =
868  desc_head;
869  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].len =
870  desc_len;
871  vhost_user_log_dirty_ring (vui, rxvq,
872  ring[rxvq->last_used_idx &
873  rxvq->qsz_mask]);
874 
875  rxvq->last_avail_idx++;
876  rxvq->last_used_idx++;
877  hdr->num_buffers++;
878  desc_len = 0;
879 
880  if (PREDICT_FALSE
881  (rxvq->last_avail_idx == rxvq->avail->idx))
882  {
883  //Dequeue queued descriptors for this packet
884  rxvq->last_used_idx -= hdr->num_buffers - 1;
885  rxvq->last_avail_idx -= hdr->num_buffers - 1;
886  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
887  goto done;
888  }
889 
890  desc_table = rxvq->desc;
891  desc_head = desc_index =
892  rxvq->avail->ring[rxvq->last_avail_idx & rxvq->qsz_mask];
893  if (PREDICT_FALSE
894  (rxvq->desc[desc_head].flags & VRING_DESC_F_INDIRECT))
895  {
896  //It is seriously unlikely that a driver will put indirect descriptor
897  //after non-indirect descriptor.
898  if (PREDICT_FALSE
899  (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
900  {
901  error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
902  goto done;
903  }
904  if (PREDICT_FALSE
905  (!(desc_table =
906  map_guest_mem (vui,
907  rxvq->desc[desc_index].addr,
908  &map_hint))))
909  {
910  error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
911  goto done;
912  }
913  desc_index = 0;
914  }
915  buffer_map_addr = desc_table[desc_index].addr;
916  buffer_len = desc_table[desc_index].len;
917  }
918  else
919  {
920  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
921  goto done;
922  }
923  }
924 
925  {
926  ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N);
927  vhost_copy_t *cpy = &cpu->copy[copy_len];
928  copy_len++;
929  cpy->len = bytes_left;
930  cpy->len = (cpy->len > buffer_len) ? buffer_len : cpy->len;
931  cpy->dst = buffer_map_addr;
932  cpy->src = (uword) vlib_buffer_get_current (current_b0) +
933  current_b0->current_length - bytes_left;
934 
935  bytes_left -= cpy->len;
936  buffer_len -= cpy->len;
937  buffer_map_addr += cpy->len;
938  desc_len += cpy->len;
939 
940  CLIB_PREFETCH (&rxvq->desc, CLIB_CACHE_LINE_BYTES, LOAD);
941  }
942 
943  // Check if vlib buffer has more data. If not, get more or break.
944  if (PREDICT_TRUE (!bytes_left))
945  {
946  if (PREDICT_FALSE
947  (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT))
948  {
949  current_b0 = vlib_get_buffer (vm, current_b0->next_buffer);
950  bytes_left = current_b0->current_length;
951  }
952  else
953  {
954  //End of packet
955  break;
956  }
957  }
958  }
959 
960  //Move from available to used ring
961  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].id = desc_head;
962  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].len = desc_len;
963  vhost_user_log_dirty_ring (vui, rxvq,
964  ring[rxvq->last_used_idx & rxvq->qsz_mask]);
965  rxvq->last_avail_idx++;
966  rxvq->last_used_idx++;
967 
968  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
969  {
970  cpu->current_trace->hdr = cpu->tx_headers[tx_headers_len - 1];
971  }
972 
973  n_left--; //At the end for error counting when 'goto done' is invoked
974 
975  /*
976  * Do the copy periodically to prevent
977  * cpu->copy array overflow and corrupt memory
978  */
980  {
981  if (PREDICT_FALSE (vhost_user_tx_copy (vui, cpu->copy, copy_len,
982  &map_hint)))
983  {
984  vlib_error_count (vm, node->node_index,
985  VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
986  }
987  copy_len = 0;
988 
989  /* give buffers back to driver */
991  rxvq->used->idx = rxvq->last_used_idx;
992  vhost_user_log_dirty_ring (vui, rxvq, idx);
993  }
994  buffers++;
995  }
996 
997 done:
998  //Do the memory copies
999  if (PREDICT_FALSE (vhost_user_tx_copy (vui, cpu->copy, copy_len,
1000  &map_hint)))
1001  {
1002  vlib_error_count (vm, node->node_index,
1003  VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
1004  }
1005 
1007  rxvq->used->idx = rxvq->last_used_idx;
1008  vhost_user_log_dirty_ring (vui, rxvq, idx);
1009 
1010  /*
1011  * When n_left is set, error is always set to something too.
1012  * In case error is due to lack of remaining buffers, we go back up and
1013  * retry.
1014  * The idea is that it is better to waste some time on packets
1015  * that have been processed already than dropping them and get
1016  * more fresh packets with a good likelihood that they will be dropped too.
1017  * This technique also gives more time to VM driver to pick-up packets.
1018  * In case the traffic flows from physical to virtual interfaces, this
1019  * technique will end-up leveraging the physical NIC buffer in order to
1020  * absorb the VM's CPU jitter.
1021  */
1022  if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry)
1023  {
1024  retry--;
1025  goto retry;
1026  }
1027 
1028  /* interrupt (call) handling */
1029  if ((rxvq->callfd_idx != ~0) &&
1031  {
1032  rxvq->n_since_last_int += frame->n_vectors - n_left;
1033 
1034  if (rxvq->n_since_last_int > vum->coalesce_frames)
1035  vhost_user_send_call (vm, rxvq);
1036  }
1037 
1038  vhost_user_vring_unlock (vui, qid);
1039 
1040 done3:
1041  if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
1042  {
1043  vlib_error_count (vm, node->node_index, error, n_left);
1047  thread_index, vui->sw_if_index, n_left);
1048  }
1049 
1050  vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors);
1051  return frame->n_vectors;
1052 }
1053 
1054 static __clib_unused clib_error_t *
1057 {
1058  vlib_main_t *vm = vnm->vlib_main;
1059  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
1061  vhost_user_intf_t *vui =
1063  vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
1064 
1065  if ((mode == VNET_HW_INTERFACE_RX_MODE_INTERRUPT) ||
1067  {
1068  if (txvq->kickfd_idx == ~0)
1069  {
1070  // We cannot support interrupt mode if the driver opts out
1071  return clib_error_return (0, "Driver does not support interrupt");
1072  }
1074  {
1075  vum->ifq_count++;
1076  // Start the timer if this is the first encounter on interrupt
1077  // interface/queue
1078  if ((vum->ifq_count == 1) &&
1079  (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
1083  }
1084  }
1085  else if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
1086  {
1087  if (((txvq->mode == VNET_HW_INTERFACE_RX_MODE_INTERRUPT) ||
1089  vum->ifq_count)
1090  {
1091  vum->ifq_count--;
1092  // Stop the timer if there is no more interrupt interface/queue
1093  if ((vum->ifq_count == 0) &&
1094  (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
1098  }
1099  }
1100 
1101  txvq->mode = mode;
1104  else if ((mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE) ||
1106  txvq->used->flags = 0;
1107  else
1108  {
1109  vu_log_err (vui, "unhandled mode %d changed for if %d queue %d", mode,
1110  hw_if_index, qid);
1111  return clib_error_return (0, "unsupported");
1112  }
1113 
1114  return 0;
1115 }
1116 
1117 static __clib_unused clib_error_t *
1119  u32 flags)
1120 {
1121  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
1123  vhost_user_intf_t *vui =
1125  u8 link_old, link_new;
1126 
1127  link_old = vui_is_link_up (vui);
1128 
1129  vui->admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1130 
1131  link_new = vui_is_link_up (vui);
1132 
1133  if (link_old != link_new)
1134  vnet_hw_interface_set_flags (vnm, vui->hw_if_index, link_new ?
1136 
1137  return /* no error */ 0;
1138 }
1139 
1140 /* *INDENT-OFF* */
1142  .name = "vhost-user",
1143  .tx_function_n_errors = VHOST_USER_TX_FUNC_N_ERROR,
1144  .tx_function_error_strings = vhost_user_tx_func_error_strings,
1145  .format_device_name = format_vhost_user_interface_name,
1146  .name_renumber = vhost_user_name_renumber,
1147  .admin_up_down_function = vhost_user_interface_admin_up_down,
1148  .rx_mode_change_function = vhost_user_interface_rx_mode_change,
1149  .format_tx_trace = format_vhost_trace,
1150 };
1151 
1152 /* *INDENT-ON* */
1153 
1154 /*
1155  * fd.io coding-style-patch-verification: ON
1156  *
1157  * Local Variables:
1158  * eval: (c-set-style "gnu")
1159  * End:
1160  */
#define VRING_EVENT_F_DISABLE
Definition: virtio_std.h:81
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
static __clib_unused u8 * format_vhost_user_interface_name(u8 *s, va_list *args)
static __clib_unused int vhost_user_name_renumber(vnet_hw_interface_t *hi, u32 new_dev_instance)
static __clib_unused clib_error_t * vhost_user_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
static_always_inline int vhost_user_vring_try_lock(vhost_user_intf_t *vui, u32 qid)
Try once to lock the vring.
vring_desc_t * desc
Definition: vhost_user.h:170
u32 virtio_ring_flags
The device index.
Definition: vhost_user.h:272
virtio_net_hdr_mrg_rxbuf_t hdr
Length of the first data descriptor.
Definition: vhost_user.h:274
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
Definition: vhost_user.h:307
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:103
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:518
#define VIRTIO_NET_HDR_F_NEEDS_CSUM
Definition: virtio_std.h:134
#define vnet_buffer2(b)
Definition: buffer.h:482
vhost_user_tx_func_error_t
#define foreach_vhost_user_tx_func_error
vnet_interface_main_t interface_main
Definition: vnet.h:59
#define PREDICT_TRUE(x)
Definition: clib.h:121
static __clib_unused char * vhost_user_tx_func_error_strings[]
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define VHOST_USER_TX_COPY_THRESHOLD
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
vring_used_elem_t ring[0]
Definition: virtio_std.h:113
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vring_packed_desc_t * packed_desc
Definition: vhost_user.h:171
vhost_copy_t copy[VHOST_USER_COPY_ARRAY_N]
Definition: vhost_user.h:286
vring_avail_t * avail
Definition: vhost_user.h:175
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:249
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
vl_api_address_t src
Definition: gre.api:54
#define VHOST_USER_EVENT_START_TIMER
Definition: vhost_user.h:212
vlib_main_t * vm
Definition: in2out_ed.c:1582
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:78
static_always_inline void vhost_user_advance_last_avail_idx(vhost_user_vring_t *vring)
static_always_inline void vhost_user_vring_lock(vhost_user_intf_t *vui, u32 qid)
Spin until the vring is successfully locked.
struct _vnet_device_class vnet_device_class_t
struct _tcp_header tcp_header_t
vring_used_t * used
Definition: vhost_user.h:180
vhost_vring_addr_t addr
Definition: vhost_user.h:111
vhost_trace_t * current_trace
Definition: vhost_user.h:290
unsigned char u8
Definition: types.h:56
static_always_inline void vhost_user_dequeue_descs(vhost_user_vring_t *rxvq, virtio_net_hdr_mrg_rxbuf_t *hdr, u16 *n_descs_processed)
#define VIRTIO_FEATURE(X)
Definition: virtio_std.h:69
#define vu_log_debug(dev, f,...)
Definition: vhost_user.h:38
vnet_hw_interface_rx_mode
Definition: interface.h:53
#define VRING_AVAIL_F_NO_INTERRUPT
Definition: virtio_std.h:85
#define static_always_inline
Definition: clib.h:108
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:476
static_always_inline u32 vhost_user_tx_copy(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
#define VHOST_VRING_IDX_TX(qid)
Definition: vhost_user.h:27
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
static_always_inline u8 * format_vhost_trace(u8 *s, va_list *va)
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:316
unsigned int u32
Definition: types.h:88
#define clib_atomic_test_and_set(a)
Definition: atomics.h:42
bool is_ip6
Definition: ip.api:43
VNET_DEVICE_CLASS(vhost_user_device_class)
#define VRING_USED_F_NO_NOTIFY
Definition: virtio_std.h:84
#define VHOST_USER_COPY_ARRAY_N
Definition: vhost_user.h:278
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
vlib_main_t * vlib_main
Definition: vnet.h:86
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:1015
#define clib_atomic_release(a)
Definition: atomics.h:43
static_always_inline void vhost_user_tx_trace_packed(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *rxvq)
static_always_inline void vhost_user_dequeue_chained_descs(vhost_user_vring_t *rxvq, u16 *n_descs_processed)
unsigned short u16
Definition: types.h:57
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define PREDICT_FALSE(x)
Definition: clib.h:120
vl_api_ip4_address_t ip4
Definition: one.api:376
vhost_user_main_t vhost_user_main
Definition: vhost_user.c:56
vnet_main_t vnet_main
Definition: misc.c:43
static_always_inline uword vhost_user_device_class_packed(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:880
u32 node_index
Node index.
Definition: node.h:487
vl_api_address_t dst
Definition: gre.api:55
static_always_inline void vhost_user_vring_unlock(vhost_user_intf_t *vui, u32 qid)
Unlock the vring lock.
static_always_inline u8 vui_is_link_up(vhost_user_intf_t *vui)
#define VIRTIO_NET_HDR_GSO_TCPV4
Definition: virtio_std.h:138
vl_api_tunnel_mode_t mode
Definition: gre.api:48
static_always_inline void vhost_user_mark_desc_available(vlib_main_t *vm, vhost_user_vring_t *rxvq, u16 *n_descs_processed, u8 chained, vlib_frame_t *frame, u32 n_left)
u8 len
Definition: ip_types.api:92
#define VHOST_VRING_IDX_RX(qid)
Definition: vhost_user.h:26
vnet_device_class_t vhost_user_device_class
u32 * show_dev_instance_by_real_dev_instance
Definition: vhost_user.h:301
u16 device_index
The interface queue index (Not the virtio vring idx)
Definition: vhost_user.h:271
vhost_user_intf_t * vhost_user_interfaces
Definition: vhost_user.h:300
u16 n_vectors
Definition: node.h:396
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
static __clib_unused clib_error_t * vhost_user_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
static_always_inline u64 vhost_user_is_packed_ring_supported(vhost_user_intf_t *vui)
#define VHOST_USER_EVENT_STOP_TIMER
Definition: vhost_user.h:213
u16 first_desc_len
Runtime queue flags.
Definition: vhost_user.h:273
static_always_inline u8 vhost_user_packed_desc_available(vhost_user_vring_t *vring, u16 idx)
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1582
static_always_inline void vhost_user_advance_last_avail_table_idx(vhost_user_intf_t *vui, vhost_user_vring_t *vring, u8 chained)
#define ASSERT(truth)
#define vu_log_err(dev, f,...)
Definition: vhost_user.h:51
static_always_inline void vhost_user_send_call(vlib_main_t *vm, vhost_user_vring_t *vq)
volatile u32 * vring_locks[VHOST_VRING_MAX_N]
Definition: vhost_user.h:242
vring_desc_event_t * avail_event
Definition: vhost_user.h:176
vlib_node_registration_t vhost_user_send_interrupt_node
(constructor) VLIB_REGISTER_NODE (vhost_user_send_interrupt_node)
Definition: vhost_user.c:53
#define VRING_DESC_F_NEXT
Definition: virtio_std.h:73
vl_api_ip4_address_t hi
Definition: arp.api:37
static_always_inline void vhost_user_tx_trace(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *rxvq)
static_always_inline void vnet_generic_header_offset_parser(vlib_buffer_t *b0, generic_header_offset_t *gho, int is_l2, int is_ip4, int is_ip6)
u16 ring[0]
Definition: virtio_std.h:99
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
#define VIRTIO_NET_HDR_GSO_NONE
Definition: virtio_std.h:137
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
Definition: vhost_user.h:241
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1583
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:498
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:297
static_always_inline void vhost_user_log_dirty_pages_2(vhost_user_intf_t *vui, u64 addr, u64 len, u8 is_host_address)
#define vhost_user_log_dirty_ring(vui, vq, member)
#define VRING_DESC_F_USED
Definition: virtio_std.h:78
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:132
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:577
static_always_inline void vhost_user_advance_last_used_idx(vhost_user_vring_t *vring)
virtio_net_hdr_mrg_rxbuf_t tx_headers[VLIB_FRAME_SIZE]
Definition: vhost_user.h:285
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:556
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define VIRTIO_NET_HDR_GSO_UDP
Definition: virtio_std.h:139
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static u16 ip4_header_checksum(ip4_header_t *i)
Definition: ip4_packet.h:314
#define VRING_DESC_F_INDIRECT
Definition: virtio_std.h:75
#define VIRTIO_NET_HDR_GSO_TCPV6
Definition: virtio_std.h:140
static_always_inline void vhost_user_handle_tx_offload(vhost_user_intf_t *vui, vlib_buffer_t *b, virtio_net_hdr_t *hdr)
#define VRING_DESC_F_AVAIL
Definition: virtio_std.h:77