FD.io VPP  v18.07.1-19-g511ce25
Vector Packet Processing
vhost_user_input.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * vhost-user-input
4  *
5  * Copyright (c) 2014-2018 Cisco and/or its affiliates.
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at:
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *------------------------------------------------------------------
18  */
19 
20 #include <fcntl.h> /* for open */
21 #include <sys/ioctl.h>
22 #include <sys/socket.h>
23 #include <sys/un.h>
24 #include <sys/stat.h>
25 #include <sys/types.h>
26 #include <sys/uio.h> /* for iovec */
27 #include <netinet/in.h>
28 #include <sys/vfs.h>
29 
30 #include <linux/if_arp.h>
31 #include <linux/if_tun.h>
32 
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
35 
36 #include <vnet/ip/ip.h>
37 
38 #include <vnet/ethernet/ethernet.h>
39 #include <vnet/devices/devices.h>
40 #include <vnet/feature/feature.h>
41 
44 
45 /*
46  * When an RX queue is down but active, received packets
47  * must be discarded. This value controls up to how many
48  * packets will be discarded during each round.
49  */
50 #define VHOST_USER_DOWN_DISCARD_COUNT 256
51 
52 /*
53  * When the number of available buffers gets under this threshold,
54  * RX node will start discarding packets.
55  */
56 #define VHOST_USER_RX_BUFFER_STARVATION 32
57 
58 /*
59  * On the receive side, the host should free descriptors as soon
60  * as possible in order to avoid TX drop in the VM.
61  * This value controls the number of copy operations that are stacked
62  * before copy is done for all and descriptors are given back to
63  * the guest.
64  * The value 64 was obtained by testing (48 and 128 were not as good).
65  */
66 #define VHOST_USER_RX_COPY_THRESHOLD 64
67 
69 
70 #define foreach_vhost_user_input_func_error \
71  _(NO_ERROR, "no error") \
72  _(NO_BUFFER, "no available buffer") \
73  _(MMAP_FAIL, "mmap failure") \
74  _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \
75  _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
76  _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")
77 
78 typedef enum
79 {
80 #define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f,
82 #undef _
85 
86 static __clib_unused char *vhost_user_input_func_error_strings[] = {
87 #define _(n,s) s,
89 #undef _
90 };
91 
94  vhost_user_intf_t * vui, u16 qid,
96 {
98  u32 last_avail_idx = txvq->last_avail_idx;
99  u32 desc_current = txvq->avail->ring[last_avail_idx & txvq->qsz_mask];
100  vring_desc_t *hdr_desc = 0;
101  virtio_net_hdr_mrg_rxbuf_t *hdr;
102  u32 hint = 0;
103 
104  memset (t, 0, sizeof (*t));
105  t->device_index = vui - vum->vhost_user_interfaces;
106  t->qid = qid;
107 
108  hdr_desc = &txvq->desc[desc_current];
109  if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
110  {
111  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
112  /* Header is the first here */
113  hdr_desc = map_guest_mem (vui, txvq->desc[desc_current].addr, &hint);
114  }
115  if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT)
116  {
117  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
118  }
119  if (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
120  !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT))
121  {
122  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
123  }
124 
125  t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
126 
127  if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint)))
128  {
129  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR;
130  }
131  else
132  {
133  u32 len = vui->virtio_net_hdr_sz;
134  memcpy (&t->hdr, hdr, len > hdr_desc->len ? hdr_desc->len : len);
135  }
136 }
137 
140  u16 copy_len, u32 * map_hint)
141 {
142  void *src0, *src1, *src2, *src3;
143  if (PREDICT_TRUE (copy_len >= 4))
144  {
145  if (PREDICT_FALSE (!(src2 = map_guest_mem (vui, cpy[0].src, map_hint))))
146  return 1;
147  if (PREDICT_FALSE (!(src3 = map_guest_mem (vui, cpy[1].src, map_hint))))
148  return 1;
149 
150  while (PREDICT_TRUE (copy_len >= 4))
151  {
152  src0 = src2;
153  src1 = src3;
154 
155  if (PREDICT_FALSE
156  (!(src2 = map_guest_mem (vui, cpy[2].src, map_hint))))
157  return 1;
158  if (PREDICT_FALSE
159  (!(src3 = map_guest_mem (vui, cpy[3].src, map_hint))))
160  return 1;
161 
162  CLIB_PREFETCH (src2, 64, LOAD);
163  CLIB_PREFETCH (src3, 64, LOAD);
164 
165  clib_memcpy ((void *) cpy[0].dst, src0, cpy[0].len);
166  clib_memcpy ((void *) cpy[1].dst, src1, cpy[1].len);
167  copy_len -= 2;
168  cpy += 2;
169  }
170  }
171  while (copy_len)
172  {
173  if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint))))
174  return 1;
175  clib_memcpy ((void *) cpy->dst, src0, cpy->len);
176  copy_len -= 1;
177  cpy += 1;
178  }
179  return 0;
180 }
181 
182 /**
183  * Try to discard packets from the tx ring (VPP RX path).
184  * Returns the number of discarded packets.
185  */
188  vhost_user_intf_t * vui,
189  vhost_user_vring_t * txvq, u32 discard_max)
190 {
191  /*
192  * On the RX side, each packet corresponds to one descriptor
193  * (it is the same whether it is a shallow descriptor, chained, or indirect).
194  * Therefore, discarding a packet is like discarding a descriptor.
195  */
196  u32 discarded_packets = 0;
197  u32 avail_idx = txvq->avail->idx;
198  while (discarded_packets != discard_max)
199  {
200  if (avail_idx == txvq->last_avail_idx)
201  goto out;
202 
203  u16 desc_chain_head =
204  txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask];
205  txvq->last_avail_idx++;
206  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id =
207  desc_chain_head;
208  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0;
209  vhost_user_log_dirty_ring (vui, txvq,
210  ring[txvq->last_used_idx & txvq->qsz_mask]);
211  txvq->last_used_idx++;
212  discarded_packets++;
213  }
214 
215 out:
217  txvq->used->idx = txvq->last_used_idx;
218  vhost_user_log_dirty_ring (vui, txvq, idx);
219  return discarded_packets;
220 }
221 
222 /*
223  * In case of overflow, we need to rewind the array of allocated buffers.
224  */
225 static __clib_unused void
227  vhost_cpu_t * cpu, vlib_buffer_t * b_head)
228 {
229  u32 bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
230  vlib_buffer_t *b_current = vlib_get_buffer (vm, bi_current);
231  b_current->current_length = 0;
232  b_current->flags = 0;
233  while (b_current != b_head)
234  {
235  cpu->rx_buffers_len++;
236  bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
237  b_current = vlib_get_buffer (vm, bi_current);
238  b_current->current_length = 0;
239  b_current->flags = 0;
240  }
241  cpu->rx_buffers_len++;
242 }
243 
244 static __clib_unused u32
246  vhost_user_main_t * vum,
247  vhost_user_intf_t * vui,
248  u16 qid, vlib_node_runtime_t * node,
250 {
251  vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
252  u16 n_rx_packets = 0;
253  u32 n_rx_bytes = 0;
254  u16 n_left;
255  u32 n_left_to_next, *to_next;
257  u32 n_trace = vlib_get_trace_count (vm, node);
258  u32 map_hint = 0;
259  u16 thread_index = vm->thread_index;
260  u16 copy_len = 0;
261 
262  /* The descriptor table is not ready yet */
263  if (PREDICT_FALSE (txvq->avail == 0))
264  return 0;
265 
266  {
267  /* do we have pending interrupts ? */
268  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
269  f64 now = vlib_time_now (vm);
270 
271  if ((txvq->n_since_last_int) && (txvq->int_deadline < now))
272  vhost_user_send_call (vm, txvq);
273 
274  if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now))
275  vhost_user_send_call (vm, rxvq);
276  }
277 
278  /*
279  * For adaptive mode, it is optimized to reduce interrupts.
280  * If the scheduler switches the input node to polling due
281  * to burst of traffic, we tell the driver no interrupt.
282  * When the traffic subsides, the scheduler switches the node back to
283  * interrupt mode. We must tell the driver we want interrupt.
284  */
286  {
287  if ((node->flags &
289  !(node->flags &
291  /* Tell driver we want notification */
292  txvq->used->flags = 0;
293  else
294  /* Tell driver we don't want notification */
295  txvq->used->flags = VRING_USED_F_NO_NOTIFY;
296  }
297 
298  if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
299  return 0;
300 
301  n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
302 
303  /* nothing to do */
304  if (PREDICT_FALSE (n_left == 0))
305  return 0;
306 
307  if (PREDICT_FALSE (!vui->admin_up || !(txvq->enabled)))
308  {
309  /*
310  * Discard input packet if interface is admin down or vring is not
311  * enabled.
312  * "For example, for a networking device, in the disabled state
313  * client must not supply any new RX packets, but must process
314  * and discard any TX packets."
315  */
316  vhost_user_rx_discard_packet (vm, vui, txvq,
318  return 0;
319  }
320 
321  if (PREDICT_FALSE (n_left == (txvq->qsz_mask + 1)))
322  {
323  /*
324  * Informational error logging when VPP is not
325  * receiving packets fast enough.
326  */
327  vlib_error_count (vm, node->node_index,
328  VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
329  }
330 
331  if (n_left > VLIB_FRAME_SIZE)
332  n_left = VLIB_FRAME_SIZE;
333 
334  /*
335  * For small packets (<2kB), we will not need more than one vlib buffer
336  * per packet. In case packets are bigger, we will just yeld at some point
337  * in the loop and come back later. This is not an issue as for big packet,
338  * processing cost really comes from the memory copy.
339  * The assumption is that big packets will fit in 40 buffers.
340  */
341  if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len < n_left + 1 ||
342  vum->cpus[thread_index].rx_buffers_len < 40))
343  {
344  u32 curr_len = vum->cpus[thread_index].rx_buffers_len;
345  vum->cpus[thread_index].rx_buffers_len +=
347  vum->cpus[thread_index].rx_buffers +
348  curr_len,
349  VHOST_USER_RX_BUFFERS_N - curr_len,
351 
352  if (PREDICT_FALSE
353  (vum->cpus[thread_index].rx_buffers_len <
355  {
356  /* In case of buffer starvation, discard some packets from the queue
357  * and log the event.
358  * We keep doing best effort for the remaining packets. */
359  u32 flush = (n_left + 1 > vum->cpus[thread_index].rx_buffers_len) ?
360  n_left + 1 - vum->cpus[thread_index].rx_buffers_len : 1;
361  flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush);
362 
363  n_left -= flush;
365  interface_main.sw_if_counters +
368  vui->sw_if_index, flush);
369 
371  VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
372  }
373  }
374 
375  while (n_left > 0)
376  {
377  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
378 
379  while (n_left > 0 && n_left_to_next > 0)
380  {
381  vlib_buffer_t *b_head, *b_current;
382  u32 bi_current;
383  u16 desc_current;
384  u32 desc_data_offset;
385  vring_desc_t *desc_table = txvq->desc;
386 
387  if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len <= 1))
388  {
389  /* Not enough rx_buffers
390  * Note: We yeld on 1 so we don't need to do an additional
391  * check for the next buffer prefetch.
392  */
393  n_left = 0;
394  break;
395  }
396 
397  desc_current =
398  txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask];
399  vum->cpus[thread_index].rx_buffers_len--;
400  bi_current = (vum->cpus[thread_index].rx_buffers)
401  [vum->cpus[thread_index].rx_buffers_len];
402  b_head = b_current = vlib_get_buffer (vm, bi_current);
403  to_next[0] = bi_current; //We do that now so we can forget about bi_current
404  to_next++;
405  n_left_to_next--;
406 
408  (vum->
409  cpus[thread_index].rx_buffers)
410  [vum->cpus[thread_index].
411  rx_buffers_len - 1], LOAD);
412 
413  /* Just preset the used descriptor id and length for later */
414  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id =
415  desc_current;
416  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0;
417  vhost_user_log_dirty_ring (vui, txvq,
418  ring[txvq->last_used_idx &
419  txvq->qsz_mask]);
420 
421  /* The buffer should already be initialized */
423  b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
424 
425  if (PREDICT_FALSE (n_trace))
426  {
427  //TODO: next_index is not exactly known at that point
428  vlib_trace_buffer (vm, node, next_index, b_head,
429  /* follow_chain */ 0);
430  vhost_trace_t *t0 =
431  vlib_add_trace (vm, node, b_head, sizeof (t0[0]));
432  vhost_user_rx_trace (t0, vui, qid, b_head, txvq);
433  n_trace--;
434  vlib_set_trace_count (vm, node, n_trace);
435  }
436 
437  /* This depends on the setup but is very consistent
438  * So I think the CPU branch predictor will make a pretty good job
439  * at optimizing the decision. */
440  if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
441  {
442  desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr,
443  &map_hint);
444  desc_current = 0;
445  if (PREDICT_FALSE (desc_table == 0))
446  {
447  vlib_error_count (vm, node->node_index,
448  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
449  goto out;
450  }
451  }
452 
453  if (PREDICT_TRUE (vui->is_any_layout) ||
454  (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT)))
455  {
456  /* ANYLAYOUT or single buffer */
457  desc_data_offset = vui->virtio_net_hdr_sz;
458  }
459  else
460  {
461  /* CSR case without ANYLAYOUT, skip 1st buffer */
462  desc_data_offset = desc_table[desc_current].len;
463  }
464 
465  while (1)
466  {
467  /* Get more input if necessary. Or end of packet. */
468  if (desc_data_offset == desc_table[desc_current].len)
469  {
470  if (PREDICT_FALSE (desc_table[desc_current].flags &
471  VIRTQ_DESC_F_NEXT))
472  {
473  desc_current = desc_table[desc_current].next;
474  desc_data_offset = 0;
475  }
476  else
477  {
478  goto out;
479  }
480  }
481 
482  /* Get more output if necessary. Or end of packet. */
483  if (PREDICT_FALSE
484  (b_current->current_length == VLIB_BUFFER_DATA_SIZE))
485  {
486  if (PREDICT_FALSE
487  (vum->cpus[thread_index].rx_buffers_len == 0))
488  {
489  /* Cancel speculation */
490  to_next--;
491  n_left_to_next++;
492 
493  /*
494  * Checking if there are some left buffers.
495  * If not, just rewind the used buffers and stop.
496  * Note: Scheduled copies are not cancelled. This is
497  * not an issue as they would still be valid. Useless,
498  * but valid.
499  */
501  &vum->cpus
502  [thread_index],
503  b_head);
504  n_left = 0;
505  goto stop;
506  }
507 
508  /* Get next output */
509  vum->cpus[thread_index].rx_buffers_len--;
510  u32 bi_next =
511  (vum->cpus[thread_index].rx_buffers)[vum->cpus
512  [thread_index].rx_buffers_len];
513  b_current->next_buffer = bi_next;
514  b_current->flags |= VLIB_BUFFER_NEXT_PRESENT;
515  bi_current = bi_next;
516  b_current = vlib_get_buffer (vm, bi_current);
517  }
518 
519  /* Prepare a copy order executed later for the data */
520  vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len];
521  copy_len++;
522  u32 desc_data_l =
523  desc_table[desc_current].len - desc_data_offset;
524  cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length;
525  cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
526  cpy->dst = (uword) (vlib_buffer_get_current (b_current) +
527  b_current->current_length);
528  cpy->src = desc_table[desc_current].addr + desc_data_offset;
529 
530  desc_data_offset += cpy->len;
531 
532  b_current->current_length += cpy->len;
534  }
535 
536  out:
537  CLIB_PREFETCH (&n_left, sizeof (n_left), LOAD);
538 
539  n_rx_bytes += b_head->total_length_not_including_first_buffer;
540  n_rx_packets++;
541 
543  b_head->current_length;
544 
545  /* consume the descriptor and return it as used */
546  txvq->last_avail_idx++;
547  txvq->last_used_idx++;
548 
550 
551  vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
552  vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0;
553  b_head->error = 0;
554 
555  {
557 
558  /* redirect if feature path enabled */
560  b_head);
561 
562  u32 bi = to_next[-1]; //Cannot use to_next[-1] in the macro
563  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
564  to_next, n_left_to_next,
565  bi, next0);
566  }
567 
568  n_left--;
569 
570  /*
571  * Although separating memory copies from virtio ring parsing
572  * is beneficial, we can offer to perform the copies from time
573  * to time in order to free some space in the ring.
574  */
576  {
577  if (PREDICT_FALSE
578  (vhost_user_input_copy (vui, vum->cpus[thread_index].copy,
579  copy_len, &map_hint)))
580  {
581  vlib_error_count (vm, node->node_index,
582  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
583  }
584  copy_len = 0;
585 
586  /* give buffers back to driver */
588  txvq->used->idx = txvq->last_used_idx;
589  vhost_user_log_dirty_ring (vui, txvq, idx);
590  }
591  }
592  stop:
593  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
594  }
595 
596  /* Do the memory copies */
597  if (PREDICT_FALSE
598  (vhost_user_input_copy (vui, vum->cpus[thread_index].copy,
599  copy_len, &map_hint)))
600  {
601  vlib_error_count (vm, node->node_index,
602  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
603  }
604 
605  /* give buffers back to driver */
607  txvq->used->idx = txvq->last_used_idx;
608  vhost_user_log_dirty_ring (vui, txvq, idx);
609 
610  /* interrupt (call) handling */
611  if ((txvq->callfd_idx != ~0) &&
612  !(txvq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
613  {
614  txvq->n_since_last_int += n_rx_packets;
615 
616  if (txvq->n_since_last_int > vum->coalesce_frames)
617  vhost_user_send_call (vm, txvq);
618  }
619 
620  /* increase rx counters */
624  vlib_get_thread_index (), vui->sw_if_index, n_rx_packets, n_rx_bytes);
625 
626  vnet_device_increment_rx_packets (thread_index, n_rx_packets);
627 
628  return n_rx_packets;
629 }
630 
632  vlib_node_runtime_t * node,
633  vlib_frame_t * frame)
634 {
636  uword n_rx_packets = 0;
637  vhost_user_intf_t *vui;
639  (vnet_device_input_runtime_t *) node->runtime_data;
641 
643  {
644  if (clib_smp_swap (&dq->interrupt_pending, 0) ||
645  (node->state == VLIB_NODE_STATE_POLLING))
646  {
647  vui =
648  pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance);
649  n_rx_packets = vhost_user_if_input (vm, vum, vui, dq->queue_id, node,
650  dq->mode);
651  }
652  }
653 
654  return n_rx_packets;
655 }
656 
657 #ifndef CLIB_MARCH_VARIANT
658 /* *INDENT-OFF* */
660  .type = VLIB_NODE_TYPE_INPUT,
661  .name = "vhost-user-input",
662  .sibling_of = "device-input",
663 
664  /* Will be enabled if/when hardware is detected. */
665  .state = VLIB_NODE_STATE_DISABLED,
666 
667  .format_buffer = format_ethernet_header_with_length,
668  .format_trace = format_vhost_trace,
669 
670  .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR,
671  .error_strings = vhost_user_input_func_error_strings,
672 };
673 /* *INDENT-ON* */
674 #endif
675 
676 /*
677  * fd.io coding-style-patch-verification: ON
678  *
679  * Local Variables:
680  * eval: (c-set-style "gnu")
681  * End:
682  */
static void vnet_device_increment_rx_packets(u32 thread_index, u64 count)
Definition: devices.h:110
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
vring_desc_t * desc
Definition: vhost_user.h:232
u32 virtio_ring_flags
The device index.
Definition: vhost_user.h:312
virtio_net_hdr_mrg_rxbuf_t hdr
Length of the first data descriptor.
Definition: vhost_user.h:314
vhost_user_input_func_error_t
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:143
#define clib_smp_swap(addr, new)
Definition: smp.h:45
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
Definition: vhost_user.h:345
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:213
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define PREDICT_TRUE(x)
Definition: clib.h:106
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:228
vhost_copy_t copy[VHOST_USER_COPY_ARRAY_N]
Definition: vhost_user.h:327
vring_avail_t * avail
Definition: vhost_user.h:233
u32 thread_index
Definition: main.h:179
#define foreach_vhost_user_input_func_error
#define VRING_AVAIL_F_NO_INTERRUPT
Definition: vhost_user.h:46
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:79
#define VHOST_USER_DOWN_DISCARD_COUNT
#define VLIB_NODE_FN(node)
Definition: node.h:173
static_always_inline u32 vhost_user_rx_discard_packet(vlib_main_t *vm, vhost_user_intf_t *vui, vhost_user_vring_t *txvq, u32 discard_max)
Try to discard packets from the tx ring (VPP RX path).
vring_used_t * used
Definition: vhost_user.h:234
static __clib_unused void vhost_user_input_rewind_buffers(vlib_main_t *vm, vhost_cpu_t *cpu, vlib_buffer_t *b_head)
double f64
Definition: types.h:142
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:104
vnet_hw_interface_rx_mode
Definition: interface.h:51
#define static_always_inline
Definition: clib.h:93
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:324
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:811
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
#define VHOST_VRING_IDX_TX(qid)
Definition: vhost_user.h:24
#define VRING_USED_F_NO_NOTIFY
Definition: vhost_user.h:45
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:297
static_always_inline u8 * format_vhost_trace(u8 *s, va_list *va)
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:364
static_always_inline void vhost_user_rx_trace(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *txvq)
vlib_node_registration_t vhost_user_input_node
(constructor) VLIB_REGISTER_NODE (vhost_user_input_node)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
#define VHOST_USER_RX_BUFFER_STARVATION
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
unsigned short u16
Definition: types.h:57
#define VIRTQ_DESC_F_INDIRECT
Definition: vhost_user.h:28
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:202
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:439
#define PREDICT_FALSE(x)
Definition: clib.h:105
vhost_user_main_t vhost_user_main
Definition: vhost_user.c:56
vnet_main_t vnet_main
Definition: misc.c:43
u32 node_index
Node index.
Definition: node.h:473
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:135
u32 flags
Definition: vhost_user.h:110
#define VHOST_VRING_IDX_RX(qid)
Definition: vhost_user.h:23
u8 * format_ethernet_header_with_length(u8 *s, va_list *args)
Definition: format.c:91
u16 device_index
The interface queue index (Not the virtio vring idx)
Definition: vhost_user.h:311
vhost_user_intf_t * vhost_user_interfaces
Definition: vhost_user.h:338
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:77
vlib_main_t * vm
Definition: buffer.c:294
#define clib_memcpy(a, b, c)
Definition: string.h:75
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
u16 first_desc_len
Runtime queue flags.
Definition: vhost_user.h:313
#define VLIB_BUFFER_DATA_SIZE
Definition: buffer.h:51
u32 rx_buffers[VHOST_USER_RX_BUFFERS_N]
Definition: vhost_user.h:324
static __clib_unused char * vhost_user_input_func_error_strings[]
#define VHOST_USER_RX_BUFFERS_N
Definition: vhost_user.h:318
#define VIRTQ_DESC_F_NEXT
Definition: vhost_user.h:27
static_always_inline void vhost_user_send_call(vlib_main_t *vm, vhost_user_vring_t *vq)
static_always_inline u32 vhost_user_input_copy(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, vlib_buffer_free_list_index_t index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:431
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
static __clib_unused u32 vhost_user_if_input(vlib_main_t *vm, vhost_user_main_t *vum, vhost_user_intf_t *vui, u16 qid, vlib_node_runtime_t *node, vnet_hw_interface_rx_mode mode)
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:152
#define VHOST_USER_RX_COPY_THRESHOLD
struct _vlib_node_registration vlib_node_registration_t
Definition: defs.h:47
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
Definition: vhost_user.h:287
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:546
u64 uword
Definition: types.h:112
#define vhost_user_log_dirty_ring(vui, vq, member)
u32 rx_buffers_len
Definition: vhost_user.h:323
#define vnet_buffer(b)
Definition: buffer.h:360
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:250
#define vec_foreach(var, vec)
Vector iterator.
u16 flags
Copy of main node flags.
Definition: node.h:486
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:109
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:159
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:298
Definition: defs.h:46