FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
vhost_user_input.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * vhost-user-input
4  *
5  * Copyright (c) 2014-2018 Cisco and/or its affiliates.
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at:
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *------------------------------------------------------------------
18  */
19 
20 #include <fcntl.h> /* for open */
21 #include <sys/ioctl.h>
22 #include <sys/socket.h>
23 #include <sys/un.h>
24 #include <sys/stat.h>
25 #include <sys/types.h>
26 #include <sys/uio.h> /* for iovec */
27 #include <netinet/in.h>
28 #include <sys/vfs.h>
29 
30 #include <linux/if_arp.h>
31 #include <linux/if_tun.h>
32 
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
35 
36 #include <vnet/ip/ip.h>
37 
38 #include <vnet/ethernet/ethernet.h>
39 #include <vnet/devices/devices.h>
40 #include <vnet/feature/feature.h>
41 
44 
45 /*
46  * When an RX queue is down but active, received packets
47  * must be discarded. This value controls up to how many
48  * packets will be discarded during each round.
49  */
50 #define VHOST_USER_DOWN_DISCARD_COUNT 256
51 
52 /*
53  * When the number of available buffers gets under this threshold,
54  * RX node will start discarding packets.
55  */
56 #define VHOST_USER_RX_BUFFER_STARVATION 32
57 
58 /*
59  * On the receive side, the host should free descriptors as soon
60  * as possible in order to avoid TX drop in the VM.
61  * This value controls the number of copy operations that are stacked
62  * before copy is done for all and descriptors are given back to
63  * the guest.
64  * The value 64 was obtained by testing (48 and 128 were not as good).
65  */
66 #define VHOST_USER_RX_COPY_THRESHOLD 64
67 
69 
70 #define foreach_vhost_user_input_func_error \
71  _(NO_ERROR, "no error") \
72  _(NO_BUFFER, "no available buffer") \
73  _(MMAP_FAIL, "mmap failure") \
74  _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \
75  _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
76  _(NOT_READY, "vhost interface not ready or down") \
77  _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")
78 
79 typedef enum
80 {
81 #define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f,
83 #undef _
86 
87 static __clib_unused char *vhost_user_input_func_error_strings[] = {
88 #define _(n,s) s,
90 #undef _
91 };
92 
95  vhost_user_intf_t * vui, u16 qid,
97  u16 last_avail_idx)
98 {
100  u32 desc_current = txvq->avail->ring[last_avail_idx & txvq->qsz_mask];
101  vring_desc_t *hdr_desc = 0;
102  virtio_net_hdr_mrg_rxbuf_t *hdr;
103  u32 hint = 0;
104 
105  clib_memset (t, 0, sizeof (*t));
106  t->device_index = vui - vum->vhost_user_interfaces;
107  t->qid = qid;
108 
109  hdr_desc = &txvq->desc[desc_current];
110  if (txvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT)
111  {
112  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
113  /* Header is the first here */
114  hdr_desc = map_guest_mem (vui, txvq->desc[desc_current].addr, &hint);
115  }
116  if (txvq->desc[desc_current].flags & VRING_DESC_F_NEXT)
117  {
118  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
119  }
120  if (!(txvq->desc[desc_current].flags & VRING_DESC_F_NEXT) &&
121  !(txvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT))
122  {
123  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
124  }
125 
126  t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
127 
128  if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint)))
129  {
130  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR;
131  }
132  else
133  {
134  u32 len = vui->virtio_net_hdr_sz;
135  memcpy (&t->hdr, hdr, len > hdr_desc->len ? hdr_desc->len : len);
136  }
137 }
138 
141  u16 copy_len, u32 * map_hint)
142 {
143  void *src0, *src1, *src2, *src3;
144  if (PREDICT_TRUE (copy_len >= 4))
145  {
146  if (PREDICT_FALSE (!(src2 = map_guest_mem (vui, cpy[0].src, map_hint))))
147  return 1;
148  if (PREDICT_FALSE (!(src3 = map_guest_mem (vui, cpy[1].src, map_hint))))
149  return 1;
150 
151  while (PREDICT_TRUE (copy_len >= 4))
152  {
153  src0 = src2;
154  src1 = src3;
155 
156  if (PREDICT_FALSE
157  (!(src2 = map_guest_mem (vui, cpy[2].src, map_hint))))
158  return 1;
159  if (PREDICT_FALSE
160  (!(src3 = map_guest_mem (vui, cpy[3].src, map_hint))))
161  return 1;
162 
163  CLIB_PREFETCH (src2, 64, LOAD);
164  CLIB_PREFETCH (src3, 64, LOAD);
165 
166  clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len);
167  clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len);
168  copy_len -= 2;
169  cpy += 2;
170  }
171  }
172  while (copy_len)
173  {
174  if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint))))
175  return 1;
176  clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len);
177  copy_len -= 1;
178  cpy += 1;
179  }
180  return 0;
181 }
182 
183 /**
184  * Try to discard packets from the tx ring (VPP RX path).
185  * Returns the number of discarded packets.
186  */
189  vhost_user_intf_t * vui,
190  vhost_user_vring_t * txvq, u32 discard_max)
191 {
192  /*
193  * On the RX side, each packet corresponds to one descriptor
194  * (it is the same whether it is a shallow descriptor, chained, or indirect).
195  * Therefore, discarding a packet is like discarding a descriptor.
196  */
197  u32 discarded_packets = 0;
198  u32 avail_idx = txvq->avail->idx;
199  u16 mask = txvq->qsz_mask;
200  u16 last_avail_idx = txvq->last_avail_idx;
201  u16 last_used_idx = txvq->last_used_idx;
202  while (discarded_packets != discard_max)
203  {
204  if (avail_idx == last_avail_idx)
205  goto out;
206 
207  u16 desc_chain_head = txvq->avail->ring[last_avail_idx & mask];
208  last_avail_idx++;
209  txvq->used->ring[last_used_idx & mask].id = desc_chain_head;
210  txvq->used->ring[last_used_idx & mask].len = 0;
211  vhost_user_log_dirty_ring (vui, txvq, ring[last_used_idx & mask]);
212  last_used_idx++;
213  discarded_packets++;
214  }
215 
216 out:
217  txvq->last_avail_idx = last_avail_idx;
218  txvq->last_used_idx = last_used_idx;
220  txvq->used->idx = txvq->last_used_idx;
221  vhost_user_log_dirty_ring (vui, txvq, idx);
222  return discarded_packets;
223 }
224 
225 /*
226  * In case of overflow, we need to rewind the array of allocated buffers.
227  */
230  vhost_cpu_t * cpu, vlib_buffer_t * b_head)
231 {
232  u32 bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
233  vlib_buffer_t *b_current = vlib_get_buffer (vm, bi_current);
234  b_current->current_length = 0;
235  b_current->flags = 0;
236  while (b_current != b_head)
237  {
238  cpu->rx_buffers_len++;
239  bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
240  b_current = vlib_get_buffer (vm, bi_current);
241  b_current->current_length = 0;
242  b_current->flags = 0;
243  }
244  cpu->rx_buffers_len++;
245 }
246 
249  virtio_net_hdr_t * hdr)
250 {
251  u8 l4_hdr_sz = 0;
252  u8 l4_proto = 0;
253  ethernet_header_t *eh = (ethernet_header_t *) b0_data;
254  u16 ethertype = clib_net_to_host_u16 (eh->type);
255  u16 l2hdr_sz = sizeof (ethernet_header_t);
256 
257  if (ethernet_frame_is_tagged (ethertype))
258  {
259  ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
260 
261  ethertype = clib_net_to_host_u16 (vlan->type);
262  l2hdr_sz += sizeof (*vlan);
263  if (ethertype == ETHERNET_TYPE_VLAN)
264  {
265  vlan++;
266  ethertype = clib_net_to_host_u16 (vlan->type);
267  l2hdr_sz += sizeof (*vlan);
268  }
269  }
270  vnet_buffer (b0)->l2_hdr_offset = 0;
271  vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz;
272  vnet_buffer (b0)->l4_hdr_offset = hdr->csum_start;
273  b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
274  VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
275  VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
276 
277  if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4))
278  {
279  ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
280  l4_proto = ip4->protocol;
281  b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
282  }
283  else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
284  {
285  ip6_header_t *ip6 = (ip6_header_t *) (b0_data + l2hdr_sz);
286  l4_proto = ip6->protocol;
287  b0->flags |= VNET_BUFFER_F_IS_IP6;
288  }
289 
290  if (l4_proto == IP_PROTOCOL_TCP)
291  {
292  tcp_header_t *tcp = (tcp_header_t *)
293  (b0_data + vnet_buffer (b0)->l4_hdr_offset);
294  l4_hdr_sz = tcp_header_bytes (tcp);
295  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
296  }
297  else if (l4_proto == IP_PROTOCOL_UDP)
298  {
299  l4_hdr_sz = sizeof (udp_header_t);
300  b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
301  }
302 
303  if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
304  {
305  vnet_buffer2 (b0)->gso_size = hdr->gso_size;
306  vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
307  b0->flags |= VNET_BUFFER_F_GSO;
308  }
309  else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4)
310  {
311  vnet_buffer2 (b0)->gso_size = hdr->gso_size;
312  vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
313  b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
314  }
315  else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV6)
316  {
317  vnet_buffer2 (b0)->gso_size = hdr->gso_size;
318  vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
319  b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
320  }
321 }
322 
325  vhost_user_vring_t * rxvq)
326 {
327  f64 now = vlib_time_now (vm);
328 
329  if ((txvq->n_since_last_int) && (txvq->int_deadline < now))
330  vhost_user_send_call (vm, txvq);
331 
332  if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now))
333  vhost_user_send_call (vm, rxvq);
334 }
335 
338  vhost_user_intf_t * vui,
339  u32 * current_config_index, u32 * next_index,
340  u32 ** to_next, u32 * n_left_to_next)
341 {
343  u8 feature_arc_idx = fm->device_input_feature_arc_index;
344 
345  if (PREDICT_FALSE (vnet_have_features (feature_arc_idx, vui->sw_if_index)))
346  {
348  cm = &fm->feature_config_mains[feature_arc_idx];
349  *current_config_index = vec_elt (cm->config_index_by_sw_if_index,
350  vui->sw_if_index);
351  vnet_get_config_data (&cm->config_main, current_config_index,
352  next_index, 0);
353  }
354 
355  vlib_get_new_next_frame (vm, node, *next_index, *to_next, *n_left_to_next);
356 
357  if (*next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
358  {
359  /* give some hints to ethernet-input */
360  vlib_next_frame_t *nf;
361  vlib_frame_t *f;
363  nf = vlib_node_runtime_get_next_frame (vm, node, *next_index);
364  f = vlib_get_frame (vm, nf->frame);
366 
367  ef = vlib_frame_scalar_args (f);
368  ef->sw_if_index = vui->sw_if_index;
369  ef->hw_if_index = vui->hw_if_index;
371  }
372 }
373 
376  vhost_user_main_t * vum,
377  vhost_user_intf_t * vui,
379  vnet_hw_interface_rx_mode mode, u8 enable_csum)
380 {
381  vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
383  u16 n_rx_packets = 0;
384  u32 n_rx_bytes = 0;
385  u16 n_left;
386  u32 n_left_to_next, *to_next;
388  u32 n_trace = vlib_get_trace_count (vm, node);
389  u32 buffer_data_size = vlib_buffer_get_default_data_size (vm);
390  u32 map_hint = 0;
391  vhost_cpu_t *cpu = &vum->cpus[vm->thread_index];
392  u16 copy_len = 0;
393  u8 feature_arc_idx = fm->device_input_feature_arc_index;
394  u32 current_config_index = ~(u32) 0;
395  u16 mask = txvq->qsz_mask;
396 
397  /* The descriptor table is not ready yet */
398  if (PREDICT_FALSE (txvq->avail == 0))
399  goto done;
400 
401  {
402  /* do we have pending interrupts ? */
403  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
404  vhost_user_input_do_interrupt (vm, txvq, rxvq);
405  }
406 
407  /*
408  * For adaptive mode, it is optimized to reduce interrupts.
409  * If the scheduler switches the input node to polling due
410  * to burst of traffic, we tell the driver no interrupt.
411  * When the traffic subsides, the scheduler switches the node back to
412  * interrupt mode. We must tell the driver we want interrupt.
413  */
415  {
416  if ((node->flags &
418  !(node->flags &
420  /* Tell driver we want notification */
421  txvq->used->flags = 0;
422  else
423  /* Tell driver we don't want notification */
425  }
426 
427  if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
428  goto done;
429 
430  n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
431 
432  /* nothing to do */
433  if (PREDICT_FALSE (n_left == 0))
434  goto done;
435 
436  if (PREDICT_FALSE (!vui->admin_up || !(txvq->enabled)))
437  {
438  /*
439  * Discard input packet if interface is admin down or vring is not
440  * enabled.
441  * "For example, for a networking device, in the disabled state
442  * client must not supply any new RX packets, but must process
443  * and discard any TX packets."
444  */
445  vhost_user_rx_discard_packet (vm, vui, txvq,
447  goto done;
448  }
449 
450  if (PREDICT_FALSE (n_left == (mask + 1)))
451  {
452  /*
453  * Informational error logging when VPP is not
454  * receiving packets fast enough.
455  */
456  vlib_error_count (vm, node->node_index,
457  VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
458  }
459 
460  if (n_left > VLIB_FRAME_SIZE)
461  n_left = VLIB_FRAME_SIZE;
462 
463  /*
464  * For small packets (<2kB), we will not need more than one vlib buffer
465  * per packet. In case packets are bigger, we will just yield at some point
466  * in the loop and come back later. This is not an issue as for big packet,
467  * processing cost really comes from the memory copy.
468  * The assumption is that big packets will fit in 40 buffers.
469  */
470  if (PREDICT_FALSE (cpu->rx_buffers_len < n_left + 1 ||
471  cpu->rx_buffers_len < 40))
472  {
473  u32 curr_len = cpu->rx_buffers_len;
474  cpu->rx_buffers_len +=
475  vlib_buffer_alloc (vm, cpu->rx_buffers + curr_len,
476  VHOST_USER_RX_BUFFERS_N - curr_len);
477 
478  if (PREDICT_FALSE
480  {
481  /* In case of buffer starvation, discard some packets from the queue
482  * and log the event.
483  * We keep doing best effort for the remaining packets. */
484  u32 flush = (n_left + 1 > cpu->rx_buffers_len) ?
485  n_left + 1 - cpu->rx_buffers_len : 1;
486  flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush);
487 
488  n_left -= flush;
490  interface_main.sw_if_counters +
492  vm->thread_index, vui->sw_if_index,
493  flush);
494 
496  VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
497  }
498  }
499 
500  vhost_user_input_setup_frame (vm, node, vui, &current_config_index,
501  &next_index, &to_next, &n_left_to_next);
502 
503  u16 last_avail_idx = txvq->last_avail_idx;
504  u16 last_used_idx = txvq->last_used_idx;
505 
506  while (n_left > 0)
507  {
508  vlib_buffer_t *b_head, *b_current;
509  u32 bi_current;
510  u16 desc_current;
511  u32 desc_data_offset;
512  vring_desc_t *desc_table = txvq->desc;
513 
514  if (PREDICT_FALSE (cpu->rx_buffers_len <= 1))
515  {
516  /* Not enough rx_buffers
517  * Note: We yeld on 1 so we don't need to do an additional
518  * check for the next buffer prefetch.
519  */
520  n_left = 0;
521  break;
522  }
523 
524  desc_current = txvq->avail->ring[last_avail_idx & mask];
525  cpu->rx_buffers_len--;
526  bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
527  b_head = b_current = vlib_get_buffer (vm, bi_current);
528  to_next[0] = bi_current; //We do that now so we can forget about bi_current
529  to_next++;
530  n_left_to_next--;
531 
533  (vm, cpu->rx_buffers[cpu->rx_buffers_len - 1], LOAD);
534 
535  /* Just preset the used descriptor id and length for later */
536  txvq->used->ring[last_used_idx & mask].id = desc_current;
537  txvq->used->ring[last_used_idx & mask].len = 0;
538  vhost_user_log_dirty_ring (vui, txvq, ring[last_used_idx & mask]);
539 
540  /* The buffer should already be initialized */
542  b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
543 
544  if (PREDICT_FALSE (n_trace))
545  {
546  vlib_trace_buffer (vm, node, next_index, b_head,
547  /* follow_chain */ 0);
548  vhost_trace_t *t0 =
549  vlib_add_trace (vm, node, b_head, sizeof (t0[0]));
550  vhost_user_rx_trace (t0, vui, qid, b_head, txvq, last_avail_idx);
551  n_trace--;
552  vlib_set_trace_count (vm, node, n_trace);
553  }
554 
555  /* This depends on the setup but is very consistent
556  * So I think the CPU branch predictor will make a pretty good job
557  * at optimizing the decision. */
558  if (txvq->desc[desc_current].flags & VRING_DESC_F_INDIRECT)
559  {
560  desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr,
561  &map_hint);
562  desc_current = 0;
563  if (PREDICT_FALSE (desc_table == 0))
564  {
565  vlib_error_count (vm, node->node_index,
566  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
567  goto out;
568  }
569  }
570 
571  desc_data_offset = vui->virtio_net_hdr_sz;
572 
573  if (enable_csum)
574  {
575  virtio_net_hdr_mrg_rxbuf_t *hdr;
576  u8 *b_data;
577  u16 current;
578 
579  hdr = map_guest_mem (vui, desc_table[desc_current].addr, &map_hint);
580  if (PREDICT_FALSE (hdr == 0))
581  {
582  vlib_error_count (vm, node->node_index,
583  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
584  goto out;
585  }
586  if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
587  {
588  if ((desc_data_offset == desc_table[desc_current].len) &&
589  (desc_table[desc_current].flags & VRING_DESC_F_NEXT))
590  {
591  current = desc_table[desc_current].next;
592  b_data = map_guest_mem (vui, desc_table[current].addr,
593  &map_hint);
594  if (PREDICT_FALSE (b_data == 0))
595  {
596  vlib_error_count (vm, node->node_index,
597  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL,
598  1);
599  goto out;
600  }
601  }
602  else
603  b_data = (u8 *) hdr + desc_data_offset;
604 
605  vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr);
606  }
607  }
608 
609  while (1)
610  {
611  /* Get more input if necessary. Or end of packet. */
612  if (desc_data_offset == desc_table[desc_current].len)
613  {
614  if (PREDICT_FALSE (desc_table[desc_current].flags &
616  {
617  desc_current = desc_table[desc_current].next;
618  desc_data_offset = 0;
619  }
620  else
621  {
622  goto out;
623  }
624  }
625 
626  /* Get more output if necessary. Or end of packet. */
627  if (PREDICT_FALSE (b_current->current_length == buffer_data_size))
628  {
629  if (PREDICT_FALSE (cpu->rx_buffers_len == 0))
630  {
631  /* Cancel speculation */
632  to_next--;
633  n_left_to_next++;
634 
635  /*
636  * Checking if there are some left buffers.
637  * If not, just rewind the used buffers and stop.
638  * Note: Scheduled copies are not cancelled. This is
639  * not an issue as they would still be valid. Useless,
640  * but valid.
641  */
642  vhost_user_input_rewind_buffers (vm, cpu, b_head);
643  n_left = 0;
644  goto stop;
645  }
646 
647  /* Get next output */
648  cpu->rx_buffers_len--;
649  u32 bi_next = cpu->rx_buffers[cpu->rx_buffers_len];
650  b_current->next_buffer = bi_next;
651  b_current->flags |= VLIB_BUFFER_NEXT_PRESENT;
652  bi_current = bi_next;
653  b_current = vlib_get_buffer (vm, bi_current);
654  }
655 
656  /* Prepare a copy order executed later for the data */
657  ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N);
658  vhost_copy_t *cpy = &cpu->copy[copy_len];
659  copy_len++;
660  u32 desc_data_l = desc_table[desc_current].len - desc_data_offset;
661  cpy->len = buffer_data_size - b_current->current_length;
662  cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
663  cpy->dst = (uword) (vlib_buffer_get_current (b_current) +
664  b_current->current_length);
665  cpy->src = desc_table[desc_current].addr + desc_data_offset;
666 
667  desc_data_offset += cpy->len;
668 
669  b_current->current_length += cpy->len;
671  }
672 
673  out:
674 
675  n_rx_bytes += b_head->total_length_not_including_first_buffer;
676  n_rx_packets++;
677 
679  b_head->current_length;
680 
681  /* consume the descriptor and return it as used */
682  last_avail_idx++;
683  last_used_idx++;
684 
686 
687  vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
688  vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0;
689  b_head->error = 0;
690 
691  if (current_config_index != ~(u32) 0)
692  {
693  b_head->current_config_index = current_config_index;
694  vnet_buffer (b_head)->feature_arc_index = feature_arc_idx;
695  }
696 
697  n_left--;
698 
699  /*
700  * Although separating memory copies from virtio ring parsing
701  * is beneficial, we can offer to perform the copies from time
702  * to time in order to free some space in the ring.
703  */
705  {
706  if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy,
707  copy_len, &map_hint)))
708  {
709  vlib_error_count (vm, node->node_index,
710  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
711  }
712  copy_len = 0;
713 
714  /* give buffers back to driver */
716  txvq->used->idx = last_used_idx;
717  vhost_user_log_dirty_ring (vui, txvq, idx);
718  }
719  }
720 stop:
721  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
722 
723  txvq->last_used_idx = last_used_idx;
724  txvq->last_avail_idx = last_avail_idx;
725 
726  /* Do the memory copies */
727  if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, copy_len,
728  &map_hint)))
729  {
730  vlib_error_count (vm, node->node_index,
731  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
732  }
733 
734  /* give buffers back to driver */
736  txvq->used->idx = txvq->last_used_idx;
737  vhost_user_log_dirty_ring (vui, txvq, idx);
738 
739  /* interrupt (call) handling */
740  if ((txvq->callfd_idx != ~0) &&
742  {
743  txvq->n_since_last_int += n_rx_packets;
744 
745  if (txvq->n_since_last_int > vum->coalesce_frames)
746  vhost_user_send_call (vm, txvq);
747  }
748 
749  /* increase rx counters */
753  n_rx_packets, n_rx_bytes);
754 
756 
757 done:
758  return n_rx_packets;
759 }
760 
763  vhost_user_vring_t * txvq, u16 desc_head,
764  u16 n_descs_processed)
765 {
766  vring_packed_desc_t *desc_table = txvq->packed_desc;
767  u16 desc_idx;
768  u16 mask = txvq->qsz_mask;
769 
770  for (desc_idx = 0; desc_idx < n_descs_processed; desc_idx++)
771  {
772  if (txvq->used_wrap_counter)
773  desc_table[(desc_head + desc_idx) & mask].flags |=
775  else
776  desc_table[(desc_head + desc_idx) & mask].flags &=
779  }
780 }
781 
784  u16 qid, vhost_user_vring_t * txvq,
785  u16 desc_current)
786 {
788  vring_packed_desc_t *hdr_desc;
789  virtio_net_hdr_mrg_rxbuf_t *hdr;
790  u32 hint = 0;
791 
792  clib_memset (t, 0, sizeof (*t));
793  t->device_index = vui - vum->vhost_user_interfaces;
794  t->qid = qid;
795 
796  hdr_desc = &txvq->packed_desc[desc_current];
797  if (txvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT)
798  {
799  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
800  /* Header is the first here */
801  hdr_desc = map_guest_mem (vui, txvq->packed_desc[desc_current].addr,
802  &hint);
803  }
804  if (txvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT)
805  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
806 
807  if (!(txvq->packed_desc[desc_current].flags & VRING_DESC_F_NEXT) &&
808  !(txvq->packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT))
809  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
810 
811  t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
812 
813  if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint)))
814  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR;
815  else
816  {
817  u32 len = vui->virtio_net_hdr_sz;
818  clib_memcpy_fast (&t->hdr, hdr,
819  len > hdr_desc->len ? hdr_desc->len : len);
820  }
821 }
822 
825  vhost_user_intf_t * vui,
826  vhost_user_vring_t * txvq,
827  u32 discard_max)
828 {
829  u32 discarded_packets = 0;
830  u16 mask = txvq->qsz_mask;
831  u16 desc_current, desc_head;
832 
833  desc_head = desc_current = txvq->last_used_idx & mask;
834 
835  /*
836  * On the RX side, each packet corresponds to one descriptor
837  * (it is the same whether it is a shallow descriptor, chained, or indirect).
838  * Therefore, discarding a packet is like discarding a descriptor.
839  */
840  while ((discarded_packets != discard_max) &&
841  vhost_user_packed_desc_available (txvq, desc_current))
842  {
844  discarded_packets++;
845  desc_current = (desc_current + 1) & mask;
846  }
847 
848  if (PREDICT_TRUE (discarded_packets))
849  vhost_user_mark_desc_consumed (vui, txvq, desc_head, discarded_packets);
850  return (discarded_packets);
851 }
852 
855  u16 copy_len, u32 * map_hint)
856 {
857  void *src0, *src1, *src2, *src3, *src4, *src5, *src6, *src7;
858  u8 bad;
859  u32 rc = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
860 
861  if (PREDICT_TRUE (copy_len >= 8))
862  {
863  src4 = map_guest_mem (vui, cpy[0].src, map_hint);
864  src5 = map_guest_mem (vui, cpy[1].src, map_hint);
865  src6 = map_guest_mem (vui, cpy[2].src, map_hint);
866  src7 = map_guest_mem (vui, cpy[3].src, map_hint);
867  bad = (src4 == 0) + (src5 == 0) + (src6 == 0) + (src7 == 0);
868  if (PREDICT_FALSE (bad))
869  goto one_by_one;
870  CLIB_PREFETCH (src4, 64, LOAD);
871  CLIB_PREFETCH (src5, 64, LOAD);
872  CLIB_PREFETCH (src6, 64, LOAD);
873  CLIB_PREFETCH (src7, 64, LOAD);
874 
875  while (PREDICT_TRUE (copy_len >= 8))
876  {
877  src0 = src4;
878  src1 = src5;
879  src2 = src6;
880  src3 = src7;
881 
882  src4 = map_guest_mem (vui, cpy[4].src, map_hint);
883  src5 = map_guest_mem (vui, cpy[5].src, map_hint);
884  src6 = map_guest_mem (vui, cpy[6].src, map_hint);
885  src7 = map_guest_mem (vui, cpy[7].src, map_hint);
886  bad = (src4 == 0) + (src5 == 0) + (src6 == 0) + (src7 == 0);
887  if (PREDICT_FALSE (bad))
888  break;
889 
890  CLIB_PREFETCH (src4, 64, LOAD);
891  CLIB_PREFETCH (src5, 64, LOAD);
892  CLIB_PREFETCH (src6, 64, LOAD);
893  CLIB_PREFETCH (src7, 64, LOAD);
894 
895  clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len);
896  clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len);
897  clib_memcpy_fast ((void *) cpy[2].dst, src2, cpy[2].len);
898  clib_memcpy_fast ((void *) cpy[3].dst, src3, cpy[3].len);
899  copy_len -= 4;
900  cpy += 4;
901  }
902  }
903 
904 one_by_one:
905  while (copy_len)
906  {
907  if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint))))
908  {
909  rc = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
910  break;
911  }
912  clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len);
913  copy_len -= 1;
914  cpy += 1;
915  }
916  return rc;
917 }
918 
921  vring_packed_desc_t * desc_table, u16 desc_current,
922  u16 mask, vlib_buffer_t * b_head, u32 * map_hint)
923 {
924  u32 rc = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
925  virtio_net_hdr_mrg_rxbuf_t *hdr;
926  u8 *b_data;
927  u32 desc_data_offset = vui->virtio_net_hdr_sz;
928 
929  hdr = map_guest_mem (vui, desc_table[desc_current].addr, map_hint);
930  if (PREDICT_FALSE (hdr == 0))
931  rc = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
932  else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
933  {
934  if (desc_data_offset == desc_table[desc_current].len)
935  {
936  desc_current = (desc_current + 1) & mask;
937  b_data =
938  map_guest_mem (vui, desc_table[desc_current].addr, map_hint);
939  if (PREDICT_FALSE (b_data == 0))
940  rc = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
941  else
942  vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr);
943  }
944  else
945  {
946  b_data = (u8 *) hdr + desc_data_offset;
947  vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr);
948  }
949  }
950 
951  return rc;
952 }
953 
955 vhost_user_compute_buffers_required (u32 desc_len, u32 buffer_data_size)
956 {
957  div_t result;
958  u32 buffers_required;
959 
960  if (PREDICT_TRUE (buffer_data_size == 2048))
961  {
962  buffers_required = desc_len >> 11;
963  if ((desc_len & 2047) != 0)
964  buffers_required++;
965  return (buffers_required);
966  }
967 
968  if (desc_len < buffer_data_size)
969  return 1;
970 
971  result = div (desc_len, buffer_data_size);
972  if (result.rem)
973  buffers_required = result.quot + 1;
974  else
975  buffers_required = result.quot;
976 
977  return (buffers_required);
978 }
979 
982  vhost_user_vring_t * txvq,
983  u32 buffer_data_size, u16 desc_current,
984  u32 * map_hint)
985 {
986  vring_packed_desc_t *desc_table = txvq->packed_desc;
987  u32 desc_len = 0;
988  u16 desc_data_offset = vui->virtio_net_hdr_sz;
989  u16 desc_idx = desc_current;
990  u32 n_descs;
991 
992  n_descs = desc_table[desc_idx].len >> 4;
993  desc_table = map_guest_mem (vui, desc_table[desc_idx].addr, map_hint);
994  if (PREDICT_FALSE (desc_table == 0))
995  return 0;
996 
997  for (desc_idx = 0; desc_idx < n_descs; desc_idx++)
998  desc_len += desc_table[desc_idx].len;
999 
1000  if (PREDICT_TRUE (desc_len > desc_data_offset))
1001  desc_len -= desc_data_offset;
1002 
1003  return vhost_user_compute_buffers_required (desc_len, buffer_data_size);
1004 }
1005 
1008  vhost_user_vring_t * txvq,
1009  u32 buffer_data_size, u16 * current,
1010  u16 * n_left)
1011 {
1012  vring_packed_desc_t *desc_table = txvq->packed_desc;
1013  u32 desc_len = 0;
1014  u16 mask = txvq->qsz_mask;
1015 
1016  while (desc_table[*current].flags & VRING_DESC_F_NEXT)
1017  {
1018  desc_len += desc_table[*current].len;
1019  (*n_left)++;
1020  *current = (*current + 1) & mask;
1022  }
1023  desc_len += desc_table[*current].len;
1024  (*n_left)++;
1025  *current = (*current + 1) & mask;
1027 
1028  if (PREDICT_TRUE (desc_len > vui->virtio_net_hdr_sz))
1029  desc_len -= vui->virtio_net_hdr_sz;
1030 
1031  return vhost_user_compute_buffers_required (desc_len, buffer_data_size);
1032 }
1033 
1035 vhost_user_assemble_packet (vring_packed_desc_t * desc_table,
1036  u16 * desc_idx, vlib_buffer_t * b_head,
1037  vlib_buffer_t ** b_current, u32 ** next,
1038  vlib_buffer_t *** b, u32 * bi_current,
1039  vhost_cpu_t * cpu, u16 * copy_len,
1040  u32 * buffers_used, u32 buffers_required,
1041  u32 * desc_data_offset, u32 buffer_data_size,
1042  u16 mask)
1043 {
1044  u32 desc_data_l;
1045 
1046  while (*desc_data_offset < desc_table[*desc_idx].len)
1047  {
1048  /* Get more output if necessary. Or end of packet. */
1049  if (PREDICT_FALSE ((*b_current)->current_length == buffer_data_size))
1050  {
1051  /* Get next output */
1052  u32 bi_next = **next;
1053  (*next)++;
1054  (*b_current)->next_buffer = bi_next;
1055  (*b_current)->flags |= VLIB_BUFFER_NEXT_PRESENT;
1056  *bi_current = bi_next;
1057  *b_current = **b;
1058  (*b)++;
1059  (*buffers_used)++;
1060  ASSERT (*buffers_used <= buffers_required);
1061  }
1062 
1063  /* Prepare a copy order executed later for the data */
1064  ASSERT (*copy_len < VHOST_USER_COPY_ARRAY_N);
1065  vhost_copy_t *cpy = &cpu->copy[*copy_len];
1066  (*copy_len)++;
1067  desc_data_l = desc_table[*desc_idx].len - *desc_data_offset;
1068  cpy->len = buffer_data_size - (*b_current)->current_length;
1069  cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
1070  cpy->dst = (uword) (vlib_buffer_get_current (*b_current) +
1071  (*b_current)->current_length);
1072  cpy->src = desc_table[*desc_idx].addr + *desc_data_offset;
1073 
1074  *desc_data_offset += cpy->len;
1075 
1076  (*b_current)->current_length += cpy->len;
1078  }
1079  *desc_idx = (*desc_idx + 1) & mask;;
1080  *desc_data_offset = 0;
1081 }
1082 
1085  vhost_user_intf_t * vui, u16 qid,
1087  vnet_hw_interface_rx_mode mode, u8 enable_csum)
1088 {
1089  vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
1091  u8 feature_arc_idx = fm->device_input_feature_arc_index;
1092  u16 n_rx_packets = 0;
1093  u32 n_rx_bytes = 0;
1094  u16 n_left = 0;
1095  u32 buffers_required = 0;
1096  u32 n_left_to_next, *to_next;
1098  u32 n_trace = vlib_get_trace_count (vm, node);
1099  u32 buffer_data_size = vlib_buffer_get_default_data_size (vm);
1100  u32 map_hint = 0;
1101  vhost_cpu_t *cpu = &vum->cpus[vm->thread_index];
1102  u16 copy_len = 0;
1103  u32 current_config_index = ~0;
1104  u16 mask = txvq->qsz_mask;
1105  u16 desc_current, desc_head, last_used_idx;
1106  vring_packed_desc_t *desc_table = 0;
1107  u32 n_descs_processed = 0;
1108  u32 rv;
1109  vlib_buffer_t **b;
1110  u32 *next;
1111  u32 buffers_used = 0;
1112  u16 current, n_descs_to_process;
1113 
1114  /* The descriptor table is not ready yet */
1115  if (PREDICT_FALSE (txvq->packed_desc == 0))
1116  goto done;
1117 
1118  /* do we have pending interrupts ? */
1119  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
1120  vhost_user_input_do_interrupt (vm, txvq, rxvq);
1121 
1122  /*
1123  * For adaptive mode, it is optimized to reduce interrupts.
1124  * If the scheduler switches the input node to polling due
1125  * to burst of traffic, we tell the driver no interrupt.
1126  * When the traffic subsides, the scheduler switches the node back to
1127  * interrupt mode. We must tell the driver we want interrupt.
1128  */
1130  {
1131  if ((node->flags &
1133  !(node->flags &
1135  /* Tell driver we want notification */
1136  txvq->used_event->flags = 0;
1137  else
1138  /* Tell driver we don't want notification */
1139  txvq->used_event->flags = VRING_EVENT_F_DISABLE;
1140  }
1141 
1142  last_used_idx = txvq->last_used_idx & mask;
1143  desc_head = desc_current = last_used_idx;
1144 
1145  if (vhost_user_packed_desc_available (txvq, desc_current) == 0)
1146  goto done;
1147 
1148  if (PREDICT_FALSE (!vui->admin_up || !vui->is_ready || !(txvq->enabled)))
1149  {
1150  /*
1151  * Discard input packet if interface is admin down or vring is not
1152  * enabled.
1153  * "For example, for a networking device, in the disabled state
1154  * client must not supply any new RX packets, but must process
1155  * and discard any TX packets."
1156  */
1157  rv = vhost_user_rx_discard_packet_packed (vm, vui, txvq,
1160  VHOST_USER_INPUT_FUNC_ERROR_NOT_READY, rv);
1161  goto done;
1162  }
1163 
1164  vhost_user_input_setup_frame (vm, node, vui, &current_config_index,
1165  &next_index, &to_next, &n_left_to_next);
1166 
1167  /*
1168  * Compute n_left and total buffers needed
1169  */
1170  desc_table = txvq->packed_desc;
1171  current = desc_current;
1172  while (vhost_user_packed_desc_available (txvq, current) &&
1173  (n_left < VLIB_FRAME_SIZE))
1174  {
1175  if (desc_table[current].flags & VRING_DESC_F_INDIRECT)
1176  {
1177  buffers_required +=
1178  vhost_user_compute_indirect_desc_len (vui, txvq, buffer_data_size,
1179  current, &map_hint);
1180  n_left++;
1181  current = (current + 1) & mask;
1183  }
1184  else
1185  {
1186  buffers_required +=
1187  vhost_user_compute_chained_desc_len (vui, txvq, buffer_data_size,
1188  &current, &n_left);
1189  }
1190  }
1191 
1192  /* Something is broken if we need more than 10000 buffers */
1193  if (PREDICT_FALSE ((buffers_required == 0) || (buffers_required > 10000)))
1194  {
1195  rv = vhost_user_rx_discard_packet_packed (vm, vui, txvq, n_left);
1197  VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, rv);
1198  goto done;
1199  }
1200 
1201  vec_validate (cpu->to_next_list, buffers_required);
1202  rv = vlib_buffer_alloc (vm, cpu->to_next_list, buffers_required);
1203  if (PREDICT_FALSE (rv != buffers_required))
1204  {
1205  vlib_buffer_free (vm, cpu->to_next_list, rv);
1206  rv = vhost_user_rx_discard_packet_packed (vm, vui, txvq, n_left);
1208  VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, rv);
1209  goto done;
1210  }
1211 
1212  next = cpu->to_next_list;
1213  vec_validate (cpu->rx_buffers_pdesc, buffers_required);
1214  vlib_get_buffers (vm, next, cpu->rx_buffers_pdesc, buffers_required);
1215  b = cpu->rx_buffers_pdesc;
1216  n_descs_processed = n_left;
1217 
1218  while (n_left)
1219  {
1220  vlib_buffer_t *b_head, *b_current;
1221  u32 bi_current;
1222  u32 desc_data_offset;
1223  u16 desc_idx = desc_current;
1224  u32 n_descs;
1225 
1226  desc_table = txvq->packed_desc;
1227  to_next[0] = bi_current = next[0];
1228  b_head = b_current = b[0];
1229  b++;
1230  buffers_used++;
1231  ASSERT (buffers_used <= buffers_required);
1232  to_next++;
1233  next++;
1234  n_left_to_next--;
1235 
1236  /* The buffer should already be initialized */
1238  b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1239  desc_data_offset = vui->virtio_net_hdr_sz;
1240  n_descs_to_process = 1;
1241 
1242  if (desc_table[desc_idx].flags & VRING_DESC_F_INDIRECT)
1243  {
1244  n_descs = desc_table[desc_idx].len >> 4;
1245  desc_table = map_guest_mem (vui, desc_table[desc_idx].addr,
1246  &map_hint);
1247  desc_idx = 0;
1248  if (PREDICT_FALSE (desc_table == 0) ||
1249  (enable_csum &&
1250  (PREDICT_FALSE
1252  (vui, desc_table, desc_idx, mask, b_head,
1253  &map_hint) != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))))
1254  {
1255  vlib_error_count (vm, node->node_index,
1256  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
1257  to_next--;
1258  next--;
1259  n_left_to_next++;
1260  buffers_used--;
1261  b--;
1262  goto out;
1263  }
1264  while (n_descs)
1265  {
1266  vhost_user_assemble_packet (desc_table, &desc_idx, b_head,
1267  &b_current, &next, &b, &bi_current,
1268  cpu, &copy_len, &buffers_used,
1269  buffers_required, &desc_data_offset,
1270  buffer_data_size, mask);
1271  n_descs--;
1272  }
1273  }
1274  else
1275  {
1276  if (enable_csum)
1277  {
1278  rv = vhost_user_do_offload (vui, desc_table, desc_idx, mask,
1279  b_head, &map_hint);
1280  if (PREDICT_FALSE (rv != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
1281  {
1282  vlib_error_count (vm, node->node_index, rv, 1);
1283  to_next--;
1284  next--;
1285  n_left_to_next++;
1286  buffers_used--;
1287  b--;
1288  goto out;
1289  }
1290  }
1291  /*
1292  * For chained descriptor, we process all chains in a single while
1293  * loop. So count how many descriptors in the chain.
1294  */
1295  n_descs_to_process = 1;
1296  while (desc_table[desc_idx].flags & VRING_DESC_F_NEXT)
1297  {
1298  vhost_user_assemble_packet (desc_table, &desc_idx, b_head,
1299  &b_current, &next, &b, &bi_current,
1300  cpu, &copy_len, &buffers_used,
1301  buffers_required, &desc_data_offset,
1302  buffer_data_size, mask);
1303  n_descs_to_process++;
1304  }
1305  vhost_user_assemble_packet (desc_table, &desc_idx, b_head,
1306  &b_current, &next, &b, &bi_current,
1307  cpu, &copy_len, &buffers_used,
1308  buffers_required, &desc_data_offset,
1309  buffer_data_size, mask);
1310  }
1311 
1312  n_rx_bytes += b_head->total_length_not_including_first_buffer;
1313  n_rx_packets++;
1314 
1316  b_head->current_length;
1317 
1319 
1320  vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
1321  vnet_buffer (b_head)->sw_if_index[VLIB_TX] = ~0;
1322  b_head->error = 0;
1323 
1324  if (current_config_index != ~0)
1325  {
1326  b_head->current_config_index = current_config_index;
1327  vnet_buffer (b_head)->feature_arc_index = feature_arc_idx;
1328  }
1329 
1330  out:
1331  ASSERT (n_left >= n_descs_to_process);
1332  n_left -= n_descs_to_process;
1333 
1334  /* advance to next descrptor */
1335  desc_current = (desc_current + n_descs_to_process) & mask;
1336 
1337  /*
1338  * Although separating memory copies from virtio ring parsing
1339  * is beneficial, we can offer to perform the copies from time
1340  * to time in order to free some space in the ring.
1341  */
1342  if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD))
1343  {
1344  rv = vhost_user_input_copy_packed (vui, cpu->copy, copy_len,
1345  &map_hint);
1346  if (PREDICT_FALSE (rv != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
1347  vlib_error_count (vm, node->node_index, rv, 1);
1348  copy_len = 0;
1349  }
1350  }
1351  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1352 
1353  /* Do the memory copies */
1354  rv = vhost_user_input_copy_packed (vui, cpu->copy, copy_len, &map_hint);
1355  if (PREDICT_FALSE (rv != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
1356  vlib_error_count (vm, node->node_index, rv, 1);
1357 
1358  /* Must do the tracing before giving buffers back to driver */
1359  if (PREDICT_FALSE (n_trace))
1360  {
1361  u32 left = n_rx_packets;
1362 
1363  b = cpu->rx_buffers_pdesc;
1364  while (n_trace && left)
1365  {
1366  vhost_trace_t *t0;
1367 
1368  vlib_trace_buffer (vm, node, next_index, b[0],
1369  /* follow_chain */ 0);
1370  t0 = vlib_add_trace (vm, node, b[0], sizeof (t0[0]));
1371  b++;
1372  vhost_user_rx_trace_packed (t0, vui, qid, txvq, last_used_idx);
1373  last_used_idx = (last_used_idx + 1) & mask;
1374  n_trace--;
1375  left--;
1376  vlib_set_trace_count (vm, node, n_trace);
1377  }
1378  }
1379 
1380  /*
1381  * Give buffers back to driver.
1382  */
1383  vhost_user_mark_desc_consumed (vui, txvq, desc_head, n_descs_processed);
1384 
1385  /* interrupt (call) handling */
1386  if ((txvq->callfd_idx != ~0) &&
1387  (txvq->avail_event->flags != VRING_EVENT_F_DISABLE))
1388  {
1389  txvq->n_since_last_int += n_rx_packets;
1390  if (txvq->n_since_last_int > vum->coalesce_frames)
1391  vhost_user_send_call (vm, txvq);
1392  }
1393 
1394  /* increase rx counters */
1398  n_rx_packets, n_rx_bytes);
1399 
1400  vnet_device_increment_rx_packets (vm->thread_index, n_rx_packets);
1401 
1402  if (PREDICT_FALSE (buffers_used < buffers_required))
1403  vlib_buffer_free (vm, next, buffers_required - buffers_used);
1404 
1405 done:
1406  return n_rx_packets;
1407 }
1408 
1411  vlib_frame_t * frame)
1412 {
1414  uword n_rx_packets = 0;
1415  vhost_user_intf_t *vui;
1417  (vnet_device_input_runtime_t *) node->runtime_data;
1419 
1421  {
1422  if ((node->state == VLIB_NODE_STATE_POLLING) ||
1423  clib_atomic_swap_acq_n (&dq->interrupt_pending, 0))
1424  {
1425  vui =
1426  pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance);
1428  {
1429  if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM))
1430  n_rx_packets += vhost_user_if_input_packed (vm, vum, vui,
1431  dq->queue_id, node,
1432  dq->mode, 1);
1433  else
1434  n_rx_packets += vhost_user_if_input_packed (vm, vum, vui,
1435  dq->queue_id, node,
1436  dq->mode, 0);
1437  }
1438  else
1439  {
1440  if (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM))
1441  n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id,
1442  node, dq->mode, 1);
1443  else
1444  n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id,
1445  node, dq->mode, 0);
1446  }
1447  }
1448  }
1449 
1450  return n_rx_packets;
1451 }
1452 
1453 /* *INDENT-OFF* */
1455  .type = VLIB_NODE_TYPE_INPUT,
1456  .name = "vhost-user-input",
1457  .sibling_of = "device-input",
1459 
1460  /* Will be enabled if/when hardware is detected. */
1461  .state = VLIB_NODE_STATE_DISABLED,
1462 
1463  .format_buffer = format_ethernet_header_with_length,
1464  .format_trace = format_vhost_trace,
1465 
1466  .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR,
1467  .error_strings = vhost_user_input_func_error_strings,
1468 };
1469 /* *INDENT-ON* */
1470 
1471 /*
1472  * fd.io coding-style-patch-verification: ON
1473  *
1474  * Local Variables:
1475  * eval: (c-set-style "gnu")
1476  * End:
1477  */
vnet_config_main_t config_main
Definition: feature.h:82
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:509
#define VRING_EVENT_F_DISABLE
Definition: virtio_std.h:81
u32 * to_next_list
Definition: vhost_user.h:292
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
static void vnet_device_increment_rx_packets(u32 thread_index, u64 count)
Definition: devices.h:110
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
vring_desc_t * desc
Definition: vhost_user.h:170
u32 virtio_ring_flags
The device index.
Definition: vhost_user.h:272
virtio_net_hdr_mrg_rxbuf_t hdr
Length of the first data descriptor.
Definition: vhost_user.h:274
vhost_user_input_func_error_t
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:193
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
Definition: vhost_user.h:307
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:103
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
vring_desc_event_t * used_event
Definition: vhost_user.h:181
#define VIRTIO_NET_HDR_F_NEEDS_CSUM
Definition: virtio_std.h:134
static_always_inline void vhost_user_assemble_packet(vring_packed_desc_t *desc_table, u16 *desc_idx, vlib_buffer_t *b_head, vlib_buffer_t **b_current, u32 **next, vlib_buffer_t ***b, u32 *bi_current, vhost_cpu_t *cpu, u16 *copy_len, u32 *buffers_used, u32 buffers_required, u32 *desc_data_offset, u32 buffer_data_size, u16 mask)
#define vnet_buffer2(b)
Definition: buffer.h:482
vnet_interface_main_t interface_main
Definition: vnet.h:59
#define PREDICT_TRUE(x)
Definition: clib.h:121
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:135
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
vring_used_elem_t ring[0]
Definition: virtio_std.h:113
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define VLIB_NODE_FLAG_TRACE_SUPPORTED
Definition: node.h:305
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:333
vring_packed_desc_t * packed_desc
Definition: vhost_user.h:171
vhost_copy_t copy[VHOST_USER_COPY_ARRAY_N]
Definition: vhost_user.h:286
vring_avail_t * avail
Definition: vhost_user.h:175
u32 thread_index
Definition: main.h:249
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:269
#define foreach_vhost_user_input_func_error
vl_api_address_t src
Definition: gre.api:54
vlib_main_t * vm
Definition: in2out_ed.c:1582
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:78
static_always_inline void vhost_user_advance_last_avail_idx(vhost_user_vring_t *vring)
static_always_inline int vnet_have_features(u8 arc, u32 sw_if_index)
Definition: feature.h:251
#define VHOST_USER_DOWN_DISCARD_COUNT
#define VLIB_NODE_FN(node)
Definition: node.h:202
static_always_inline u32 vhost_user_compute_buffers_required(u32 desc_len, u32 buffer_data_size)
static_always_inline u32 vhost_user_rx_discard_packet(vlib_main_t *vm, vhost_user_intf_t *vui, vhost_user_vring_t *txvq, u32 discard_max)
Try to discard packets from the tx ring (VPP RX path).
u16 mask
Definition: flow_types.api:52
struct _tcp_header tcp_header_t
vring_used_t * used
Definition: vhost_user.h:180
vhost_vring_addr_t addr
Definition: vhost_user.h:111
unsigned char u8
Definition: types.h:56
#define VIRTIO_FEATURE(X)
Definition: virtio_std.h:69
double f64
Definition: types.h:142
#define fm
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:136
static_always_inline u32 vhost_user_if_input(vlib_main_t *vm, vhost_user_main_t *vum, vhost_user_intf_t *vui, u16 qid, vlib_node_runtime_t *node, vnet_hw_interface_rx_mode mode, u8 enable_csum)
vnet_hw_interface_rx_mode
Definition: interface.h:53
#define VRING_AVAIL_F_NO_INTERRUPT
Definition: virtio_std.h:85
static_always_inline u32 vhost_user_compute_chained_desc_len(vhost_user_intf_t *vui, vhost_user_vring_t *txvq, u32 buffer_data_size, u16 *current, u16 *n_left)
#define static_always_inline
Definition: clib.h:108
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:476
static_always_inline void vhost_user_rx_trace_packed(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vhost_user_vring_t *txvq, u16 desc_current)
vl_api_ip6_address_t ip6
Definition: one.api:424
#define ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX
Definition: ethernet.h:52
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:881
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
#define VHOST_VRING_IDX_TX(qid)
Definition: vhost_user.h:27
#define vlib_get_new_next_frame(vm, node, next_index, vectors, n_vectors_left)
Definition: node_funcs.h:396
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:303
static_always_inline u8 * format_vhost_trace(u8 *s, va_list *va)
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:377
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:317
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
#define VRING_USED_F_NO_NOTIFY
Definition: virtio_std.h:84
vnet_crypto_main_t * cm
Definition: quic_crypto.c:53
#define VHOST_USER_COPY_ARRAY_N
Definition: vhost_user.h:278
vlib_node_registration_t vhost_user_input_node
(constructor) VLIB_REGISTER_NODE (vhost_user_input_node)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
#define VHOST_USER_RX_BUFFER_STARVATION
static __clib_warn_unused_result u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:677
unsigned short u16
Definition: types.h:57
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
static void * vnet_get_config_data(vnet_config_main_t *cm, u32 *config_index, u32 *next_index, u32 n_data_bytes)
Definition: config.h:123
static_always_inline void vhost_user_handle_rx_offload(vlib_buffer_t *b0, u8 *b0_data, virtio_net_hdr_t *hdr)
static_always_inline void vhost_user_mark_desc_consumed(vhost_user_intf_t *vui, vhost_user_vring_t *txvq, u16 desc_head, u16 n_descs_processed)
#define PREDICT_FALSE(x)
Definition: clib.h:120
vl_api_ip4_address_t ip4
Definition: one.api:376
vhost_user_main_t vhost_user_main
Definition: vhost_user.c:56
vnet_main_t vnet_main
Definition: misc.c:43
u32 node_index
Node index.
Definition: node.h:487
vl_api_address_t dst
Definition: gre.api:55
#define VIRTIO_NET_HDR_GSO_TCPV4
Definition: virtio_std.h:138
vl_api_tunnel_mode_t mode
Definition: gre.api:48
u8 len
Definition: ip_types.api:92
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:96
#define VHOST_VRING_IDX_RX(qid)
Definition: vhost_user.h:26
u8 * format_ethernet_header_with_length(u8 *s, va_list *args)
Definition: format.c:97
u16 device_index
The interface queue index (Not the virtio vring idx)
Definition: vhost_user.h:271
vhost_user_intf_t * vhost_user_interfaces
Definition: vhost_user.h:300
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
static_always_inline void vhost_user_input_rewind_buffers(vlib_main_t *vm, vhost_cpu_t *cpu, vlib_buffer_t *b_head)
static_always_inline u64 vhost_user_is_packed_ring_supported(vhost_user_intf_t *vui)
static void * vlib_frame_scalar_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:311
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:483
u16 first_desc_len
Runtime queue flags.
Definition: vhost_user.h:273
static_always_inline u8 vhost_user_packed_desc_available(vhost_user_vring_t *vring, u16 idx)
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1582
static_always_inline u32 vhost_user_compute_indirect_desc_len(vhost_user_intf_t *vui, vhost_user_vring_t *txvq, u32 buffer_data_size, u16 desc_current, u32 *map_hint)
u32 rx_buffers[VHOST_USER_RX_BUFFERS_N]
Definition: vhost_user.h:283
static __clib_unused char * vhost_user_input_func_error_strings[]
#define ASSERT(truth)
#define VHOST_USER_RX_BUFFERS_N
Definition: vhost_user.h:277
static_always_inline u32 vhost_user_rx_discard_packet_packed(vlib_main_t *vm, vhost_user_intf_t *vui, vhost_user_vring_t *txvq, u32 discard_max)
vlib_frame_t * frame
Definition: node.h:405
#define clib_atomic_swap_acq_n(a, b)
Definition: atomics.h:51
u16 flags
Definition: node.h:387
static_always_inline void vhost_user_send_call(vlib_main_t *vm, vhost_user_vring_t *vq)
static_always_inline u32 vhost_user_input_copy(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
vlib_buffer_t ** rx_buffers_pdesc
Definition: vhost_user.h:293
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:78
vring_desc_event_t * avail_event
Definition: vhost_user.h:176
#define VRING_DESC_F_NEXT
Definition: virtio_std.h:73
#define vec_elt(v, i)
Get vector value at index i.
u8 device_input_feature_arc_index
Feature arc index for device-input.
Definition: feature.h:112
u16 ring[0]
Definition: virtio_std.h:99
#define VHOST_USER_RX_COPY_THRESHOLD
struct _vlib_node_registration vlib_node_registration_t
Definition: defs.h:47
static_always_inline u32 vhost_user_do_offload(vhost_user_intf_t *vui, vring_packed_desc_t *desc_table, u16 desc_current, u16 mask, vlib_buffer_t *b_head, u32 *map_hint)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
static_always_inline void vhost_user_input_setup_frame(vlib_main_t *vm, vlib_node_runtime_t *node, vhost_user_intf_t *vui, u32 *current_config_index, u32 *next_index, u32 **to_next, u32 *n_left_to_next)
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
Definition: vhost_user.h:241
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1583
static_always_inline u32 vhost_user_input_copy_packed(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:492
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static_always_inline void vhost_user_rx_trace(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *txvq, u16 last_avail_idx)
static_always_inline void vhost_user_input_do_interrupt(vlib_main_t *vm, vhost_user_vring_t *txvq, vhost_user_vring_t *rxvq)
left
#define vhost_user_log_dirty_ring(vui, vq, member)
u32 rx_buffers_len
Definition: vhost_user.h:282
#define vnet_buffer(b)
Definition: buffer.h:417
static_always_inline u32 vhost_user_if_input_packed(vlib_main_t *vm, vhost_user_main_t *vum, vhost_user_intf_t *vui, u16 qid, vlib_node_runtime_t *node, vnet_hw_interface_rx_mode mode, u8 enable_csum)
static int tcp_header_bytes(tcp_header_t *t)
Definition: tcp_packet.h:93
#define VRING_DESC_F_USED
Definition: virtio_std.h:78
#define vec_foreach(var, vec)
Vector iterator.
u16 flags
Copy of main node flags.
Definition: node.h:500
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:577
static_always_inline void vhost_user_advance_last_used_idx(vhost_user_vring_t *vring)
static void vlib_frame_no_append(vlib_frame_t *f)
Definition: node_funcs.h:277
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:209
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:280
vnet_feature_config_main_t * feature_config_mains
feature config main objects
Definition: feature.h:100
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
#define VIRTIO_NET_HDR_GSO_UDP
Definition: virtio_std.h:139
vnet_feature_main_t feature_main
Definition: feature.c:18
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:304
Definition: defs.h:46
#define VRING_DESC_F_INDIRECT
Definition: virtio_std.h:75
#define VIRTIO_NET_HDR_GSO_TCPV6
Definition: virtio_std.h:140
#define VRING_DESC_F_AVAIL
Definition: virtio_std.h:77