FD.io VPP  v18.07-rc0-415-g6c78436
Vector Packet Processing
vhost-user.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * vhost.c - vhost-user
4  *
5  * Copyright (c) 2014 Cisco and/or its affiliates.
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at:
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *------------------------------------------------------------------
18  */
19 
20 #include <fcntl.h> /* for open */
21 #include <sys/ioctl.h>
22 #include <sys/socket.h>
23 #include <sys/un.h>
24 #include <sys/stat.h>
25 #include <sys/types.h>
26 #include <sys/uio.h> /* for iovec */
27 #include <netinet/in.h>
28 #include <sys/vfs.h>
29 
30 #include <linux/if_arp.h>
31 #include <linux/if_tun.h>
32 
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
35 
36 #include <vnet/ip/ip.h>
37 
38 #include <vnet/ethernet/ethernet.h>
39 #include <vnet/devices/devices.h>
40 #include <vnet/feature/feature.h>
41 
43 
44 /**
45  * @file
46  * @brief vHost User Device Driver.
47  *
48  * This file contains the source code for vHost User interface.
49  */
50 
51 
52 #define VHOST_DEBUG_VQ 0
53 
54 #define DBG_SOCK(args...) \
55  { \
56  vhost_user_main_t *_vum = &vhost_user_main; \
57  if (_vum->debug) \
58  clib_warning(args); \
59  };
60 
61 #if VHOST_DEBUG_VQ == 1
62 #define DBG_VQ(args...) clib_warning(args);
63 #else
64 #define DBG_VQ(args...)
65 #endif
66 
67 /*
68  * When an RX queue is down but active, received packets
69  * must be discarded. This value controls up to how many
70  * packets will be discarded during each round.
71  */
72 #define VHOST_USER_DOWN_DISCARD_COUNT 256
73 
74 /*
75  * When the number of available buffers gets under this threshold,
76  * RX node will start discarding packets.
77  */
78 #define VHOST_USER_RX_BUFFER_STARVATION 32
79 
80 /*
81  * On the receive side, the host should free descriptors as soon
82  * as possible in order to avoid TX drop in the VM.
83  * This value controls the number of copy operations that are stacked
84  * before copy is done for all and descriptors are given back to
85  * the guest.
86  * The value 64 was obtained by testing (48 and 128 were not as good).
87  */
88 #define VHOST_USER_RX_COPY_THRESHOLD 64
89 /*
90  * On the transmit side, we keep processing the buffers from vlib in the while
91  * loop and prepare the copy order to be executed later. However, the static
92  * array which we keep the copy order is limited to VHOST_USER_COPY_ARRAY_N
93  * entries. In order to not corrupt memory, we have to do the copy when the
94  * static array reaches the copy threshold. We subtract 40 in case the code
95  * goes into the inner loop for a maximum of 64k frames which may require
96  * more array entries.
97  */
98 #define VHOST_USER_TX_COPY_THRESHOLD (VHOST_USER_COPY_ARRAY_N - 40)
99 
100 #define UNIX_GET_FD(unixfd_idx) ({ \
101  typeof(unixfd_idx) __unixfd_idx = (unixfd_idx); \
102  (__unixfd_idx != ~0) ? \
103  pool_elt_at_index (file_main.file_pool, \
104  __unixfd_idx)->file_descriptor : -1; })
105 
106 #define foreach_virtio_trace_flags \
107  _ (SIMPLE_CHAINED, 0, "Simple descriptor chaining") \
108  _ (SINGLE_DESC, 1, "Single descriptor packet") \
109  _ (INDIRECT, 2, "Indirect descriptor") \
110  _ (MAP_ERROR, 4, "Memory mapping error")
111 
112 typedef enum
113 {
114 #define _(n,i,s) VIRTIO_TRACE_F_##n,
116 #undef _
118 
120 
121 #define foreach_vhost_user_tx_func_error \
122  _(NONE, "no error") \
123  _(NOT_READY, "vhost vring not ready") \
124  _(DOWN, "vhost interface is down") \
125  _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \
126  _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \
127  _(MMAP_FAIL, "mmap failure") \
128  _(INDIRECT_OVERFLOW, "indirect descriptor table overflow")
129 
130 typedef enum
131 {
132 #define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f,
134 #undef _
137 
139 #define _(n,s) s,
141 #undef _
142 };
143 
144 #define foreach_vhost_user_input_func_error \
145  _(NO_ERROR, "no error") \
146  _(NO_BUFFER, "no available buffer") \
147  _(MMAP_FAIL, "mmap failure") \
148  _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \
149  _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
150  _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")
151 
152 typedef enum
153 {
154 #define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f,
156 #undef _
159 
161 #define _(n,s) s,
163 #undef _
164 };
165 
166 /* *INDENT-OFF* */
167 static vhost_user_main_t vhost_user_main = {
168  .mtu_bytes = 1518,
169 };
170 
171 VNET_HW_INTERFACE_CLASS (vhost_interface_class, static) = {
172  .name = "vhost-user",
173 };
174 /* *INDENT-ON* */
175 
176 static u8 *
178 {
179  u32 i = va_arg (*args, u32);
180  u32 show_dev_instance = ~0;
182 
184  show_dev_instance = vum->show_dev_instance_by_real_dev_instance[i];
185 
186  if (show_dev_instance != ~0)
187  i = show_dev_instance;
188 
189  s = format (s, "VirtualEthernet0/0/%d", i);
190  return s;
191 }
192 
193 static int
195 {
196  // FIXME: check if the new dev instance is already used
199  hi->dev_instance, ~0);
200 
202  new_dev_instance;
203 
204  DBG_SOCK ("renumbered vhost-user interface dev_instance %d to %d",
205  hi->dev_instance, new_dev_instance);
206 
207  return 0;
208 }
209 
212 {
213  int i = *hint;
214  if (PREDICT_TRUE ((vui->regions[i].guest_phys_addr <= addr) &&
215  ((vui->regions[i].guest_phys_addr +
216  vui->regions[i].memory_size) > addr)))
217  {
218  return (void *) (vui->region_mmap_addr[i] + addr -
219  vui->regions[i].guest_phys_addr);
220  }
221 #if __SSE4_2__
222  __m128i rl, rh, al, ah, r;
223  al = _mm_set1_epi64x (addr + 1);
224  ah = _mm_set1_epi64x (addr);
225 
226  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]);
227  rl = _mm_cmpgt_epi64 (al, rl);
228  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]);
229  rh = _mm_cmpgt_epi64 (rh, ah);
230  r = _mm_and_si128 (rl, rh);
231 
232  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]);
233  rl = _mm_cmpgt_epi64 (al, rl);
234  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]);
235  rh = _mm_cmpgt_epi64 (rh, ah);
236  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22);
237 
238  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]);
239  rl = _mm_cmpgt_epi64 (al, rl);
240  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]);
241  rh = _mm_cmpgt_epi64 (rh, ah);
242  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44);
243 
244  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]);
245  rl = _mm_cmpgt_epi64 (al, rl);
246  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]);
247  rh = _mm_cmpgt_epi64 (rh, ah);
248  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88);
249 
250  r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800));
251  i = count_trailing_zeros (_mm_movemask_epi8 (r) |
253 
254  if (i < vui->nregions)
255  {
256  *hint = i;
257  return (void *) (vui->region_mmap_addr[i] + addr -
258  vui->regions[i].guest_phys_addr);
259  }
260 #elif __aarch64__ && __ARM_NEON
261  uint64x2_t al, ah, rl, rh, r;
262  uint32_t u32 = 0;
263 
264  al = vdupq_n_u64 (addr + 1);
265  ah = vdupq_n_u64 (addr);
266 
267  /*First Iteration */
268  rl = vld1q_u64 (&vui->region_guest_addr_lo[0]);
269  rl = vcgtq_u64 (al, rl);
270  rh = vld1q_u64 (&vui->region_guest_addr_hi[0]);
271  rh = vcgtq_u64 (rh, ah);
272  r = vandq_u64 (rl, rh);
273  u32 |= (vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 0) & 0x1);
274  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 8) & 0x1) << 1);
275 
276  if (u32)
277  {
278  i = count_trailing_zeros (u32);
279  goto vhost_map_guest_mem_done;
280  }
281 
282  /*Second Iteration */
283  rl = vld1q_u64 (&vui->region_guest_addr_lo[2]);
284  rl = vcgtq_u64 (al, rl);
285  rh = vld1q_u64 (&vui->region_guest_addr_hi[2]);
286  rh = vcgtq_u64 (rh, ah);
287  r = vandq_u64 (rl, rh);
288  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 0) & 0x1) << 2);
289  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 8) & 0x1) << 3);
290 
291  if (u32)
292  {
293  i = count_trailing_zeros (u32);
294  goto vhost_map_guest_mem_done;
295  }
296 
297  /*Third Iteration */
298  rl = vld1q_u64 (&vui->region_guest_addr_lo[4]);
299  rl = vcgtq_u64 (al, rl);
300  rh = vld1q_u64 (&vui->region_guest_addr_hi[4]);
301  rh = vcgtq_u64 (rh, ah);
302  r = vandq_u64 (rl, rh);
303  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 0) & 0x1) << 4);
304  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 8) & 0x1) << 5);
305 
306  if (u32)
307  {
308  i = count_trailing_zeros (u32);
309  goto vhost_map_guest_mem_done;
310  }
311 
312  /*Fourth Iteration */
313  rl = vld1q_u64 (&vui->region_guest_addr_lo[6]);
314  rl = vcgtq_u64 (al, rl);
315  rh = vld1q_u64 (&vui->region_guest_addr_hi[6]);
316  rh = vcgtq_u64 (rh, ah);
317  r = vandq_u64 (rl, rh);
318  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 0) & 0x1) << 6);
319  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 8) & 0x1) << 7);
320 
322 
323 vhost_map_guest_mem_done:
324  if (i < vui->nregions)
325  {
326  *hint = i;
327  return (void *) (vui->region_mmap_addr[i] + addr -
328  vui->regions[i].guest_phys_addr);
329  }
330 #else
331  for (i = 0; i < vui->nregions; i++)
332  {
333  if ((vui->regions[i].guest_phys_addr <= addr) &&
334  ((vui->regions[i].guest_phys_addr + vui->regions[i].memory_size) >
335  addr))
336  {
337  *hint = i;
338  return (void *) (vui->region_mmap_addr[i] + addr -
339  vui->regions[i].guest_phys_addr);
340  }
341  }
342 #endif
343  DBG_VQ ("failed to map guest mem addr %llx", addr);
344  *hint = 0;
345  return 0;
346 }
347 
348 static inline void *
350 {
351  int i;
352  for (i = 0; i < vui->nregions; i++)
353  {
354  if ((vui->regions[i].userspace_addr <= addr) &&
355  ((vui->regions[i].userspace_addr + vui->regions[i].memory_size) >
356  addr))
357  {
358  return (void *) (vui->region_mmap_addr[i] + addr -
359  vui->regions[i].userspace_addr);
360  }
361  }
362  return 0;
363 }
364 
365 static long
367 {
368  struct statfs s;
369  fstatfs (fd, &s);
370  return s.f_bsize;
371 }
372 
373 static void
375 {
376  int i, r;
377  for (i = 0; i < vui->nregions; i++)
378  {
379  if (vui->region_mmap_addr[i] != MAP_FAILED)
380  {
381 
382  long page_sz = get_huge_page_size (vui->region_mmap_fd[i]);
383 
384  ssize_t map_sz = (vui->regions[i].memory_size +
385  vui->regions[i].mmap_offset +
386  page_sz - 1) & ~(page_sz - 1);
387 
388  r =
389  munmap (vui->region_mmap_addr[i] - vui->regions[i].mmap_offset,
390  map_sz);
391 
392  DBG_SOCK
393  ("unmap memory region %d addr 0x%lx len 0x%lx page_sz 0x%x", i,
394  vui->region_mmap_addr[i], map_sz, page_sz);
395 
396  vui->region_mmap_addr[i] = MAP_FAILED;
397 
398  if (r == -1)
399  {
400  clib_warning ("failed to unmap memory region (errno %d)",
401  errno);
402  }
403  close (vui->region_mmap_fd[i]);
404  }
405  }
406  vui->nregions = 0;
407 }
408 
409 static void
411 {
412  //Let's try to assign one queue to each thread
413  u32 qid = 0;
414  u32 thread_index = 0;
415  vui->use_tx_spinlock = 0;
416  while (1)
417  {
418  for (qid = 0; qid < VHOST_VRING_MAX_N / 2; qid++)
419  {
420  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
421  if (!rxvq->started || !rxvq->enabled)
422  continue;
423 
424  vui->per_cpu_tx_qid[thread_index] = qid;
425  thread_index++;
426  if (thread_index == vlib_get_thread_main ()->n_vlib_mains)
427  return;
428  }
429  //We need to loop, meaning the spinlock has to be used
430  vui->use_tx_spinlock = 1;
431  if (thread_index == 0)
432  {
433  //Could not find a single valid one
434  for (thread_index = 0;
435  thread_index < vlib_get_thread_main ()->n_vlib_mains;
436  thread_index++)
437  {
438  vui->per_cpu_tx_qid[thread_index] = 0;
439  }
440  return;
441  }
442  }
443 }
444 
445 /**
446  * @brief Unassign existing interface/queue to thread mappings and re-assign
447  * new interface/queue to thread mappings
448  */
449 static void
451 {
453  vhost_user_intf_t *vui;
454  vhost_user_vring_t *txvq;
455  vnet_main_t *vnm = vnet_get_main ();
456  u32 qid;
457  int rv;
458  u16 *queue;
459 
460  // Scrap all existing mappings for all interfaces/queues
461  /* *INDENT-OFF* */
462  pool_foreach (vui, vum->vhost_user_interfaces, {
463  vec_foreach (queue, vui->rx_queues)
464  {
465  rv = vnet_hw_interface_unassign_rx_thread (vnm, vui->hw_if_index,
466  *queue);
467  if (rv)
468  clib_warning ("Warning: unable to unassign interface %d, "
469  "queue %d: rc=%d", vui->hw_if_index, *queue, rv);
470  }
472  });
473  /* *INDENT-ON* */
474 
475  // Create the rx_queues for all interfaces
476  /* *INDENT-OFF* */
477  pool_foreach (vui, vum->vhost_user_interfaces, {
478  for (qid = 0; qid < VHOST_VRING_MAX_N / 2; qid++)
479  {
480  txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
481  if (txvq->started)
482  {
483  if (txvq->mode == VNET_HW_INTERFACE_RX_MODE_UNKNOWN)
484  /* Set polling as the default */
485  txvq->mode = VNET_HW_INTERFACE_RX_MODE_POLLING;
486  vec_add1 (vui->rx_queues, qid);
487  }
488  }
489  });
490  /* *INDENT-ON* */
491 
492  // Assign new mappings for all interfaces/queues
493  /* *INDENT-OFF* */
494  pool_foreach (vui, vum->vhost_user_interfaces, {
495  vnet_hw_interface_set_input_node (vnm, vui->hw_if_index,
496  vhost_user_input_node.index);
497  vec_foreach (queue, vui->rx_queues)
498  {
499  vnet_hw_interface_assign_rx_thread (vnm, vui->hw_if_index, *queue,
500  ~0);
501  txvq = &vui->vrings[VHOST_VRING_IDX_TX (*queue)];
502  rv = vnet_hw_interface_set_rx_mode (vnm, vui->hw_if_index, *queue,
503  txvq->mode);
504  if (rv)
505  clib_warning ("Warning: unable to set rx mode for interface %d, "
506  "queue %d: rc=%d", vui->hw_if_index, *queue, rv);
507  }
508  });
509  /* *INDENT-ON* */
510 }
511 
512 /** @brief Returns whether at least one TX and one RX vring are enabled */
513 int
515 {
516  int i, found[2] = { }; //RX + TX
517 
518  for (i = 0; i < VHOST_VRING_MAX_N; i++)
519  if (vui->vrings[i].started && vui->vrings[i].enabled)
520  found[i & 1] = 1;
521 
522  return found[0] && found[1];
523 }
524 
525 static void
527 {
528  /* if we have pointers to descriptor table, go up */
529  int is_up = vhost_user_intf_ready (vui);
530  if (is_up != vui->is_up)
531  {
532  DBG_SOCK ("interface %d %s", vui->sw_if_index,
533  is_up ? "ready" : "down");
536  0);
537  vui->is_up = is_up;
538  }
541 }
542 
543 static void
545 {
546  u32 qid;
547  vnet_main_t *vnm = vnet_get_main ();
548 
549  qid = ifq & 0xff;
550  if ((qid & 1) == 0)
551  /* Only care about the odd number, or TX, virtqueue */
552  return;
553 
554  if (vhost_user_intf_ready (vui))
555  // qid >> 1 is to convert virtqueue number to vring queue index
557 }
558 
559 static clib_error_t *
561 {
562  __attribute__ ((unused)) int n;
563  u8 buff[8];
564 
565  n = read (uf->file_descriptor, ((char *) &buff), 8);
566 
567  return 0;
568 }
569 
570 static clib_error_t *
572 {
573  __attribute__ ((unused)) int n;
574  u8 buff[8];
575  vhost_user_intf_t *vui =
576  pool_elt_at_index (vhost_user_main.vhost_user_interfaces,
577  uf->private_data >> 8);
578  u32 qid = uf->private_data & 0xff;
579 
580  n = read (uf->file_descriptor, ((char *) &buff), 8);
581  DBG_SOCK ("if %d KICK queue %d", uf->private_data >> 8, qid);
582  if (!vui->vrings[qid].started ||
583  (vhost_user_intf_ready (vui) != vui->is_up))
584  {
586  vui->vrings[qid].started = 1;
589  }
590 
592  return 0;
593 }
594 
595 /**
596  * @brief Try once to lock the vring
597  * @return 0 on success, non-zero on failure.
598  */
599 static inline int
601 {
602  return __sync_lock_test_and_set (vui->vring_locks[qid], 1);
603 }
604 
605 /**
606  * @brief Spin until the vring is successfully locked
607  */
608 static inline void
610 {
611  while (vhost_user_vring_try_lock (vui, qid))
612  ;
613 }
614 
615 /**
616  * @brief Unlock the vring lock
617  */
618 static inline void
620 {
621  *vui->vring_locks[qid] = 0;
622 }
623 
624 static inline void
626 {
627  vhost_user_vring_t *vring = &vui->vrings[qid];
628  memset (vring, 0, sizeof (*vring));
629  vring->kickfd_idx = ~0;
630  vring->callfd_idx = ~0;
631  vring->errfd = -1;
632 
633  /*
634  * We have a bug with some qemu 2.5, and this may be a fix.
635  * Feel like interpretation holy text, but this is from vhost-user.txt.
636  * "
637  * One queue pair is enabled initially. More queues are enabled
638  * dynamically, by sending message VHOST_USER_SET_VRING_ENABLE.
639  * "
640  * Don't know who's right, but this is what DPDK does.
641  */
642  if (qid == 0 || qid == 1)
643  vring->enabled = 1;
644 }
645 
646 static inline void
648 {
649  vhost_user_vring_t *vring = &vui->vrings[qid];
650  if (vring->kickfd_idx != ~0)
651  {
653  vring->kickfd_idx);
654  clib_file_del (&file_main, uf);
655  vring->kickfd_idx = ~0;
656  }
657  if (vring->callfd_idx != ~0)
658  {
660  vring->callfd_idx);
661  clib_file_del (&file_main, uf);
662  vring->callfd_idx = ~0;
663  }
664  if (vring->errfd != -1)
665  {
666  close (vring->errfd);
667  vring->errfd = -1;
668  }
669  vhost_user_vring_init (vui, qid);
670 }
671 
672 static inline void
674 {
675  vnet_main_t *vnm = vnet_get_main ();
676  int q;
677 
679 
680  if (vui->clib_file_index != ~0)
681  {
683  vui->clib_file_index = ~0;
684  }
685 
686  vui->is_up = 0;
687 
688  for (q = 0; q < VHOST_VRING_MAX_N; q++)
689  vhost_user_vring_close (vui, q);
690 
691  unmap_all_mem_regions (vui);
692  DBG_SOCK ("interface ifindex %d disconnected", vui->sw_if_index);
693 }
694 
695 #define VHOST_LOG_PAGE 0x1000
698  u64 addr, u64 len, u8 is_host_address)
699 {
700  if (PREDICT_TRUE (vui->log_base_addr == 0
701  || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL))))
702  {
703  return;
704  }
705  if (is_host_address)
706  {
707  addr = pointer_to_uword (map_user_mem (vui, (uword) addr));
708  }
709  if (PREDICT_FALSE ((addr + len - 1) / VHOST_LOG_PAGE / 8 >= vui->log_size))
710  {
711  DBG_SOCK ("vhost_user_log_dirty_pages(): out of range\n");
712  return;
713  }
714 
716  u64 page = addr / VHOST_LOG_PAGE;
717  while (page * VHOST_LOG_PAGE < addr + len)
718  {
719  ((u8 *) vui->log_base_addr)[page / 8] |= 1 << page % 8;
720  page++;
721  }
722 }
723 
726 {
727  vhost_user_log_dirty_pages_2 (vui, addr, len, 0);
728 }
729 
730 #define vhost_user_log_dirty_ring(vui, vq, member) \
731  if (PREDICT_FALSE(vq->log_used)) { \
732  vhost_user_log_dirty_pages(vui, vq->log_guest_addr + STRUCT_OFFSET_OF(vring_used_t, member), \
733  sizeof(vq->used->member)); \
734  }
735 
736 static clib_error_t *
738 {
739  int n, i;
740  int fd, number_of_fds = 0;
741  int fds[VHOST_MEMORY_MAX_NREGIONS];
742  vhost_user_msg_t msg;
743  struct msghdr mh;
744  struct iovec iov[1];
746  vhost_user_intf_t *vui;
747  struct cmsghdr *cmsg;
748  u8 q;
749  clib_file_t template = { 0 };
750  vnet_main_t *vnm = vnet_get_main ();
751 
752  vui = pool_elt_at_index (vum->vhost_user_interfaces, uf->private_data);
753 
754  char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS * sizeof (int))];
755 
756  memset (&mh, 0, sizeof (mh));
757  memset (control, 0, sizeof (control));
758 
759  for (i = 0; i < VHOST_MEMORY_MAX_NREGIONS; i++)
760  fds[i] = -1;
761 
762  /* set the payload */
763  iov[0].iov_base = (void *) &msg;
764  iov[0].iov_len = VHOST_USER_MSG_HDR_SZ;
765 
766  mh.msg_iov = iov;
767  mh.msg_iovlen = 1;
768  mh.msg_control = control;
769  mh.msg_controllen = sizeof (control);
770 
771  n = recvmsg (uf->file_descriptor, &mh, 0);
772 
773  /* Stop workers to avoid end of the world */
775 
776  if (n != VHOST_USER_MSG_HDR_SZ)
777  {
778  if (n == -1)
779  {
780  DBG_SOCK ("recvmsg returned error %d %s", errno, strerror (errno));
781  }
782  else
783  {
784  DBG_SOCK ("n (%d) != VHOST_USER_MSG_HDR_SZ (%d)",
786  }
787  goto close_socket;
788  }
789 
790  if (mh.msg_flags & MSG_CTRUNC)
791  {
792  DBG_SOCK ("MSG_CTRUNC is set");
793  goto close_socket;
794  }
795 
796  cmsg = CMSG_FIRSTHDR (&mh);
797 
798  if (cmsg && (cmsg->cmsg_len > 0) && (cmsg->cmsg_level == SOL_SOCKET) &&
799  (cmsg->cmsg_type == SCM_RIGHTS) &&
800  (cmsg->cmsg_len - CMSG_LEN (0) <=
801  VHOST_MEMORY_MAX_NREGIONS * sizeof (int)))
802  {
803  number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int);
804  clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int));
805  }
806 
807  /* version 1, no reply bit set */
808  if ((msg.flags & 7) != 1)
809  {
810  DBG_SOCK ("malformed message received. closing socket");
811  goto close_socket;
812  }
813 
814  {
815  int rv;
816  rv =
817  read (uf->file_descriptor, ((char *) &msg) + VHOST_USER_MSG_HDR_SZ,
818  msg.size);
819  if (rv < 0)
820  {
821  DBG_SOCK ("read failed %s", strerror (errno));
822  goto close_socket;
823  }
824  else if (rv != msg.size)
825  {
826  DBG_SOCK ("message too short (read %dB should be %dB)", rv, msg.size);
827  goto close_socket;
828  }
829  }
830 
831  switch (msg.request)
832  {
834  msg.flags |= 4;
835  msg.u64 = (1ULL << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
836  (1ULL << FEAT_VIRTIO_NET_F_CTRL_VQ) |
837  (1ULL << FEAT_VIRTIO_F_ANY_LAYOUT) |
838  (1ULL << FEAT_VIRTIO_F_INDIRECT_DESC) |
839  (1ULL << FEAT_VHOST_F_LOG_ALL) |
840  (1ULL << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) |
841  (1ULL << FEAT_VIRTIO_NET_F_MQ) |
842  (1ULL << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) |
843  (1ULL << FEAT_VIRTIO_F_VERSION_1);
844  msg.u64 &= vui->feature_mask;
845  msg.size = sizeof (msg.u64);
846  DBG_SOCK ("if %d msg VHOST_USER_GET_FEATURES - reply 0x%016llx",
847  vui->hw_if_index, msg.u64);
848  break;
849 
851  DBG_SOCK ("if %d msg VHOST_USER_SET_FEATURES features 0x%016llx",
852  vui->hw_if_index, msg.u64);
853 
854  vui->features = msg.u64;
855 
856  if (vui->features &
857  ((1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
858  (1ULL << FEAT_VIRTIO_F_VERSION_1)))
859  vui->virtio_net_hdr_sz = 12;
860  else
861  vui->virtio_net_hdr_sz = 10;
862 
863  vui->is_any_layout =
864  (vui->features & (1 << FEAT_VIRTIO_F_ANY_LAYOUT)) ? 1 : 0;
865 
868  vui->is_up = 0;
869 
870  /*for (q = 0; q < VHOST_VRING_MAX_N; q++)
871  vhost_user_vring_close(&vui->vrings[q]); */
872 
873  break;
874 
876  DBG_SOCK ("if %d msg VHOST_USER_SET_MEM_TABLE nregions %d",
877  vui->hw_if_index, msg.memory.nregions);
878 
879  if ((msg.memory.nregions < 1) ||
880  (msg.memory.nregions > VHOST_MEMORY_MAX_NREGIONS))
881  {
882 
883  DBG_SOCK ("number of mem regions must be between 1 and %i",
884  VHOST_MEMORY_MAX_NREGIONS);
885 
886  goto close_socket;
887  }
888 
889  if (msg.memory.nregions != number_of_fds)
890  {
891  DBG_SOCK ("each memory region must have FD");
892  goto close_socket;
893  }
894  unmap_all_mem_regions (vui);
895  for (i = 0; i < msg.memory.nregions; i++)
896  {
897  clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i],
898  sizeof (vhost_user_memory_region_t));
899 
900  long page_sz = get_huge_page_size (fds[i]);
901 
902  /* align size to page */
903  ssize_t map_sz = (vui->regions[i].memory_size +
904  vui->regions[i].mmap_offset +
905  page_sz - 1) & ~(page_sz - 1);
906 
907  vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
908  MAP_SHARED, fds[i], 0);
909  vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr;
910  vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr +
911  vui->regions[i].memory_size;
912 
913  DBG_SOCK
914  ("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "
915  "page_sz 0x%x", i, map_sz, fds[i], vui->region_mmap_addr[i],
916  page_sz);
917 
918  if (vui->region_mmap_addr[i] == MAP_FAILED)
919  {
920  clib_warning ("failed to map memory. errno is %d", errno);
921  goto close_socket;
922  }
923  vui->region_mmap_addr[i] += vui->regions[i].mmap_offset;
924  vui->region_mmap_fd[i] = fds[i];
925 
926  vui->nregions++;
927  }
928  break;
929 
931  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_NUM idx %d num %d",
932  vui->hw_if_index, msg.state.index, msg.state.num);
933 
934  if ((msg.state.num > 32768) || /* maximum ring size is 32768 */
935  (msg.state.num == 0) || /* it cannot be zero */
936  ((msg.state.num - 1) & msg.state.num)) /* must be power of 2 */
937  goto close_socket;
938  vui->vrings[msg.state.index].qsz_mask = msg.state.num - 1;
939  break;
940 
942  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ADDR idx %d",
943  vui->hw_if_index, msg.state.index);
944 
945  if (msg.state.index >= VHOST_VRING_MAX_N)
946  {
947  DBG_SOCK ("invalid vring index VHOST_USER_SET_VRING_ADDR:"
948  " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
949  goto close_socket;
950  }
951 
952  if (msg.size < sizeof (msg.addr))
953  {
954  DBG_SOCK ("vhost message is too short (%d < %d)",
955  msg.size, sizeof (msg.addr));
956  goto close_socket;
957  }
958 
959  vui->vrings[msg.state.index].desc = (vring_desc_t *)
960  map_user_mem (vui, msg.addr.desc_user_addr);
961  vui->vrings[msg.state.index].used = (vring_used_t *)
962  map_user_mem (vui, msg.addr.used_user_addr);
963  vui->vrings[msg.state.index].avail = (vring_avail_t *)
964  map_user_mem (vui, msg.addr.avail_user_addr);
965 
966  if ((vui->vrings[msg.state.index].desc == NULL) ||
967  (vui->vrings[msg.state.index].used == NULL) ||
968  (vui->vrings[msg.state.index].avail == NULL))
969  {
970  DBG_SOCK ("failed to map user memory for hw_if_index %d",
971  vui->hw_if_index);
972  goto close_socket;
973  }
974 
975  vui->vrings[msg.state.index].log_guest_addr = msg.addr.log_guest_addr;
976  vui->vrings[msg.state.index].log_used =
977  (msg.addr.flags & (1 << VHOST_VRING_F_LOG)) ? 1 : 0;
978 
979  /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
980  the ring is initialized in an enabled state. */
981  if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES)))
982  {
983  vui->vrings[msg.state.index].enabled = 1;
984  }
985 
986  vui->vrings[msg.state.index].last_used_idx =
987  vui->vrings[msg.state.index].last_avail_idx =
988  vui->vrings[msg.state.index].used->idx;
989 
990  /* tell driver that we don't want interrupts */
991  vui->vrings[msg.state.index].used->flags = VRING_USED_F_NO_NOTIFY;
992  break;
993 
995  DBG_SOCK ("if %d msg VHOST_USER_SET_OWNER", vui->hw_if_index);
996  break;
997 
999  DBG_SOCK ("if %d msg VHOST_USER_RESET_OWNER", vui->hw_if_index);
1000  break;
1001 
1003  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_CALL %d",
1004  vui->hw_if_index, msg.u64);
1005 
1006  q = (u8) (msg.u64 & 0xFF);
1007 
1008  /* if there is old fd, delete and close it */
1009  if (vui->vrings[q].callfd_idx != ~0)
1010  {
1012  vui->vrings[q].callfd_idx);
1013  clib_file_del (&file_main, uf);
1014  vui->vrings[q].callfd_idx = ~0;
1015  }
1016 
1017  if (!(msg.u64 & VHOST_USER_VRING_NOFD_MASK))
1018  {
1019  if (number_of_fds != 1)
1020  {
1021  DBG_SOCK ("More than one fd received !");
1022  goto close_socket;
1023  }
1024 
1025  template.read_function = vhost_user_callfd_read_ready;
1026  template.file_descriptor = fds[0];
1027  template.private_data =
1028  ((vui - vhost_user_main.vhost_user_interfaces) << 8) + q;
1029  vui->vrings[q].callfd_idx = clib_file_add (&file_main, &template);
1030  }
1031  else
1032  vui->vrings[q].callfd_idx = ~0;
1033  break;
1034 
1036  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_KICK %d",
1037  vui->hw_if_index, msg.u64);
1038 
1039  q = (u8) (msg.u64 & 0xFF);
1040 
1041  if (vui->vrings[q].kickfd_idx != ~0)
1042  {
1044  vui->vrings[q].kickfd_idx);
1045  clib_file_del (&file_main, uf);
1046  vui->vrings[q].kickfd_idx = ~0;
1047  }
1048 
1049  if (!(msg.u64 & VHOST_USER_VRING_NOFD_MASK))
1050  {
1051  if (number_of_fds != 1)
1052  {
1053  DBG_SOCK ("More than one fd received !");
1054  goto close_socket;
1055  }
1056 
1057  template.read_function = vhost_user_kickfd_read_ready;
1058  template.file_descriptor = fds[0];
1059  template.private_data =
1060  (((uword) (vui - vhost_user_main.vhost_user_interfaces)) << 8) +
1061  q;
1062  vui->vrings[q].kickfd_idx = clib_file_add (&file_main, &template);
1063  }
1064  else
1065  {
1066  //When no kickfd is set, the queue is initialized as started
1067  vui->vrings[q].kickfd_idx = ~0;
1068  vui->vrings[q].started = 1;
1069  }
1070 
1071  break;
1072 
1074  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ERR %d",
1075  vui->hw_if_index, msg.u64);
1076 
1077  q = (u8) (msg.u64 & 0xFF);
1078 
1079  if (vui->vrings[q].errfd != -1)
1080  close (vui->vrings[q].errfd);
1081 
1082  if (!(msg.u64 & VHOST_USER_VRING_NOFD_MASK))
1083  {
1084  if (number_of_fds != 1)
1085  goto close_socket;
1086 
1087  vui->vrings[q].errfd = fds[0];
1088  }
1089  else
1090  vui->vrings[q].errfd = -1;
1091 
1092  break;
1093 
1095  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_BASE idx %d num %d",
1096  vui->hw_if_index, msg.state.index, msg.state.num);
1097 
1098  vui->vrings[msg.state.index].last_avail_idx = msg.state.num;
1099  break;
1100 
1102  if (msg.state.index >= VHOST_VRING_MAX_N)
1103  {
1104  DBG_SOCK ("invalid vring index VHOST_USER_GET_VRING_BASE:"
1105  " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
1106  goto close_socket;
1107  }
1108 
1109  /*
1110  * Copy last_avail_idx from the vring before closing it because
1111  * closing the vring also initializes the vring last_avail_idx
1112  */
1113  msg.state.num = vui->vrings[msg.state.index].last_avail_idx;
1114  msg.flags |= 4;
1115  msg.size = sizeof (msg.state);
1116 
1117  /* Spec says: Client must [...] stop ring upon receiving VHOST_USER_GET_VRING_BASE. */
1118  vhost_user_vring_close (vui, msg.state.index);
1119  DBG_SOCK ("if %d msg VHOST_USER_GET_VRING_BASE idx %d num %d",
1120  vui->hw_if_index, msg.state.index, msg.state.num);
1121  break;
1122 
1123  case VHOST_USER_NONE:
1124  DBG_SOCK ("if %d msg VHOST_USER_NONE", vui->hw_if_index);
1125 
1126  break;
1127 
1129  {
1130  DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_BASE", vui->hw_if_index);
1131 
1132  if (msg.size != sizeof (msg.log))
1133  {
1134  DBG_SOCK
1135  ("invalid msg size for VHOST_USER_SET_LOG_BASE: %d instead of %d",
1136  msg.size, sizeof (msg.log));
1137  goto close_socket;
1138  }
1139 
1140  if (!
1142  {
1143  DBG_SOCK
1144  ("VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but VHOST_USER_SET_LOG_BASE received");
1145  goto close_socket;
1146  }
1147 
1148  fd = fds[0];
1149  /* align size to page */
1150  long page_sz = get_huge_page_size (fd);
1151  ssize_t map_sz =
1152  (msg.log.size + msg.log.offset + page_sz - 1) & ~(page_sz - 1);
1153 
1154  vui->log_base_addr = mmap (0, map_sz, PROT_READ | PROT_WRITE,
1155  MAP_SHARED, fd, 0);
1156 
1157  DBG_SOCK
1158  ("map log region addr 0 len 0x%lx off 0x%lx fd %d mapped 0x%lx",
1159  map_sz, msg.log.offset, fd, vui->log_base_addr);
1160 
1161  if (vui->log_base_addr == MAP_FAILED)
1162  {
1163  clib_warning ("failed to map memory. errno is %d", errno);
1164  goto close_socket;
1165  }
1166 
1167  vui->log_base_addr += msg.log.offset;
1168  vui->log_size = msg.log.size;
1169 
1170  msg.flags |= 4;
1171  msg.size = sizeof (msg.u64);
1172 
1173  break;
1174  }
1175 
1176  case VHOST_USER_SET_LOG_FD:
1177  DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_FD", vui->hw_if_index);
1178 
1179  break;
1180 
1182  msg.flags |= 4;
1183  msg.u64 = (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |
1184  (1 << VHOST_USER_PROTOCOL_F_MQ);
1185  msg.size = sizeof (msg.u64);
1186  DBG_SOCK
1187  ("if %d msg VHOST_USER_GET_PROTOCOL_FEATURES - reply 0x%016llx",
1188  vui->hw_if_index, msg.u64);
1189  break;
1190 
1192  DBG_SOCK
1193  ("if %d msg VHOST_USER_SET_PROTOCOL_FEATURES features 0x%016llx",
1194  vui->hw_if_index, msg.u64);
1195 
1196  vui->protocol_features = msg.u64;
1197 
1198  break;
1199 
1201  msg.flags |= 4;
1202  msg.u64 = VHOST_VRING_MAX_N;
1203  msg.size = sizeof (msg.u64);
1204  DBG_SOCK ("if %d msg VHOST_USER_GET_QUEUE_NUM - reply %d",
1205  vui->hw_if_index, msg.u64);
1206  break;
1207 
1209  DBG_SOCK ("if %d VHOST_USER_SET_VRING_ENABLE: %s queue %d",
1210  vui->hw_if_index, msg.state.num ? "enable" : "disable",
1211  msg.state.index);
1212  if (msg.state.index >= VHOST_VRING_MAX_N)
1213  {
1214  DBG_SOCK ("invalid vring index VHOST_USER_SET_VRING_ENABLE:"
1215  " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
1216  goto close_socket;
1217  }
1218 
1219  vui->vrings[msg.state.index].enabled = msg.state.num;
1220  break;
1221 
1222  default:
1223  DBG_SOCK ("unknown vhost-user message %d received. closing socket",
1224  msg.request);
1225  goto close_socket;
1226  }
1227 
1228  /* if we need to reply */
1229  if (msg.flags & 4)
1230  {
1231  n =
1232  send (uf->file_descriptor, &msg, VHOST_USER_MSG_HDR_SZ + msg.size, 0);
1233  if (n != (msg.size + VHOST_USER_MSG_HDR_SZ))
1234  {
1235  DBG_SOCK ("could not send message response");
1236  goto close_socket;
1237  }
1238  }
1239 
1242  return 0;
1243 
1244 close_socket:
1248  return 0;
1249 }
1250 
1251 static clib_error_t *
1253 {
1256  vhost_user_intf_t *vui =
1258 
1259  DBG_SOCK ("socket error on if %d", vui->sw_if_index);
1264  return 0;
1265 }
1266 
1267 static clib_error_t *
1269 {
1270  int client_fd, client_len;
1271  struct sockaddr_un client;
1272  clib_file_t template = { 0 };
1274  vhost_user_intf_t *vui;
1275 
1277 
1278  client_len = sizeof (client);
1279  client_fd = accept (uf->file_descriptor,
1280  (struct sockaddr *) &client,
1281  (socklen_t *) & client_len);
1282 
1283  if (client_fd < 0)
1284  return clib_error_return_unix (0, "accept");
1285 
1286  if (vui->clib_file_index != ~0)
1287  {
1288  DBG_SOCK ("Close client socket for vhost interface %d, fd %d",
1289  vui->sw_if_index, UNIX_GET_FD (vui->clib_file_index));
1291  }
1292 
1293  DBG_SOCK ("New client socket for vhost interface %d, fd %d",
1294  vui->sw_if_index, client_fd);
1295  template.read_function = vhost_user_socket_read;
1296  template.error_function = vhost_user_socket_error;
1297  template.file_descriptor = client_fd;
1298  template.private_data = vui - vhost_user_main.vhost_user_interfaces;
1299  vui->clib_file_index = clib_file_add (&file_main, &template);
1300  return 0;
1301 }
1302 
1303 static clib_error_t *
1305 {
1306  clib_error_t *error;
1309 
1310  error = vlib_call_init_function (vm, ip4_init);
1311  if (error)
1312  return error;
1313 
1314  vum->coalesce_frames = 32;
1315  vum->coalesce_time = 1e-3;
1316 
1317  vec_validate (vum->cpus, tm->n_vlib_mains - 1);
1318 
1319  vhost_cpu_t *cpu;
1320  vec_foreach (cpu, vum->cpus)
1321  {
1322  /* This is actually not necessary as validate already zeroes it
1323  * Just keeping the loop here for later because I am lazy. */
1324  cpu->rx_buffers_len = 0;
1325  }
1326 
1327  vum->random = random_default_seed ();
1328 
1330 
1331  return 0;
1332 }
1333 
1335 
1336 static u8 *
1337 format_vhost_trace (u8 * s, va_list * va)
1338 {
1339  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
1340  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
1341  CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
1343  vhost_trace_t *t = va_arg (*va, vhost_trace_t *);
1345  t->device_index);
1346 
1348 
1349  u32 indent = format_get_indent (s);
1350 
1351  s = format (s, "%U %U queue %d\n", format_white_space, indent,
1352  format_vnet_sw_interface_name, vnm, sw, t->qid);
1353 
1354  s = format (s, "%U virtio flags:\n", format_white_space, indent);
1355 #define _(n,i,st) \
1356  if (t->virtio_ring_flags & (1 << VIRTIO_TRACE_F_##n)) \
1357  s = format (s, "%U %s %s\n", format_white_space, indent, #n, st);
1359 #undef _
1360  s = format (s, "%U virtio_net_hdr first_desc_len %u\n",
1361  format_white_space, indent, t->first_desc_len);
1362 
1363  s = format (s, "%U flags 0x%02x gso_type %u\n",
1364  format_white_space, indent,
1365  t->hdr.hdr.flags, t->hdr.hdr.gso_type);
1366 
1367  if (vui->virtio_net_hdr_sz == 12)
1368  s = format (s, "%U num_buff %u",
1369  format_white_space, indent, t->hdr.num_buffers);
1370 
1371  return s;
1372 }
1373 
1374 void
1376  vhost_user_intf_t * vui, u16 qid,
1377  vlib_buffer_t * b, vhost_user_vring_t * txvq)
1378 {
1380  u32 last_avail_idx = txvq->last_avail_idx;
1381  u32 desc_current = txvq->avail->ring[last_avail_idx & txvq->qsz_mask];
1382  vring_desc_t *hdr_desc = 0;
1383  virtio_net_hdr_mrg_rxbuf_t *hdr;
1384  u32 hint = 0;
1385 
1386  memset (t, 0, sizeof (*t));
1387  t->device_index = vui - vum->vhost_user_interfaces;
1388  t->qid = qid;
1389 
1390  hdr_desc = &txvq->desc[desc_current];
1391  if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
1392  {
1393  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
1394  /* Header is the first here */
1395  hdr_desc = map_guest_mem (vui, txvq->desc[desc_current].addr, &hint);
1396  }
1397  if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT)
1398  {
1399  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
1400  }
1401  if (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
1402  !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT))
1403  {
1404  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
1405  }
1406 
1407  t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
1408 
1409  if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint)))
1410  {
1411  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR;
1412  }
1413  else
1414  {
1415  u32 len = vui->virtio_net_hdr_sz;
1416  memcpy (&t->hdr, hdr, len > hdr_desc->len ? hdr_desc->len : len);
1417  }
1418 }
1419 
1420 static inline void
1422 {
1424  u64 x = 1;
1425  int fd = UNIX_GET_FD (vq->callfd_idx);
1426  int rv;
1427 
1428  rv = write (fd, &x, sizeof (x));
1429  if (rv <= 0)
1430  {
1432  ("Error: Could not write to unix socket for callfd %d", fd);
1433  return;
1434  }
1435 
1436  vq->n_since_last_int = 0;
1437  vq->int_deadline = vlib_time_now (vm) + vum->coalesce_time;
1438 }
1439 
1442  u16 copy_len, u32 * map_hint)
1443 {
1444  void *src0, *src1, *src2, *src3;
1445  if (PREDICT_TRUE (copy_len >= 4))
1446  {
1447  if (PREDICT_FALSE (!(src2 = map_guest_mem (vui, cpy[0].src, map_hint))))
1448  return 1;
1449  if (PREDICT_FALSE (!(src3 = map_guest_mem (vui, cpy[1].src, map_hint))))
1450  return 1;
1451 
1452  while (PREDICT_TRUE (copy_len >= 4))
1453  {
1454  src0 = src2;
1455  src1 = src3;
1456 
1457  if (PREDICT_FALSE
1458  (!(src2 = map_guest_mem (vui, cpy[2].src, map_hint))))
1459  return 1;
1460  if (PREDICT_FALSE
1461  (!(src3 = map_guest_mem (vui, cpy[3].src, map_hint))))
1462  return 1;
1463 
1464  CLIB_PREFETCH (src2, 64, LOAD);
1465  CLIB_PREFETCH (src3, 64, LOAD);
1466 
1467  clib_memcpy ((void *) cpy[0].dst, src0, cpy[0].len);
1468  clib_memcpy ((void *) cpy[1].dst, src1, cpy[1].len);
1469  copy_len -= 2;
1470  cpy += 2;
1471  }
1472  }
1473  while (copy_len)
1474  {
1475  if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint))))
1476  return 1;
1477  clib_memcpy ((void *) cpy->dst, src0, cpy->len);
1478  copy_len -= 1;
1479  cpy += 1;
1480  }
1481  return 0;
1482 }
1483 
1484 /**
1485  * Try to discard packets from the tx ring (VPP RX path).
1486  * Returns the number of discarded packets.
1487  */
1488 u32
1490  vhost_user_intf_t * vui,
1491  vhost_user_vring_t * txvq, u32 discard_max)
1492 {
1493  /*
1494  * On the RX side, each packet corresponds to one descriptor
1495  * (it is the same whether it is a shallow descriptor, chained, or indirect).
1496  * Therefore, discarding a packet is like discarding a descriptor.
1497  */
1498  u32 discarded_packets = 0;
1499  u32 avail_idx = txvq->avail->idx;
1500  while (discarded_packets != discard_max)
1501  {
1502  if (avail_idx == txvq->last_avail_idx)
1503  goto out;
1504 
1505  u16 desc_chain_head =
1506  txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask];
1507  txvq->last_avail_idx++;
1508  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id =
1509  desc_chain_head;
1510  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0;
1511  vhost_user_log_dirty_ring (vui, txvq,
1512  ring[txvq->last_used_idx & txvq->qsz_mask]);
1513  txvq->last_used_idx++;
1514  discarded_packets++;
1515  }
1516 
1517 out:
1519  txvq->used->idx = txvq->last_used_idx;
1520  vhost_user_log_dirty_ring (vui, txvq, idx);
1521  return discarded_packets;
1522 }
1523 
1524 /*
1525  * In case of overflow, we need to rewind the array of allocated buffers.
1526  */
1527 static void
1529  vhost_cpu_t * cpu, vlib_buffer_t * b_head)
1530 {
1531  u32 bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
1532  vlib_buffer_t *b_current = vlib_get_buffer (vm, bi_current);
1533  b_current->current_length = 0;
1534  b_current->flags = 0;
1535  while (b_current != b_head)
1536  {
1537  cpu->rx_buffers_len++;
1538  bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
1539  b_current = vlib_get_buffer (vm, bi_current);
1540  b_current->current_length = 0;
1541  b_current->flags = 0;
1542  }
1543  cpu->rx_buffers_len++;
1544 }
1545 
1546 static u32
1548  vhost_user_main_t * vum,
1549  vhost_user_intf_t * vui,
1550  u16 qid, vlib_node_runtime_t * node,
1552 {
1553  vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
1554  u16 n_rx_packets = 0;
1555  u32 n_rx_bytes = 0;
1556  u16 n_left;
1557  u32 n_left_to_next, *to_next;
1559  u32 n_trace = vlib_get_trace_count (vm, node);
1560  u32 map_hint = 0;
1561  u16 thread_index = vlib_get_thread_index ();
1562  u16 copy_len = 0;
1563 
1564  {
1565  /* do we have pending interrupts ? */
1566  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
1567  f64 now = vlib_time_now (vm);
1568 
1569  if ((txvq->n_since_last_int) && (txvq->int_deadline < now))
1570  vhost_user_send_call (vm, txvq);
1571 
1572  if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now))
1573  vhost_user_send_call (vm, rxvq);
1574  }
1575 
1576  /*
1577  * For adaptive mode, it is optimized to reduce interrupts.
1578  * If the scheduler switches the input node to polling due
1579  * to burst of traffic, we tell the driver no interrupt.
1580  * When the traffic subsides, the scheduler switches the node back to
1581  * interrupt mode. We must tell the driver we want interrupt.
1582  */
1584  {
1585  if ((node->flags &
1587  !(node->flags &
1589  /* Tell driver we want notification */
1590  txvq->used->flags = 0;
1591  else
1592  /* Tell driver we don't want notification */
1593  txvq->used->flags = VRING_USED_F_NO_NOTIFY;
1594  }
1595 
1596  if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
1597  return 0;
1598 
1599  n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
1600 
1601  /* nothing to do */
1602  if (PREDICT_FALSE (n_left == 0))
1603  return 0;
1604 
1605  if (PREDICT_FALSE (!vui->admin_up || !(txvq->enabled)))
1606  {
1607  /*
1608  * Discard input packet if interface is admin down or vring is not
1609  * enabled.
1610  * "For example, for a networking device, in the disabled state
1611  * client must not supply any new RX packets, but must process
1612  * and discard any TX packets."
1613  */
1614  vhost_user_rx_discard_packet (vm, vui, txvq,
1616  return 0;
1617  }
1618 
1619  if (PREDICT_FALSE (n_left == (txvq->qsz_mask + 1)))
1620  {
1621  /*
1622  * Informational error logging when VPP is not
1623  * receiving packets fast enough.
1624  */
1625  vlib_error_count (vm, node->node_index,
1626  VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
1627  }
1628 
1629  if (n_left > VLIB_FRAME_SIZE)
1630  n_left = VLIB_FRAME_SIZE;
1631 
1632  /*
1633  * For small packets (<2kB), we will not need more than one vlib buffer
1634  * per packet. In case packets are bigger, we will just yeld at some point
1635  * in the loop and come back later. This is not an issue as for big packet,
1636  * processing cost really comes from the memory copy.
1637  * The assumption is that big packets will fit in 40 buffers.
1638  */
1639  if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len < n_left + 1 ||
1640  vum->cpus[thread_index].rx_buffers_len < 40))
1641  {
1642  u32 curr_len = vum->cpus[thread_index].rx_buffers_len;
1643  vum->cpus[thread_index].rx_buffers_len +=
1645  vum->cpus[thread_index].rx_buffers +
1646  curr_len,
1647  VHOST_USER_RX_BUFFERS_N - curr_len,
1649 
1650  if (PREDICT_FALSE
1651  (vum->cpus[thread_index].rx_buffers_len <
1653  {
1654  /* In case of buffer starvation, discard some packets from the queue
1655  * and log the event.
1656  * We keep doing best effort for the remaining packets. */
1657  u32 flush = (n_left + 1 > vum->cpus[thread_index].rx_buffers_len) ?
1658  n_left + 1 - vum->cpus[thread_index].rx_buffers_len : 1;
1659  flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush);
1660 
1661  n_left -= flush;
1663  interface_main.sw_if_counters +
1666  vui->sw_if_index, flush);
1667 
1669  VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
1670  }
1671  }
1672 
1673  while (n_left > 0)
1674  {
1675  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1676 
1677  while (n_left > 0 && n_left_to_next > 0)
1678  {
1679  vlib_buffer_t *b_head, *b_current;
1680  u32 bi_current;
1681  u16 desc_current;
1682  u32 desc_data_offset;
1683  vring_desc_t *desc_table = txvq->desc;
1684 
1685  if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len <= 1))
1686  {
1687  /* Not enough rx_buffers
1688  * Note: We yeld on 1 so we don't need to do an additional
1689  * check for the next buffer prefetch.
1690  */
1691  n_left = 0;
1692  break;
1693  }
1694 
1695  desc_current =
1696  txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask];
1697  vum->cpus[thread_index].rx_buffers_len--;
1698  bi_current = (vum->cpus[thread_index].rx_buffers)
1699  [vum->cpus[thread_index].rx_buffers_len];
1700  b_head = b_current = vlib_get_buffer (vm, bi_current);
1701  to_next[0] = bi_current; //We do that now so we can forget about bi_current
1702  to_next++;
1703  n_left_to_next--;
1704 
1706  (vum->
1707  cpus[thread_index].rx_buffers)
1708  [vum->cpus[thread_index].
1709  rx_buffers_len - 1], LOAD);
1710 
1711  /* Just preset the used descriptor id and length for later */
1712  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id =
1713  desc_current;
1714  txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0;
1715  vhost_user_log_dirty_ring (vui, txvq,
1716  ring[txvq->last_used_idx &
1717  txvq->qsz_mask]);
1718 
1719  /* The buffer should already be initialized */
1721  b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1722 
1723  if (PREDICT_FALSE (n_trace))
1724  {
1725  //TODO: next_index is not exactly known at that point
1726  vlib_trace_buffer (vm, node, next_index, b_head,
1727  /* follow_chain */ 0);
1728  vhost_trace_t *t0 =
1729  vlib_add_trace (vm, node, b_head, sizeof (t0[0]));
1730  vhost_user_rx_trace (t0, vui, qid, b_head, txvq);
1731  n_trace--;
1732  vlib_set_trace_count (vm, node, n_trace);
1733  }
1734 
1735  /* This depends on the setup but is very consistent
1736  * So I think the CPU branch predictor will make a pretty good job
1737  * at optimizing the decision. */
1738  if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
1739  {
1740  desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr,
1741  &map_hint);
1742  desc_current = 0;
1743  if (PREDICT_FALSE (desc_table == 0))
1744  {
1745  vlib_error_count (vm, node->node_index,
1746  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
1747  goto out;
1748  }
1749  }
1750 
1751  if (PREDICT_TRUE (vui->is_any_layout) ||
1752  (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT)))
1753  {
1754  /* ANYLAYOUT or single buffer */
1755  desc_data_offset = vui->virtio_net_hdr_sz;
1756  }
1757  else
1758  {
1759  /* CSR case without ANYLAYOUT, skip 1st buffer */
1760  desc_data_offset = desc_table[desc_current].len;
1761  }
1762 
1763  while (1)
1764  {
1765  /* Get more input if necessary. Or end of packet. */
1766  if (desc_data_offset == desc_table[desc_current].len)
1767  {
1768  if (PREDICT_FALSE (desc_table[desc_current].flags &
1769  VIRTQ_DESC_F_NEXT))
1770  {
1771  desc_current = desc_table[desc_current].next;
1772  desc_data_offset = 0;
1773  }
1774  else
1775  {
1776  goto out;
1777  }
1778  }
1779 
1780  /* Get more output if necessary. Or end of packet. */
1781  if (PREDICT_FALSE
1782  (b_current->current_length == VLIB_BUFFER_DATA_SIZE))
1783  {
1784  if (PREDICT_FALSE
1785  (vum->cpus[thread_index].rx_buffers_len == 0))
1786  {
1787  /* Cancel speculation */
1788  to_next--;
1789  n_left_to_next++;
1790 
1791  /*
1792  * Checking if there are some left buffers.
1793  * If not, just rewind the used buffers and stop.
1794  * Note: Scheduled copies are not cancelled. This is
1795  * not an issue as they would still be valid. Useless,
1796  * but valid.
1797  */
1799  &vum->cpus
1800  [thread_index],
1801  b_head);
1802  n_left = 0;
1803  goto stop;
1804  }
1805 
1806  /* Get next output */
1807  vum->cpus[thread_index].rx_buffers_len--;
1808  u32 bi_next =
1809  (vum->cpus[thread_index].rx_buffers)[vum->cpus
1810  [thread_index].rx_buffers_len];
1811  b_current->next_buffer = bi_next;
1812  b_current->flags |= VLIB_BUFFER_NEXT_PRESENT;
1813  bi_current = bi_next;
1814  b_current = vlib_get_buffer (vm, bi_current);
1815  }
1816 
1817  /* Prepare a copy order executed later for the data */
1818  vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len];
1819  copy_len++;
1820  u32 desc_data_l =
1821  desc_table[desc_current].len - desc_data_offset;
1822  cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length;
1823  cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
1824  cpy->dst = (uword) (vlib_buffer_get_current (b_current) +
1825  b_current->current_length);
1826  cpy->src = desc_table[desc_current].addr + desc_data_offset;
1827 
1828  desc_data_offset += cpy->len;
1829 
1830  b_current->current_length += cpy->len;
1832  }
1833 
1834  out:
1835  CLIB_PREFETCH (&n_left, sizeof (n_left), LOAD);
1836 
1837  n_rx_bytes += b_head->total_length_not_including_first_buffer;
1838  n_rx_packets++;
1839 
1841  b_head->current_length;
1842 
1843  /* consume the descriptor and return it as used */
1844  txvq->last_avail_idx++;
1845  txvq->last_used_idx++;
1846 
1848 
1849  vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
1850  vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1851  b_head->error = 0;
1852 
1853  {
1855 
1856  /* redirect if feature path enabled */
1858  b_head);
1859 
1860  u32 bi = to_next[-1]; //Cannot use to_next[-1] in the macro
1861  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1862  to_next, n_left_to_next,
1863  bi, next0);
1864  }
1865 
1866  n_left--;
1867 
1868  /*
1869  * Although separating memory copies from virtio ring parsing
1870  * is beneficial, we can offer to perform the copies from time
1871  * to time in order to free some space in the ring.
1872  */
1873  if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD))
1874  {
1875  if (PREDICT_FALSE
1876  (vhost_user_input_copy (vui, vum->cpus[thread_index].copy,
1877  copy_len, &map_hint)))
1878  {
1879  vlib_error_count (vm, node->node_index,
1880  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
1881  }
1882  copy_len = 0;
1883 
1884  /* give buffers back to driver */
1886  txvq->used->idx = txvq->last_used_idx;
1887  vhost_user_log_dirty_ring (vui, txvq, idx);
1888  }
1889  }
1890  stop:
1891  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1892  }
1893 
1894  /* Do the memory copies */
1895  if (PREDICT_FALSE
1896  (vhost_user_input_copy (vui, vum->cpus[thread_index].copy,
1897  copy_len, &map_hint)))
1898  {
1899  vlib_error_count (vm, node->node_index,
1900  VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
1901  }
1902 
1903  /* give buffers back to driver */
1905  txvq->used->idx = txvq->last_used_idx;
1906  vhost_user_log_dirty_ring (vui, txvq, idx);
1907 
1908  /* interrupt (call) handling */
1909  if ((txvq->callfd_idx != ~0) &&
1910  !(txvq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
1911  {
1912  txvq->n_since_last_int += n_rx_packets;
1913 
1914  if (txvq->n_since_last_int > vum->coalesce_frames)
1915  vhost_user_send_call (vm, txvq);
1916  }
1917 
1918  /* increase rx counters */
1922  vlib_get_thread_index (), vui->sw_if_index, n_rx_packets, n_rx_bytes);
1923 
1924  vnet_device_increment_rx_packets (thread_index, n_rx_packets);
1925 
1926  return n_rx_packets;
1927 }
1928 
1929 static uword
1931  vlib_node_runtime_t * node, vlib_frame_t * f)
1932 {
1934  uword n_rx_packets = 0;
1935  vhost_user_intf_t *vui;
1939 
1941  {
1942  if (clib_smp_swap (&dq->interrupt_pending, 0) ||
1943  (node->state == VLIB_NODE_STATE_POLLING))
1944  {
1945  vui =
1946  pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance);
1947  n_rx_packets = vhost_user_if_input (vm, vum, vui, dq->queue_id, node,
1948  dq->mode);
1949  }
1950  }
1951 
1952  return n_rx_packets;
1953 }
1954 
1955 /* *INDENT-OFF* */
1957  .function = vhost_user_input,
1958  .type = VLIB_NODE_TYPE_INPUT,
1959  .name = "vhost-user-input",
1960  .sibling_of = "device-input",
1961 
1962  /* Will be enabled if/when hardware is detected. */
1963  .state = VLIB_NODE_STATE_DISABLED,
1964 
1965  .format_buffer = format_ethernet_header_with_length,
1966  .format_trace = format_vhost_trace,
1967 
1968  .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR,
1969  .error_strings = vhost_user_input_func_error_strings,
1970 };
1971 
1973 /* *INDENT-ON* */
1974 
1975 
1976 void
1978  vhost_user_intf_t * vui, u16 qid,
1979  vlib_buffer_t * b, vhost_user_vring_t * rxvq)
1980 {
1982  u32 last_avail_idx = rxvq->last_avail_idx;
1983  u32 desc_current = rxvq->avail->ring[last_avail_idx & rxvq->qsz_mask];
1984  vring_desc_t *hdr_desc = 0;
1985  u32 hint = 0;
1986 
1987  memset (t, 0, sizeof (*t));
1988  t->device_index = vui - vum->vhost_user_interfaces;
1989  t->qid = qid;
1990 
1991  hdr_desc = &rxvq->desc[desc_current];
1992  if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
1993  {
1994  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
1995  /* Header is the first here */
1996  hdr_desc = map_guest_mem (vui, rxvq->desc[desc_current].addr, &hint);
1997  }
1998  if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT)
1999  {
2000  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
2001  }
2002  if (!(rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
2003  !(rxvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT))
2004  {
2005  t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
2006  }
2007 
2008  t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
2009 }
2010 
2013  u16 copy_len, u32 * map_hint)
2014 {
2015  void *dst0, *dst1, *dst2, *dst3;
2016  if (PREDICT_TRUE (copy_len >= 4))
2017  {
2018  if (PREDICT_FALSE (!(dst2 = map_guest_mem (vui, cpy[0].dst, map_hint))))
2019  return 1;
2020  if (PREDICT_FALSE (!(dst3 = map_guest_mem (vui, cpy[1].dst, map_hint))))
2021  return 1;
2022  while (PREDICT_TRUE (copy_len >= 4))
2023  {
2024  dst0 = dst2;
2025  dst1 = dst3;
2026 
2027  if (PREDICT_FALSE
2028  (!(dst2 = map_guest_mem (vui, cpy[2].dst, map_hint))))
2029  return 1;
2030  if (PREDICT_FALSE
2031  (!(dst3 = map_guest_mem (vui, cpy[3].dst, map_hint))))
2032  return 1;
2033 
2034  CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD);
2035  CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD);
2036 
2037  clib_memcpy (dst0, (void *) cpy[0].src, cpy[0].len);
2038  clib_memcpy (dst1, (void *) cpy[1].src, cpy[1].len);
2039 
2040  vhost_user_log_dirty_pages_2 (vui, cpy[0].dst, cpy[0].len, 1);
2041  vhost_user_log_dirty_pages_2 (vui, cpy[1].dst, cpy[1].len, 1);
2042  copy_len -= 2;
2043  cpy += 2;
2044  }
2045  }
2046  while (copy_len)
2047  {
2048  if (PREDICT_FALSE (!(dst0 = map_guest_mem (vui, cpy->dst, map_hint))))
2049  return 1;
2050  clib_memcpy (dst0, (void *) cpy->src, cpy->len);
2051  vhost_user_log_dirty_pages_2 (vui, cpy->dst, cpy->len, 1);
2052  copy_len -= 1;
2053  cpy += 1;
2054  }
2055  return 0;
2056 }
2057 
2058 
2059 static uword
2061  vlib_node_runtime_t * node, vlib_frame_t * frame)
2062 {
2063  u32 *buffers = vlib_frame_args (frame);
2064  u32 n_left = frame->n_vectors;
2066  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
2067  vhost_user_intf_t *vui =
2069  u32 qid = ~0;
2070  vhost_user_vring_t *rxvq;
2071  u8 error;
2072  u32 thread_index = vlib_get_thread_index ();
2073  u32 map_hint = 0;
2074  u8 retry = 8;
2075  u16 copy_len;
2076  u16 tx_headers_len;
2077 
2078  if (PREDICT_FALSE (!vui->admin_up))
2079  {
2080  error = VHOST_USER_TX_FUNC_ERROR_DOWN;
2081  goto done3;
2082  }
2083 
2084  if (PREDICT_FALSE (!vui->is_up))
2085  {
2086  error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
2087  goto done3;
2088  }
2089 
2090  qid =
2092  (vui->per_cpu_tx_qid, thread_index));
2093  rxvq = &vui->vrings[qid];
2094  if (PREDICT_FALSE (vui->use_tx_spinlock))
2095  vhost_user_vring_lock (vui, qid);
2096 
2097 retry:
2098  error = VHOST_USER_TX_FUNC_ERROR_NONE;
2099  tx_headers_len = 0;
2100  copy_len = 0;
2101  while (n_left > 0)
2102  {
2103  vlib_buffer_t *b0, *current_b0;
2104  u16 desc_head, desc_index, desc_len;
2105  vring_desc_t *desc_table;
2106  uword buffer_map_addr;
2107  u32 buffer_len;
2108  u16 bytes_left;
2109 
2110  if (PREDICT_TRUE (n_left > 1))
2111  vlib_prefetch_buffer_with_index (vm, buffers[1], LOAD);
2112 
2113  b0 = vlib_get_buffer (vm, buffers[0]);
2114 
2115  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2116  {
2117  vum->cpus[thread_index].current_trace =
2118  vlib_add_trace (vm, node, b0,
2119  sizeof (*vum->cpus[thread_index].current_trace));
2120  vhost_user_tx_trace (vum->cpus[thread_index].current_trace,
2121  vui, qid / 2, b0, rxvq);
2122  }
2123 
2124  if (PREDICT_FALSE (rxvq->last_avail_idx == rxvq->avail->idx))
2125  {
2126  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
2127  goto done;
2128  }
2129 
2130  desc_table = rxvq->desc;
2131  desc_head = desc_index =
2132  rxvq->avail->ring[rxvq->last_avail_idx & rxvq->qsz_mask];
2133 
2134  /* Go deeper in case of indirect descriptor
2135  * I don't know of any driver providing indirect for RX. */
2136  if (PREDICT_FALSE (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT))
2137  {
2138  if (PREDICT_FALSE
2139  (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
2140  {
2141  error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
2142  goto done;
2143  }
2144  if (PREDICT_FALSE
2145  (!(desc_table =
2146  map_guest_mem (vui, rxvq->desc[desc_index].addr,
2147  &map_hint))))
2148  {
2149  error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
2150  goto done;
2151  }
2152  desc_index = 0;
2153  }
2154 
2155  desc_len = vui->virtio_net_hdr_sz;
2156  buffer_map_addr = desc_table[desc_index].addr;
2157  buffer_len = desc_table[desc_index].len;
2158 
2159  {
2160  // Get a header from the header array
2161  virtio_net_hdr_mrg_rxbuf_t *hdr =
2162  &vum->cpus[thread_index].tx_headers[tx_headers_len];
2163  tx_headers_len++;
2164  hdr->hdr.flags = 0;
2165  hdr->hdr.gso_type = 0;
2166  hdr->num_buffers = 1; //This is local, no need to check
2167 
2168  // Prepare a copy order executed later for the header
2169  vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len];
2170  copy_len++;
2171  cpy->len = vui->virtio_net_hdr_sz;
2172  cpy->dst = buffer_map_addr;
2173  cpy->src = (uword) hdr;
2174  }
2175 
2176  buffer_map_addr += vui->virtio_net_hdr_sz;
2177  buffer_len -= vui->virtio_net_hdr_sz;
2178  bytes_left = b0->current_length;
2179  current_b0 = b0;
2180  while (1)
2181  {
2182  if (buffer_len == 0)
2183  { //Get new output
2184  if (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT)
2185  {
2186  //Next one is chained
2187  desc_index = desc_table[desc_index].next;
2188  buffer_map_addr = desc_table[desc_index].addr;
2189  buffer_len = desc_table[desc_index].len;
2190  }
2191  else if (vui->virtio_net_hdr_sz == 12) //MRG is available
2192  {
2193  virtio_net_hdr_mrg_rxbuf_t *hdr =
2194  &vum->cpus[thread_index].tx_headers[tx_headers_len - 1];
2195 
2196  //Move from available to used buffer
2197  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].id =
2198  desc_head;
2199  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].len =
2200  desc_len;
2201  vhost_user_log_dirty_ring (vui, rxvq,
2202  ring[rxvq->last_used_idx &
2203  rxvq->qsz_mask]);
2204 
2205  rxvq->last_avail_idx++;
2206  rxvq->last_used_idx++;
2207  hdr->num_buffers++;
2208  desc_len = 0;
2209 
2210  if (PREDICT_FALSE
2211  (rxvq->last_avail_idx == rxvq->avail->idx))
2212  {
2213  //Dequeue queued descriptors for this packet
2214  rxvq->last_used_idx -= hdr->num_buffers - 1;
2215  rxvq->last_avail_idx -= hdr->num_buffers - 1;
2216  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
2217  goto done;
2218  }
2219 
2220  desc_table = rxvq->desc;
2221  desc_head = desc_index =
2222  rxvq->avail->ring[rxvq->last_avail_idx & rxvq->qsz_mask];
2223  if (PREDICT_FALSE
2224  (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT))
2225  {
2226  //It is seriously unlikely that a driver will put indirect descriptor
2227  //after non-indirect descriptor.
2228  if (PREDICT_FALSE
2229  (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
2230  {
2231  error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
2232  goto done;
2233  }
2234  if (PREDICT_FALSE
2235  (!(desc_table =
2236  map_guest_mem (vui,
2237  rxvq->desc[desc_index].addr,
2238  &map_hint))))
2239  {
2240  error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
2241  goto done;
2242  }
2243  desc_index = 0;
2244  }
2245  buffer_map_addr = desc_table[desc_index].addr;
2246  buffer_len = desc_table[desc_index].len;
2247  }
2248  else
2249  {
2250  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
2251  goto done;
2252  }
2253  }
2254 
2255  {
2256  vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len];
2257  copy_len++;
2258  cpy->len = bytes_left;
2259  cpy->len = (cpy->len > buffer_len) ? buffer_len : cpy->len;
2260  cpy->dst = buffer_map_addr;
2261  cpy->src = (uword) vlib_buffer_get_current (current_b0) +
2262  current_b0->current_length - bytes_left;
2263 
2264  bytes_left -= cpy->len;
2265  buffer_len -= cpy->len;
2266  buffer_map_addr += cpy->len;
2267  desc_len += cpy->len;
2268 
2269  CLIB_PREFETCH (&rxvq->desc, CLIB_CACHE_LINE_BYTES, LOAD);
2270  }
2271 
2272  // Check if vlib buffer has more data. If not, get more or break.
2273  if (PREDICT_TRUE (!bytes_left))
2274  {
2275  if (PREDICT_FALSE
2276  (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT))
2277  {
2278  current_b0 = vlib_get_buffer (vm, current_b0->next_buffer);
2279  bytes_left = current_b0->current_length;
2280  }
2281  else
2282  {
2283  //End of packet
2284  break;
2285  }
2286  }
2287  }
2288 
2289  //Move from available to used ring
2290  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].id = desc_head;
2291  rxvq->used->ring[rxvq->last_used_idx & rxvq->qsz_mask].len = desc_len;
2292  vhost_user_log_dirty_ring (vui, rxvq,
2293  ring[rxvq->last_used_idx & rxvq->qsz_mask]);
2294  rxvq->last_avail_idx++;
2295  rxvq->last_used_idx++;
2296 
2297  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2298  {
2299  vum->cpus[thread_index].current_trace->hdr =
2300  vum->cpus[thread_index].tx_headers[tx_headers_len - 1];
2301  }
2302 
2303  n_left--; //At the end for error counting when 'goto done' is invoked
2304 
2305  /*
2306  * Do the copy periodically to prevent
2307  * vum->cpus[thread_index].copy array overflow and corrupt memory
2308  */
2309  if (PREDICT_FALSE (copy_len >= VHOST_USER_TX_COPY_THRESHOLD))
2310  {
2311  if (PREDICT_FALSE
2312  (vhost_user_tx_copy (vui, vum->cpus[thread_index].copy,
2313  copy_len, &map_hint)))
2314  {
2315  vlib_error_count (vm, node->node_index,
2316  VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
2317  }
2318  copy_len = 0;
2319 
2320  /* give buffers back to driver */
2322  rxvq->used->idx = rxvq->last_used_idx;
2323  vhost_user_log_dirty_ring (vui, rxvq, idx);
2324  }
2325  buffers++;
2326  }
2327 
2328 done:
2329  //Do the memory copies
2330  if (PREDICT_FALSE
2331  (vhost_user_tx_copy (vui, vum->cpus[thread_index].copy,
2332  copy_len, &map_hint)))
2333  {
2334  vlib_error_count (vm, node->node_index,
2335  VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
2336  }
2337 
2339  rxvq->used->idx = rxvq->last_used_idx;
2340  vhost_user_log_dirty_ring (vui, rxvq, idx);
2341 
2342  /*
2343  * When n_left is set, error is always set to something too.
2344  * In case error is due to lack of remaining buffers, we go back up and
2345  * retry.
2346  * The idea is that it is better to waste some time on packets
2347  * that have been processed already than dropping them and get
2348  * more fresh packets with a good likelyhood that they will be dropped too.
2349  * This technique also gives more time to VM driver to pick-up packets.
2350  * In case the traffic flows from physical to virtual interfaces, this
2351  * technique will end-up leveraging the physical NIC buffer in order to
2352  * absorb the VM's CPU jitter.
2353  */
2354  if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry)
2355  {
2356  retry--;
2357  goto retry;
2358  }
2359 
2360  /* interrupt (call) handling */
2361  if ((rxvq->callfd_idx != ~0) &&
2362  !(rxvq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
2363  {
2364  rxvq->n_since_last_int += frame->n_vectors - n_left;
2365 
2366  if (rxvq->n_since_last_int > vum->coalesce_frames)
2367  vhost_user_send_call (vm, rxvq);
2368  }
2369 
2370  vhost_user_vring_unlock (vui, qid);
2371 
2372 done3:
2373  if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
2374  {
2375  vlib_error_count (vm, node->node_index, error, n_left);
2379  thread_index, vui->sw_if_index, n_left);
2380  }
2381 
2382  vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
2383  return frame->n_vectors;
2384 }
2385 
2386 static uword
2389 {
2390  vhost_user_intf_t *vui;
2391  f64 timeout = 3153600000.0 /* 100 years */ ;
2392  uword event_type, *event_data = 0;
2394  u16 *queue;
2395  f64 now, poll_time_remaining;
2396  f64 next_timeout;
2397  u8 stop_timer = 0;
2398 
2399  while (1)
2400  {
2401  poll_time_remaining =
2403  event_type = vlib_process_get_events (vm, &event_data);
2404  vec_reset_length (event_data);
2405 
2406  /*
2407  * Use the remaining timeout if it is less than coalesce time to avoid
2408  * resetting the existing timer in the middle of expiration
2409  */
2410  timeout = poll_time_remaining;
2411  if (vlib_process_suspend_time_is_zero (timeout) ||
2412  (timeout > vum->coalesce_time))
2413  timeout = vum->coalesce_time;
2414 
2415  now = vlib_time_now (vm);
2416  switch (event_type)
2417  {
2419  stop_timer = 1;
2420  break;
2421 
2423  stop_timer = 0;
2424  if (!vlib_process_suspend_time_is_zero (poll_time_remaining))
2425  break;
2426  /* fall through */
2427 
2428  case ~0:
2429  /* *INDENT-OFF* */
2430  pool_foreach (vui, vum->vhost_user_interfaces, {
2431  next_timeout = timeout;
2432  vec_foreach (queue, vui->rx_queues)
2433  {
2434  vhost_user_vring_t *rxvq =
2435  &vui->vrings[VHOST_VRING_IDX_RX (*queue)];
2436  vhost_user_vring_t *txvq =
2437  &vui->vrings[VHOST_VRING_IDX_TX (*queue)];
2438 
2439  if (txvq->n_since_last_int)
2440  {
2441  if (now >= txvq->int_deadline)
2442  vhost_user_send_call (vm, txvq);
2443  else
2444  next_timeout = txvq->int_deadline - now;
2445  }
2446 
2447  if (rxvq->n_since_last_int)
2448  {
2449  if (now >= rxvq->int_deadline)
2450  vhost_user_send_call (vm, rxvq);
2451  else
2452  next_timeout = rxvq->int_deadline - now;
2453  }
2454 
2455  if ((next_timeout < timeout) && (next_timeout > 0.0))
2456  timeout = next_timeout;
2457  }
2458  });
2459  /* *INDENT-ON* */
2460  break;
2461 
2462  default:
2463  clib_warning ("BUG: unhandled event type %d", event_type);
2464  break;
2465  }
2466  /* No less than 1 millisecond */
2467  if (timeout < 1e-3)
2468  timeout = 1e-3;
2469  if (stop_timer)
2470  timeout = 3153600000.0;
2471  }
2472  return 0;
2473 }
2474 
2475 /* *INDENT-OFF* */
2478  .type = VLIB_NODE_TYPE_PROCESS,
2479  .name = "vhost-user-send-interrupt-process",
2480 };
2481 /* *INDENT-ON* */
2482 
2483 static clib_error_t *
2485  u32 qid, vnet_hw_interface_rx_mode mode)
2486 {
2487  vlib_main_t *vm = vnm->vlib_main;
2488  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
2490  vhost_user_intf_t *vui =
2492  vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
2493 
2494  if ((mode == VNET_HW_INTERFACE_RX_MODE_INTERRUPT) ||
2496  {
2497  if (txvq->kickfd_idx == ~0)
2498  {
2499  // We cannot support interrupt mode if the driver opts out
2500  return clib_error_return (0, "Driver does not support interrupt");
2501  }
2503  {
2504  vum->ifq_count++;
2505  // Start the timer if this is the first encounter on interrupt
2506  // interface/queue
2507  if ((vum->ifq_count == 1) &&
2508  (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
2512  }
2513  }
2514  else if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
2515  {
2516  if (((txvq->mode == VNET_HW_INTERFACE_RX_MODE_INTERRUPT) ||
2518  vum->ifq_count)
2519  {
2520  vum->ifq_count--;
2521  // Stop the timer if there is no more interrupt interface/queue
2522  if ((vum->ifq_count == 0) &&
2523  (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
2527  }
2528  }
2529 
2530  txvq->mode = mode;
2532  txvq->used->flags = VRING_USED_F_NO_NOTIFY;
2533  else if ((mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE) ||
2535  txvq->used->flags = 0;
2536  else
2537  {
2538  clib_warning ("BUG: unhandled mode %d changed for if %d queue %d", mode,
2539  hw_if_index, qid);
2540  return clib_error_return (0, "unsupported");
2541  }
2542 
2543  return 0;
2544 }
2545 
2546 static clib_error_t *
2548  u32 flags)
2549 {
2550  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
2552  vhost_user_intf_t *vui =
2554  u32 hw_flags = 0;
2555  vui->admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
2556  hw_flags = vui->admin_up ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0;
2557 
2558  vnet_hw_interface_set_flags (vnm, vui->hw_if_index, hw_flags);
2559 
2560  return /* no error */ 0;
2561 }
2562 
2563 /* *INDENT-OFF* */
2564 VNET_DEVICE_CLASS (vhost_user_dev_class,static) = {
2565  .name = "vhost-user",
2566  .tx_function = vhost_user_tx,
2567  .tx_function_n_errors = VHOST_USER_TX_FUNC_N_ERROR,
2568  .tx_function_error_strings = vhost_user_tx_func_error_strings,
2569  .format_device_name = format_vhost_user_interface_name,
2570  .name_renumber = vhost_user_name_renumber,
2571  .admin_up_down_function = vhost_user_interface_admin_up_down,
2572  .rx_mode_change_function = vhost_user_interface_rx_mode_change,
2573  .format_tx_trace = format_vhost_trace,
2574 };
2575 
2577  vhost_user_tx)
2578 /* *INDENT-ON* */
2579 
2580 static uword
2581 vhost_user_process (vlib_main_t * vm,
2583 {
2585  vhost_user_intf_t *vui;
2586  struct sockaddr_un sun;
2587  int sockfd;
2588  clib_file_t template = { 0 };
2589  f64 timeout = 3153600000.0 /* 100 years */ ;
2590  uword *event_data = 0;
2591 
2592  sockfd = -1;
2593  sun.sun_family = AF_UNIX;
2595  template.error_function = vhost_user_socket_error;
2596 
2597  while (1)
2598  {
2600  vlib_process_get_events (vm, &event_data);
2601  vec_reset_length (event_data);
2602 
2603  timeout = 3.0;
2604 
2605  /* *INDENT-OFF* */
2606  pool_foreach (vui, vum->vhost_user_interfaces, {
2607 
2608  if (vui->unix_server_index == ~0) { //Nothing to do for server sockets
2609  if (vui->clib_file_index == ~0)
2610  {
2611  if ((sockfd < 0) &&
2612  ((sockfd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0))
2613  {
2614  /*
2615  * 1st time error or new error for this interface,
2616  * spit out the message and record the error
2617  */
2618  if (!vui->sock_errno || (vui->sock_errno != errno))
2619  {
2620  clib_unix_warning
2621  ("Error: Could not open unix socket for %s",
2622  vui->sock_filename);
2623  vui->sock_errno = errno;
2624  }
2625  continue;
2626  }
2627 
2628  /* try to connect */
2629  strncpy (sun.sun_path, (char *) vui->sock_filename,
2630  sizeof (sun.sun_path) - 1);
2631 
2632  /* Avoid hanging VPP if the other end does not accept */
2633  if (fcntl(sockfd, F_SETFL, O_NONBLOCK) < 0)
2634  clib_unix_warning ("fcntl");
2635 
2636  if (connect (sockfd, (struct sockaddr *) &sun,
2637  sizeof (struct sockaddr_un)) == 0)
2638  {
2639  /* Set the socket to blocking as it was before */
2640  if (fcntl(sockfd, F_SETFL, 0) < 0)
2641  clib_unix_warning ("fcntl2");
2642 
2643  vui->sock_errno = 0;
2644  template.file_descriptor = sockfd;
2645  template.private_data =
2646  vui - vhost_user_main.vhost_user_interfaces;
2647  vui->clib_file_index = clib_file_add (&file_main, &template);
2648 
2649  /* This sockfd is considered consumed */
2650  sockfd = -1;
2651  }
2652  else
2653  {
2654  vui->sock_errno = errno;
2655  }
2656  }
2657  else
2658  {
2659  /* check if socket is alive */
2660  int error = 0;
2661  socklen_t len = sizeof (error);
2662  int fd = UNIX_GET_FD(vui->clib_file_index);
2663  int retval =
2664  getsockopt (fd, SOL_SOCKET, SO_ERROR, &error, &len);
2665 
2666  if (retval)
2667  {
2668  DBG_SOCK ("getsockopt returned %d", retval);
2669  vhost_user_if_disconnect (vui);
2670  }
2671  }
2672  }
2673  });
2674  /* *INDENT-ON* */
2675  }
2676  return 0;
2677 }
2678 
2679 /* *INDENT-OFF* */
2681  .function = vhost_user_process,
2682  .type = VLIB_NODE_TYPE_PROCESS,
2683  .name = "vhost-user-process",
2684 };
2685 /* *INDENT-ON* */
2686 
2687 /**
2688  * Disables and reset interface structure.
2689  * It can then be either init again, or removed from used interfaces.
2690  */
2691 static void
2693 {
2694  int q;
2696 
2697  // disconnect interface sockets
2700 
2701  for (q = 0; q < VHOST_VRING_MAX_N; q++)
2702  {
2703  clib_mem_free ((void *) vui->vring_locks[q]);
2704  }
2705 
2706  if (vui->unix_server_index != ~0)
2707  {
2708  //Close server socket
2710  vui->unix_server_index);
2711  clib_file_del (&file_main, uf);
2712  vui->unix_server_index = ~0;
2713  unlink (vui->sock_filename);
2714  }
2715 
2717  &vui->if_index);
2718 }
2719 
2720 int
2722 {
2724  vhost_user_intf_t *vui;
2725  int rv = 0;
2726  vnet_hw_interface_t *hwif;
2727  u16 *queue;
2728 
2729  if (!(hwif = vnet_get_sup_hw_interface (vnm, sw_if_index)) ||
2730  hwif->dev_class_index != vhost_user_dev_class.index)
2731  return VNET_API_ERROR_INVALID_SW_IF_INDEX;
2732 
2733  DBG_SOCK ("Deleting vhost-user interface %s (instance %d)",
2734  hwif->name, hwif->dev_instance);
2735 
2737 
2738  vec_foreach (queue, vui->rx_queues)
2739  {
2740  vhost_user_vring_t *txvq;
2741 
2742  txvq = &vui->vrings[VHOST_VRING_IDX_TX (*queue)];
2743  if ((vum->ifq_count > 0) &&
2746  {
2747  vum->ifq_count--;
2748  // Stop the timer if there is no more interrupt interface/queue
2749  if ((vum->ifq_count == 0) &&
2750  (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
2751  {
2755  break;
2756  }
2757  }
2758  }
2759 
2760  // Disable and reset interface
2761  vhost_user_term_if (vui);
2762 
2763  // Reset renumbered iface
2764  if (hwif->dev_instance <
2767 
2768  // Delete ethernet interface
2770 
2771  // Back to pool
2772  pool_put (vum->vhost_user_interfaces, vui);
2773 
2774  return rv;
2775 }
2776 
2777 static clib_error_t *
2779 {
2780  vnet_main_t *vnm = vnet_get_main ();
2782  vhost_user_intf_t *vui;
2783 
2785  /* *INDENT-OFF* */
2786  pool_foreach (vui, vum->vhost_user_interfaces, {
2787  vhost_user_delete_if (vnm, vm, vui->sw_if_index);
2788  });
2789  /* *INDENT-ON* */
2791  return 0;
2792 }
2793 
2795 
2796 /**
2797  * Open server unix socket on specified sock_filename.
2798  */
2799 static int
2800 vhost_user_init_server_sock (const char *sock_filename, int *sock_fd)
2801 {
2802  int rv = 0;
2803  struct sockaddr_un un = { };
2804  int fd;
2805  /* create listening socket */
2806  if ((fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
2807  return VNET_API_ERROR_SYSCALL_ERROR_1;
2808 
2809  un.sun_family = AF_UNIX;
2810  strncpy ((char *) un.sun_path, (char *) sock_filename,
2811  sizeof (un.sun_path) - 1);
2812 
2813  /* remove if exists */
2814  unlink ((char *) sock_filename);
2815 
2816  if (bind (fd, (struct sockaddr *) &un, sizeof (un)) == -1)
2817  {
2818  rv = VNET_API_ERROR_SYSCALL_ERROR_2;
2819  goto error;
2820  }
2821 
2822  if (listen (fd, 1) == -1)
2823  {
2824  rv = VNET_API_ERROR_SYSCALL_ERROR_3;
2825  goto error;
2826  }
2827 
2828  *sock_fd = fd;
2829  return 0;
2830 
2831 error:
2832  close (fd);
2833  return rv;
2834 }
2835 
2836 /**
2837  * Create ethernet interface for vhost user interface.
2838  */
2839 static void
2841  vhost_user_intf_t * vui, u8 * hwaddress)
2842 {
2844  u8 hwaddr[6];
2845  clib_error_t *error;
2846 
2847  /* create hw and sw interface */
2848  if (hwaddress)
2849  {
2850  clib_memcpy (hwaddr, hwaddress, 6);
2851  }
2852  else
2853  {
2854  random_u32 (&vum->random);
2855  clib_memcpy (hwaddr + 2, &vum->random, sizeof (vum->random));
2856  hwaddr[0] = 2;
2857  hwaddr[1] = 0xfe;
2858  }
2859 
2861  (vnm,
2862  vhost_user_dev_class.index,
2863  vui - vum->vhost_user_interfaces /* device instance */ ,
2864  hwaddr /* ethernet address */ ,
2865  &vui->hw_if_index, 0 /* flag change */ );
2866 
2867  if (error)
2868  clib_error_report (error);
2869 
2870  vnet_sw_interface_set_mtu (vnm, vui->sw_if_index, 9000);
2871 }
2872 
2873 /*
2874  * Initialize vui with specified attributes
2875  */
2876 static void
2878  vhost_user_intf_t * vui,
2879  int server_sock_fd,
2880  const char *sock_filename,
2881  u64 feature_mask, u32 * sw_if_index)
2882 {
2883  vnet_sw_interface_t *sw;
2884  int q;
2886  vnet_hw_interface_t *hw;
2887 
2888  hw = vnet_get_hw_interface (vnm, vui->hw_if_index);
2889  sw = vnet_get_hw_sw_interface (vnm, vui->hw_if_index);
2890  if (server_sock_fd != -1)
2891  {
2892  clib_file_t template = { 0 };
2894  template.file_descriptor = server_sock_fd;
2895  template.private_data = vui - vum->vhost_user_interfaces; //hw index
2896  vui->unix_server_index = clib_file_add (&file_main, &template);
2897  }
2898  else
2899  {
2900  vui->unix_server_index = ~0;
2901  }
2902 
2903  vui->sw_if_index = sw->sw_if_index;
2904  strncpy (vui->sock_filename, sock_filename,
2905  ARRAY_LEN (vui->sock_filename) - 1);
2906  vui->sock_errno = 0;
2907  vui->is_up = 0;
2908  vui->feature_mask = feature_mask;
2909  vui->clib_file_index = ~0;
2910  vui->log_base_addr = 0;
2911  vui->if_index = vui - vum->vhost_user_interfaces;
2913  &vui->if_index, 0);
2914 
2915  for (q = 0; q < VHOST_VRING_MAX_N; q++)
2916  vhost_user_vring_init (vui, q);
2917 
2920 
2921  if (sw_if_index)
2922  *sw_if_index = vui->sw_if_index;
2923 
2924  for (q = 0; q < VHOST_VRING_MAX_N; q++)
2925  {
2928  memset ((void *) vui->vring_locks[q], 0, CLIB_CACHE_LINE_BYTES);
2929  }
2930 
2932  vlib_get_thread_main ()->n_vlib_mains - 1);
2934 }
2935 
2936 int
2938  const char *sock_filename,
2939  u8 is_server,
2940  u32 * sw_if_index,
2941  u64 feature_mask,
2942  u8 renumber, u32 custom_dev_instance, u8 * hwaddr)
2943 {
2944  vhost_user_intf_t *vui = NULL;
2945  u32 sw_if_idx = ~0;
2946  int rv = 0;
2947  int server_sock_fd = -1;
2949  uword *if_index;
2950 
2951  if (sock_filename == NULL || !(strlen (sock_filename) > 0))
2952  {
2953  return VNET_API_ERROR_INVALID_ARGUMENT;
2954  }
2955 
2956  if_index = mhash_get (&vum->if_index_by_sock_name, (void *) sock_filename);
2957  if (if_index)
2958  {
2959  if (sw_if_index)
2960  {
2961  vui = &vum->vhost_user_interfaces[*if_index];
2962  *sw_if_index = vui->sw_if_index;
2963  }
2964  return VNET_API_ERROR_IF_ALREADY_EXISTS;
2965  }
2966 
2967  if (is_server)
2968  {
2969  if ((rv =
2970  vhost_user_init_server_sock (sock_filename, &server_sock_fd)) != 0)
2971  {
2972  return rv;
2973  }
2974  }
2975 
2976  pool_get (vhost_user_main.vhost_user_interfaces, vui);
2977 
2978  vhost_user_create_ethernet (vnm, vm, vui, hwaddr);
2979  vhost_user_vui_init (vnm, vui, server_sock_fd, sock_filename,
2980  feature_mask, &sw_if_idx);
2981 
2982  if (renumber)
2983  vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
2984 
2985  if (sw_if_index)
2986  *sw_if_index = sw_if_idx;
2987 
2988  // Process node must connect
2990 
2991  return rv;
2992 }
2993 
2994 int
2996  const char *sock_filename,
2997  u8 is_server,
2998  u32 sw_if_index,
2999  u64 feature_mask, u8 renumber, u32 custom_dev_instance)
3000 {
3002  vhost_user_intf_t *vui = NULL;
3003  u32 sw_if_idx = ~0;
3004  int server_sock_fd = -1;
3005  int rv = 0;
3006  vnet_hw_interface_t *hwif;
3007  uword *if_index;
3008 
3009  if (!(hwif = vnet_get_sup_hw_interface (vnm, sw_if_index)) ||
3010  hwif->dev_class_index != vhost_user_dev_class.index)
3011  return VNET_API_ERROR_INVALID_SW_IF_INDEX;
3012 
3013  if (sock_filename == NULL || !(strlen (sock_filename) > 0))
3014  return VNET_API_ERROR_INVALID_ARGUMENT;
3015 
3017 
3018  /*
3019  * Disallow changing the interface to have the same path name
3020  * as other interface
3021  */
3022  if_index = mhash_get (&vum->if_index_by_sock_name, (void *) sock_filename);
3023  if (if_index && (*if_index != vui->if_index))
3024  return VNET_API_ERROR_IF_ALREADY_EXISTS;
3025 
3026  // First try to open server socket
3027  if (is_server)
3028  if ((rv = vhost_user_init_server_sock (sock_filename,
3029  &server_sock_fd)) != 0)
3030  return rv;
3031 
3032  vhost_user_term_if (vui);
3033  vhost_user_vui_init (vnm, vui, server_sock_fd,
3034  sock_filename, feature_mask, &sw_if_idx);
3035 
3036  if (renumber)
3037  vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
3038 
3039  // Process node must connect
3041 
3042  return rv;
3043 }
3044 
3045 clib_error_t *
3047  unformat_input_t * input,
3048  vlib_cli_command_t * cmd)
3049 {
3050  unformat_input_t _line_input, *line_input = &_line_input;
3051  u8 *sock_filename = NULL;
3052  u32 sw_if_index;
3053  u8 is_server = 0;
3054  u64 feature_mask = (u64) ~ (0ULL);
3055  u8 renumber = 0;
3056  u32 custom_dev_instance = ~0;
3057  u8 hwaddr[6];
3058  u8 *hw = NULL;
3059  clib_error_t *error = NULL;
3060 
3061  /* Get a line of input. */
3062  if (!unformat_user (input, unformat_line_input, line_input))
3063  return 0;
3064 
3065  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
3066  {
3067  if (unformat (line_input, "socket %s", &sock_filename))
3068  ;
3069  else if (unformat (line_input, "server"))
3070  is_server = 1;
3071  else if (unformat (line_input, "feature-mask 0x%llx", &feature_mask))
3072  ;
3073  else
3074  if (unformat
3075  (line_input, "hwaddr %U", unformat_ethernet_address, hwaddr))
3076  hw = hwaddr;
3077  else if (unformat (line_input, "renumber %d", &custom_dev_instance))
3078  {
3079  renumber = 1;
3080  }
3081  else
3082  {
3083  error = clib_error_return (0, "unknown input `%U'",
3084  format_unformat_error, line_input);
3085  goto done;
3086  }
3087  }
3088 
3089  vnet_main_t *vnm = vnet_get_main ();
3090 
3091  int rv;
3092  if ((rv = vhost_user_create_if (vnm, vm, (char *) sock_filename,
3093  is_server, &sw_if_index, feature_mask,
3094  renumber, custom_dev_instance, hw)))
3095  {
3096  error = clib_error_return (0, "vhost_user_create_if returned %d", rv);
3097  goto done;
3098  }
3099 
3101  sw_if_index);
3102 
3103 done:
3104  vec_free (sock_filename);
3105  unformat_free (line_input);
3106 
3107  return error;
3108 }
3109 
3110 clib_error_t *
3112  unformat_input_t * input,
3113  vlib_cli_command_t * cmd)
3114 {
3115  unformat_input_t _line_input, *line_input = &_line_input;
3116  u32 sw_if_index = ~0;
3117  vnet_main_t *vnm = vnet_get_main ();
3118  clib_error_t *error = NULL;
3119 
3120  /* Get a line of input. */
3121  if (!unformat_user (input, unformat_line_input, line_input))
3122  return 0;
3123 
3124  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
3125  {
3126  if (unformat (line_input, "sw_if_index %d", &sw_if_index))
3127  ;
3128  else if (unformat
3129  (line_input, "%U", unformat_vnet_sw_interface, vnm,
3130  &sw_if_index))
3131  {
3132  vnet_hw_interface_t *hwif =
3133  vnet_get_sup_hw_interface (vnm, sw_if_index);
3134  if (hwif == NULL ||
3135  vhost_user_dev_class.index != hwif->dev_class_index)
3136  {
3137  error = clib_error_return (0, "Not a vhost interface");
3138  goto done;
3139  }
3140  }
3141  else
3142  {
3143  error = clib_error_return (0, "unknown input `%U'",
3144  format_unformat_error, line_input);
3145  goto done;
3146  }
3147  }
3148 
3149  vhost_user_delete_if (vnm, vm, sw_if_index);
3150 
3151 done:
3152  unformat_free (line_input);
3153 
3154  return error;
3155 }
3156 
3157 int
3159  vhost_user_intf_details_t ** out_vuids)
3160 {
3161  int rv = 0;
3163  vhost_user_intf_t *vui;
3164  vhost_user_intf_details_t *r_vuids = NULL;
3166  u32 *hw_if_indices = 0;
3168  u8 *s = NULL;
3169  int i;
3170 
3171  if (!out_vuids)
3172  return -1;
3173 
3175  vec_add1 (hw_if_indices, vui->hw_if_index);
3176  );
3177 
3178  for (i = 0; i < vec_len (hw_if_indices); i++)
3179  {
3180  hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
3182 
3183  vec_add2 (r_vuids, vuid, 1);
3184  vuid->sw_if_index = vui->sw_if_index;
3185  vuid->virtio_net_hdr_sz = vui->virtio_net_hdr_sz;
3186  vuid->features = vui->features;
3187  vuid->num_regions = vui->nregions;
3188  vuid->is_server = vui->unix_server_index != ~0;
3189  vuid->sock_errno = vui->sock_errno;
3190  strncpy ((char *) vuid->sock_filename, (char *) vui->sock_filename,
3191  ARRAY_LEN (vuid->sock_filename) - 1);
3192 
3193  s = format (s, "%v%c", hi->name, 0);
3194 
3195  strncpy ((char *) vuid->if_name, (char *) s,
3196  ARRAY_LEN (vuid->if_name) - 1);
3197  _vec_len (s) = 0;
3198  }
3199 
3200  vec_free (s);
3201  vec_free (hw_if_indices);
3202 
3203  *out_vuids = r_vuids;
3204 
3205  return rv;
3206 }
3207 
3208 clib_error_t *
3210  unformat_input_t * input,
3211  vlib_cli_command_t * cmd)
3212 {
3213  clib_error_t *error = 0;
3214  vnet_main_t *vnm = vnet_get_main ();
3216  vhost_user_intf_t *vui;
3217  u32 hw_if_index, *hw_if_indices = 0;
3219  u16 *queue;
3220  u32 ci;
3221  int i, j, q;
3222  int show_descr = 0;
3223  struct feat_struct
3224  {
3225  u8 bit;
3226  char *str;
3227  };
3228  struct feat_struct *feat_entry;
3229 
3230  static struct feat_struct feat_array[] = {
3231 #define _(s,b) { .str = #s, .bit = b, },
3233 #undef _
3234  {.str = NULL}
3235  };
3236 
3237 #define foreach_protocol_feature \
3238  _(VHOST_USER_PROTOCOL_F_MQ) \
3239  _(VHOST_USER_PROTOCOL_F_LOG_SHMFD)
3240 
3241  static struct feat_struct proto_feat_array[] = {
3242 #define _(s) { .str = #s, .bit = s},
3244 #undef _
3245  {.str = NULL}
3246  };
3247 
3248  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3249  {
3250  if (unformat
3251  (input, "%U", unformat_vnet_hw_interface, vnm, &hw_if_index))
3252  {
3253  hi = vnet_get_hw_interface (vnm, hw_if_index);
3254  if (vhost_user_dev_class.index != hi->dev_class_index)
3255  {
3256  error = clib_error_return (0, "unknown input `%U'",
3257  format_unformat_error, input);
3258  goto done;
3259  }
3260  vec_add1 (hw_if_indices, hw_if_index);
3261  }
3262  else if (unformat (input, "descriptors") || unformat (input, "desc"))
3263  show_descr = 1;
3264  else
3265  {
3266  error = clib_error_return (0, "unknown input `%U'",
3267  format_unformat_error, input);
3268  goto done;
3269  }
3270  }
3271  if (vec_len (hw_if_indices) == 0)
3272  {
3274  vec_add1 (hw_if_indices, vui->hw_if_index);
3275  );
3276  }
3277  vlib_cli_output (vm, "Virtio vhost-user interfaces");
3278  vlib_cli_output (vm, "Global:\n coalesce frames %d time %e",
3279  vum->coalesce_frames, vum->coalesce_time);
3280  vlib_cli_output (vm, " number of rx virtqueues in interrupt mode: %d",
3281  vum->ifq_count);
3282 
3283  for (i = 0; i < vec_len (hw_if_indices); i++)
3284  {
3285  hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
3287  vlib_cli_output (vm, "Interface: %s (ifindex %d)",
3288  hi->name, hw_if_indices[i]);
3289 
3290  vlib_cli_output (vm, "virtio_net_hdr_sz %d\n"
3291  " features mask (0x%llx): \n"
3292  " features (0x%llx): \n",
3293  vui->virtio_net_hdr_sz, vui->feature_mask,
3294  vui->features);
3295 
3296  feat_entry = (struct feat_struct *) &feat_array;
3297  while (feat_entry->str)
3298  {
3299  if (vui->features & (1ULL << feat_entry->bit))
3300  vlib_cli_output (vm, " %s (%d)", feat_entry->str,
3301  feat_entry->bit);
3302  feat_entry++;
3303  }
3304 
3305  vlib_cli_output (vm, " protocol features (0x%llx)",
3306  vui->protocol_features);
3307  feat_entry = (struct feat_struct *) &proto_feat_array;
3308  while (feat_entry->str)
3309  {
3310  if (vui->protocol_features & (1ULL << feat_entry->bit))
3311  vlib_cli_output (vm, " %s (%d)", feat_entry->str,
3312  feat_entry->bit);
3313  feat_entry++;
3314  }
3315 
3316  vlib_cli_output (vm, "\n");
3317 
3318  vlib_cli_output (vm, " socket filename %s type %s errno \"%s\"\n\n",
3319  vui->sock_filename,
3320  (vui->unix_server_index != ~0) ? "server" : "client",
3321  strerror (vui->sock_errno));
3322 
3323  vlib_cli_output (vm, " rx placement: ");
3324 
3325  vec_foreach (queue, vui->rx_queues)
3326  {
3327  vnet_main_t *vnm = vnet_get_main ();
3328  uword thread_index;
3330 
3331  thread_index = vnet_get_device_input_thread_index (vnm,
3332  vui->hw_if_index,
3333  *queue);
3334  vnet_hw_interface_get_rx_mode (vnm, vui->hw_if_index, *queue, &mode);
3335  vlib_cli_output (vm, " thread %d on vring %d, %U\n",
3336  thread_index, VHOST_VRING_IDX_TX (*queue),
3338  }
3339 
3340  vlib_cli_output (vm, " tx placement: %s\n",
3341  vui->use_tx_spinlock ? "spin-lock" : "lock-free");
3342 
3344  {
3345  vlib_cli_output (vm, " thread %d on vring %d\n", ci,
3346  VHOST_VRING_IDX_RX (vui->per_cpu_tx_qid[ci]));
3347  }
3348 
3349  vlib_cli_output (vm, "\n");
3350 
3351  vlib_cli_output (vm, " Memory regions (total %d)\n", vui->nregions);
3352 
3353  if (vui->nregions)
3354  {
3355  vlib_cli_output (vm,
3356  " region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr\n");
3357  vlib_cli_output (vm,
3358  " ====== ===== ================== ================== ================== ================== ==================\n");
3359  }
3360  for (j = 0; j < vui->nregions; j++)
3361  {
3362  vlib_cli_output (vm,
3363  " %d %-5d 0x%016lx 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
3364  j, vui->region_mmap_fd[j],
3365  vui->regions[j].guest_phys_addr,
3366  vui->regions[j].memory_size,
3367  vui->regions[j].userspace_addr,
3368  vui->regions[j].mmap_offset,
3370  }
3371  for (q = 0; q < VHOST_VRING_MAX_N; q++)
3372  {
3373  if (!vui->vrings[q].started)
3374  continue;
3375 
3376  vlib_cli_output (vm, "\n Virtqueue %d (%s%s)\n", q,
3377  (q & 1) ? "RX" : "TX",
3378  vui->vrings[q].enabled ? "" : " disabled");
3379 
3380  vlib_cli_output (vm,
3381  " qsz %d last_avail_idx %d last_used_idx %d\n",
3382  vui->vrings[q].qsz_mask + 1,
3383  vui->vrings[q].last_avail_idx,
3384  vui->vrings[q].last_used_idx);
3385 
3386  if (vui->vrings[q].avail && vui->vrings[q].used)
3387  vlib_cli_output (vm,
3388  " avail.flags %x avail.idx %d used.flags %x used.idx %d\n",
3389  vui->vrings[q].avail->flags,
3390  vui->vrings[q].avail->idx,
3391  vui->vrings[q].used->flags,
3392  vui->vrings[q].used->idx);
3393 
3394  int kickfd = UNIX_GET_FD (vui->vrings[q].kickfd_idx);
3395  int callfd = UNIX_GET_FD (vui->vrings[q].callfd_idx);
3396  vlib_cli_output (vm, " kickfd %d callfd %d errfd %d\n",
3397  kickfd, callfd, vui->vrings[q].errfd);
3398 
3399  if (show_descr)
3400  {
3401  vlib_cli_output (vm, "\n descriptor table:\n");
3402  vlib_cli_output (vm,
3403  " id addr len flags next user_addr\n");
3404  vlib_cli_output (vm,
3405  " ===== ================== ===== ====== ===== ==================\n");
3406  for (j = 0; j < vui->vrings[q].qsz_mask + 1; j++)
3407  {
3408  u32 mem_hint = 0;
3409  vlib_cli_output (vm,
3410  " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
3411  j, vui->vrings[q].desc[j].addr,
3412  vui->vrings[q].desc[j].len,
3413  vui->vrings[q].desc[j].flags,
3414  vui->vrings[q].desc[j].next,
3416  (vui,
3417  vui->vrings[q].desc[j].
3418  addr, &mem_hint)));
3419  }
3420  }
3421  }
3422  vlib_cli_output (vm, "\n");
3423  }
3424 done:
3425  vec_free (hw_if_indices);
3426  return error;
3427 }
3428 
3429 /*
3430  * CLI functions
3431  */
3432 
3433 /*?
3434  * Create a vHost User interface. Once created, a new virtual interface
3435  * will exist with the name '<em>VirtualEthernet0/0/x</em>', where '<em>x</em>'
3436  * is the next free index.
3437  *
3438  * There are several parameters associated with a vHost interface:
3439  *
3440  * - <b>socket <socket-filename></b> - Name of the linux socket used by hypervisor
3441  * and VPP to manage the vHost interface. If in '<em>server</em>' mode, VPP will
3442  * create the socket if it does not already exist. If in '<em>client</em>' mode,
3443  * hypervisor will create the socket if it does not already exist. The VPP code
3444  * is indifferent to the file location. However, if SELinux is enabled, then the
3445  * socket needs to be created in '<em>/var/run/vpp/</em>'.
3446  *
3447  * - <b>server</b> - Optional flag to indicate that VPP should be the server for
3448  * the linux socket. If not provided, VPP will be the client. In '<em>server</em>'
3449  * mode, the VM can be reset without tearing down the vHost Interface. In
3450  * '<em>client</em>' mode, VPP can be reset without bringing down the VM and
3451  * tearing down the vHost Interface.
3452  *
3453  * - <b>feature-mask <hex></b> - Optional virtio/vhost feature set negotiated at
3454  * startup. <b>This is intended for degugging only.</b> It is recommended that this
3455  * parameter not be used except by experienced users. By default, all supported
3456  * features will be advertised. Otherwise, provide the set of features desired.
3457  * - 0x000008000 (15) - VIRTIO_NET_F_MRG_RXBUF
3458  * - 0x000020000 (17) - VIRTIO_NET_F_CTRL_VQ
3459  * - 0x000200000 (21) - VIRTIO_NET_F_GUEST_ANNOUNCE
3460  * - 0x000400000 (22) - VIRTIO_NET_F_MQ
3461  * - 0x004000000 (26) - VHOST_F_LOG_ALL
3462  * - 0x008000000 (27) - VIRTIO_F_ANY_LAYOUT
3463  * - 0x010000000 (28) - VIRTIO_F_INDIRECT_DESC
3464  * - 0x040000000 (30) - VHOST_USER_F_PROTOCOL_FEATURES
3465  * - 0x100000000 (32) - VIRTIO_F_VERSION_1
3466  *
3467  * - <b>hwaddr <mac-addr></b> - Optional ethernet address, can be in either
3468  * X:X:X:X:X:X unix or X.X.X cisco format.
3469  *
3470  * - <b>renumber <dev_instance></b> - Optional parameter which allows the instance
3471  * in the name to be specified. If instance already exists, name will be used
3472  * anyway and multiple instances will have the same name. Use with caution.
3473  *
3474  * @cliexpar
3475  * Example of how to create a vhost interface with VPP as the client and all features enabled:
3476  * @cliexstart{create vhost-user socket /var/run/vpp/vhost1.sock}
3477  * VirtualEthernet0/0/0
3478  * @cliexend
3479  * Example of how to create a vhost interface with VPP as the server and with just
3480  * multiple queues enabled:
3481  * @cliexstart{create vhost-user socket /var/run/vpp/vhost2.sock server feature-mask 0x40400000}
3482  * VirtualEthernet0/0/1
3483  * @cliexend
3484  * Once the vHost interface is created, enable the interface using:
3485  * @cliexcmd{set interface state VirtualEthernet0/0/0 up}
3486 ?*/
3487 /* *INDENT-OFF* */
3488 VLIB_CLI_COMMAND (vhost_user_connect_command, static) = {
3489  .path = "create vhost-user",
3490  .short_help = "create vhost-user socket <socket-filename> [server] "
3491  "[feature-mask <hex>] [hwaddr <mac-addr>] [renumber <dev_instance>] ",
3492  .function = vhost_user_connect_command_fn,
3493 };
3494 /* *INDENT-ON* */
3495 
3496 /*?
3497  * Delete a vHost User interface using the interface name or the
3498  * software interface index. Use the '<em>show interface</em>'
3499  * command to determine the software interface index. On deletion,
3500  * the linux socket will not be deleted.
3501  *
3502  * @cliexpar
3503  * Example of how to delete a vhost interface by name:
3504  * @cliexcmd{delete vhost-user VirtualEthernet0/0/1}
3505  * Example of how to delete a vhost interface by software interface index:
3506  * @cliexcmd{delete vhost-user sw_if_index 1}
3507 ?*/
3508 /* *INDENT-OFF* */
3509 VLIB_CLI_COMMAND (vhost_user_delete_command, static) = {
3510  .path = "delete vhost-user",
3511  .short_help = "delete vhost-user {<interface> | sw_if_index <sw_idx>}",
3512  .function = vhost_user_delete_command_fn,
3513 };
3514 
3515 /*?
3516  * Display the attributes of a single vHost User interface (provide interface
3517  * name), multiple vHost User interfaces (provide a list of interface names seperated
3518  * by spaces) or all Vhost User interfaces (omit an interface name to display all
3519  * vHost interfaces).
3520  *
3521  * @cliexpar
3522  * @parblock
3523  * Example of how to display a vhost interface:
3524  * @cliexstart{show vhost-user VirtualEthernet0/0/0}
3525  * Virtio vhost-user interfaces
3526  * Global:
3527  * coalesce frames 32 time 1e-3
3528  * Interface: VirtualEthernet0/0/0 (ifindex 1)
3529  * virtio_net_hdr_sz 12
3530  * features mask (0xffffffffffffffff):
3531  * features (0x50408000):
3532  * VIRTIO_NET_F_MRG_RXBUF (15)
3533  * VIRTIO_NET_F_MQ (22)
3534  * VIRTIO_F_INDIRECT_DESC (28)
3535  * VHOST_USER_F_PROTOCOL_FEATURES (30)
3536  * protocol features (0x3)
3537  * VHOST_USER_PROTOCOL_F_MQ (0)
3538  * VHOST_USER_PROTOCOL_F_LOG_SHMFD (1)
3539  *
3540  * socket filename /var/run/vpp/vhost1.sock type client errno "Success"
3541  *
3542  * rx placement:
3543  * thread 1 on vring 1
3544  * thread 1 on vring 5
3545  * thread 2 on vring 3
3546  * thread 2 on vring 7
3547  * tx placement: spin-lock
3548  * thread 0 on vring 0
3549  * thread 1 on vring 2
3550  * thread 2 on vring 0
3551  *
3552  * Memory regions (total 2)
3553  * region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr
3554  * ====== ===== ================== ================== ================== ================== ==================
3555  * 0 60 0x0000000000000000 0x00000000000a0000 0x00002aaaaac00000 0x0000000000000000 0x00002aab2b400000
3556  * 1 61 0x00000000000c0000 0x000000003ff40000 0x00002aaaaacc0000 0x00000000000c0000 0x00002aababcc0000
3557  *
3558  * Virtqueue 0 (TX)
3559  * qsz 256 last_avail_idx 0 last_used_idx 0
3560  * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
3561  * kickfd 62 callfd 64 errfd -1
3562  *
3563  * Virtqueue 1 (RX)
3564  * qsz 256 last_avail_idx 0 last_used_idx 0
3565  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
3566  * kickfd 65 callfd 66 errfd -1
3567  *
3568  * Virtqueue 2 (TX)
3569  * qsz 256 last_avail_idx 0 last_used_idx 0
3570  * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
3571  * kickfd 63 callfd 70 errfd -1
3572  *
3573  * Virtqueue 3 (RX)
3574  * qsz 256 last_avail_idx 0 last_used_idx 0
3575  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
3576  * kickfd 72 callfd 74 errfd -1
3577  *
3578  * Virtqueue 4 (TX disabled)
3579  * qsz 256 last_avail_idx 0 last_used_idx 0
3580  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
3581  * kickfd 76 callfd 78 errfd -1
3582  *
3583  * Virtqueue 5 (RX disabled)
3584  * qsz 256 last_avail_idx 0 last_used_idx 0
3585  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
3586  * kickfd 80 callfd 82 errfd -1
3587  *
3588  * Virtqueue 6 (TX disabled)
3589  * qsz 256 last_avail_idx 0 last_used_idx 0
3590  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
3591  * kickfd 84 callfd 86 errfd -1
3592  *
3593  * Virtqueue 7 (RX disabled)
3594  * qsz 256 last_avail_idx 0 last_used_idx 0
3595  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
3596  * kickfd 88 callfd 90 errfd -1
3597  *
3598  * @cliexend
3599  *
3600  * The optional '<em>descriptors</em>' parameter will display the same output as
3601  * the previous example but will include the descriptor table for each queue.
3602  * The output is truncated below:
3603  * @cliexstart{show vhost-user VirtualEthernet0/0/0 descriptors}
3604  * Virtio vhost-user interfaces
3605  * Global:
3606  * coalesce frames 32 time 1e-3
3607  * Interface: VirtualEthernet0/0/0 (ifindex 1)
3608  * virtio_net_hdr_sz 12
3609  * features mask (0xffffffffffffffff):
3610  * features (0x50408000):
3611  * VIRTIO_NET_F_MRG_RXBUF (15)
3612  * VIRTIO_NET_F_MQ (22)
3613  * :
3614  * Virtqueue 0 (TX)
3615  * qsz 256 last_avail_idx 0 last_used_idx 0
3616  * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
3617  * kickfd 62 callfd 64 errfd -1
3618  *
3619  * descriptor table:
3620  * id addr len flags next user_addr
3621  * ===== ================== ===== ====== ===== ==================
3622  * 0 0x0000000010b6e974 2060 0x0002 1 0x00002aabbc76e974
3623  * 1 0x0000000010b6e034 2060 0x0002 2 0x00002aabbc76e034
3624  * 2 0x0000000010b6d6f4 2060 0x0002 3 0x00002aabbc76d6f4
3625  * 3 0x0000000010b6cdb4 2060 0x0002 4 0x00002aabbc76cdb4
3626  * 4 0x0000000010b6c474 2060 0x0002 5 0x00002aabbc76c474
3627  * 5 0x0000000010b6bb34 2060 0x0002 6 0x00002aabbc76bb34
3628  * 6 0x0000000010b6b1f4 2060 0x0002 7 0x00002aabbc76b1f4
3629  * 7 0x0000000010b6a8b4 2060 0x0002 8 0x00002aabbc76a8b4
3630  * 8 0x0000000010b69f74 2060 0x0002 9 0x00002aabbc769f74
3631  * 9 0x0000000010b69634 2060 0x0002 10 0x00002aabbc769634
3632  * 10 0x0000000010b68cf4 2060 0x0002 11 0x00002aabbc768cf4
3633  * :
3634  * 249 0x0000000000000000 0 0x0000 250 0x00002aab2b400000
3635  * 250 0x0000000000000000 0 0x0000 251 0x00002aab2b400000
3636  * 251 0x0000000000000000 0 0x0000 252 0x00002aab2b400000
3637  * 252 0x0000000000000000 0 0x0000 253 0x00002aab2b400000
3638  * 253 0x0000000000000000 0 0x0000 254 0x00002aab2b400000
3639  * 254 0x0000000000000000 0 0x0000 255 0x00002aab2b400000
3640  * 255 0x0000000000000000 0 0x0000 32768 0x00002aab2b400000
3641  *
3642  * Virtqueue 1 (RX)
3643  * qsz 256 last_avail_idx 0 last_used_idx 0
3644  * :
3645  * @cliexend
3646  * @endparblock
3647 ?*/
3648 /* *INDENT-OFF* */
3649 VLIB_CLI_COMMAND (show_vhost_user_command, static) = {
3650  .path = "show vhost-user",
3651  .short_help = "show vhost-user [<interface> [<interface> [..]]] [descriptors]",
3652  .function = show_vhost_user_command_fn,
3653 };
3654 /* *INDENT-ON* */
3655 
3656 clib_error_t *
3658  unformat_input_t * input,
3659  vlib_cli_command_t * cmd)
3660 {
3661  unformat_input_t _line_input, *line_input = &_line_input;
3662  clib_error_t *error = NULL;
3664  u8 onoff = 0;
3665  u8 input_found = 0;
3666 
3667  /* Get a line of input. */
3668  if (!unformat_user (input, unformat_line_input, line_input))
3669  return clib_error_return (0, "missing argument");
3670 
3671  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
3672  {
3673  if (input_found)
3674  {
3675  error = clib_error_return (0, "unknown input `%U'",
3676  format_unformat_error, line_input);
3677  goto done;
3678  }
3679 
3680  if (unformat (line_input, "on"))
3681  {
3682  input_found = 1;
3683  onoff = 1;
3684  }
3685  else if (unformat (line_input, "off"))
3686  {
3687  input_found = 1;
3688  onoff = 0;
3689  }
3690  else
3691  {
3692  error = clib_error_return (0, "unknown input `%U'",
3693  format_unformat_error, line_input);
3694  goto done;
3695  }
3696  }
3697 
3698  vum->debug = onoff;
3699 
3700 done:
3701  unformat_free (line_input);
3702 
3703  return error;
3704 }
3705 
3706 /* *INDENT-OFF* */
3707 VLIB_CLI_COMMAND (debug_vhost_user_command, static) = {
3708  .path = "debug vhost-user",
3709  .short_help = "debug vhost-user <on | off>",
3710  .function = debug_vhost_user_command_fn,
3711 };
3712 /* *INDENT-ON* */
3713 
3714 static clib_error_t *
3716 {
3718 
3719  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3720  {
3721  if (unformat (input, "coalesce-frames %d", &vum->coalesce_frames))
3722  ;
3723  else if (unformat (input, "coalesce-time %f", &vum->coalesce_time))
3724  ;
3725  else if (unformat (input, "dont-dump-memory"))
3726  vum->dont_dump_vhost_user_memory = 1;
3727  else
3728  return clib_error_return (0, "unknown input `%U'",
3729  format_unformat_error, input);
3730  }
3731 
3732  return 0;
3733 }
3734 
3735 /* vhost-user { ... } configuration. */
3736 VLIB_CONFIG_FUNCTION (vhost_user_config, "vhost-user");
3737 
3738 void
3740 {
3742  vhost_user_intf_t *vui;
3743 
3744  if (vum->dont_dump_vhost_user_memory)
3745  {
3747  unmap_all_mem_regions (vui);
3748  );
3749  }
3750 }
3751 
3752 /*
3753  * fd.io coding-style-patch-verification: ON
3754  *
3755  * Local Variables:
3756  * eval: (c-set-style "gnu")
3757  * End:
3758  */
unformat_function_t unformat_vnet_hw_interface
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
static clib_error_t * vhost_user_init(vlib_main_t *vm)
Definition: vhost-user.c:1304
static void vhost_user_vring_close(vhost_user_intf_t *vui, u32 qid)
Definition: vhost-user.c:647
static void vnet_device_increment_rx_packets(u32 thread_index, u64 count)
Definition: devices.h:110
vmrglw vmrglh hi
static void vhost_user_if_disconnect(vhost_user_intf_t *vui)
Definition: vhost-user.c:673
#define vec_foreach_index(var, v)
Iterate over vector indices.
static void clib_file_del(clib_file_main_t *um, clib_file_t *f)
Definition: file.h:109
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
vring_desc_t * desc
Definition: vhost-user.h:199
#define VRING_AVAIL_F_NO_INTERRUPT
Definition: vhost-user.h:46
#define CLIB_UNUSED(x)
Definition: clib.h:79
u32 virtio_ring_flags
The device index.
Definition: vhost-user.h:274
virtio_net_hdr_mrg_rxbuf_t hdr
Length of the first data descriptor.
Definition: vhost-user.h:276
static uword random_default_seed(void)
Default random seed (unix/linux user-mode)
Definition: random.h:91
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:541
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:143
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
#define clib_smp_swap(addr, new)
Definition: smp.h:45
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
Definition: vhost-user.h:307
static void vhost_user_create_ethernet(vnet_main_t *vnm, vlib_main_t *vm, vhost_user_intf_t *vui, u8 *hwaddress)
Create ethernet interface for vhost user interface.
Definition: vhost-user.c:2840
#define VHOST_USER_DOWN_DISCARD_COUNT
Definition: vhost-user.c:72
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:213
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:534
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:500
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:319
#define VHOST_VRING_IDX_TX(qid)
Definition: vhost-user.h:24
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
void vhost_user_rx_trace(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *txvq)
Definition: vhost-user.c:1375
u64 region_guest_addr_hi[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost-user.h:242
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define PREDICT_TRUE(x)
Definition: clib.h:106
unsigned long u64
Definition: types.h:89
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define NULL
Definition: clib.h:55
uword mhash_unset(mhash_t *h, void *key, uword *old_value)
Definition: mhash.c:353
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:225
#define foreach_virtio_trace_flags
Definition: vhost-user.c:106
vhost_copy_t copy[VHOST_USER_COPY_ARRAY_N]
Definition: vhost-user.h:289
static void vhost_user_term_if(vhost_user_intf_t *vui)
Disables and reset interface structure.
Definition: vhost-user.c:2692
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
vring_avail_t * avail
Definition: vhost-user.h:200
static uword vhost_user_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
Definition: vhost-user.c:1930
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define VHOST_USER_EVENT_START_TIMER
Definition: vhost-user.h:217
u32 file_descriptor
Definition: file.h:54
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
int vnet_interface_name_renumber(u32 sw_if_index, u32 new_show_dev_instance)
Definition: interface.c:1373
static_always_inline u32 vhost_user_input_copy(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
Definition: vhost-user.c:1441
#define VHOST_USER_MSG_HDR_SZ
Definition: vhost-user.h:20
static clib_error_t * vhost_user_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: vhost-user.c:2547
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:983
static u32 format_get_indent(u8 *s)
Definition: format.h:72
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
clib_error_t * show_vhost_user_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost-user.c:3209
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:79
unformat_function_t unformat_vnet_sw_interface
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:458
static char * vhost_user_input_func_error_strings[]
Definition: vhost-user.c:160
static char * vhost_user_tx_func_error_strings[]
Definition: vhost-user.c:138
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:227
vring_used_t * used
Definition: vhost-user.h:201
format_function_t format_vnet_sw_if_index_name
vhost_trace_t * current_trace
Definition: vhost-user.h:293
unsigned char u8
Definition: types.h:56
static uword vlib_process_suspend_time_is_zero(f64 dt)
Returns TRUE if a process suspend time is less than 10us.
Definition: node_funcs.h:436
#define count_trailing_zeros(x)
Definition: clib.h:133
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static int vhost_user_name_renumber(vnet_hw_interface_t *hi, u32 new_dev_instance)
Definition: vhost-user.c:194
clib_file_function_t * read_function
Definition: file.h:67
double f64
Definition: types.h:142
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:212
static void vhost_user_vui_init(vnet_main_t *vnm, vhost_user_intf_t *vui, int server_sock_fd, const char *sock_filename, u64 feature_mask, u32 *sw_if_index)
Definition: vhost-user.c:2877
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:104
static clib_error_t * vhost_user_socket_error(clib_file_t *uf)
Definition: vhost-user.c:1252
clib_file_t * file_pool
Definition: file.h:88
#define VHOST_VRING_F_LOG
Definition: vhost-user.h:33
vnet_hw_interface_rx_mode
Definition: interface.h:51
VNET_DEVICE_CLASS(vhost_user_dev_class, static)
static clib_error_t * vhost_user_socket_read(clib_file_t *uf)
Definition: vhost-user.c:737
static u8 * format_vhost_user_interface_name(u8 *s, va_list *args)
Definition: vhost-user.c:177
#define static_always_inline
Definition: clib.h:93
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:440
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:324
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:156
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:542
static clib_error_t * ip4_init(vlib_main_t *vm)
Definition: ip4_input.c:357
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:810
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
void * log_base_addr
Definition: vhost-user.h:252
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
#define foreach_protocol_feature
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
clib_file_main_t file_main
Definition: main.c:63
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:297
vhost_user_tx_func_error_t
Definition: vhost-user.c:130
static void unmap_all_mem_regions(vhost_user_intf_t *vui)
Definition: vhost-user.c:374
static void vhost_user_set_interrupt_pending(vhost_user_intf_t *vui, u32 ifq)
Definition: vhost-user.c:544
unsigned int u32
Definition: types.h:88
vhost_user_input_func_error_t
Definition: vhost-user.c:152
#define vlib_call_init_function(vm, x)
Definition: init.h:227
#define VHOST_USER_TX_COPY_THRESHOLD
Definition: vhost-user.c:98
#define VLIB_FRAME_SIZE
Definition: node.h:364
#define UNIX_GET_FD(unixfd_idx)
Definition: vhost-user.c:100
unformat_function_t unformat_line_input
Definition: format.h:281
#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn)
Definition: node.h:194
static int vhost_user_init_server_sock(const char *sock_filename, int *sock_fd)
Open server unix socket on specified sock_filename.
Definition: vhost-user.c:2800
static uword vhost_user_send_interrupt_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: vhost-user.c:2387
VLIB_DEVICE_TX_FUNCTION_MULTIARCH(vhost_user_dev_class, vhost_user_tx)
Definition: vhost-user.c:2576
#define VHOST_USER_EVENT_STOP_TIMER
Definition: vhost-user.h:218
static void vhost_user_vring_unlock(vhost_user_intf_t *vui, u32 qid)
Unlock the vring lock.
Definition: vhost-user.c:619
format_function_t format_vnet_sw_interface_name
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:461
static_always_inline uword vnet_get_device_input_thread_index(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:127
u16 state
Input node state.
Definition: node.h:488
vlib_main_t * vlib_main
Definition: vnet.h:80
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:952
int vhost_user_delete_if(vnet_main_t *vnm, vlib_main_t *vm, u32 sw_if_index)
Definition: vhost-user.c:2721
static void * map_user_mem(vhost_user_intf_t *vui, uword addr)
Definition: vhost-user.c:349
u32 random
Pseudo random iterator.
Definition: vhost-user.h:310
static_always_inline void vhost_user_log_dirty_pages(vhost_user_intf_t *vui, u64 addr, u64 len)
Definition: vhost-user.c:725
uword mhash_set_mem(mhash_t *h, void *key, uword *new_value, uword *old_value)
Definition: mhash.c:271
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
#define VIRTQ_DESC_F_INDIRECT
Definition: vhost-user.h:28
#define clib_error_return_unix(e, args...)
Definition: error.h:102
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:202
int vnet_hw_interface_get_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode *mode)
Definition: devices.c:312
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:439
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:273
format_function_t format_vnet_hw_interface_rx_mode
void vhost_user_tx_trace(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *rxvq)
Definition: vhost-user.c:1977
#define PREDICT_FALSE(x)
Definition: clib.h:105
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:164
#define vhost_user_log_dirty_ring(vui, vq, member)
Definition: vhost-user.c:730
static vlib_node_registration_t vhost_user_process_node
(constructor) VLIB_REGISTER_NODE (vhost_user_process_node)
Definition: vhost-user.c:2680
void vhost_user_unmap_all(void)
Definition: vhost-user.c:3739
char sock_filename[256]
Definition: vhost-user.h:227
vnet_main_t vnet_main
Definition: misc.c:43
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:809
u32 region_mmap_fd[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost-user.h:243
static void vhost_user_send_call(vlib_main_t *vm, vhost_user_vring_t *vq)
Definition: vhost-user.c:1421
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE
Definition: interface.h:496
u32 node_index
Node index.
Definition: node.h:473
vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost-user.h:239
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
int vhost_user_dump_ifs(vnet_main_t *vnm, vlib_main_t *vm, vhost_user_intf_details_t **out_vuids)
Definition: vhost-user.c:3158
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:135
static clib_error_t * vhost_user_exit(vlib_main_t *vm)
Definition: vhost-user.c:2778
static void vhost_user_tx_thread_placement(vhost_user_intf_t *vui)
Definition: vhost-user.c:410
static void vhost_user_vring_init(vhost_user_intf_t *vui, u32 qid)
Definition: vhost-user.c:625
u8 * format_ethernet_header_with_length(u8 *s, va_list *args)
Definition: format.c:91
static vlib_node_registration_t vhost_user_send_interrupt_node
(constructor) VLIB_REGISTER_NODE (vhost_user_send_interrupt_node)
Definition: vhost-user.c:2476
u32 * show_dev_instance_by_real_dev_instance
Definition: vhost-user.h:301
int vhost_user_create_if(vnet_main_t *vnm, vlib_main_t *vm, const char *sock_filename, u8 is_server, u32 *sw_if_index, u64 feature_mask, u8 renumber, u32 custom_dev_instance, u8 *hwaddr)
Definition: vhost-user.c:2937
u16 device_index
The interface queue index (Not the virtio vring idx)
Definition: vhost-user.h:273
vhost_user_intf_t * vhost_user_interfaces
Definition: vhost-user.h:300
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
static void mhash_init_c_string(mhash_t *h, uword n_value_bytes)
Definition: mhash.h:78
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
u16 n_vectors
Definition: node.h:380
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:77
vlib_main_t * vm
Definition: buffer.c:294
static_always_inline void vhost_user_log_dirty_pages_2(vhost_user_intf_t *vui, u64 addr, u64 len, u8 is_host_address)
Definition: vhost-user.c:697
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
static int vhost_user_vring_try_lock(vhost_user_intf_t *vui, u32 qid)
Try once to lock the vring.
Definition: vhost-user.c:600
#define VLIB_MAIN_LOOP_EXIT_FUNCTION(x)
Definition: init.h:161
int vhost_user_modify_if(vnet_main_t *vnm, vlib_main_t *vm, const char *sock_filename, u8 is_server, u32 sw_if_index, u64 feature_mask, u8 renumber, u32 custom_dev_instance)
Definition: vhost-user.c:2995
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
#define VHOST_MEMORY_MAX_NREGIONS
Definition: vhost-user.h:19
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
Definition: vhost-user.c:211
clib_error_t * debug_vhost_user_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost-user.c:3657
u32 nregions
Definition: vhost-user.h:76
#define ARRAY_LEN(x)
Definition: clib.h:59
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
static uword vhost_user_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: vhost-user.c:2060
u16 first_desc_len
Runtime queue flags.
Definition: vhost-user.h:275
#define VHOST_USER_PROTOCOL_F_LOG_SHMFD
Definition: vhost-user.h:32
#define VLIB_BUFFER_DATA_SIZE
Definition: buffer.h:51
static void vhost_user_input_rewind_buffers(vlib_main_t *vm, vhost_cpu_t *cpu, vlib_buffer_t *b_head)
Definition: vhost-user.c:1528
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
#define VHOST_USER_VRING_NOFD_MASK
Definition: vhost-user.h:26
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
Definition: interface.h:660
u32 rx_buffers[VHOST_USER_RX_BUFFERS_N]
Definition: vhost-user.h:286
uword unformat_ethernet_address(unformat_input_t *input, va_list *args)
Definition: format.c:227
#define ASSERT(truth)
#define VHOST_USER_RX_BUFFER_STARVATION
Definition: vhost-user.c:78
static uword * mhash_get(mhash_t *h, const void *key)
Definition: mhash.h:110
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
static long get_huge_page_size(int fd)
Definition: vhost-user.c:366
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
clib_error_t * vhost_user_delete_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost-user.c:3111
static void clib_mem_free(void *p)
Definition: mem.h:179
#define VIRTQ_DESC_F_NEXT
Definition: vhost-user.h:27
volatile u32 * vring_locks[VHOST_VRING_MAX_N]
Definition: vhost-user.h:247
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:273
mhash_t if_index_by_sock_name
Definition: vhost-user.h:298
virtio_trace_flag_t
Definition: vhost-user.c:112
#define clib_error_report(e)
Definition: error.h:113
static void * vlib_frame_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:284
static clib_error_t * vhost_user_kickfd_read_ready(clib_file_t *uf)
Definition: vhost-user.c:571
static u8 * format_vhost_trace(u8 *s, va_list *va)
Definition: vhost-user.c:1337
#define VHOST_USER_RX_COPY_THRESHOLD
Definition: vhost-user.c:88
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static void vhost_user_vring_lock(vhost_user_intf_t *vui, u32 qid)
Spin until the vring is successfully locked.
Definition: vhost-user.c:609
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static void vhost_user_rx_thread_placement()
Unassign existing interface/queue to thread mappings and re-assign new interface/queue to thread mapp...
Definition: vhost-user.c:450
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, vlib_buffer_free_list_index_t index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:431
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
static void vhost_user_update_iface_state(vhost_user_intf_t *vui)
Definition: vhost-user.c:526
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:152
struct _vlib_node_registration vlib_node_registration_t
#define foreach_vhost_user_tx_func_error
Definition: vhost-user.c:121
void * region_mmap_addr[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost-user.h:240
Definition: defs.h:47
static u32 vhost_user_if_input(vlib_main_t *vm, vhost_user_main_t *vum, vhost_user_intf_t *vui, u16 qid, vlib_node_runtime_t *node, vnet_hw_interface_rx_mode mode)
Definition: vhost-user.c:1547
static clib_error_t * vhost_user_config(vlib_main_t *vm, unformat_input_t *input)
Definition: vhost-user.c:3715
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VRING_USED_F_NO_NOTIFY
Definition: vhost-user.h:45
#define VHOST_USER_RX_BUFFERS_N
Definition: vhost-user.h:280
static clib_error_t * vhost_user_socksvr_accept_ready(clib_file_t *uf)
Definition: vhost-user.c:1268
int vhost_user_intf_ready(vhost_user_intf_t *vui)
Returns whether at least one TX and one RX vring are enabled.
Definition: vhost-user.c:514
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
Definition: vhost-user.h:246
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:546
u64 uword
Definition: types.h:112
#define VHOST_VRING_MAX_N
Definition: vhost-user.h:22
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
#define clib_unix_warning(format, args...)
Definition: error.h:68
vlib_node_registration_t vhost_user_input_node
(constructor) VLIB_REGISTER_NODE (vhost_user_input_node)
Definition: vhost-user.c:119
void vnet_sw_interface_set_mtu(vnet_main_t *vnm, u32 sw_if_index, u32 mtu)
Definition: interface.c:676
u32 rx_buffers_len
Definition: vhost-user.h:285
#define DBG_SOCK(args...)
Definition: vhost-user.c:54
u32 vhost_user_rx_discard_packet(vlib_main_t *vm, vhost_user_intf_t *vui, vhost_user_vring_t *txvq, u32 discard_max)
Try to discard packets from the tx ring (VPP RX path).
Definition: vhost-user.c:1489
static vhost_user_main_t vhost_user_main
Definition: vhost-user.c:167
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:120
#define vnet_buffer(b)
Definition: buffer.h:360
#define DBG_VQ(args...)
Definition: vhost-user.c:64
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1508
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:250
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
u64 region_guest_addr_lo[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost-user.h:241
#define vec_foreach(var, vec)
Vector iterator.
uword private_data
Definition: file.h:64
#define foreach_vhost_user_input_func_error
Definition: vhost-user.c:144
u16 flags
Copy of main node flags.
Definition: node.h:486
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:109
Definition: file.h:51
vhost_vring_addr_t addr
Definition: vhost-user.h:83
virtio_net_hdr_mrg_rxbuf_t tx_headers[VLIB_FRAME_SIZE]
Definition: vhost-user.h:288
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:159
u32 flags
Definition: vhost-user.h:77
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:486
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
static_always_inline u32 vhost_user_tx_copy(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
Definition: vhost-user.c:2012
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:681
static clib_error_t * vhost_user_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
Definition: vhost-user.c:2484
#define VHOST_USER_PROTOCOL_F_MQ
Definition: vhost-user.h:31
#define VHOST_LOG_PAGE
Definition: vhost-user.c:695
vl_api_address_union_t un
Definition: ip_types.api:36
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
clib_error_t * vhost_user_connect_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost-user.c:3046
VNET_HW_INTERFACE_CLASS(vhost_interface_class, static)
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:298
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
Definition: defs.h:46
static clib_error_t * vhost_user_callfd_read_ready(clib_file_t *uf)
Definition: vhost-user.c:560
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
int dont_dump_vhost_user_memory
Definition: vhost-user.h:304
#define VHOST_VRING_IDX_RX(qid)
Definition: vhost-user.h:23