FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
pci.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <fcntl.h>
17 #include <sys/ioctl.h>
18 
19 #include <vppinfra/types.h>
20 #include <vlib/vlib.h>
21 #include <vlib/pci/pci.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/ip/ip4_packet.h>
24 #include <vnet/ip/ip6_packet.h>
28 
29 #define PCI_VENDOR_ID_VIRTIO 0x1af4
30 #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000
31 /* Doesn't support modern device */
32 #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041
33 
34 #define PCI_CAPABILITY_LIST 0x34
35 #define PCI_CAP_ID_VNDR 0x09
36 #define PCI_CAP_ID_MSIX 0x11
37 
38 #define PCI_MSIX_ENABLE 0x8000
39 
41  {
43  .device_id = PCI_DEVICE_ID_VIRTIO_NIC},
44  {
45  .vendor_id = PCI_VENDOR_ID_VIRTIO,
46  .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN},
47  {0},
48 };
49 
50 static u32
52  u32 flags)
53 {
54  return 0;
55 }
56 
57 static clib_error_t *
59 {
60  clib_error_t *error = 0;
61  u16 max_queue_pairs = 1;
62 
63  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
64  {
65  max_queue_pairs = vif->virtio_pci_func->get_max_queue_pairs (vm, vif);
66  }
67 
68  virtio_log_debug (vif, "max queue pair is %x", max_queue_pairs);
69  if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
70  return clib_error_return (error, "max queue pair is %x,"
71  " should be in range [1, 0x8000]",
72  max_queue_pairs);
73 
74  vif->max_queue_pairs = max_queue_pairs;
75  return error;
76 }
77 
78 static void
80 {
81  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
82  vif->virtio_pci_func->set_mac (vm, vif);
83 }
84 
85 static u32
87 {
88  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
89  {
90  vif->virtio_pci_func->get_mac (vm, vif);
91  return 0;
92  }
93  return 1;
94 }
95 
96 static u16
98 {
99  /*
100  * Minimal driver: assumes link is up
101  */
102  u16 status = 1;
103  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS))
104  status = vif->virtio_pci_func->get_device_status (vm, vif);
105  return status;
106 }
107 
108 static void
110  u16 line)
111 {
112  vnet_main_t *vnm = vnet_get_main ();
113  virtio_main_t *vim = &virtio_main;
115  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
116  line--;
117  u16 qid = line;
118 
119  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
121 }
122 
123 static void
125  u16 line)
126 {
127  vnet_main_t *vnm = vnet_get_main ();
128  virtio_main_t *vim = &virtio_main;
130  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
131 
133  {
134  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
137  }
138  else
139  {
140  vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
142  }
143 }
144 
145 static void
147 {
148  virtio_main_t *vim = &virtio_main;
150  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
151  u8 isr = 0;
152  u16 line = 0;
153 
154  isr = vif->virtio_pci_func->get_isr (vm, vif);
155 
156  /*
157  * If the lower bit is set: look through the used rings of
158  * all virtqueues for the device, to see if any progress has
159  * been made by the device which requires servicing.
160  */
161  if (isr & VIRTIO_PCI_ISR_INTR)
162  {
163  for (; line < vif->num_rxqs; line++)
164  virtio_pci_irq_queue_handler (vm, h, (line + 1));
165  }
166 
167  if (isr & VIRTIO_PCI_ISR_CONFIG)
169 }
170 
171 inline void
173 {
174  struct status_struct
175  {
176  u8 bit;
177  char *str;
178  };
179  struct status_struct *status_entry;
180  static struct status_struct status_array[] = {
181 #define _(s,b) { .str = #s, .bit = b, },
183 #undef _
184  {.str = NULL}
185  };
186 
187  vlib_cli_output (vm, " status 0x%x", vif->status);
188 
189  status_entry = (struct status_struct *) &status_array;
190  while (status_entry->str)
191  {
192  if (vif->status & status_entry->bit)
193  vlib_cli_output (vm, " %s (%x)", status_entry->str,
194  status_entry->bit);
195  status_entry++;
196  }
197 }
198 
199 static int
202 {
203  virtio_vring_t *vring = vif->cxq_vring;
205  virtio_ctrl_msg_t result;
206  u32 buffer_index;
207  vlib_buffer_t *b;
208  u16 used, next;
209  u16 sz = vring->size;
210  u16 flags = 0, first_desc_flags = 0;
211 
212  used = vring->desc_in_use;
213  next = vring->desc_next;
214  vring_packed_desc_t *d = &vring->packed_desc[next];
215 
216  if (vlib_buffer_alloc (vm, &buffer_index, 1))
217  b = vlib_get_buffer (vm, buffer_index);
218  else
219  return VIRTIO_NET_ERR;
220  /*
221  * current_data may not be initialized with 0 and may contain
222  * previous offset.
223  */
224  b->current_data = 0;
226 
227  first_desc_flags = VRING_DESC_F_NEXT;
228  if (vring->avail_wrap_counter)
229  {
230  first_desc_flags |= VRING_DESC_F_AVAIL;
231  first_desc_flags &= ~VRING_DESC_F_USED;
232  }
233  else
234  {
235  first_desc_flags &= ~VRING_DESC_F_AVAIL;
236  first_desc_flags |= VRING_DESC_F_USED;
237  }
238  d->addr = vlib_buffer_get_current_pa (vm, b);
239  d->len = sizeof (virtio_net_ctrl_hdr_t);
240  d->id = next;
241 
242  next++;
243  if (next >= sz)
244  {
245  next = 0;
246  vring->avail_wrap_counter ^= 1;
247  }
248  used++;
249 
250  d = &vring->packed_desc[next];
252  if (vring->avail_wrap_counter)
253  {
256  }
257  else
258  {
261  }
262  d->addr = vlib_buffer_get_current_pa (vm, b) +
264  d->len = len;
265  d->id = next;
266  d->flags = flags;
267 
268  next++;
269  if (next >= sz)
270  {
271  next = 0;
272  vring->avail_wrap_counter ^= 1;
273  }
274  used++;
275 
276  d = &vring->packed_desc[next];
278  if (vring->avail_wrap_counter)
279  {
282  }
283  else
284  {
287  }
288  d->addr = vlib_buffer_get_current_pa (vm, b) +
290  d->len = sizeof (data->status);
291  d->id = next;
292  d->flags = flags;
293 
294  next++;
295  if (next >= sz)
296  {
297  next = 0;
298  vring->avail_wrap_counter ^= 1;
299  }
300  used++;
301 
303  vring->packed_desc[vring->desc_next].flags = first_desc_flags;
304  vring->desc_next = next;
305  vring->desc_in_use = used;
307  if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
308  {
309  virtio_kick (vm, vring, vif);
310  }
311 
312  u16 last = vring->last_used_idx;
313  d = &vring->packed_desc[last];
314  do
315  {
316  flags = d->flags;
317  }
318  while ((flags & VRING_DESC_F_AVAIL) != (vring->used_wrap_counter << 7)
319  || (flags & VRING_DESC_F_USED) != (vring->used_wrap_counter << 15));
320 
321  last += 3;
322  if (last >= vring->size)
323  {
324  last = last - vring->size;
325  vring->used_wrap_counter ^= 1;
326  }
327  vring->desc_in_use -= 3;
328  vring->last_used_idx = last;
329 
332  sizeof (virtio_ctrl_msg_t));
333  virtio_log_debug (vif, "ctrl-queue: status %u", result.status);
334  status = result.status;
335  vlib_buffer_free (vm, &buffer_index, 1);
336  return status;
337 }
338 
339 static int
342 {
343  virtio_vring_t *vring = vif->cxq_vring;
345  virtio_ctrl_msg_t result;
346  u32 buffer_index;
347  vlib_buffer_t *b;
348  u16 used, next, avail;
349  u16 sz = vring->size;
350  u16 mask = sz - 1;
351 
352  used = vring->desc_in_use;
353  next = vring->desc_next;
354  avail = vring->avail->idx;
355  vring_desc_t *d = &vring->desc[next];
356 
357  if (vlib_buffer_alloc (vm, &buffer_index, 1))
358  b = vlib_get_buffer (vm, buffer_index);
359  else
360  return VIRTIO_NET_ERR;
361  /*
362  * current_data may not be initialized with 0 and may contain
363  * previous offset.
364  */
365  b->current_data = 0;
369  d->len = sizeof (virtio_net_ctrl_hdr_t);
370  vring->avail->ring[avail & mask] = next;
371  avail++;
372  next = (next + 1) & mask;
373  d->next = next;
374  used++;
375 
376  d = &vring->desc[next];
380  d->len = len;
381  next = (next + 1) & mask;
382  d->next = next;
383  used++;
384 
385  d = &vring->desc[next];
389  d->len = sizeof (data->status);
390  next = (next + 1) & mask;
391  used++;
392 
394  vring->avail->idx = avail;
395  vring->desc_next = next;
396  vring->desc_in_use = used;
397 
398  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
399  {
400  virtio_kick (vm, vring, vif);
401  }
402 
403  u16 last = vring->last_used_idx, n_left = 0;
404  n_left = vring->used->idx - last;
405 
406  while (n_left)
407  {
408  vring_used_elem_t *e = &vring->used->ring[last & mask];
409  u16 slot = e->id;
410 
411  d = &vring->desc[slot];
412  while (d->flags & VRING_DESC_F_NEXT)
413  {
414  used--;
415  slot = d->next;
416  d = &vring->desc[slot];
417  }
418  used--;
419  last++;
420  n_left--;
421  }
422  vring->desc_in_use = used;
423  vring->last_used_idx = last;
424 
427  sizeof (virtio_ctrl_msg_t));
428  virtio_log_debug (vif, "ctrl-queue: status %u", result.status);
429  status = result.status;
430  vlib_buffer_free (vm, &buffer_index, 1);
431  return status;
432 }
433 
434 static int
437 {
438  if (vif->is_packed)
439  return virtio_pci_send_ctrl_msg_packed (vm, vif, data, len);
440  else
441  return virtio_pci_send_ctrl_msg_split (vm, vif, data, len);
442 }
443 
444 static int
446 {
447  virtio_ctrl_msg_t offload_hdr;
449 
450  offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
451  offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
452  offload_hdr.status = VIRTIO_NET_ERR;
453  u64 offloads = 0ULL;
454  clib_memcpy (offload_hdr.data, &offloads, sizeof (offloads));
455 
456  status =
457  virtio_pci_send_ctrl_msg (vm, vif, &offload_hdr, sizeof (offloads));
458  virtio_log_debug (vif, "disable offloads");
459  vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
460  vif->virtio_pci_func->get_driver_features (vm, vif);
461  return status;
462 }
463 
464 static int
466 {
467  virtio_ctrl_msg_t csum_offload_hdr;
469 
470  csum_offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
471  csum_offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
472  csum_offload_hdr.status = VIRTIO_NET_ERR;
473  u64 offloads = 0ULL;
474  offloads |= VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM);
475  clib_memcpy (csum_offload_hdr.data, &offloads, sizeof (offloads));
476 
477  status =
478  virtio_pci_send_ctrl_msg (vm, vif, &csum_offload_hdr, sizeof (offloads));
479  virtio_log_debug (vif, "enable checksum offload");
480  vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
481  vif->features = vif->virtio_pci_func->get_driver_features (vm, vif);
482  return status;
483 }
484 
485 static int
487 {
488  virtio_ctrl_msg_t gso_hdr;
490 
491  gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
493  gso_hdr.status = VIRTIO_NET_ERR;
494  u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
495  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
496  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6);
497  clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads));
498 
499  status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads));
500  virtio_log_debug (vif, "enable gso");
501  vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
502  vif->virtio_pci_func->get_driver_features (vm, vif);
503  return status;
504 }
505 
506 static int
507 virtio_pci_offloads (vlib_main_t * vm, virtio_if_t * vif, int gso_enabled,
508  int csum_offload_enabled)
509 {
510  vnet_main_t *vnm = vnet_get_main ();
512 
513  if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
514  (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)))
515  {
516  if (gso_enabled
517  && (vif->features & (VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) |
518  VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6))))
519  {
520  if (virtio_pci_enable_gso (vm, vif))
521  {
522  virtio_log_warning (vif, "gso is not enabled");
523  }
524  else
525  {
526  vif->gso_enabled = 1;
527  vif->csum_offload_enabled = 0;
531  }
532  }
533  else if (csum_offload_enabled
534  && (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)))
535  {
537  {
538  virtio_log_warning (vif, "checksum offload is not enabled");
539  }
540  else
541  {
542  vif->csum_offload_enabled = 1;
543  vif->gso_enabled = 0;
547  }
548  }
549  else
550  {
551  if (virtio_pci_disable_offload (vm, vif))
552  {
553  virtio_log_warning (vif, "offloads are not disabled");
554  }
555  else
556  {
557  vif->csum_offload_enabled = 0;
558  vif->gso_enabled = 0;
561  }
562  }
563  }
564 
565  return 0;
566 }
567 
568 static int
570  u16 num_queues)
571 {
572  virtio_ctrl_msg_t mq_hdr;
574 
575  mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
577  mq_hdr.status = VIRTIO_NET_ERR;
578  clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues));
579 
580  status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues));
581  virtio_log_debug (vif, "multi-queue enable %u queues", num_queues);
582  return status;
583 }
584 
585 static u8
587 {
588  if (qsz < 64 || qsz > 4096)
589  return 0;
590  if ((qsz % 64) != 0)
591  return 0;
592  return 1;
593 }
594 
595 clib_error_t *
597  u16 queue_num)
598 {
599  clib_error_t *error = 0;
600  u16 queue_size = 0;
601  virtio_vring_t *vring;
602  u32 i = 0;
603  void *ptr = NULL;
604 
605  queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
606 
607  if (queue_size > 32768)
608  return clib_error_return (0, "ring size must be 32768 or lower");
609 
610  if (queue_size == 0)
611  queue_size = 256;
612 
614  vring = vec_elt_at_index (vif->cxq_vring, 0);
615 
616  i =
617  (((queue_size * sizeof (vring_packed_desc_t)) +
618  sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
619  1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t);
620 
621  ptr =
623  vif->numa_node);
624  if (!ptr)
625  return vlib_physmem_last_error (vm);
626  clib_memset (ptr, 0, i);
627 
628  vring->packed_desc = ptr;
629 
630  vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t));
631  vring->driver_event->off_wrap = 0;
632  vring->driver_event->flags = VRING_EVENT_F_DISABLE;
633 
634  vring->device_event =
635  ptr +
636  (((queue_size * sizeof (vring_packed_desc_t)) +
637  sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
638  1) & ~(VIRTIO_PCI_VRING_ALIGN - 1));
639  vring->device_event->off_wrap = 0;
640  vring->device_event->flags = 0;
641 
642  vring->queue_id = queue_num;
643  vring->size = queue_size;
644  vring->avail_wrap_counter = 1;
645  vring->used_wrap_counter = 1;
646 
647  ASSERT (vring->buffers == 0);
648 
649  virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num,
650  queue_size);
651  vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring);
652  vring->queue_notify_offset =
653  vif->notify_off_multiplier *
654  vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
655  virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
656  queue_num, vring->queue_notify_offset);
657  return error;
658 }
659 
660 clib_error_t *
662  u16 queue_num)
663 {
664  clib_error_t *error = 0;
665  u16 queue_size = 0;
666  virtio_vring_t *vring;
667  vring_t vr;
668  u32 i = 0;
669  void *ptr = NULL;
670 
671  queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
672  if (!virtio_pci_queue_size_valid (queue_size))
673  clib_warning ("queue size is not valid");
674 
675  if (!is_pow2 (queue_size))
676  return clib_error_return (0, "ring size must be power of 2");
677 
678  if (queue_size > 32768)
679  return clib_error_return (0, "ring size must be 32768 or lower");
680 
681  if (queue_size == 0)
682  queue_size = 256;
683 
685  vring = vec_elt_at_index (vif->cxq_vring, 0);
686  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
688  ptr =
690  vif->numa_node);
691  if (!ptr)
692  return vlib_physmem_last_error (vm);
693  clib_memset (ptr, 0, i);
694  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
695  vring->desc = vr.desc;
696  vring->avail = vr.avail;
697  vring->used = vr.used;
698  vring->queue_id = queue_num;
700 
701  ASSERT (vring->buffers == 0);
702 
703  vring->size = queue_size;
704  virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num,
705  queue_size);
706  vif->virtio_pci_func->setup_queue (vm, vif, queue_num, ptr);
707  vring->queue_notify_offset =
708  vif->notify_off_multiplier *
709  vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
710  virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
711  queue_num, vring->queue_notify_offset);
712 
713  return error;
714 }
715 
716 clib_error_t *
718  u16 queue_num)
719 {
720  if (vif->is_packed)
721  return virtio_pci_control_vring_packed_init (vm, vif, queue_num);
722  else
723  return virtio_pci_control_vring_split_init (vm, vif, queue_num);
724 }
725 
726 clib_error_t *
728  u16 queue_num)
729 {
731  clib_error_t *error = 0;
732  u16 queue_size = 0;
733  virtio_vring_t *vring;
734  vring_t vr;
735  u32 i = 0;
736  void *ptr = NULL;
737 
738  queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
739  if (!virtio_pci_queue_size_valid (queue_size))
740  clib_warning ("queue size is not valid");
741 
742  if (!is_pow2 (queue_size))
743  return clib_error_return (0, "ring size must be power of 2");
744 
745  if (queue_size > 32768)
746  return clib_error_return (0, "ring size must be 32768 or lower");
747 
748  if (queue_size == 0)
749  queue_size = 256;
750 
751  if (queue_num % 2)
752  {
755  vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
756  if (vif->max_queue_pairs < vtm->n_vlib_mains)
757  clib_spinlock_init (&vring->lockp);
758  }
759  else
760  {
763  vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
764  }
765  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
767  ptr =
769  vif->numa_node);
770  if (!ptr)
771  return vlib_physmem_last_error (vm);
772  clib_memset (ptr, 0, i);
773  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
774  vring->desc = vr.desc;
775  vring->avail = vr.avail;
776  vring->used = vr.used;
777  vring->queue_id = queue_num;
779  vring->flow_table = 0;
780 
781  ASSERT (vring->buffers == 0);
782  vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
783  if (queue_num % 2)
784  {
785  virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num,
786  queue_size);
787  clib_memset_u32 (vring->buffers, ~0, queue_size);
788  }
789  else
790  {
791  virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num,
792  queue_size);
793  }
794  vring->size = queue_size;
795  if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, ptr))
796  return clib_error_return (0, "error in queue address setup");
797 
798  vring->queue_notify_offset =
799  vif->notify_off_multiplier *
800  vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
801  virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
802  queue_num, vring->queue_notify_offset);
803  return error;
804 }
805 
806 clib_error_t *
808  u16 queue_num)
809 {
811  clib_error_t *error = 0;
812  u16 queue_size = 0;
813  virtio_vring_t *vring;
814  u32 i = 0;
815  void *ptr = NULL;
816 
817  queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
818 
819  if (queue_size > 32768)
820  return clib_error_return (0, "ring size must be 32768 or lower");
821 
822  if (queue_size == 0)
823  queue_size = 256;
824 
825  if (queue_num % 2)
826  {
829  vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
830  if (vif->max_queue_pairs < vtm->n_vlib_mains)
831  clib_spinlock_init (&vring->lockp);
832  }
833  else
834  {
837  vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
838  }
839 
840  i =
841  (((queue_size * sizeof (vring_packed_desc_t)) +
842  sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
843  1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t);
844 
845  ptr =
847  vif->numa_node);
848  if (!ptr)
849  return vlib_physmem_last_error (vm);
850 
851  clib_memset (ptr, 0, i);
852  vring->packed_desc = ptr;
853 
854  vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t));
855  vring->driver_event->off_wrap = 0;
856  vring->driver_event->flags = VRING_EVENT_F_DISABLE;
857 
858  vring->device_event =
859  ptr +
860  (((queue_size * sizeof (vring_packed_desc_t)) +
861  sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
862  1) & ~(VIRTIO_PCI_VRING_ALIGN - 1));
863  vring->device_event->off_wrap = 0;
864  vring->device_event->flags = 0;
865 
866  vring->queue_id = queue_num;
867 
868  vring->avail_wrap_counter = 1;
869  vring->used_wrap_counter = 1;
870 
871  ASSERT (vring->buffers == 0);
872  vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
873  if (queue_num % 2)
874  {
875  virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num,
876  queue_size);
877  clib_memset_u32 (vring->buffers, ~0, queue_size);
878  }
879  else
880  {
881  virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num,
882  queue_size);
883  }
884  vring->size = queue_size;
885  if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring))
886  return clib_error_return (0, "error in queue address setup");
887 
888  vring->queue_notify_offset =
889  vif->notify_off_multiplier *
890  vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
891  virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
892  queue_num, vring->queue_notify_offset);
893 
894  return error;
895 }
896 
897 clib_error_t *
899 {
900  if (vif->is_packed)
901  return virtio_pci_vring_packed_init (vm, vif, queue_num);
902  else
903  return virtio_pci_vring_split_init (vm, vif, queue_num);
904 }
905 
906 static void
908  u64 req_features)
909 {
910  /*
911  * if features are not requested
912  * default: all supported features
913  */
914  u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)
915  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
916  | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
917  | VIRTIO_FEATURE (VIRTIO_NET_F_MTU)
918  | VIRTIO_FEATURE (VIRTIO_NET_F_MAC)
919  | VIRTIO_FEATURE (VIRTIO_NET_F_GSO)
920  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
921  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)
922  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)
923  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4)
924  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)
925  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO)
926  | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)
927  | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)
928  | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)
929  | VIRTIO_FEATURE (VIRTIO_NET_F_MQ)
930  | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY)
931  | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
932  | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
933 
934  if (vif->is_modern)
935  supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1);
936 
937  if (vif->is_packed)
938  {
939  supported_features |=
940  (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED) |
941  VIRTIO_FEATURE (VIRTIO_F_IN_ORDER));
942  }
943 
944  if (req_features == 0)
945  {
946  req_features = supported_features;
947  }
948 
949  vif->features = req_features & vif->remote_features & supported_features;
950 
951  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MTU))
952  {
953  u16 mtu = 0;
954  mtu = vif->virtio_pci_func->get_mtu (vm, vif);
955 
956  if (mtu < 64)
957  vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MTU);
958  }
959 
960  if ((vif->features & (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED))) == 0)
961  vif->is_packed = 0;
962 
963  vif->virtio_pci_func->set_driver_features (vm, vif, vif->features);
964  vif->features = vif->virtio_pci_func->get_driver_features (vm, vif);
965 }
966 
967 void
969 {
970  vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif);
971 }
972 
973 int
975 {
976  u8 status = 0;
977 
978  /*
979  * Reset the device
980  */
981  status = vif->virtio_pci_func->device_reset (vm, vif);
982 
983  /*
984  * Set the Acknowledge status bit
985  */
986  vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK);
987 
988  /*
989  * Set the Driver status bit
990  */
991  vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER);
992 
993  /*
994  * Read the status and verify it
995  */
996  status = vif->virtio_pci_func->get_status (vm, vif);
997  if ((status & VIRTIO_CONFIG_STATUS_ACK)
998  && (status & VIRTIO_CONFIG_STATUS_DRIVER))
999  vif->status = status;
1000  else
1001  return -1;
1002 
1003  return 0;
1004 }
1005 
1006 clib_error_t *
1008 {
1009  clib_error_t *error = 0;
1010  virtio_pci_cap_t cap;
1011  u8 pos, common_cfg = 0, notify = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
1013 
1014  if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
1015  {
1016  virtio_log_error (vif, "error in reading capabilty list position");
1017  return clib_error_return (error,
1018  "error in reading capabilty list position");
1019  }
1020  while (pos)
1021  {
1022  if ((error =
1023  vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap,
1024  sizeof (cap))))
1025  {
1026  virtio_log_error (vif, "%s [%2x]",
1027  "error in reading the capability at", pos);
1028  return clib_error_return (error,
1029  "error in reading the capability at [%2x]",
1030  pos);
1031  }
1032 
1033  if (cap.cap_vndr == PCI_CAP_ID_MSIX)
1034  {
1035  u16 flags, table_size, table_size_mask = 0x07FF;
1036 
1037  if ((error =
1039  sizeof (flags))))
1040  return clib_error_return (error,
1041  "error in reading the capability at [%2x]",
1042  pos + 2);
1043 
1044  table_size = flags & table_size_mask;
1045  virtio_log_debug (vif, "flags:0x%x %s 0x%x", flags,
1046  "msix interrupt vector table-size", table_size);
1047 
1048  if (flags & PCI_MSIX_ENABLE)
1049  {
1050  virtio_log_debug (vif, "msix interrupt enabled");
1052  vif->msix_table_size = table_size;
1053  }
1054  else
1055  {
1056  virtio_log_debug (vif, "msix interrupt disabled");
1058  vif->msix_table_size = 0;
1059  }
1060  }
1061 
1062  if (cap.cap_vndr != PCI_CAP_ID_VNDR)
1063  {
1064  virtio_log_debug (vif, "[%2x] %s %2x ", pos,
1065  "skipping non VNDR cap id:", cap.cap_vndr);
1066  goto next;
1067  }
1068 
1069  virtio_log_debug (vif,
1070  "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
1071  pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
1072 
1073  if (cap.bar >= 0 && cap.bar <= 5)
1074  {
1075  vif->bar = bar[cap.bar];
1076  vif->bar_id = cap.bar;
1077  }
1078  else
1079  return clib_error_return (error, "invalid bar %u", cap.bar);
1080 
1081  switch (cap.cfg_type)
1082  {
1084  vif->common_offset = cap.offset;
1085  common_cfg = 1;
1086  break;
1088  if ((error =
1090  pos + sizeof (cap),
1091  &vif->notify_off_multiplier,
1092  sizeof
1093  (vif->notify_off_multiplier))))
1094  {
1095  virtio_log_error (vif, "notify off multiplier is not given");
1096  }
1097  else
1098  {
1099  virtio_log_debug (vif, "notify off multiplier is %u",
1100  vif->notify_off_multiplier);
1101  vif->notify_offset = cap.offset;
1102  notify = 1;
1103  }
1104  break;
1106  vif->device_offset = cap.offset;
1107  dev_cfg = 1;
1108  break;
1110  vif->isr_offset = cap.offset;
1111  isr = 1;
1112  break;
1114  if (cap.bar == 0)
1115  pci_cfg = 1;
1116  break;
1117  }
1118  next:
1119  pos = cap.cap_next;
1120  }
1121 
1122  if (common_cfg == 0 || notify == 0 || dev_cfg == 0 || isr == 0)
1123  {
1125  vif->notify_off_multiplier = 0;
1126  virtio_log_debug (vif, "legacy virtio pci device found");
1127  return error;
1128  }
1129 
1130  vif->is_modern = 1;
1132 
1133  if (!pci_cfg)
1134  {
1135  virtio_log_debug (vif, "modern virtio pci device found");
1136  }
1137  else
1138  {
1139  virtio_log_debug (vif, "transitional virtio pci device found");
1140  }
1141 
1142  return error;
1143 }
1144 
1145 static clib_error_t *
1147  virtio_pci_create_if_args_t * args, void **bar)
1148 {
1149  clib_error_t *error = 0;
1151  u8 status = 0;
1152 
1153  if ((error = virtio_pci_read_caps (vm, vif, bar)))
1154  {
1155  args->rv = VNET_API_ERROR_UNSUPPORTED;
1156  virtio_log_error (vif, "Device is not supported");
1157  return clib_error_return (error, "Device is not supported");
1158  }
1159 
1160  if (virtio_pci_reset_device (vm, vif) < 0)
1161  {
1162  args->rv = VNET_API_ERROR_INIT_FAILED;
1163  virtio_log_error (vif, "Failed to reset the device");
1164  return clib_error_return (error, "Failed to reset the device");
1165  }
1166  /*
1167  * read device features and negotiate (user) requested features
1168  */
1170  if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) ==
1171  0)
1172  {
1173  virtio_log_warning (vif, "error encountered: vhost-net backend doesn't "
1174  "support VIRTIO_RING_F_INDIRECT_DESC features");
1175  }
1176  if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)) == 0)
1177  {
1178  virtio_log_warning (vif, "error encountered: vhost-net backend doesn't "
1179  "support VIRTIO_NET_F_MRG_RXBUF features");
1180  }
1181  virtio_negotiate_features (vm, vif, args->features);
1182 
1183  /*
1184  * After FEATURE_OK, driver should not accept new feature bits
1185  */
1186  vif->virtio_pci_func->set_status (vm, vif,
1187  VIRTIO_CONFIG_STATUS_FEATURES_OK);
1188  status = vif->virtio_pci_func->get_status (vm, vif);
1189  if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
1190  {
1191  args->rv = VNET_API_ERROR_UNSUPPORTED;
1192  virtio_log_error (vif,
1193  "error encountered: Device doesn't support requested features");
1194  return clib_error_return (error,
1195  "Device doesn't support requested features");
1196  }
1197  vif->status = status;
1198 
1199  /*
1200  * get or set the mac address
1201  */
1202  if (virtio_pci_get_mac (vm, vif))
1203  {
1204  f64 now = vlib_time_now (vm);
1205  u32 rnd;
1206  rnd = (u32) (now * 1e6);
1207  rnd = random_u32 (&rnd);
1208 
1209  memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd));
1210  vif->mac_addr[0] = 2;
1211  vif->mac_addr[1] = 0xfe;
1212  virtio_pci_set_mac (vm, vif);
1213  }
1214 
1216 
1217  /*
1218  * Initialize the virtqueues
1219  */
1221  {
1222  args->rv = VNET_API_ERROR_EXCEEDED_NUMBER_OF_RANGES_CAPACITY;
1223  goto err;
1224  }
1225 
1226  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1227  {
1228  if (vif->msix_table_size <= vif->max_queue_pairs)
1229  {
1230  virtio_log_error (vif,
1231  "error MSIX lines (%u) <= Number of RXQs (%u)",
1232  vif->msix_table_size, vif->max_queue_pairs);
1233  return clib_error_return (error,
1234  "error MSIX lines (%u) <= Number of RXQs (%u)",
1235  vif->msix_table_size,
1236  vif->max_queue_pairs);
1237  }
1238  }
1239 
1240  for (int i = 0; i < vif->max_queue_pairs; i++)
1241  {
1242  if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i))))
1243  {
1244  args->rv = VNET_API_ERROR_INIT_FAILED;
1245  virtio_log_error (vif, "%s (%u) %s", "error in rxq-queue",
1246  RX_QUEUE (i), "initialization");
1247  error =
1248  clib_error_return (error, "%s (%u) %s", "error in rxq-queue",
1249  RX_QUEUE (i), "initialization");
1250  goto err;
1251  }
1252  else
1253  {
1254  vif->num_rxqs++;
1255  }
1256 
1257  if (i >= vtm->n_vlib_mains)
1258  {
1259  /*
1260  * There is 1:1 mapping between tx queue and vpp worker thread.
1261  * tx queue 0 is bind with thread index 0, tx queue 1 on thread
1262  * index 1 and so on.
1263  * Multiple worker threads can poll same tx queue when number of
1264  * workers are more than tx queues. In this case, 1:N mapping
1265  * between tx queue and vpp worker thread.
1266  */
1267  virtio_log_debug (vif, "%s %u, %s", "tx-queue: number",
1268  TX_QUEUE (i),
1269  "no VPP worker thread is available");
1270  continue;
1271  }
1272 
1273  if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i))))
1274  {
1275  args->rv = VNET_API_ERROR_INIT_FAILED;
1276  virtio_log_error (vif, "%s (%u) %s", "error in txq-queue",
1277  TX_QUEUE (i), "initialization");
1278  error =
1279  clib_error_return (error, "%s (%u) %s", "error in txq-queue",
1280  TX_QUEUE (i), "initialization");
1281  goto err;
1282  }
1283  else
1284  {
1285  vif->num_txqs++;
1286  }
1287  }
1288 
1289  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1290  {
1291  if ((error =
1293  {
1294  virtio_log_warning (vif, "%s (%u) %s", "error in control-queue",
1295  vif->max_queue_pairs * 2, "initialization");
1296  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
1297  vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ);
1298  }
1299  }
1300  else
1301  {
1302  virtio_log_debug (vif, "control queue is not available");
1303  vif->cxq_vring = NULL;
1304  }
1305 
1306  /*
1307  * set the msix interrupts
1308  */
1309  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1310  {
1311  int i, j;
1312  if (vif->virtio_pci_func->set_config_irq (vm, vif, 0) ==
1314  {
1315  virtio_log_warning (vif, "config vector 0 is not set");
1316  }
1317  else
1318  {
1319  virtio_log_debug (vif, "config msix vector is set at 0");
1320  }
1321  for (i = 0, j = 1; i < vif->max_queue_pairs; i++, j++)
1322  {
1323  if (vif->virtio_pci_func->set_queue_irq (vm, vif, j,
1324  RX_QUEUE (i)) ==
1326  {
1327  virtio_log_warning (vif, "queue (%u) vector is not set at %u",
1328  RX_QUEUE (i), j);
1329  }
1330  else
1331  {
1332  virtio_log_debug (vif, "%s (%u) %s %u", "queue",
1333  RX_QUEUE (i), "msix vector is set at", j);
1334  }
1335  }
1336  }
1337 
1338  /*
1339  * set the driver status OK
1340  */
1341  vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1342  vif->status = vif->virtio_pci_func->get_status (vm, vif);
1343 err:
1344  return error;
1345 }
1346 
1347 void
1349 {
1350  vnet_main_t *vnm = vnet_get_main ();
1351  virtio_main_t *vim = &virtio_main;
1352  virtio_if_t *vif;
1354  clib_error_t *error = 0;
1355  u32 interrupt_count = 0;
1356 
1357  /* *INDENT-OFF* */
1358  pool_foreach (vif, vim->interfaces) {
1359  if (vif->pci_addr.as_u32 == args->addr)
1360  {
1361  args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
1362  args->error =
1363  clib_error_return (error, "PCI address in use");
1364  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1365  format_vlib_pci_addr, &args->addr,
1366  " PCI address in use");
1367  return;
1368  }
1369  }
1370  /* *INDENT-ON* */
1371 
1372  pool_get (vim->interfaces, vif);
1373  vif->dev_instance = vif - vim->interfaces;
1374  vif->per_interface_next_index = ~0;
1375  vif->pci_addr.as_u32 = args->addr;
1376  if (args->virtio_flags & VIRTIO_FLAG_PACKED)
1377  vif->is_packed = 1;
1378 
1379  if ((error =
1380  vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
1382  {
1383  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1384  args->error =
1385  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1386  &vif->pci_addr);
1387  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1389  "error encountered on pci device open");
1390  pool_put (vim->interfaces, vif);
1391  return;
1392  }
1393  vif->pci_dev_handle = h;
1396  vif->type = VIRTIO_IF_TYPE_PCI;
1397 
1398  if ((error = vlib_pci_bus_master_enable (vm, h)))
1399  {
1400  virtio_log_error (vif, "error encountered on pci bus master enable");
1401  goto error;
1402  }
1403 
1404  void *bar[6];
1405  for (u32 i = 0; i <= 5; i++)
1406  {
1407 
1408  if ((error = vlib_pci_map_region (vm, h, i, &bar[i])))
1409  {
1410  virtio_log_debug (vif, "no pci map region for bar %u", i);
1411  }
1412  else
1413  {
1414  virtio_log_debug (vif, "pci map region for bar %u at %p", i,
1415  bar[i]);
1416  }
1417  }
1418 
1419  if ((error = vlib_pci_io_region (vm, h, 0)))
1420  {
1421  virtio_log_error (vif, "error encountered on pci io region");
1422  goto error;
1423  }
1424 
1425  interrupt_count = vlib_pci_get_num_msix_interrupts (vm, h);
1426  if (interrupt_count > 1)
1427  {
1428  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
1430  {
1431  args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1432  virtio_log_error (vif,
1433  "error encountered on pci register msix handler 0");
1434  goto error;
1435  }
1436 
1437  if ((error =
1438  vlib_pci_register_msix_handler (vm, h, 1, (interrupt_count - 1),
1440  {
1441  args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1442  virtio_log_error (vif,
1443  "error encountered on pci register msix handler 1");
1444  goto error;
1445  }
1446 
1447  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, interrupt_count)))
1448  {
1449  virtio_log_error (vif, "error encountered on pci enable msix irq");
1450  goto error;
1451  }
1452  vif->support_int_mode = 1;
1453  virtio_log_debug (vif, "device supports msix interrupts");
1454  }
1455  else
1456  {
1457  /*
1458  * WARN: performance will be sub-optimal.
1459  * Fall back to intX, if msix table-size is 1 or
1460  * if UIO driver is being used.
1461  */
1462  if ((error =
1464  {
1465  virtio_log_error (vif,
1466  "error encountered on pci register interrupt handler");
1467  goto error;
1468  }
1469  vif->support_int_mode = 1;
1470  virtio_log_debug (vif, "pci register interrupt handler");
1471  }
1472 
1473  if ((error = vlib_pci_intr_enable (vm, h)))
1474  {
1475  virtio_log_error (vif, "error encountered on pci interrupt enable");
1476  goto error;
1477  }
1478 
1479  if ((error = virtio_pci_device_init (vm, vif, args, bar)))
1480  {
1481  virtio_log_error (vif, "error encountered on device init");
1482  goto error;
1483  }
1484 
1485  /* create interface */
1487  vif->dev_instance, vif->mac_addr,
1488  &vif->hw_if_index,
1490 
1491  if (error)
1492  {
1493  args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1494  virtio_log_error (vif,
1495  "error encountered on ethernet register interface");
1496  goto error;
1497  }
1498 
1500  vif->sw_if_index = sw->sw_if_index;
1501  args->sw_if_index = sw->sw_if_index;
1502 
1505 
1506  if (args->virtio_flags & VIRTIO_FLAG_BUFFERING)
1507  {
1509  if (error)
1510  {
1511  args->rv = VNET_API_ERROR_INIT_FAILED;
1512  virtio_log_error (vif,
1513  "error encountered during packet buffering init");
1514  goto error;
1515  }
1516  }
1517 
1519 
1521  {
1522  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1525  }
1526  else
1528 
1529  virtio_pci_offloads (vm, vif, args->gso_enabled,
1530  args->checksum_offload_enabled);
1531 
1532  if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
1533  (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)))
1534  {
1536  virtio_log_warning (vif, "multiqueue is not set");
1537  }
1538  return;
1539 
1540 error:
1541  virtio_pci_delete_if (vm, vif);
1542  if (args->rv == 0)
1543  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1544  args->error = error;
1545 }
1546 
1547 int
1549 {
1550  vnet_main_t *vnm = vnet_get_main ();
1551  virtio_main_t *vim = &virtio_main;
1552  u32 i = 0;
1553 
1554  if (vif->type != VIRTIO_IF_TYPE_PCI)
1555  return VNET_API_ERROR_INVALID_INTERFACE;
1556 
1558 
1559  for (i = 0; i < vif->max_queue_pairs; i++)
1560  {
1561  vif->virtio_pci_func->del_queue (vm, vif, RX_QUEUE (i));
1562  vif->virtio_pci_func->del_queue (vm, vif, TX_QUEUE (i));
1563  }
1564 
1565  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1566  vif->virtio_pci_func->del_queue (vm, vif, vif->max_queue_pairs * 2);
1567 
1568  if (vif->virtio_pci_func)
1569  vif->virtio_pci_func->device_reset (vm, vif);
1570 
1571  if (vif->hw_if_index)
1572  {
1575  }
1576 
1578 
1580  {
1581  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i);
1582  if (vring->used)
1583  {
1584  virtio_free_buffers (vm, vring);
1585  }
1586  vec_free (vring->buffers);
1587  vlib_physmem_free (vm, vring->desc);
1588  }
1589 
1591  {
1592  virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i);
1593  if (vring->used)
1594  {
1595  virtio_free_buffers (vm, vring);
1596  }
1597  vec_free (vring->buffers);
1600  clib_spinlock_free (&vring->lockp);
1601  vlib_physmem_free (vm, vring->desc);
1602  }
1603 
1604  if (vif->cxq_vring != NULL)
1605  {
1606  u16 last = vif->cxq_vring->last_used_idx;
1607  u16 n_left = vif->cxq_vring->used->idx - last;
1608  while (n_left)
1609  {
1610  last++;
1611  n_left--;
1612  }
1613 
1614  vif->cxq_vring->last_used_idx = last;
1616  }
1617 
1618  vec_free (vif->rxq_vrings);
1619  vec_free (vif->txq_vrings);
1620  vec_free (vif->cxq_vring);
1621 
1622  clib_error_free (vif->error);
1623  memset (vif, 0, sizeof (*vif));
1624  pool_put (vim->interfaces, vif);
1625 
1626  return 0;
1627 }
1628 
1629 int
1631  int gso_enabled,
1632  int checksum_offload_enabled,
1633  int offloads_disabled)
1634 {
1635  if (vif->type != VIRTIO_IF_TYPE_PCI)
1636  return VNET_API_ERROR_INVALID_INTERFACE;
1637 
1638  if (gso_enabled)
1639  virtio_pci_offloads (vm, vif, 1, 0);
1640  else if (checksum_offload_enabled)
1641  virtio_pci_offloads (vm, vif, 0, 1);
1642  else if (offloads_disabled)
1643  virtio_pci_offloads (vm, vif, 0, 0);
1644 
1645  return 0;
1646 }
1647 
1648 /*
1649  * fd.io coding-style-patch-verification: ON
1650  *
1651  * Local Variables:
1652  * eval: (c-set-style "gnu")
1653  * End:
1654  */
vlib.h
clib_spinlock_init
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:65
virtio_device_class
vnet_device_class_t virtio_device_class
vlib_pci_set_private_data
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
virtio_ctrl_msg_t::status
virtio_net_ctrl_ack_t status
Definition: pci.h:170
pci.h
vring_used_elem_t
Definition: virtio_std.h:109
virtio_pci_create_if
void virtio_pci_create_if(vlib_main_t *vm, virtio_pci_create_if_args_t *args)
Definition: pci.c:1348
vlib_buffer_free
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:982
PCI_DEVICE_ID_VIRTIO_NIC_MODERN
#define PCI_DEVICE_ID_VIRTIO_NIC_MODERN
Definition: pci.c:32
virtio_pci_enable_checksum_offload
static int virtio_pci_enable_checksum_offload(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:465
virtio_pci_device_ids
static pci_device_id_t virtio_pci_device_ids[]
Definition: pci.c:40
virtio_pci_vring_split_init
clib_error_t * virtio_pci_vring_split_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:727
virtio_pci_offloads
static int virtio_pci_offloads(vlib_main_t *vm, virtio_if_t *vif, int gso_enabled, int csum_offload_enabled)
Definition: pci.c:507
VIRTIO_PCI_CAP_DEVICE_CFG
#define VIRTIO_PCI_CAP_DEVICE_CFG
Definition: pci.h:84
virtio_vring_t::flow_table
gro_flow_table_t * flow_table
Definition: virtio.h:110
VIRTIO_MSI_NO_VECTOR
#define VIRTIO_MSI_NO_VECTOR
Definition: pci.h:29
vnet_sw_interface_t
Definition: interface.h:868
VIRTIO_NET_CTRL_GUEST_OFFLOADS
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS
Definition: pci.h:74
VIRTIO_MSIX_ENABLED
@ VIRTIO_MSIX_ENABLED
Definition: pci.h:96
vlib_buffer_get_current_pa
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:494
VNET_HW_INTERFACE_CAP_SUPPORTS_L4_TX_CKSUM
#define VNET_HW_INTERFACE_CAP_SUPPORTS_L4_TX_CKSUM
Definition: interface.h:555
vnet_hw_interface_t::caps
vnet_hw_interface_capabilities_t caps
Definition: interface.h:645
virtio_pci_send_ctrl_msg
static int virtio_pci_send_ctrl_msg(vlib_main_t *vm, virtio_if_t *vif, virtio_ctrl_msg_t *data, u32 len)
Definition: pci.c:435
format_vlib_pci_addr
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
Definition: pci.c:138
TX_QUEUE_ACCESS
#define TX_QUEUE_ACCESS(X)
Definition: virtio.h:39
clib_memcpy
#define clib_memcpy(d, s, n)
Definition: string.h:197
types.h
virtio_vring_t
Definition: virtio.h:63
vring_used_t::flags
u16 flags
Definition: virtio_std.h:117
virtio.h
virtio_pci_cap_t::offset
u32 offset
Definition: pci.h:108
VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
Definition: pci.h:75
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
virtio_pci_irq_queue_handler
static void virtio_pci_irq_queue_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:109
vlib_pci_get_numa_node
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
pool_elt_at_index
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:553
vring_avail_t::ring
u16 ring[0]
Definition: virtio_std.h:105
virtio_if_t::mac_addr
u8 mac_addr[6]
Definition: virtio.h:171
virtio_if_t::flags
u32 flags
Definition: virtio.h:132
VIRTIO_RING_FLAG_MASK_INT
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:58
virtio_if_t::error
clib_error_t * error
Definition: virtio.h:163
virtio_kick
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
Definition: virtio.h:251
virtio_vring_t::queue_notify_offset
u16 queue_notify_offset
Definition: virtio.h:102
next
u16 * next
Definition: nat44_ei_out2in.c:718
PCI_CAP_ID_VNDR
#define PCI_CAP_ID_VNDR
Definition: pci.c:35
clib_error_return
#define clib_error_return(e, args...)
Definition: error.h:99
RX_QUEUE
#define RX_QUEUE(X)
Definition: virtio.h:38
virtio_pci_create_if_args_t::gso_enabled
u8 gso_enabled
Definition: pci.h:239
vlib_pci_dev_handle_t
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
vlib_pci_enable_msix_irq
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:895
virtio_if_t::per_interface_next_index
u32 per_interface_next_index
Definition: virtio.h:133
virtio_ctrl_msg_t::ctrl
virtio_net_ctrl_hdr_t ctrl
Definition: pci.h:169
virtio_pci_device_init
static clib_error_t * virtio_pci_device_init(vlib_main_t *vm, virtio_if_t *vif, virtio_pci_create_if_args_t *args, void **bar)
Definition: pci.c:1146
u16
unsigned short u16
Definition: types.h:57
virtio_pci_cap_t
Definition: pci.h:100
vring_desc_t::addr
u64 addr
Definition: virtio_std.h:95
vlib_physmem_free
static void vlib_physmem_free(vlib_main_t *vm, void *p)
Definition: physmem_funcs.h:89
pool_put
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:305
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
VRING_DESC_F_USED
#define VRING_DESC_F_USED
Definition: virtio_std.h:76
virtio_pci_enable_gso
static int virtio_pci_enable_gso(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:486
VNET_HW_INTERFACE_FLAG_LINK_UP
@ VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:509
VIRTIO_FEATURE
#define VIRTIO_FEATURE(X)
Definition: virtio_std.h:67
PCI_CAPABILITY_LIST
#define PCI_CAPABILITY_LIST
Definition: pci.c:34
vlib_pci_register_msix_handler
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:838
vring_t::avail
vring_avail_t * avail
Definition: virtio_std.h:182
VRING_DESC_F_WRITE
#define VRING_DESC_F_WRITE
Definition: virtio_std.h:72
virtio_vring_t::queue_id
u16 queue_id
Definition: virtio.h:84
virtio_if_t::numa_node
u32 numa_node
Definition: virtio.h:159
vring_used_elem_t::id
u32 id
Definition: virtio_std.h:111
virtio_main_t::log_default
vlib_log_class_t log_default
Definition: virtio.h:218
virtio_pci_get_mac
static u32 virtio_pci_get_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:86
virtio_if_t::virtio_pci_func
const virtio_pci_func_t * virtio_pci_func
Definition: virtio.h:210
virtio_pci_vring_packed_init
clib_error_t * virtio_pci_vring_packed_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:807
vlib_pci_map_region
clib_error_t * vlib_pci_map_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, void **result)
Definition: pci.c:1182
pci_device_id_t::vendor_id
u16 vendor_id
Definition: pci.h:127
virtio_vring_t::size
u16 size
Definition: virtio.h:83
virtio_pci_set_mac
static void virtio_pci_set_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:79
virtio_ctrl_msg_t::data
u8 data[1024]
Definition: pci.h:171
ethernet.h
virtio_vring_t::avail
vring_avail_t * avail
Definition: virtio.h:73
h
h
Definition: flowhash_template.h:372
error
Definition: cJSON.c:88
virtio_pci_read_caps
clib_error_t * virtio_pci_read_caps(vlib_main_t *vm, virtio_if_t *vif, void **bar)
Definition: pci.c:1007
VIRTIO_NET_S_LINK_UP
#define VIRTIO_NET_S_LINK_UP
Definition: pci.h:54
vlib_pci_intr_enable
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:239
virtio_pci_create_if_args_t::features
u64 features
Definition: pci.h:238
virtio_if_t::notify_off_multiplier
u32 notify_off_multiplier
Definition: virtio.h:198
virtio_pci_control_vring_init
clib_error_t * virtio_pci_control_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:717
virtio_if_t::num_rxqs
u16 num_rxqs
Definition: virtio.h:134
ip6_packet.h
random_u32
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
virtio_if_t::bar_id
u32 bar_id
Definition: virtio.h:197
virtio_if_t::notify_offset
u16 notify_offset
Definition: virtio.h:201
VNET_HW_INTERFACE_CAP_SUPPORTS_TX_TCP_CKSUM
@ VNET_HW_INTERFACE_CAP_SUPPORTS_TX_TCP_CKSUM
Definition: interface.h:524
virtio_if_t::msix_table_size
u16 msix_table_size
Definition: virtio.h:205
gro_flow_table_free
static_always_inline void gro_flow_table_free(gro_flow_table_t *flow_table)
Definition: gro.h:171
vlib_pci_bus_master_enable
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:271
vring_t::desc
vring_desc_t * desc
Definition: virtio_std.h:181
round_pow2
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:279
vnet_sw_interface_t::sw_if_index
u32 sw_if_index
Definition: interface.h:875
virtio_main
virtio_main_t virtio_main
Definition: virtio.c:37
STRUCT_OFFSET_OF
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:73
vlib_buffer_t::current_data
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:119
virtio_pci_queue_size_valid
static u8 virtio_pci_queue_size_valid(u16 qsz)
Definition: pci.c:586
vlib_thread_main_t::n_vlib_mains
u32 n_vlib_mains
Definition: threads.h:283
pool_foreach
#define pool_foreach(VAR, POOL)
Iterate through pool.
Definition: pool.h:534
vlib_pci_get_private_data
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
clib_memset_u32
static_always_inline void clib_memset_u32(void *p, u32 val, uword count)
Definition: string.h:349
vring_size
static_always_inline u16 vring_size(u32 num, u32 align)
Definition: virtio_std.h:200
device_status
void device_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:172
CLIB_MEMORY_STORE_BARRIER
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:140
virtio_if_t::txq_vrings
virtio_vring_t * txq_vrings
Definition: virtio.h:137
len
u8 len
Definition: ip_types.api:103
virtio_vring_t::desc_next
u16 desc_next
Definition: virtio.h:87
virtio_vring_t::used
vring_used_t * used
Definition: virtio.h:72
virtio_if_t::pci_addr
pci_addr_t pci_addr
Definition: virtio.h:196
virtio_if_t::common_offset
u16 common_offset
Definition: virtio.h:200
virtio_pci_delete_if
int virtio_pci_delete_if(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:1548
slot
u8 slot
Definition: pci_types.api:22
vring_desc_t::flags
u16 flags
Definition: virtio_std.h:97
virtio_if_t::pci_dev_handle
u32 pci_dev_handle
Definition: virtio.h:145
virtio_vring_t::lockp
clib_spinlock_t lockp
Definition: virtio.h:66
vring_desc_t
Definition: virtio_std.h:93
vlib_buffer_alloc
static __clib_warn_unused_result u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:708
vring_avail_t::flags
u16 flags
Definition: virtio_std.h:103
VRING_DESC_F_AVAIL
#define VRING_DESC_F_AVAIL
Definition: virtio_std.h:75
virtio_vring_t::desc
vring_desc_t * desc
Definition: virtio.h:71
virtio_pci_control_vring_packed_init
clib_error_t * virtio_pci_control_vring_packed_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:596
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
vnet_get_hw_interface
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface_funcs.h:44
vnet_get_main
vnet_main_t * vnet_get_main(void)
Definition: pnat_test_stubs.h:56
vlib_pci_io_region
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
Definition: pci.c:1196
virtio_pci_flag_change
static u32 virtio_pci_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: pci.c:51
VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO
@ VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO
Definition: interface.h:537
virtio_if_t::features
u64 features
Definition: virtio.h:131
virtio_if_t::device_offset
u16 device_offset
Definition: virtio.h:202
rx_queue_funcs.h
vec_validate_aligned
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:534
foreach_virtio_config_status_flags
@ foreach_virtio_config_status_flags
Definition: pci.h:49
PCI_MSIX_ENABLE
#define PCI_MSIX_ENABLE
Definition: pci.c:38
virtio_pci_cap_t::cap_vndr
u8 cap_vndr
Definition: pci.h:102
virtio_if_t::csum_offload_enabled
int csum_offload_enabled
Definition: virtio.h:139
vec_foreach_index
#define vec_foreach_index(var, v)
Iterate over vector indices.
Definition: vec_bootstrap.h:220
uword
u64 uword
Definition: types.h:112
last
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
VIRTIO_PCI_CAP_COMMON_CFG
#define VIRTIO_PCI_CAP_COMMON_CFG
Definition: pci.h:78
virtio_pci_send_ctrl_msg_packed
static int virtio_pci_send_ctrl_msg_packed(vlib_main_t *vm, virtio_if_t *vif, virtio_ctrl_msg_t *data, u32 len)
Definition: pci.c:200
VIRTIO_PCI_VRING_ALIGN
#define VIRTIO_PCI_VRING_ALIGN
Definition: pci.h:90
virtio_pci_create_if_args_t::addr
u32 addr
Definition: pci.h:232
virtio_if_t::msix_enabled
u32 msix_enabled
Definition: virtio.h:146
virtio_pci_create_if_args_t::virtio_flags
u32 virtio_flags
Definition: pci.h:242
virtio_pci_create_if_args_t::checksum_offload_enabled
u8 checksum_offload_enabled
Definition: pci.h:240
virtio_pci_create_if_args_t::error
clib_error_t * error
Definition: pci.h:243
virtio_if_t::hw_if_index
u32 hw_if_index
Definition: virtio.h:152
VRING_DESC_F_NEXT
#define VRING_DESC_F_NEXT
Definition: virtio_std.h:71
virtio_pci_create_if_args_t::rv
i32 rv
Definition: pci.h:234
virtio_set_net_hdr_size
void virtio_set_net_hdr_size(virtio_if_t *vif)
Definition: virtio.c:288
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
f64
double f64
Definition: types.h:142
clib_spinlock_free
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:72
mask
vl_api_pnat_mask_t mask
Definition: pnat.api:45
vring_desc_t::len
u32 len
Definition: virtio_std.h:96
pool_get
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:255
PCI_CAP_ID_MSIX
#define PCI_CAP_ID_MSIX
Definition: pci.c:36
vnet_hw_if_rx_queue_set_int_pending
static_always_inline void vnet_hw_if_rx_queue_set_int_pending(vnet_main_t *vnm, u32 queue_index)
Definition: rx_queue_funcs.h:51
VIRTIO_PCI_ISR_CONFIG
#define VIRTIO_PCI_ISR_CONFIG
Definition: pci.h:34
virtio_pci_create_if_args_t::sw_if_index
u32 sw_if_index
Definition: pci.h:235
vlib_pci_register_intx_handler
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
Definition: pci.c:787
CLIB_MEMORY_BARRIER
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:137
virtio_pci_control_vring_split_init
clib_error_t * virtio_pci_control_vring_split_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:661
CLIB_CACHE_LINE_BYTES
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
virtio_pci_cap_t::bar
u8 bar
Definition: pci.h:106
vlib_pci_read_write_config
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
Definition: pci.c:1034
vlib_cli_output
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:716
virtio_if_t::sw_if_index
u32 sw_if_index
Definition: virtio.h:153
vlib_pci_get_num_msix_interrupts
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:177
virtio_vring_buffering_free
static_always_inline void virtio_vring_buffering_free(vlib_main_t *vm, virtio_vring_buffering_t *buffering)
Definition: virtio_buffering.h:86
virtio_vring_t::queue_index
u32 queue_index
Definition: virtio.h:85
vring_t
Definition: virtio_std.h:178
virtio_if_t::max_queue_pairs
u16 max_queue_pairs
Definition: virtio.h:204
virtio_vring_t::driver_event
vring_desc_event_t * driver_event
Definition: virtio.h:78
virtio_pci_enable_multiqueue
static int virtio_pci_enable_multiqueue(vlib_main_t *vm, virtio_if_t *vif, u16 num_queues)
Definition: pci.c:569
data
u8 data[128]
Definition: ipsec_types.api:92
virtio_vring_t::last_used_idx
u16 last_used_idx
Definition: virtio.h:88
virtio_pci_legacy_func
const virtio_pci_func_t virtio_pci_legacy_func
Definition: virtio_pci_legacy.c:360
ip4_packet.h
virtio_pci_send_ctrl_msg_split
static int virtio_pci_send_ctrl_msg_split(vlib_main_t *vm, virtio_if_t *vif, virtio_ctrl_msg_t *data, u32 len)
Definition: pci.c:340
virtio_main_t
Definition: virtio.h:214
PCI_DEVICE_ID_VIRTIO_NIC
#define PCI_DEVICE_ID_VIRTIO_NIC
Definition: pci.c:30
VLIB_READ
@ VLIB_READ
Definition: defs.h:64
VIRTIO_NET_CTRL_MQ
#define VIRTIO_NET_CTRL_MQ
Definition: pci.h:61
virtio_vring_t::avail_wrap_counter
u16 avail_wrap_counter
Definition: virtio.h:100
vnet_hw_interface_t
Definition: interface.h:638
vnet_main_t
Definition: vnet.h:76
vec_free
#define vec_free(V)
Free vector's memory (no header).
Definition: vec.h:395
virtio_if_t::dev_instance
u32 dev_instance
Definition: virtio.h:158
VIRTIO_PCI_CAP_PCI_CFG
#define VIRTIO_PCI_CAP_PCI_CFG
Definition: pci.h:86
virtio_log_debug
#define virtio_log_debug(vif, f,...)
Definition: virtio.h:272
virtio_if_t::cxq_vring
virtio_vring_t * cxq_vring
Definition: virtio.h:195
vring_t::used
vring_used_t * used
Definition: virtio_std.h:183
VIRTIO_PCI_CAP_NOTIFY_CFG
#define VIRTIO_PCI_CAP_NOTIFY_CFG
Definition: pci.h:80
VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
Definition: pci.h:62
TX_QUEUE
#define TX_QUEUE(X)
Definition: virtio.h:37
RX_QUEUE_ACCESS
#define RX_QUEUE_ACCESS(X)
Definition: virtio.h:40
u64
unsigned long u64
Definition: types.h:89
virtio_if_t::num_txqs
u16 num_txqs
Definition: virtio.h:135
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
virtio_log_warning
#define virtio_log_warning(vif, f,...)
Definition: virtio.h:279
virtio_pci_irq_handler
static void virtio_pci_irq_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:146
vnet_get_hw_sw_interface
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface_funcs.h:72
u32
unsigned int u32
Definition: types.h:88
ethernet_delete_interface
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:393
vlib_thread_main_t
Definition: threads.h:264
virtio_pci_cap_t::length
u32 length
Definition: pci.h:109
virtio_if_t::remote_features
u64 remote_features
Definition: virtio.h:160
virtio_pci_disable_offload
static int virtio_pci_disable_offload(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:445
virtio_if_t::support_int_mode
u8 support_int_mode
Definition: virtio.h:206
virtio_vring_t::packed_desc
vring_packed_desc_t * packed_desc
Definition: virtio.h:77
pci.h
n_left
u32 n_left
Definition: interface_output.c:1078
virtio_ctrl_msg_t
Definition: pci.h:167
virtio_if_t::type
virtio_if_type_t type
Definition: virtio.h:150
vring_used_t::idx
u16 idx
Definition: virtio_std.h:118
virtio_if_t
Definition: virtio.h:128
virtio_pci_cap_t::cap_next
u8 cap_next
Definition: pci.h:103
VIRTIO_PCI_ISR_INTR
#define VIRTIO_PCI_ISR_INTR
Definition: pci.h:32
vlib_log
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:131
VIRTIO_MSIX_DISABLED
@ VIRTIO_MSIX_DISABLED
Definition: pci.h:95
virtio_pci_vring_init
clib_error_t * virtio_pci_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:898
virtio_if_t::rxq_vrings
virtio_vring_t * rxq_vrings
Definition: virtio.h:136
virtio_if_t::isr_offset
u16 isr_offset
Definition: virtio.h:203
VIRTIO_NET_ERR
#define VIRTIO_NET_ERR
Definition: pci.h:58
virtio_main_t::interfaces
virtio_if_t * interfaces
Definition: virtio.h:220
virtio_pci_create_if_args_t
Definition: pci.h:230
now
f64 now
Definition: nat44_ei_out2in.c:710
vlib_physmem_last_error
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
Definition: physmem_funcs.h:110
clib_memset
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vlib_main_t
Definition: main.h:102
virtio_free_buffers
void virtio_free_buffers(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:128
virtio_pci_irq_config_handler
static void virtio_pci_irq_config_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:124
virtio_vring_t::buffers
u32 * buffers
Definition: virtio.h:82
virtio_if_t::bar
void * bar
Definition: virtio.h:194
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
vnet_hw_interface_set_flags
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:513
vring_init
static_always_inline void vring_init(vring_t *vr, u32 num, void *p, u32 align)
Definition: virtio_std.h:187
vlib_buffer_get_current
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:257
vring_used_t::ring
vring_used_elem_t ring[0]
Definition: virtio_std.h:119
virtio_vring_t::buffering
virtio_vring_buffering_t * buffering
Definition: virtio.h:109
virtio_vring_t::desc_in_use
u16 desc_in_use
Definition: virtio.h:86
virtio_pci_enable_disable_offloads
int virtio_pci_enable_disable_offloads(vlib_main_t *vm, virtio_if_t *vif, int gso_enabled, int checksum_offload_enabled, int offloads_disabled)
Definition: pci.c:1630
VNET_HW_INTERFACE_CAP_SUPPORTS_TX_UDP_CKSUM
@ VNET_HW_INTERFACE_CAP_SUPPORTS_TX_UDP_CKSUM
Definition: interface.h:525
vlib_physmem_alloc_aligned_on_numa
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
Definition: physmem_funcs.h:63
virtio_log_error
#define virtio_log_error(vif, f,...)
Definition: virtio.h:286
virtio_pci_modern_func
const virtio_pci_func_t virtio_pci_modern_func
Definition: virtio_pci_modern.c:420
vring_desc_t::next
u16 next
Definition: virtio_std.h:98
clib_warning
#define clib_warning(format, args...)
Definition: error.h:59
virtio_set_packet_buffering
clib_error_t * virtio_set_packet_buffering(virtio_if_t *vif, u16 buffering_size)
Definition: virtio.c:203
vlib_pci_device_open
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1251
VNET_HW_INTERFACE_CAP_SUPPORTS_INT_MODE
@ VNET_HW_INTERFACE_CAP_SUPPORTS_INT_MODE
Definition: interface.h:550
clib_error_free
#define clib_error_free(e)
Definition: error.h:86
virtio_pci_get_max_virtqueue_pairs
static clib_error_t * virtio_pci_get_max_virtqueue_pairs(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:58
pci_addr_t::as_u32
u32 as_u32
Definition: virtio.h:122
vlib_time_now
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:325
virtio_vring_t::device_event
vring_desc_event_t * device_event
Definition: virtio.h:79
vlib_pci_device_close
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1311
vlib_pci_intr_disable
static clib_error_t * vlib_pci_intr_disable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:255
virtio_pci_create_if_args_t::buffering_size
u32 buffering_size
Definition: pci.h:241
virtio_vring_set_rx_queues
void virtio_vring_set_rx_queues(vlib_main_t *vm, virtio_if_t *vif)
Definition: virtio.c:238
vlib_get_thread_main
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:56
is_pow2
static uword is_pow2(uword x)
Definition: clib.h:267
virtio_pci_is_link_up
static u16 virtio_pci_is_link_up(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:97
virtio_negotiate_features
static void virtio_negotiate_features(vlib_main_t *vm, virtio_if_t *vif, u64 req_features)
Definition: pci.c:907
virtio_pci_read_device_feature
void virtio_pci_read_device_feature(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:968
virtio_pci_reset_device
int virtio_pci_reset_device(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:974
virtio_if_t::is_packed
int is_packed
Definition: virtio.h:211
PCI_VENDOR_ID_VIRTIO
#define PCI_VENDOR_ID_VIRTIO
Definition: pci.c:29
virtio_vring_t::used_wrap_counter
u16 used_wrap_counter
Definition: virtio.h:101
virtio_if_t::status
u8 status
Definition: virtio.h:207
vring_avail_t::idx
u16 idx
Definition: virtio_std.h:104
virtio_pci_cap_t::cfg_type
u8 cfg_type
Definition: pci.h:105
virtio_if_t::is_modern
u32 is_modern
Definition: virtio.h:199
ethernet_register_interface
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:348
VIRTIO_PCI_CAP_ISR_CFG
#define VIRTIO_PCI_CAP_ISR_CFG
Definition: pci.h:82
virtio_if_t::gso_enabled
int gso_enabled
Definition: virtio.h:138
virtio_net_ctrl_ack_t
u8 virtio_net_ctrl_ack_t
Definition: pci.h:165
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
pci_device_id_t
Definition: pci.h:125
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105