FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
pci.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <fcntl.h>
17 #include <sys/ioctl.h>
18 
19 #include <vppinfra/types.h>
20 #include <vlib/vlib.h>
21 #include <vlib/pci/pci.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/ip/ip4_packet.h>
24 #include <vnet/ip/ip6_packet.h>
27 
28 #define PCI_VENDOR_ID_VIRTIO 0x1af4
29 #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000
30 /* Doesn't support modern device */
31 #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041
32 
33 #define PCI_CAPABILITY_LIST 0x34
34 #define PCI_CAP_ID_VNDR 0x09
35 #define PCI_CAP_ID_MSIX 0x11
36 
37 #define PCI_MSIX_ENABLE 0x8000
38 
39 #define PCI_CONFIG_SIZE(vif) ((vif->msix_enabled == VIRTIO_MSIX_ENABLED) ? \
40  24 : 20)
41 
42 static pci_device_id_t virtio_pci_device_ids[] = {
43  {
45  .device_id = PCI_DEVICE_ID_VIRTIO_NIC},
46  {
47  .vendor_id = PCI_VENDOR_ID_VIRTIO,
48  .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN},
49  {0},
50 };
51 
52 static void
54  int len, u32 addr)
55 {
56  u32 size = 0;
58 
59  while (len > 0)
60  {
61  if (len >= 4)
62  {
63  size = 4;
64  vlib_pci_read_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
65  }
66  else if (len >= 2)
67  {
68  size = 2;
69  vlib_pci_read_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
70  }
71  else
72  {
73  size = 1;
74  vlib_pci_read_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
75  }
76  dst = (u8 *) dst + size;
77  addr += size;
78  len -= size;
79  }
80 }
81 
82 static void
84  void *src, int len, u32 addr)
85 {
86  u32 size = 0;
88 
89  while (len > 0)
90  {
91  if (len >= 4)
92  {
93  size = 4;
94  vlib_pci_write_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
95  }
96  else if (len >= 2)
97  {
98  size = 2;
99  vlib_pci_write_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
100  }
101  else
102  {
103  size = 1;
104  vlib_pci_write_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
105  }
106  src = (u8 *) src + size;
107  addr += size;
108  len -= size;
109  }
110 }
111 
112 static u64
114 {
115  u32 features;
116  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
117  &features);
118  return features;
119 }
120 
121 static u32
123 {
124  u32 feature = 0;
125  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
126  &feature);
127  vif->features = feature;
128  return feature;
129 }
130 
131 static u32
133  u64 features)
134 {
135  if ((features >> 32) != 0)
136  {
137  clib_warning ("only 32 bit features are allowed for legacy virtio!");
138  }
139  u32 feature = 0, guest_features = (u32) features;
140  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
141  &guest_features);
142  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
143  &feature);
144  return feature;
145 }
146 
147 static u8
149 {
150  u8 status = 0;
151  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
152  return status;
153 }
154 
155 static void
157 {
158  if (status != VIRTIO_CONFIG_STATUS_RESET)
159  status |= virtio_pci_legacy_get_status (vm, vif);
160  vlib_pci_write_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
161 }
162 
163 static u8
165 {
166  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_RESET);
167  return virtio_pci_legacy_get_status (vm, vif);
168 }
169 
170 static u8
172 {
173  u8 isr = 0;
174  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &isr);
175  return isr;
176 }
177 
178 static u16
180  u16 queue_id)
181 {
182  u16 queue_num = 0;
183  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
184  &queue_id);
185  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
186  &queue_num);
187  return queue_num;
188 }
189 
190 static int
192  u16 queue_id, void *p)
193 {
195  u32 addr2 = 0;
196  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
197  &queue_id);
198  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
199  (u32 *) & addr);
200  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
201  &addr2);
202  if ((u32) addr == addr2)
203  return 0;
204  return 1;
205 }
206 
207 static void
209  u16 queue_id)
210 {
211  u32 src = 0;
212  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
213  &queue_id);
214  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN, &src);
215 }
216 
217 inline void
219  u16 queue_id)
220 {
221  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
222  &queue_id);
223 }
224 
225 /* Enable one vector (0) for Link State Intrerrupt */
226 static u16
228  u16 vec)
229 {
230  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
231  &vec);
232  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
233  &vec);
234  return vec;
235 }
236 
237 static u16
239  u16 queue_id)
240 {
241  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
242  &queue_id);
243  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
244  &vec);
245  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
246  &vec);
247  return vec;
248 }
249 
250 static u32
252  u32 flags)
253 {
254  return 0;
255 }
256 
257 static clib_error_t *
259 {
260  virtio_net_config_t config;
261  clib_error_t *error = 0;
262  u16 max_queue_pairs = 1;
263 
264  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
265  {
267  sizeof (config.max_virtqueue_pairs),
269  max_virtqueue_pairs));
270  max_queue_pairs = config.max_virtqueue_pairs;
271  }
272 
273  virtio_log_debug (vif, "max queue pair is %x", max_queue_pairs);
274  if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
275  return clib_error_return (error, "max queue pair is %x,"
276  " should be in range [1, 0x8000]",
277  max_queue_pairs);
278 
279  vif->max_queue_pairs = max_queue_pairs;
280  return error;
281 }
282 
283 static void
285 {
287  sizeof (vif->mac_addr), 0);
288 }
289 
290 static u32
292 {
293  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
294  {
296  sizeof (vif->mac_addr), 0);
297  return 0;
298  }
299  return 1;
300 }
301 
302 static u16
304 {
305  /*
306  * Minimal driver: assumes link is up
307  */
308  u16 status = 1;
309  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS))
310  virtio_pci_legacy_read_config (vm, vif, &status, sizeof (status), /* mac */
312  status));
313  return status;
314 }
315 
316 static void
318  u16 line)
319 {
320  vnet_main_t *vnm = vnet_get_main ();
321  virtio_main_t *vim = &virtio_main;
322  uword pd = vlib_pci_get_private_data (vm, h);
323  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
324  line--;
325  u16 qid = line;
326 
328 }
329 
330 static void
332  u16 line)
333 {
334  vnet_main_t *vnm = vnet_get_main ();
335  virtio_main_t *vim = &virtio_main;
336  uword pd = vlib_pci_get_private_data (vm, h);
337  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
338 
339  if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
340  {
341  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
344  }
345  else
346  {
347  vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
349  }
350 }
351 
352 static void
354 {
355  virtio_main_t *vim = &virtio_main;
356  uword pd = vlib_pci_get_private_data (vm, h);
357  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
358  u8 isr = 0;
359  u16 line = 0;
360 
361  isr = virtio_pci_legacy_get_isr (vm, vif);
362 
363  /*
364  * If the lower bit is set: look through the used rings of
365  * all virtqueues for the device, to see if any progress has
366  * been made by the device which requires servicing.
367  */
368  if (isr & VIRTIO_PCI_ISR_INTR)
369  {
370  for (; line < vif->num_rxqs; line++)
371  virtio_pci_irq_queue_handler (vm, h, (line + 1));
372  }
373 
374  if (isr & VIRTIO_PCI_ISR_CONFIG)
375  virtio_pci_irq_config_handler (vm, h, line);
376 }
377 
378 inline void
380 {
381  struct status_struct
382  {
383  u8 bit;
384  char *str;
385  };
386  struct status_struct *status_entry;
387  static struct status_struct status_array[] = {
388 #define _(s,b) { .str = #s, .bit = b, },
390 #undef _
391  {.str = NULL}
392  };
393 
394  vlib_cli_output (vm, " status 0x%x", vif->status);
395 
396  status_entry = (struct status_struct *) &status_array;
397  while (status_entry->str)
398  {
399  if (vif->status & status_entry->bit)
400  vlib_cli_output (vm, " %s (%x)", status_entry->str,
401  status_entry->bit);
402  status_entry++;
403  }
404 }
405 
406 inline void
408 {
409  u32 data_u32;
410  u16 data_u16;
411  u8 data_u8;
412  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
413  &data_u32);
414  vlib_cli_output (vm, "remote features 0x%lx", data_u32);
415  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
416  &data_u32);
417  vlib_cli_output (vm, "guest features 0x%lx", data_u32);
418  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
419  &data_u32);
420  vlib_cli_output (vm, "queue address 0x%lx", data_u32);
421  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
422  &data_u16);
423  vlib_cli_output (vm, "queue size 0x%x", data_u16);
424  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
425  &data_u16);
426  vlib_cli_output (vm, "queue select 0x%x", data_u16);
427  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
428  &data_u16);
429  vlib_cli_output (vm, "queue notify 0x%x", data_u16);
430  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &data_u8);
431  vlib_cli_output (vm, "status 0x%x", data_u8);
432  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &data_u8);
433  vlib_cli_output (vm, "isr 0x%x", data_u8);
434 
435  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
436  {
437  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
438  &data_u16);
439  vlib_cli_output (vm, "config vector 0x%x", data_u16);
440  u16 queue_id = 0;
441  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
442  &queue_id);
443  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
444  &data_u16);
445  vlib_cli_output (vm, "queue vector for queue (0) 0x%x", data_u16);
446  }
447 
448  u8 mac[6];
449  virtio_pci_legacy_read_config (vm, vif, mac, sizeof (mac), 0);
450  vlib_cli_output (vm, "mac %U", format_ethernet_address, mac);
451  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to status */
452  6);
453  vlib_cli_output (vm, "link up/down status 0x%x", data_u16);
454  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16),
455  /* offset to max_virtqueue */ 8);
456  vlib_cli_output (vm, "num of virtqueue 0x%x", data_u16);
457  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to mtu */
458  10);
459  vlib_cli_output (vm, "mtu 0x%x", data_u16);
460 
461  u32 i = PCI_CONFIG_SIZE (vif) + 12, a = 4;
462  i += a;
463  i &= ~a;
464  for (; i < 64; i += 4)
465  {
466  u32 data = 0;
467  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, i, &data);
468  vlib_cli_output (vm, "0x%lx", data);
469  }
470 }
471 
473 {
474  struct virtio_net_ctrl_hdr ctrl;
475  virtio_net_ctrl_ack status;
476  u8 data[1024];
477 };
478 
479 static int
481  struct virtio_ctrl_msg *data, u32 len)
482 {
483  virtio_vring_t *vring = vif->cxq_vring;
484  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
485  struct virtio_ctrl_msg result;
486  u32 buffer_index;
487  vlib_buffer_t *b;
488  u16 used, next, avail;
489  u16 sz = vring->size;
490  u16 mask = sz - 1;
491 
492  used = vring->desc_in_use;
493  next = vring->desc_next;
494  avail = vring->avail->idx;
495  struct vring_desc *d = &vring->desc[next];
496 
497  if (vlib_buffer_alloc (vm, &buffer_index, 1))
498  b = vlib_get_buffer (vm, buffer_index);
499  else
500  return VIRTIO_NET_ERR;
501  /*
502  * current_data may not be initialized with 0 and may contain
503  * previous offset.
504  */
505  b->current_data = 0;
507  sizeof (struct virtio_ctrl_msg));
508  d->flags = VRING_DESC_F_NEXT;
509  d->addr = vlib_buffer_get_current_pa (vm, b);
510  d->len = sizeof (struct virtio_net_ctrl_hdr);
511  vring->avail->ring[avail & mask] = next;
512  avail++;
513  next = (next + 1) & mask;
514  d->next = next;
515  used++;
516 
517  d = &vring->desc[next];
518  d->flags = VRING_DESC_F_NEXT;
519  d->addr = vlib_buffer_get_current_pa (vm, b) +
520  STRUCT_OFFSET_OF (struct virtio_ctrl_msg, data);
521  d->len = len;
522  next = (next + 1) & mask;
523  d->next = next;
524  used++;
525 
526  d = &vring->desc[next];
527  d->flags = VRING_DESC_F_WRITE;
528  d->addr = vlib_buffer_get_current_pa (vm, b) +
529  STRUCT_OFFSET_OF (struct virtio_ctrl_msg, status);
530  d->len = sizeof (data->status);
531  next = (next + 1) & mask;
532  used++;
533 
535  vring->avail->idx = avail;
536  vring->desc_next = next;
537  vring->desc_in_use = used;
538 
539  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
540  {
541  virtio_kick (vm, vring, vif);
542  }
543 
544  u16 last = vring->last_used_idx, n_left = 0;
545  n_left = vring->used->idx - last;
546 
547  while (n_left)
548  {
549  struct vring_used_elem *e = &vring->used->ring[last & mask];
550  u16 slot = e->id;
551 
552  d = &vring->desc[slot];
553  while (d->flags & VRING_DESC_F_NEXT)
554  {
555  used--;
556  slot = d->next;
557  d = &vring->desc[slot];
558  }
559  used--;
560  last++;
561  n_left--;
562  }
563  vring->desc_in_use = used;
564  vring->last_used_idx = last;
565 
567  clib_memcpy (&result, vlib_buffer_get_current (b),
568  sizeof (struct virtio_ctrl_msg));
569  virtio_log_debug (vif, "ctrl-queue: status %u", result.status);
570  status = result.status;
571  vlib_buffer_free (vm, &buffer_index, 1);
572  return status;
573 }
574 
575 static int
577 {
578  struct virtio_ctrl_msg offload_hdr;
579  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
580 
581  offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
582  offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
583  offload_hdr.status = VIRTIO_NET_ERR;
584  u64 offloads = 0ULL;
585  clib_memcpy (offload_hdr.data, &offloads, sizeof (offloads));
586 
587  status =
588  virtio_pci_send_ctrl_msg (vm, vif, &offload_hdr, sizeof (offloads));
589  virtio_log_debug (vif, "disable offloads");
592  return status;
593 }
594 
595 static int
597 {
598  struct virtio_ctrl_msg csum_offload_hdr;
599  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
600 
601  csum_offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
602  csum_offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
603  csum_offload_hdr.status = VIRTIO_NET_ERR;
604  u64 offloads = 0ULL;
605  offloads |= VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM);
606  clib_memcpy (csum_offload_hdr.data, &offloads, sizeof (offloads));
607 
608  status =
609  virtio_pci_send_ctrl_msg (vm, vif, &csum_offload_hdr, sizeof (offloads));
610  virtio_log_debug (vif, "enable checksum offload");
613  return status;
614 }
615 
616 static int
618 {
619  struct virtio_ctrl_msg gso_hdr;
620  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
621 
622  gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
624  gso_hdr.status = VIRTIO_NET_ERR;
625  u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
626  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
627  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6);
628  clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads));
629 
630  status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads));
631  virtio_log_debug (vif, "enable gso");
634  return status;
635 }
636 
637 static int
638 virtio_pci_offloads (vlib_main_t * vm, virtio_if_t * vif, int gso_enabled,
639  int csum_offload_enabled)
640 {
641  vnet_main_t *vnm = vnet_get_main ();
643 
644  if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
646  {
647  if (gso_enabled
648  && (vif->features & (VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) |
649  VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6))))
650  {
651  if (virtio_pci_enable_gso (vm, vif))
652  {
653  virtio_log_warning (vif, "gso is not enabled");
654  }
655  else
656  {
657  vif->gso_enabled = 1;
658  vif->csum_offload_enabled = 0;
661  }
662  }
663  else if (csum_offload_enabled
664  && (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)))
665  {
667  {
668  virtio_log_warning (vif, "checksum offload is not enabled");
669  }
670  else
671  {
672  vif->csum_offload_enabled = 1;
673  vif->gso_enabled = 0;
675  hw->flags |=
677  }
678  }
679  else
680  {
681  if (virtio_pci_disable_offload (vm, vif))
682  {
683  virtio_log_warning (vif, "offloads are not disabled");
684  }
685  else
686  {
687  vif->csum_offload_enabled = 0;
688  vif->gso_enabled = 0;
691  }
692  }
693  }
694 
695  return 0;
696 }
697 
698 static int
700  u16 num_queues)
701 {
702  struct virtio_ctrl_msg mq_hdr;
703  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
704 
705  mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
707  mq_hdr.status = VIRTIO_NET_ERR;
708  clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues));
709 
710  status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues));
711  virtio_log_debug (vif, "multi-queue enable %u queues", num_queues);
712  return status;
713 }
714 
715 static u8
717 {
718  if (qsz < 64 || qsz > 4096)
719  return 0;
720  if ((qsz % 64) != 0)
721  return 0;
722  return 1;
723 }
724 
725 clib_error_t *
727  u16 queue_num)
728 {
729  clib_error_t *error = 0;
730  u16 queue_size = 0;
731  virtio_vring_t *vring;
732  struct vring vr;
733  u32 i = 0;
734  void *ptr = NULL;
735 
736  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
737  if (!virtio_pci_queue_size_valid (queue_size))
738  clib_warning ("queue size is not valid");
739 
740  if (!is_pow2 (queue_size))
741  return clib_error_return (0, "ring size must be power of 2");
742 
743  if (queue_size > 32768)
744  return clib_error_return (0, "ring size must be 32768 or lower");
745 
746  if (queue_size == 0)
747  queue_size = 256;
748 
750  vring = vec_elt_at_index (vif->cxq_vring, 0);
751  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
753  ptr =
755  vif->numa_node);
756  if (!ptr)
757  return vlib_physmem_last_error (vm);
758  clib_memset (ptr, 0, i);
759  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
760  vring->desc = vr.desc;
761  vring->avail = vr.avail;
762  vring->used = vr.used;
763  vring->queue_id = queue_num;
764  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
765 
766  ASSERT (vring->buffers == 0);
767 
768  vring->size = queue_size;
769  virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num,
770  queue_size);
771  virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr);
772  vring->kick_fd = -1;
773 
774  return error;
775 }
776 
777 clib_error_t *
779 {
780  clib_error_t *error = 0;
781  u16 queue_size = 0;
782  virtio_vring_t *vring;
783  struct vring vr;
784  u32 i = 0;
785  void *ptr = NULL;
786 
787  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
788  if (!virtio_pci_queue_size_valid (queue_size))
789  clib_warning ("queue size is not valid");
790 
791  if (!is_pow2 (queue_size))
792  return clib_error_return (0, "ring size must be power of 2");
793 
794  if (queue_size > 32768)
795  return clib_error_return (0, "ring size must be 32768 or lower");
796 
797  if (queue_size == 0)
798  queue_size = 256;
799 
800  if (queue_num % 2)
801  {
804  vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
805  clib_spinlock_init (&vring->lockp);
806  }
807  else
808  {
811  vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
812  }
813  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
815  ptr =
817  vif->numa_node);
818  if (!ptr)
819  return vlib_physmem_last_error (vm);
820  clib_memset (ptr, 0, i);
821  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
822  vring->desc = vr.desc;
823  vring->avail = vr.avail;
824  vring->used = vr.used;
825  vring->queue_id = queue_num;
826  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
827 
828  ASSERT (vring->buffers == 0);
829  vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
830  if (queue_num % 2)
831  {
832  virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num,
833  queue_size);
834  clib_memset_u32 (vring->buffers, ~0, queue_size);
835  }
836  else
837  {
838  virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num,
839  queue_size);
840  }
841  vring->size = queue_size;
842  if (virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr))
843  return clib_error_return (0, "error in queue address setup");
844 
845  vring->kick_fd = -1;
846  return error;
847 }
848 
849 static void
851  u64 req_features)
852 {
853  /*
854  * if features are not requested
855  * default: all supported features
856  */
857  u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)
858  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
861  | VIRTIO_FEATURE (VIRTIO_NET_F_MAC)
862  | VIRTIO_FEATURE (VIRTIO_NET_F_GSO)
863  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
864  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)
865  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)
866  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4)
867  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)
868  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO)
869  | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)
870  | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)
871  | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)
872  | VIRTIO_FEATURE (VIRTIO_NET_F_MQ)
873  | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY)
874  | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
875  | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
876 
877  if (req_features == 0)
878  {
879  req_features = supported_features;
880  }
881 
882  vif->features = req_features & vif->remote_features & supported_features;
883 
885  {
886  virtio_net_config_t config;
887  virtio_pci_legacy_read_config (vm, vif, &config.mtu,
888  sizeof (config.mtu),
890  mtu));
891  if (config.mtu < 64)
893  }
894 
895  vif->features =
897 }
898 
899 void
901 {
903 }
904 
905 int
907 {
908  u8 status = 0;
909 
910  /*
911  * Reset the device
912  */
913  status = virtio_pci_legacy_reset (vm, vif);
914 
915  /*
916  * Set the Acknowledge status bit
917  */
918  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK);
919 
920  /*
921  * Set the Driver status bit
922  */
923  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER);
924 
925  /*
926  * Read the status and verify it
927  */
928  status = virtio_pci_legacy_get_status (vm, vif);
929  if (!
930  ((status & VIRTIO_CONFIG_STATUS_ACK)
931  && (status & VIRTIO_CONFIG_STATUS_DRIVER)))
932  return -1;
933  vif->status = status;
934 
935  return 0;
936 }
937 
938 clib_error_t *
940 {
941  clib_error_t *error = 0;
942  struct virtio_pci_cap cap;
943  u8 pos, common_cfg = 0, notify_base = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
945 
946  if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
947  {
948  virtio_log_error (vif, "error in reading capabilty list position");
949  clib_error_return (error, "error in reading capabilty list position");
950  }
951  while (pos)
952  {
953  if ((error =
954  vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap,
955  sizeof (cap))))
956  {
957  virtio_log_error (vif, "%s [%2x]",
958  "error in reading the capability at", pos);
959  clib_error_return (error,
960  "error in reading the capability at [%2x]", pos);
961  }
962 
963  if (cap.cap_vndr == PCI_CAP_ID_MSIX)
964  {
965  u16 flags, table_size, table_size_mask = 0x07FF;
966 
967  if ((error =
968  vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags,
969  sizeof (flags))))
970  clib_error_return (error,
971  "error in reading the capability at [%2x]",
972  pos + 2);
973 
974  table_size = flags & table_size_mask;
975  virtio_log_debug (vif, "flags:0x%x %s 0x%x", flags,
976  "msix interrupt vector table-size", table_size);
977 
978  if (flags & PCI_MSIX_ENABLE)
979  {
980  virtio_log_debug (vif, "msix interrupt enabled");
982  vif->msix_table_size = table_size;
983  }
984  else
985  {
986  virtio_log_debug (vif, "msix interrupt disabled");
988  vif->msix_table_size = 0;
989  }
990  }
991 
992  if (cap.cap_vndr != PCI_CAP_ID_VNDR)
993  {
994  virtio_log_debug (vif, "[%2x] %s %2x ", pos,
995  "skipping non VNDR cap id:", cap.cap_vndr);
996  goto next;
997  }
998 
999  virtio_log_debug (vif,
1000  "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
1001  pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
1002  switch (cap.cfg_type)
1003  {
1005  common_cfg = 1;
1006  break;
1008  notify_base = 1;
1009  break;
1011  dev_cfg = 1;
1012  break;
1014  isr = 1;
1015  break;
1017  if (cap.bar == 0)
1018  pci_cfg = 1;
1019  break;
1020  }
1021  next:
1022  pos = cap.cap_next;
1023  }
1024 
1025  if (common_cfg == 0 || notify_base == 0 || dev_cfg == 0 || isr == 0)
1026  {
1027  virtio_log_debug (vif, "legacy virtio pci device found");
1028  return error;
1029  }
1030 
1031  if (!pci_cfg)
1032  clib_error_return (error, "modern virtio pci device found");
1033 
1034  virtio_log_debug (vif, "transitional virtio pci device found");
1035  return error;
1036 }
1037 
1038 static clib_error_t *
1041 {
1042  clib_error_t *error = 0;
1044  u8 status = 0;
1045 
1046  if ((error = virtio_pci_read_caps (vm, vif)))
1047  {
1048  args->rv = VNET_API_ERROR_UNSUPPORTED;
1049  virtio_log_error (vif, "Device is not supported");
1050  clib_error_return (error, "Device is not supported");
1051  }
1052 
1053  if (virtio_pci_reset_device (vm, vif) < 0)
1054  {
1055  args->rv = VNET_API_ERROR_INIT_FAILED;
1056  virtio_log_error (vif, "Failed to reset the device");
1057  clib_error_return (error, "Failed to reset the device");
1058  }
1059  /*
1060  * read device features and negotiate (user) requested features
1061  */
1063  if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) ==
1064  0)
1065  {
1066  virtio_log_warning (vif, "error encountered: vhost-net backend doesn't "
1067  "support VIRTIO_RING_F_INDIRECT_DESC features");
1068  }
1069  if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)) == 0)
1070  {
1071  virtio_log_warning (vif, "error encountered: vhost-net backend doesn't "
1072  "support VIRTIO_NET_F_MRG_RXBUF features");
1073  }
1074  virtio_negotiate_features (vm, vif, args->features);
1075 
1076  /*
1077  * After FEATURE_OK, driver should not accept new feature bits
1078  */
1079  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1080  status = virtio_pci_legacy_get_status (vm, vif);
1081  if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
1082  {
1083  args->rv = VNET_API_ERROR_UNSUPPORTED;
1084  virtio_log_error (vif,
1085  "error encountered: Device doesn't support requested features");
1086  clib_error_return (error, "Device doesn't support requested features");
1087  }
1088  vif->status = status;
1089 
1090  /*
1091  * get or set the mac address
1092  */
1093  if (virtio_pci_get_mac (vm, vif))
1094  {
1095  f64 now = vlib_time_now (vm);
1096  u32 rnd;
1097  rnd = (u32) (now * 1e6);
1098  rnd = random_u32 (&rnd);
1099 
1100  memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd));
1101  vif->mac_addr[0] = 2;
1102  vif->mac_addr[1] = 0xfe;
1103  virtio_pci_set_mac (vm, vif);
1104  }
1105 
1107 
1108  /*
1109  * Initialize the virtqueues
1110  */
1111  if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif)))
1112  {
1113  args->rv = VNET_API_ERROR_EXCEEDED_NUMBER_OF_RANGES_CAPACITY;
1114  goto err;
1115  }
1116 
1117  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1118  {
1119  if (vif->msix_table_size <= vif->max_queue_pairs)
1120  {
1121  virtio_log_error (vif,
1122  "error MSIX lines (%u) <= Number of RXQs (%u)",
1123  vif->msix_table_size, vif->max_queue_pairs);
1124  return clib_error_return (error,
1125  "error MSIX lines (%u) <= Number of RXQs (%u)",
1126  vif->msix_table_size,
1127  vif->max_queue_pairs);
1128  }
1129  }
1130 
1131  for (int i = 0; i < vif->max_queue_pairs; i++)
1132  {
1133  if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i))))
1134  {
1135  args->rv = VNET_API_ERROR_INIT_FAILED;
1136  virtio_log_error (vif, "%s (%u) %s", "error in rxq-queue",
1137  RX_QUEUE (i), "initialization");
1138  clib_error_return (error, "%s (%u) %s", "error in rxq-queue",
1139  RX_QUEUE (i), "initialization");
1140  }
1141  else
1142  {
1143  vif->num_rxqs++;
1144  }
1145 
1146  if (i >= vtm->n_vlib_mains)
1147  {
1148  /*
1149  * There is 1:1 mapping between tx queue and vpp worker thread.
1150  * tx queue 0 is bind with thread index 0, tx queue 1 on thread
1151  * index 1 and so on.
1152  * Multiple worker threads can poll same tx queue when number of
1153  * workers are more than tx queues. In this case, 1:N mapping
1154  * between tx queue and vpp worker thread.
1155  */
1156  virtio_log_debug (vif, "%s %u, %s", "tx-queue: number",
1157  TX_QUEUE (i),
1158  "no VPP worker thread is available");
1159  continue;
1160  }
1161 
1162  if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i))))
1163  {
1164  args->rv = VNET_API_ERROR_INIT_FAILED;
1165  virtio_log_error (vif, "%s (%u) %s", "error in txq-queue",
1166  TX_QUEUE (i), "initialization");
1167  clib_error_return (error, "%s (%u) %s", "error in txq-queue",
1168  TX_QUEUE (i), "initialization");
1169  }
1170  else
1171  {
1172  vif->num_txqs++;
1173  }
1174  }
1175 
1176  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1177  {
1178  if ((error =
1179  virtio_pci_control_vring_init (vm, vif, vif->max_queue_pairs * 2)))
1180  {
1181  virtio_log_warning (vif, "%s (%u) %s", "error in control-queue",
1182  vif->max_queue_pairs * 2, "initialization");
1183  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
1184  vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ);
1185  }
1186  }
1187  else
1188  {
1189  virtio_log_debug (vif, "control queue is not available");
1190  vif->cxq_vring = NULL;
1191  }
1192 
1193  /*
1194  * set the msix interrupts
1195  */
1196  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1197  {
1198  int i, j;
1199  if (virtio_pci_legacy_set_config_irq (vm, vif, 0) ==
1200  VIRTIO_MSI_NO_VECTOR)
1201  {
1202  virtio_log_warning (vif, "config vector 0 is not set");
1203  }
1204  else
1205  {
1206  virtio_log_debug (vif, "config msix vector is set at 0");
1207  }
1208  for (i = 0, j = 1; i < vif->max_queue_pairs; i++, j++)
1209  {
1210  if (virtio_pci_legacy_set_queue_irq (vm, vif, j, RX_QUEUE (i)) ==
1211  VIRTIO_MSI_NO_VECTOR)
1212  {
1213  virtio_log_warning (vif, "queue (%u) vector is not set at %u",
1214  RX_QUEUE (i), j);
1215  }
1216  else
1217  {
1218  virtio_log_debug (vif, "%s (%u) %s %u", "queue",
1219  RX_QUEUE (i), "msix vector is set at", j);
1220  }
1221  }
1222  }
1223 
1224  /*
1225  * set the driver status OK
1226  */
1227  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1228  vif->status = virtio_pci_legacy_get_status (vm, vif);
1229 err:
1230  return error;
1231 }
1232 
1233 void
1235 {
1236  vnet_main_t *vnm = vnet_get_main ();
1237  virtio_main_t *vim = &virtio_main;
1238  virtio_if_t *vif;
1240  clib_error_t *error = 0;
1241  u32 interrupt_count = 0;
1242 
1243  /* *INDENT-OFF* */
1244  pool_foreach (vif, vim->interfaces, ({
1245  if (vif->pci_addr.as_u32 == args->addr)
1246  {
1247  args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
1248  args->error =
1249  clib_error_return (error, "PCI address in use");
1250  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1251  format_vlib_pci_addr, &args->addr,
1252  " PCI address in use");
1253  return;
1254  }
1255  }));
1256  /* *INDENT-ON* */
1257 
1258  pool_get (vim->interfaces, vif);
1259  vif->dev_instance = vif - vim->interfaces;
1260  vif->per_interface_next_index = ~0;
1261  vif->pci_addr.as_u32 = args->addr;
1262 
1263  if ((error =
1264  vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
1265  virtio_pci_device_ids, &h)))
1266  {
1267  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1268  args->error =
1269  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1270  &vif->pci_addr);
1271  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1272  format_vlib_pci_addr, &vif->pci_addr,
1273  "error encountered on pci device open");
1274  pool_put (vim->interfaces, vif);
1275  return;
1276  }
1277  vif->pci_dev_handle = h;
1278  vlib_pci_set_private_data (vm, h, vif->dev_instance);
1279  vif->numa_node = vlib_pci_get_numa_node (vm, h);
1280  vif->type = VIRTIO_IF_TYPE_PCI;
1281 
1282  if ((error = vlib_pci_bus_master_enable (vm, h)))
1283  {
1284  virtio_log_error (vif, "error encountered on pci bus master enable");
1285  goto error;
1286  }
1287 
1288  if ((error = vlib_pci_io_region (vm, h, 0)))
1289  {
1290  virtio_log_error (vif, "error encountered on pci io region");
1291  goto error;
1292  }
1293 
1294  interrupt_count = vlib_pci_get_num_msix_interrupts (vm, h);
1295  if (interrupt_count > 1)
1296  {
1297  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
1299  {
1300  args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1301  virtio_log_error (vif,
1302  "error encountered on pci register msix handler 0");
1303  goto error;
1304  }
1305 
1306  if ((error =
1307  vlib_pci_register_msix_handler (vm, h, 1, (interrupt_count - 1),
1309  {
1310  args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1311  virtio_log_error (vif,
1312  "error encountered on pci register msix handler 1");
1313  goto error;
1314  }
1315 
1316  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, interrupt_count)))
1317  {
1318  virtio_log_error (vif, "error encountered on pci enable msix irq");
1319  goto error;
1320  }
1321  vif->support_int_mode = 1;
1322  virtio_log_debug (vif, "device supports msix interrupts");
1323  }
1324  else if (interrupt_count == 1)
1325  {
1326  /*
1327  * if msix table-size is 1, fall back to intX.
1328  */
1329  if ((error =
1331  {
1332  virtio_log_error (vif,
1333  "error encountered on pci register interrupt handler");
1334  goto error;
1335  }
1336  vif->support_int_mode = 1;
1337  virtio_log_debug (vif, "pci register interrupt handler");
1338  }
1339  else
1340  {
1341  /*
1342  * WARN: intX is showing some weird behaviour.
1343  * Please don't use interrupt mode with UIO driver.
1344  */
1345  vif->support_int_mode = 0;
1346  virtio_log_debug (vif, "driver is configured in poll mode only");
1347  }
1348 
1349  if ((error = vlib_pci_intr_enable (vm, h)))
1350  {
1351  virtio_log_error (vif, "error encountered on pci interrupt enable");
1352  goto error;
1353  }
1354 
1355  if ((error = virtio_pci_device_init (vm, vif, args)))
1356  {
1357  virtio_log_error (vif, "error encountered on device init");
1358  goto error;
1359  }
1360 
1361  /* create interface */
1363  vif->dev_instance, vif->mac_addr,
1364  &vif->hw_if_index,
1366 
1367  if (error)
1368  {
1369  args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1370  virtio_log_error (vif,
1371  "error encountered on ethernet register interface");
1372  goto error;
1373  }
1374 
1375  vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index);
1376  vif->sw_if_index = sw->sw_if_index;
1377  args->sw_if_index = sw->sw_if_index;
1378 
1379  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
1381  vnet_hw_interface_set_input_node (vnm, vif->hw_if_index,
1382  virtio_input_node.index);
1383  u32 i = 0;
1384  vec_foreach_index (i, vif->rxq_vrings)
1385  {
1386  vnet_hw_interface_assign_rx_thread (vnm, vif->hw_if_index, i, ~0);
1388  /* Set default rx mode to POLLING */
1389  vnet_hw_interface_set_rx_mode (vnm, vif->hw_if_index, i,
1391  }
1392  if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
1393  {
1394  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1395  vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
1397  }
1398  else
1399  vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1400 
1401  virtio_pci_offloads (vm, vif, args->gso_enabled,
1402  args->checksum_offload_enabled);
1403 
1404  if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
1405  (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)))
1406  {
1407  if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs))
1408  virtio_log_warning (vif, "multiqueue is not set");
1409  }
1410  return;
1411 
1412 error:
1413  virtio_pci_delete_if (vm, vif);
1414  if (args->rv == 0)
1415  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1416  args->error = error;
1417 }
1418 
1419 int
1421 {
1422  vnet_main_t *vnm = vnet_get_main ();
1423  virtio_main_t *vim = &virtio_main;
1424  u32 i = 0;
1425 
1426  if (vif->type != VIRTIO_IF_TYPE_PCI)
1427  return VNET_API_ERROR_INVALID_INTERFACE;
1428 
1430 
1431  for (i = 0; i < vif->max_queue_pairs; i++)
1432  {
1433  virtio_pci_legacy_del_queue (vm, vif, RX_QUEUE (i));
1434  virtio_pci_legacy_del_queue (vm, vif, TX_QUEUE (i));
1435  }
1436 
1437  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1438  virtio_pci_legacy_del_queue (vm, vif, vif->max_queue_pairs * 2);
1439 
1440  virtio_pci_legacy_reset (vm, vif);
1441 
1442  if (vif->hw_if_index)
1443  {
1445  vec_foreach_index (i, vif->rxq_vrings)
1446  {
1448  }
1450  }
1451 
1453 
1454  vec_foreach_index (i, vif->rxq_vrings)
1455  {
1456  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i);
1457  if (vring->kick_fd != -1)
1458  close (vring->kick_fd);
1459  if (vring->used)
1460  {
1461  virtio_free_rx_buffers (vm, vring);
1462  }
1463  vec_free (vring->buffers);
1464  vlib_physmem_free (vm, vring->desc);
1465  }
1466 
1467  vec_foreach_index (i, vif->txq_vrings)
1468  {
1469  virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i);
1470  if (vring->kick_fd != -1)
1471  close (vring->kick_fd);
1472  if (vring->used)
1473  {
1474  virtio_free_used_desc (vm, vring);
1475  }
1476  vec_free (vring->buffers);
1477  clib_spinlock_free (&vring->lockp);
1478  vlib_physmem_free (vm, vring->desc);
1479  }
1480 
1481  if (vif->cxq_vring != NULL)
1482  {
1483  u16 last = vif->cxq_vring->last_used_idx;
1484  u16 n_left = vif->cxq_vring->used->idx - last;
1485  while (n_left)
1486  {
1487  last++;
1488  n_left--;
1489  }
1490 
1491  vif->cxq_vring->last_used_idx = last;
1492  vlib_physmem_free (vm, vif->cxq_vring->desc);
1493  }
1494 
1495  vec_free (vif->rxq_vrings);
1496  vec_free (vif->txq_vrings);
1497  vec_free (vif->cxq_vring);
1498 
1499  clib_error_free (vif->error);
1500  memset (vif, 0, sizeof (*vif));
1501  pool_put (vim->interfaces, vif);
1502 
1503  return 0;
1504 }
1505 
1506 int
1508  int gso_enabled,
1509  int checksum_offload_enabled,
1510  int offloads_disabled)
1511 {
1512  if (vif->type != VIRTIO_IF_TYPE_PCI)
1513  return VNET_API_ERROR_INVALID_INTERFACE;
1514 
1515  if (gso_enabled)
1516  virtio_pci_offloads (vm, vif, 1, 0);
1517  else if (checksum_offload_enabled)
1518  virtio_pci_offloads (vm, vif, 0, 1);
1519  else if (offloads_disabled)
1520  virtio_pci_offloads (vm, vif, 0, 0);
1521 
1522  return 0;
1523 }
1524 
1525 /*
1526  * fd.io coding-style-patch-verification: ON
1527  *
1528  * Local Variables:
1529  * eval: (c-set-style "gnu")
1530  * End:
1531  */
static u8 virtio_pci_legacy_get_isr(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:171
struct vring_used * used
Definition: virtio.h:110
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
Definition: node.c:442
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:463
#define vec_foreach_index(var, v)
Iterate over vector indices.
static int virtio_pci_disable_offload(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:576
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:239
void virtio_set_net_hdr_size(virtio_if_t *vif)
Definition: virtio.c:240
virtio_if_t * interfaces
Definition: virtio.h:195
clib_error_t * virtio_pci_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:778
static int virtio_pci_legacy_setup_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id, void *p)
Definition: pci.c:191
static u32 virtio_pci_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: pci.c:251
static void virtio_pci_legacy_del_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:208
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:271
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
Definition: physmem_funcs.h:63
#define VIRTIO_PCI_CAP_ISR_CFG
Definition: pci.h:125
vl_api_mac_address_t mac
Definition: l2.api:502
a
Definition: bitmap.h:538
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
u16 max_virtqueue_pairs
Definition: virtio.h:100
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:378
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
static u16 virtio_pci_legacy_get_queue_num(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:179
#define VIRTIO_MSI_QUEUE_VECTOR
Definition: pci.h:36
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
int gso_enabled
Definition: virtio.h:183
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1286
#define VIRTIO_PCI_STATUS
Definition: pci.h:31
static void virtio_pci_irq_queue_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:317
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:133
#define VIRTIO_PCI_QUEUE_PFN
Definition: pci.h:27
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static u8 virtio_pci_legacy_get_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:148
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:291
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:177
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
static clib_error_t * vlib_pci_intr_disable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:255
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t virtio_device_class
static clib_error_t * virtio_pci_device_init(vlib_main_t *vm, virtio_if_t *vif, virtio_pci_create_if_args_t *args)
Definition: pci.c:1039
vl_api_address_t src
Definition: gre.api:54
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
void virtio_vring_set_numa_node(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
Definition: virtio.c:224
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:69
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:894
#define VIRTIO_MSI_CONFIG_VECTOR
Definition: pci.h:35
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:518
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:252
vhost_vring_addr_t addr
Definition: vhost_user.h:254
#define VIRTIO_PCI_HOST_FEATURES
Definition: pci.h:25
static int virtio_pci_enable_gso(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:617
static u64 virtio_pci_legacy_get_host_features(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:113
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS
Definition: pci.h:117
static void virtio_pci_set_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:284
static int virtio_pci_send_ctrl_msg(vlib_main_t *vm, virtio_if_t *vif, struct virtio_ctrl_msg *data, u32 len)
Definition: pci.c:480
unsigned char u8
Definition: types.h:56
void virtio_pci_create_if(vlib_main_t *vm, virtio_pci_create_if_args_t *args)
Definition: pci.c:1234
double f64
Definition: types.h:142
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:70
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define clib_memcpy(d, s, n)
Definition: string.h:180
static void virtio_pci_irq_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:353
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
Definition: pci.c:1028
u32 msix_enabled
Definition: virtio.h:159
static u32 virtio_pci_legacy_set_guest_features(vlib_main_t *vm, virtio_if_t *vif, u64 features)
Definition: pci.c:132
u16 msix_table_size
Definition: virtio.h:171
#define VIRTIO_PCI_CAP_COMMON_CFG
Definition: pci.h:121
struct vring_avail * avail
Definition: virtio.h:111
u64 features
Definition: virtio.h:163
int virtio_pci_reset_device(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:906
u32 hw_if_index
Definition: virtio.h:146
u8 * format_ethernet_address(u8 *s, va_list *args)
Definition: format.c:44
static u8 virtio_pci_queue_size_valid(u16 qsz)
Definition: pci.c:716
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:513
static u16 virtio_pci_legacy_set_queue_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec, u16 queue_id)
Definition: pci.c:238
#define virtio_log_error(vif, f,...)
Definition: virtio.h:250
#define PCI_DEVICE_ID_VIRTIO_NIC
Definition: pci.c:29
#define TX_QUEUE_ACCESS(X)
Definition: virtio.h:79
void device_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:379
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
static void virtio_negotiate_features(vlib_main_t *vm, virtio_if_t *vif, u64 req_features)
Definition: pci.c:850
vnet_hw_interface_flags_t flags
Definition: interface.h:526
void virtio_free_rx_buffers(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:138
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
clib_spinlock_t lockp
Definition: virtio.h:112
#define clib_error_return(e, args...)
Definition: error.h:99
#define VIRTIO_PCI_CAP_NOTIFY_CFG
Definition: pci.h:123
unsigned int u32
Definition: types.h:88
static void virtio_pci_legacy_read_config(vlib_main_t *vm, virtio_if_t *vif, void *dst, int len, u32 addr)
Definition: pci.c:53
#define virtio_log_warning(vif, f,...)
Definition: virtio.h:243
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
Definition: pci.h:118
u16 queue_id
Definition: virtio.h:119
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
u16 num_txqs
Definition: virtio.h:170
u32 pci_dev_handle
Definition: virtio.h:160
#define TX_QUEUE(X)
Definition: virtio.h:77
static void virtio_pci_legacy_set_status(vlib_main_t *vm, virtio_if_t *vif, u8 status)
Definition: pci.c:156
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
#define VIRTIO_PCI_GUEST_FEATURES
Definition: pci.h:26
#define VIRTIO_NET_F_MTU
Definition: pci.h:105
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
unsigned short u16
Definition: types.h:57
#define VIRTIO_PCI_ISR_CONFIG
Definition: pci.h:48
u64 size
Definition: vhost_user.h:150
vec_header_t h
Definition: buffer.c:322
struct virtio_net_ctrl_hdr ctrl
Definition: pci.c:474
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:302
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:837
#define PCI_MSIX_ENABLE
Definition: pci.c:37
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
static u16 virtio_pci_legacy_set_config_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec)
Definition: pci.c:227
void virtio_pci_read_device_feature(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:900
vl_api_address_t dst
Definition: gre.api:55
static u32 virtio_pci_legacy_get_guest_features(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:122
clib_error_t * virtio_pci_read_caps(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:939
static clib_error_t * virtio_pci_get_max_virtqueue_pairs(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:258
#define PCI_CAPABILITY_LIST
Definition: pci.c:33
vlib_main_t * vm
Definition: in2out_ed.c:1599
int virtio_pci_enable_disable_offloads(vlib_main_t *vm, virtio_if_t *vif, int gso_enabled, int checksum_offload_enabled, int offloads_disabled)
Definition: pci.c:1507
#define virtio_log_debug(vif, f,...)
Definition: virtio.h:236
u8 status
Definition: virtio.h:172
u8 len
Definition: ip_types.api:92
u16 desc_next
Definition: virtio.h:114
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
Definition: pci.c:141
u8 slot
Definition: pci_types.api:22
#define VIRTIO_PCI_VRING_ALIGN
Definition: pci.h:133
u16 num_rxqs
Definition: virtio.h:169
virtio_vring_t * rxq_vrings
Definition: virtio.h:161
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
u16 last_used_idx
Definition: virtio.h:124
u32 flags
Definition: vhost_user.h:248
#define VIRTIO_PCI_QUEUE_NOTIFY
Definition: pci.h:30
#define VIRTIO_PCI_ISR_INTR
Definition: pci.h:46
static void vlib_physmem_free(vlib_main_t *vm, void *p)
Definition: physmem_funcs.h:89
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:104
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
#define clib_warning(format, args...)
Definition: error.h:59
static u16 virtio_pci_is_link_up(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:303
void debug_device_config_space(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:407
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:87
u8 mac_addr[6]
Definition: virtio.h:173
static void virtio_pci_legacy_write_config(vlib_main_t *vm, virtio_if_t *vif, void *src, int len, u32 addr)
Definition: pci.c:83
u32 flags
Definition: virtio.h:143
clib_error_t * virtio_pci_control_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:726
clib_error_t * error
Definition: virtio.h:166
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:256
void virtio_pci_legacy_notify_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:218
int virtio_pci_delete_if(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:1420
u32 numa_node
Definition: virtio.h:148
virtio_if_type_t type
Definition: virtio.h:150
void virtio_free_used_desc(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:175
#define PCI_DEVICE_ID_VIRTIO_NIC_MODERN
Definition: pci.c:31
#define PCI_CONFIG_SIZE(vif)
Definition: pci.c:39
#define ASSERT(truth)
static int virtio_pci_offloads(vlib_main_t *vm, virtio_if_t *vif, int gso_enabled, int csum_offload_enabled)
Definition: pci.c:638
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:689
u64 remote_features
Definition: virtio.h:163
u8 data[128]
Definition: ipsec_types.api:89
#define VIRTIO_FEATURE(X)
Definition: virtio.h:75
static int virtio_pci_enable_checksum_offload(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:596
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
Definition: pci.c:786
virtio_main_t virtio_main
Definition: virtio.c:37
static uword is_pow2(uword x)
Definition: clib.h:250
#define VIRTIO_PCI_QUEUE_SEL
Definition: pci.h:29
Definition: defs.h:64
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
#define RX_QUEUE_ACCESS(X)
Definition: virtio.h:80
virtio_net_ctrl_ack status
Definition: pci.c:475
int csum_offload_enabled
Definition: virtio.h:184
#define VIRTIO_PCI_CAP_DEVICE_CFG
Definition: pci.h:127
#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
Definition: pci.h:104
#define PCI_CAP_ID_VNDR
Definition: pci.c:34
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:331
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:498
VLIB buffer representation.
Definition: buffer.h:102
u8 data[1024]
Definition: pci.c:476
u64 uword
Definition: types.h:112
static void virtio_pci_irq_config_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:331
#define clib_error_free(e)
Definition: error.h:86
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
Definition: pci.c:1171
u32 * buffers
Definition: virtio.h:123
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
static u8 virtio_pci_legacy_reset(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:164
virtio_vring_t * cxq_vring
Definition: virtio.h:187
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
struct vring_desc * desc
Definition: virtio.h:109
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:130
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
Definition: devices.c:253
#define VIRTIO_PCI_CAP_PCI_CFG
Definition: pci.h:129
u16 max_queue_pairs
Definition: virtio.h:168
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define PCI_VENDOR_ID_VIRTIO
Definition: pci.c:28
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:677
#define PCI_CAP_ID_MSIX
Definition: pci.c:35
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1226
#define RX_QUEUE(X)
Definition: virtio.h:78
static u32 virtio_pci_get_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:291
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
#define VIRTIO_NET_CTRL_MQ
Definition: vhost_user.h:47
#define VIRTIO_PCI_QUEUE_NUM
Definition: pci.h:28
u16 vendor_id
Definition: pci.h:127
static int virtio_pci_enable_multiqueue(vlib_main_t *vm, virtio_if_t *vif, u16 num_queues)
Definition: pci.c:699
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
Definition: vhost_user.h:48
u16 desc_in_use
Definition: virtio.h:113
virtio_vring_t * txq_vrings
Definition: virtio.h:162
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT
Definition: pci.h:131
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
Definition: virtio.h:221
#define VIRTIO_PCI_ISR
Definition: pci.h:32
static_always_inline void clib_memset_u32(void *p, u32 val, uword count)
Definition: string.h:332