FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
pci.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * pci.c: Linux user space PCI bus management.
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <vppinfra/linux/sysfs.h>
41 
42 #include <vlib/vlib.h>
43 #include <vlib/pci/pci.h>
44 #include <vlib/unix/unix.h>
45 #include <vlib/linux/vfio.h>
46 
47 #include <sys/types.h>
48 #include <sys/stat.h>
49 #include <fcntl.h>
50 #include <dirent.h>
51 #include <sys/ioctl.h>
52 #include <net/if.h>
53 #include <linux/ethtool.h>
54 #include <linux/sockios.h>
55 #include <linux/vfio.h>
56 #include <sys/eventfd.h>
57 
58 static const char *sysfs_pci_dev_path = "/sys/bus/pci/devices";
59 static const char *sysfs_pci_drv_path = "/sys/bus/pci/drivers";
60 static char *sysfs_mod_vfio_noiommu =
61  "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode";
62 
63 #define pci_log_debug(vm, dev, f, ...) \
64  vlib_log(VLIB_LOG_LEVEL_DEBUG, pci_main.log_default, "%U: " f, \
65  format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__)
66 #define pci_log_err(vm, dev, f, ...) \
67  vlib_log(VLIB_LOG_LEVEL_ERR, pci_main.log_default, "%U: " f, \
68  format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__)
69 
70 typedef struct
71 {
72  int fd;
73  void *addr;
74  size_t size;
76 
77 typedef struct
78 {
79  int fd;
81  union
82  {
85  };
87 
88 typedef enum
89 {
94 
95 typedef struct
96 {
99  vlib_pci_addr_t addr;
101 
102  /* Resource file descriptors. */
104 
105  /* File descriptor for config space read/write. */
108 
109  /* Device File descriptor */
110  int fd;
111 
112  /* read/write file descriptor for io bar */
113  int io_fd;
115 
116  /* Minor device for uio device. */
118 
119  /* Interrupt handlers */
122 
123  /* private data */
125 
127 
129 
130 /* Pool of PCI devices. */
131 typedef struct
132 {
135 
137 
139 
140 static linux_pci_device_t *
142 {
144  return pool_elt_at_index (lpm->linux_pci_devices, h);
145 }
146 
147 uword
149 {
151  return d->private_data;
152 }
153 
154 void
156  uword private_data)
157 {
159  d->private_data = private_data;
160 }
161 
162 vlib_pci_addr_t *
164 {
166  return &d->addr;
167 }
168 
169 u32
171 {
173  return d->numa_node;
174 }
175 
176 u32
178 {
180 
182  {
183  struct vfio_irq_info ii = { 0 };
184 
185  ii.argsz = sizeof (struct vfio_irq_info);
186  ii.index = VFIO_PCI_MSIX_IRQ_INDEX;
187  if (ioctl (d->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
188  return 0;
189  return ii.count;
190  }
191  return 0;
192 }
193 
194 /* Call to allocate/initialize the pci subsystem.
195  This is not an init function so that users can explicitly enable
196  pci only when it's needed. */
198 
200 
203  clib_error_t ** error)
204 {
205  clib_error_t *err;
207  u8 *f = 0;
208  u32 tmp;
209  int fd;
210  u8 *tmpstr;
211  clib_bitmap_t *bmp = 0;
212 
214  clib_memset (di, 0, sizeof (vlib_pci_device_info_t));
215  di->addr.as_u32 = addr->as_u32;
216 
217  u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path,
219 
220  f = format (0, "%v/config%c", dev_dir_name, 0);
221  fd = open ((char *) f, O_RDWR);
222 
223  /* Try read-only access if write fails. */
224  if (fd < 0)
225  fd = open ((char *) f, O_RDONLY);
226 
227  if (fd < 0)
228  {
229  err = clib_error_return_unix (0, "open `%s'", f);
230  goto error;
231  }
232 
233  /* You can only read more that 64 bytes of config space as root; so we try to
234  read the full space but fall back to just the first 64 bytes. */
235  if (read (fd, &di->config_data, sizeof (di->config_data)) <
236  sizeof (di->config0))
237  {
238  err = clib_error_return_unix (0, "read `%s'", f);
239  close (fd);
240  goto error;
241  }
242 
243  {
244  static pci_config_header_t all_ones;
245  if (all_ones.vendor_id == 0)
246  clib_memset (&all_ones, ~0, sizeof (all_ones));
247 
248  if (!memcmp (&di->config0.header, &all_ones, sizeof (all_ones)))
249  {
250  err = clib_error_return (0, "invalid PCI config for `%s'", f);
251  close (fd);
252  goto error;
253  }
254  }
255 
256  if (di->config0.header.header_type == 0)
258  else
260 
261  di->numa_node = -1;
263  f = format (f, "%v/numa_node%c", dev_dir_name, 0);
264  err = clib_sysfs_read ((char *) f, "%d", &di->numa_node);
265  if (err)
266  {
267  di->numa_node = -1;
268  clib_error_free (err);
269  }
270  if (di->numa_node == -1)
271  {
272  /* if '/sys/bus/pci/devices/<device id>/numa_node' returns -1 and
273  it is a SMP system, set numa_node to 0. */
274  if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U",
275  unformat_bitmap_list, &bmp)))
276  clib_error_free (err);
277  if (clib_bitmap_count_set_bits (bmp) == 1)
278  di->numa_node = 0;
279  }
280 
282  f = format (f, "%v/class%c", dev_dir_name, 0);
283  err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
284  if (err)
285  goto error;
286  di->device_class = tmp >> 8;
287 
289  f = format (f, "%v/vendor%c", dev_dir_name, 0);
290  err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
291  if (err)
292  goto error;
293  di->vendor_id = tmp;
294 
296  f = format (f, "%v/device%c", dev_dir_name, 0);
297  err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
298  if (err)
299  goto error;
300  di->device_id = tmp;
301 
303  f = format (f, "%v/driver%c", dev_dir_name, 0);
304  di->driver_name = clib_sysfs_link_to_name ((char *) f);
305  if (!di->driver_name)
306  di->driver_name = format (0, "<NONE>%c", 0);
307 
308  di->iommu_group = -1;
310  f = format (f, "%v/iommu_group%c", dev_dir_name, 0);
311  tmpstr = clib_sysfs_link_to_name ((char *) f);
312  if (tmpstr)
313  {
314  di->iommu_group = atoi ((char *) tmpstr);
315  vec_free (tmpstr);
316  }
317 
319  f = format (f, "%v/iommu_group/name%c", dev_dir_name, 0);
320  err = clib_sysfs_read ((char *) f, "%s", &tmpstr);
321  if (err == 0)
322  {
323  if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0)
325  vec_free (tmpstr);
326  }
327  else
328  clib_error_free (err);
329 
330  close (fd);
331 
333  f = format (f, "%v/vpd%c", dev_dir_name, 0);
334  fd = open ((char *) f, O_RDONLY);
335  if (fd >= 0)
336  {
337  while (1)
338  {
339  u8 tag[3];
340  u8 *data = 0;
341  uword len;
342 
343  if (read (fd, &tag, 3) != 3)
344  break;
345 
346  if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91)
347  break;
348 
349  len = (tag[2] << 8) | tag[1];
350  vec_validate (data, len);
351 
352  if (read (fd, data, len) != len)
353  {
354  vec_free (data);
355  break;
356  }
357  if (tag[0] == 0x82)
358  di->product_name = data;
359  else if (tag[0] == 0x90)
360  di->vpd_r = data;
361  else if (tag[0] == 0x91)
362  di->vpd_w = data;
363 
364  data = 0;
365  }
366  close (fd);
367  }
368 
369  goto done;
370 
371 error:
373  di = 0;
374 
375 done:
376  vec_free (bmp);
377  vec_free (f);
378  vec_free (dev_dir_name);
379  if (error)
380  *error = err;
381  else
382  clib_error_free (err);
383  return di;
384 }
385 
386 static int
388 {
389  struct stat s = { 0 };
390  if (stat (path, &s) == -1)
391  return 0;
392 
393  return S_ISDIR (s.st_mode);
394 }
395 
396 clib_error_t *
397 vlib_pci_bind_to_uio (vlib_main_t * vm, vlib_pci_addr_t * addr,
398  char *uio_drv_name)
399 {
400  clib_error_t *error = 0;
401  u8 *s = 0, *driver_name = 0;
402  DIR *dir = 0;
403  struct dirent *e;
405  int fd, clear_driver_override = 0;
406  u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path,
408 
410 
411  if (error)
412  return error;
413 
414  if (strncmp ("auto", uio_drv_name, 5) == 0)
415  {
416  int vfio_pci_loaded = 0;
417 
418  if (directory_exists ("/sys/module/vfio_pci"))
419  vfio_pci_loaded = 1;
420 
421  if (di->iommu_group != -1)
422  {
423  /* device is bound to IOMMU group */
424  if (!vfio_pci_loaded)
425  {
426  error = clib_error_return (0, "Skipping PCI device %U: device "
427  "is bound to IOMMU group and "
428  "vfio-pci driver is not loaded",
430  goto done;
431  }
432  else
433  uio_drv_name = "vfio-pci";
434  }
435  else
436  {
437  /* device is not bound to IOMMU group so we have multiple options */
438  if (vfio_pci_loaded &&
440  uio_drv_name = "vfio-pci";
441  else if (directory_exists ("/sys/module/uio_pci_generic"))
442  uio_drv_name = "uio_pci_generic";
443  else if (directory_exists ("/sys/module/igb_uio"))
444  uio_drv_name = "igb_uio";
445  else
446  {
448  error = clib_error_return (0, "Skipping PCI device %U: missing "
449  "kernel VFIO or UIO driver",
451  goto done;
452  }
454  }
455  }
456 
457  s = format (s, "%v/driver%c", dev_dir_name, 0);
458  driver_name = clib_sysfs_link_to_name ((char *) s);
459  vec_reset_length (s);
460 
461  if (driver_name &&
462  ((strcmp ("vfio-pci", (char *) driver_name) == 0) ||
463  (strcmp ("uio_pci_generic", (char *) driver_name) == 0) ||
464  (strcmp ("igb_uio", (char *) driver_name) == 0)))
465  goto done;
466 
467  /* walk trough all linux interfaces and if interface belonging to
468  this device is founf check if interface is admin up */
469  dir = opendir ("/sys/class/net");
470  s = format (s, "%U%c", format_vlib_pci_addr, addr, 0);
471 
472  if (!dir)
473  {
474  error = clib_error_return (0, "Skipping PCI device %U: failed to "
475  "read /sys/class/net",
477  goto done;
478  }
479 
480  fd = socket (PF_INET, SOCK_DGRAM, 0);
481  if (fd < 0)
482  {
483  error = clib_error_return_unix (0, "socket");
484  goto done;
485  }
486 
487  while ((e = readdir (dir)))
488  {
489  struct ifreq ifr;
490  struct ethtool_drvinfo drvinfo;
491 
492  if (e->d_name[0] == '.') /* skip . and .. */
493  continue;
494 
495  clib_memset (&ifr, 0, sizeof ifr);
496  clib_memset (&drvinfo, 0, sizeof drvinfo);
497  ifr.ifr_data = (char *) &drvinfo;
498  clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
499 
500  drvinfo.cmd = ETHTOOL_GDRVINFO;
501  if (ioctl (fd, SIOCETHTOOL, &ifr) < 0)
502  {
503  /* Some interfaces (eg "lo") don't support this ioctl */
504  if ((errno != ENOTSUP) && (errno != ENODEV))
505  clib_unix_warning ("ioctl fetch intf %s bus info error",
506  e->d_name);
507  continue;
508  }
509 
510  if (strcmp ((char *) s, drvinfo.bus_info))
511  continue;
512 
513  clib_memset (&ifr, 0, sizeof (ifr));
514  clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
515 
516  if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
517  {
518  error = clib_error_return_unix (0, "ioctl fetch intf %s flags",
519  e->d_name);
520  close (fd);
521  goto done;
522  }
523 
524  if (ifr.ifr_flags & IFF_UP)
525  {
526  vlib_log (VLIB_LOG_LEVEL_WARNING, pci_main.log_default,
527  "Skipping PCI device %U as host "
528  "interface %s is up", format_vlib_pci_addr, addr,
529  e->d_name);
530  close (fd);
531  goto done;
532  }
533  }
534 
535  close (fd);
536  vec_reset_length (s);
537 
538  s = format (s, "%v/driver/unbind%c", dev_dir_name, 0);
539  clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr);
540  vec_reset_length (s);
541 
542  s = format (s, "%v/driver_override%c", dev_dir_name, 0);
543  if (access ((char *) s, F_OK) == 0)
544  {
545  clib_sysfs_write ((char *) s, "%s", uio_drv_name);
546  clear_driver_override = 1;
547  }
548  else
549  {
550  vec_reset_length (s);
551  s = format (s, "%s/%s/new_id%c", sysfs_pci_drv_path, uio_drv_name, 0);
552  clib_sysfs_write ((char *) s, "0x%04x 0x%04x", di->vendor_id,
553  di->device_id);
554  }
555  vec_reset_length (s);
556 
557  s = format (s, "%s/%s/bind%c", sysfs_pci_drv_path, uio_drv_name, 0);
558  clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr);
559  vec_reset_length (s);
560 
561  if (clear_driver_override)
562  {
563  s = format (s, "%v/driver_override%c", dev_dir_name, 0);
564  clib_sysfs_write ((char *) s, "%c", 0);
565  vec_reset_length (s);
566  }
567 
568 done:
569  closedir (dir);
570  vec_free (s);
571  vec_free (dev_dir_name);
572  vec_free (driver_name);
573  return error;
574 }
575 
576 
577 static clib_error_t *
578 scan_uio_dir (void *arg, u8 * path_name, u8 * file_name)
579 {
580  linux_pci_device_t *l = arg;
581  unformat_input_t input;
582 
583  unformat_init_string (&input, (char *) file_name, vec_len (file_name));
584 
585  if (!unformat (&input, "uio%d", &l->uio_minor))
586  abort ();
587 
588  unformat_free (&input);
589  return 0;
590 }
591 
592 static clib_error_t *
594  u32 count, u32 flags, int *efds)
595 {
596  int data_len = efds ? count * sizeof (int) : 0;
597  u8 buf[sizeof (struct vfio_irq_set) + data_len];
598  struct vfio_irq_info ii = { 0 };
599  struct vfio_irq_set *irq_set = (struct vfio_irq_set *) buf;
600 
601 
602  ii.argsz = sizeof (struct vfio_irq_info);
603  ii.index = index;
604 
605  if (ioctl (p->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
606  return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) "
607  "'%U'", format_vlib_pci_addr, &p->addr);
608 
609  pci_log_debug (vm, p, "%s index:%u count:%u flags: %s%s%s%s(0x%x)",
610  __func__, ii.index, ii.count,
611  ii.flags & VFIO_IRQ_INFO_EVENTFD ? "eventfd " : "",
612  ii.flags & VFIO_IRQ_INFO_MASKABLE ? "maskable " : "",
613  ii.flags & VFIO_IRQ_INFO_AUTOMASKED ? "automasked " : "",
614  ii.flags & VFIO_IRQ_INFO_NORESIZE ? "noresize " : "",
615  ii.flags);
616 
617  if (ii.count < start + count)
618  return clib_error_return_unix (0, "vfio_set_irq: unexistng interrupt on "
619  "'%U'", format_vlib_pci_addr, &p->addr);
620 
621 
622  if (efds)
623  {
624  flags |= VFIO_IRQ_SET_DATA_EVENTFD;
625  clib_memcpy_fast (&irq_set->data, efds, data_len);
626  }
627  else
628  flags |= VFIO_IRQ_SET_DATA_NONE;
629 
630  ASSERT ((flags & (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD)) !=
631  (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD));
632 
633  irq_set->argsz = sizeof (struct vfio_irq_set) + data_len;
634  irq_set->index = index;
635  irq_set->start = start;
636  irq_set->count = count;
637  irq_set->flags = flags;
638 
639  if (ioctl (p->fd, VFIO_DEVICE_SET_IRQS, irq_set) < 0)
640  return clib_error_return_unix (0, "%U:ioctl(VFIO_DEVICE_SET_IRQS) "
641  "[index = %u, start = %u, count = %u, "
642  "flags = 0x%x]",
644  index, start, count, flags);
645  return 0;
646 }
647 
648 static clib_error_t *
650 {
652  int __attribute__ ((unused)) rv;
655  linux_pci_irq_t *irq = &p->intx_irq;
656 
657  u32 icount;
658  rv = read (uf->file_descriptor, &icount, 4);
659 
660  if (irq->intx_handler)
661  irq->intx_handler (vm, h);
662 
664 
665  return /* no error */ 0;
666 }
667 
668 static clib_error_t *
670 {
671  return vfio_set_irqs (vm, d, VFIO_PCI_INTX_IRQ_INDEX, 0, 1,
672  VFIO_IRQ_SET_ACTION_UNMASK, 0);
673 }
674 
675 static clib_error_t *
677 {
678  u32 error_index = (u32) uf->private_data;
679 
680  return clib_error_return (0, "pci device %d: error", error_index);
681 }
682 
683 static clib_error_t *
685 {
687  int __attribute__ ((unused)) rv;
689  u16 line = uf->private_data & 0xffff;
691  linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, line);
692 
693  u64 icount;
694  rv = read (uf->file_descriptor, &icount, sizeof (icount));
695 
696  if (irq->msix_handler)
697  irq->msix_handler (vm, h, line);
698 
699  return /* no error */ 0;
700 }
701 
702 static clib_error_t *
704 {
706  int __attribute__ ((unused)) rv;
709  linux_pci_irq_t *irq = &p->intx_irq;
710 
711  u64 icount;
712  rv = read (uf->file_descriptor, &icount, sizeof (icount));
713 
714  if (irq->intx_handler)
715  irq->intx_handler (vm, h);
716 
718 
719  return /* no error */ 0;
720 }
721 
722 static clib_error_t *
724 {
725  u32 error_index = (u32) uf->private_data;
726 
727  return clib_error_return (0, "pci device %d: error", error_index);
728 }
729 
730 static clib_error_t *
733 {
735  clib_error_t *err = 0;
736  u8 *s = 0;
737 
738  p->fd = -1;
740 
741  s = format (s, "%s/%U/config%c", sysfs_pci_dev_path,
742  format_vlib_pci_addr, &di->addr, 0);
743 
744  p->config_fd = open ((char *) s, O_RDWR);
745  p->config_offset = 0;
746  vec_reset_length (s);
747 
748  if (p->config_fd == -1)
749  {
750  err = clib_error_return_unix (0, "open '%s'", s);
751  goto error;
752  }
753 
754  s = format (0, "%s/%U/uio%c", sysfs_pci_dev_path,
755  format_vlib_pci_addr, &di->addr, 0);
756  foreach_directory_file ((char *) s, scan_uio_dir, p, /* scan_dirs */
757  1);
758  vec_reset_length (s);
759 
760  s = format (s, "/dev/uio%d%c", p->uio_minor, 0);
761  p->fd = open ((char *) s, O_RDWR);
762  if (p->fd < 0)
763  {
764  err = clib_error_return_unix (0, "open '%s'", s);
765  goto error;
766  }
767 
768  if (r && r->interrupt_handler)
769  vlib_pci_register_intx_handler (vm, p->handle, r->interrupt_handler);
770 
771  if (r && r->init_function)
772  err = r->init_function (lpm->vlib_main, p->handle);
773 
774 error:
775  vec_free (s);
776  if (err)
777  {
778  if (p->config_fd != -1)
779  close (p->config_fd);
780  if (p->fd != -1)
781  close (p->fd);
782  }
783  return err;
784 }
785 
786 clib_error_t *
788  pci_intx_handler_function_t * intx_handler)
789 {
791  clib_file_t t = { 0 };
792  linux_pci_irq_t *irq = &p->intx_irq;
793  ASSERT (irq->fd == -1);
794 
796  {
797  struct vfio_irq_info ii = { 0 };
798  ii.argsz = sizeof (struct vfio_irq_info);
799  ii.index = VFIO_PCI_INTX_IRQ_INDEX;
800  if (ioctl (p->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
801  return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) '"
802  "%U'", format_vlib_pci_addr, &p->addr);
803  pci_log_debug (vm, p, "%s index:%u count:%u flags: %s%s%s%s(0x%x)",
804  __func__, ii.index, ii.count,
805  ii.flags & VFIO_IRQ_INFO_EVENTFD ? "eventfd " : "",
806  ii.flags & VFIO_IRQ_INFO_MASKABLE ? "maskable " : "",
807  ii.flags & VFIO_IRQ_INFO_AUTOMASKED ? "automasked " : "",
808  ii.flags & VFIO_IRQ_INFO_NORESIZE ? "noresize " : "",
809  ii.flags);
810  if (ii.count != 1)
811  return clib_error_return (0, "INTx interrupt does not exist on device"
812  "'%U'", format_vlib_pci_addr, &p->addr);
813 
814  irq->fd = eventfd (0, EFD_NONBLOCK);
815  if (irq->fd == -1)
816  return clib_error_return_unix (0, "eventfd");
817 
818  t.file_descriptor = irq->fd;
820  }
821  else if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
822  {
823  t.file_descriptor = p->fd;
825  }
826  else
827  return 0;
828 
830  t.private_data = p->handle;
831  t.description = format (0, "PCI %U INTx", format_vlib_pci_addr, &p->addr);
833  irq->intx_handler = intx_handler;
834  return 0;
835 }
836 
837 clib_error_t *
839  u32 start, u32 count,
840  pci_msix_handler_function_t * msix_handler)
841 {
842  clib_error_t *err = 0;
844  u32 i;
845 
847  return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
848  "support");
849 
850  /* *INDENT-OFF* */
852  { .fd = -1});
853  /* *INDENT-ON* */
854 
855  for (i = start; i < start + count; i++)
856  {
857  clib_file_t t = { 0 };
859  ASSERT (irq->fd == -1);
860 
861  irq->fd = eventfd (0, EFD_NONBLOCK);
862  if (irq->fd == -1)
863  {
864  err = clib_error_return_unix (0, "eventfd");
865  goto error;
866  }
867 
869  t.file_descriptor = irq->fd;
871  t.private_data = p->handle << 16 | i;
872  t.description = format (0, "PCI %U MSI-X #%u", format_vlib_pci_addr,
873  &p->addr, i);
875  irq->msix_handler = msix_handler;
876  }
877 
878  return 0;
879 
880 error:
881  while (i-- > start)
882  {
884  if (irq->fd != -1)
885  {
887  close (irq->fd);
888  irq->fd = -1;
889  }
890  }
891  return err;
892 }
893 
894 clib_error_t *
896  u16 start, u16 count)
897 {
899  int fds[count];
900  int i;
901 
903  return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
904  "support");
905 
906  for (i = start; i < start + count; i++)
907  {
909  fds[i] = irq->fd;
910  }
911 
912  return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
913  VFIO_IRQ_SET_ACTION_TRIGGER, fds);
914 }
915 
916 uword
918  u16 index)
919 {
922  if (irq->fd == -1)
923  return ~0;
924  return irq->clib_file_index;
925 }
926 
927 clib_error_t *
929  u16 start, u16 count)
930 {
932  int i, fds[count];
933 
935  return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
936  "support");
937 
938  for (i = start; i < start + count; i++)
939  fds[i] = -1;
940 
941  return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
942  VFIO_IRQ_SET_ACTION_TRIGGER, fds);
943 }
944 
945 static clib_error_t *
948 {
950  struct vfio_device_info device_info = { 0 };
951  struct vfio_region_info reg = { 0 };
952  clib_error_t *err = 0;
953  u8 *s = 0;
954  int is_noiommu;
955 
957 
958  if ((err = linux_vfio_group_get_device_fd (&p->addr, &p->fd, &is_noiommu)))
959  return err;
960 
961  if (is_noiommu == 0)
962  p->supports_va_dma = 1;
963 
964  device_info.argsz = sizeof (device_info);
965  if (ioctl (p->fd, VFIO_DEVICE_GET_INFO, &device_info) < 0)
966  {
967  err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'",
968  format_vlib_pci_addr, &di->addr);
969  goto error;
970  }
971 
972  reg.argsz = sizeof (struct vfio_region_info);
973  reg.index = VFIO_PCI_CONFIG_REGION_INDEX;
974  if (ioctl (p->fd, VFIO_DEVICE_GET_REGION_INFO, &reg) < 0)
975  {
976  err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'",
977  format_vlib_pci_addr, &di->addr);
978  goto error;
979  }
980 
981  pci_log_debug (vm, p, "%s %U", __func__, format_vfio_region_info, &reg);
982 
983  p->config_offset = reg.offset;
984  p->config_fd = p->fd;
985 
986  /* reset if device supports it */
987  if (device_info.flags & VFIO_DEVICE_FLAGS_RESET)
988  if (ioctl (p->fd, VFIO_DEVICE_RESET) < 0)
989  {
990  err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_RESET) '%U'",
991  format_vlib_pci_addr, &di->addr);
992  goto error;
993  }
994 
995  if (r && r->interrupt_handler)
996  {
997  vlib_pci_register_intx_handler (vm, p->handle, r->interrupt_handler);
999  }
1000 
1001  if (p->supports_va_dma)
1002  {
1003  vlib_buffer_pool_t *bp;
1004  /* *INDENT-OFF* */
1006  {
1007  u32 i;
1008  vlib_physmem_map_t *pm;
1010  for (i = 0; i < pm->n_pages; i++)
1011  vfio_map_physmem_page (vm, pm->base + (i << pm->log2_page_size));
1012  }
1013  /* *INDENT-ON* */
1014  }
1015 
1016  if (r && r->init_function)
1017  err = r->init_function (lpm->vlib_main, p->handle);
1018 
1019 error:
1020  vec_free (s);
1021  if (err)
1022  {
1023  if (p->fd != -1)
1024  close (p->fd);
1025  if (p->config_fd != -1 && p->config_fd != p->fd)
1026  close (p->config_fd);
1027  p->config_fd = p->fd = -1;
1028  }
1029  return err;
1030 }
1031 
1032 /* Configuration space read/write. */
1033 clib_error_t *
1035  vlib_read_or_write_t read_or_write,
1036  uword address, void *data, u32 n_bytes)
1037 {
1039  int n;
1040 
1041  if (read_or_write == VLIB_READ)
1042  n = pread (p->config_fd, data, n_bytes, p->config_offset + address);
1043  else
1044  n = pwrite (p->config_fd, data, n_bytes, p->config_offset + address);
1045 
1046  if (n != n_bytes)
1047  return clib_error_return_unix (0, "%s",
1048  read_or_write == VLIB_READ
1049  ? "read" : "write");
1050 
1051  return 0;
1052 }
1053 
1054 static clib_error_t *
1056  u64 * size, u64 * offset)
1057 {
1059  clib_error_t *error = 0;
1060  int _fd = -1;
1061  u64 _size = 0, _offset = 0;
1062 
1063  ASSERT (bar <= 5);
1064 
1065  error = 0;
1066 
1067  if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
1068  {
1069  u8 *file_name;
1070  struct stat stat_buf;
1071  file_name = format (0, "%s/%U/resource%d%c", sysfs_pci_dev_path,
1072  format_vlib_pci_addr, &p->addr, bar, 0);
1073 
1074  _fd = open ((char *) file_name, O_RDWR);
1075  if (_fd < 0)
1076  {
1077  error = clib_error_return_unix (0, "open `%s'", file_name);
1078  vec_free (file_name);
1079  return error;
1080  }
1081 
1082  if (fstat (_fd, &stat_buf) < 0)
1083  {
1084  error = clib_error_return_unix (0, "fstat `%s'", file_name);
1085  vec_free (file_name);
1086  close (_fd);
1087  return error;
1088  }
1089 
1090  vec_free (file_name);
1091  _size = stat_buf.st_size;
1092  _offset = 0;
1093  }
1094  else if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
1095  {
1096  struct vfio_region_info *r;
1097  u32 sz = sizeof (struct vfio_region_info);
1098  again:
1099  r = clib_mem_alloc (sz);
1100  clib_memset (r, 0, sz);
1101  r->argsz = sz;
1102  r->index = bar;
1103  if (ioctl (p->fd, VFIO_DEVICE_GET_REGION_INFO, r) < 0)
1104  return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) "
1105  "'%U'", format_vlib_pci_addr,
1106  &p->addr);
1107  if (sz != r->argsz)
1108  {
1109  sz = r->argsz;
1110  clib_mem_free (r);
1111  goto again;
1112  }
1113  _fd = p->fd;
1114  _size = r->size;
1115  _offset = r->offset;
1116  pci_log_debug (vm, p, "%s %U", __func__, format_vfio_region_info, r);
1117  clib_mem_free (r);
1118  }
1119  else
1120  ASSERT (0);
1121 
1122  *fd = _fd;
1123  *size = _size;
1124  *offset = _offset;
1125 
1126  return error;
1127 }
1128 
1129 static clib_error_t *
1131  u32 bar, u8 * addr, void **result)
1132 {
1134  int fd = -1;
1136  int flags = MAP_SHARED;
1137  u64 size = 0, offset = 0;
1138  u16 command;
1139 
1140  pci_log_debug (vm, p, "map region %u to va %p", bar, addr);
1141 
1142  if ((error = vlib_pci_read_config_u16 (vm, h, 4, &command)))
1143  return error;
1144 
1145  if (!(command & PCI_COMMAND_MEMORY))
1146  {
1147  pci_log_debug (vm, p, "setting memory enable bit");
1148  command |= PCI_COMMAND_MEMORY;
1149  if ((error = vlib_pci_write_config_u16 (vm, h, 4, &command)))
1150  return error;
1151  }
1152 
1153  if ((error = vlib_pci_region (vm, h, bar, &fd, &size, &offset)))
1154  return error;
1155 
1156  if (p->type == LINUX_PCI_DEVICE_TYPE_UIO && addr != 0)
1157  flags |= MAP_FIXED;
1158 
1159  *result = clib_mem_vm_map_shared (addr, size, fd, offset,
1160  "PCIe %U region %u", format_vlib_pci_addr,
1161  vlib_pci_get_addr (vm, h), bar);
1162  if (*result == CLIB_MEM_VM_MAP_FAILED)
1163  {
1164  error = clib_error_return_unix (0, "mmap `BAR%u'", bar);
1165  if (p->type == LINUX_PCI_DEVICE_TYPE_UIO && (fd != -1))
1166  close (fd);
1167  return error;
1168  }
1169 
1170  /* *INDENT-OFF* */
1172  (linux_pci_region_t) { .fd = -1});
1173  /* *INDENT-ON* */
1174  if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
1175  p->regions[bar].fd = fd;
1176  p->regions[bar].addr = *result;
1177  p->regions[bar].size = size;
1178  return 0;
1179 }
1180 
1181 clib_error_t *
1183  void **result)
1184 {
1185  return (vlib_pci_map_region_int (vm, h, resource, 0 /* addr */ , result));
1186 }
1187 
1188 clib_error_t *
1190  u32 resource, u8 * addr, void **result)
1191 {
1192  return (vlib_pci_map_region_int (vm, h, resource, addr, result));
1193 }
1194 
1195 clib_error_t *
1197 {
1199  clib_error_t *error = 0;
1200  int fd = -1;
1201  u64 size = 0, offset = 0;
1202 
1203  if ((error = vlib_pci_region (vm, h, resource, &fd, &size, &offset)))
1204  return error;
1205 
1206  p->io_fd = fd;
1207  p->io_offset = offset;
1208  return error;
1209 }
1210 
1211 clib_error_t *
1213  vlib_read_or_write_t read_or_write,
1214  uword offset, void *data, u32 length)
1215 {
1217  int n = 0;
1218 
1219  if (read_or_write == VLIB_READ)
1220  n = pread (p->io_fd, data, length, p->io_offset + offset);
1221  else
1222  n = pwrite (p->io_fd, data, length, p->io_offset + offset);
1223 
1224  if (n != length)
1225  return clib_error_return_unix (0, "%s",
1226  read_or_write == VLIB_READ
1227  ? "read" : "write");
1228  return 0;
1229 }
1230 
1231 clib_error_t *
1233 {
1235 
1236  if (!p->supports_va_dma)
1237  return 0;
1238 
1239  return vfio_map_physmem_page (vm, ptr);
1240 }
1241 
1242 int
1244 {
1246 
1247  return p->supports_va_dma != 0;
1248 }
1249 
1250 clib_error_t *
1251 vlib_pci_device_open (vlib_main_t * vm, vlib_pci_addr_t * addr,
1252  pci_device_id_t ids[], vlib_pci_dev_handle_t * handle)
1253 {
1256  linux_pci_device_t *p;
1257  clib_error_t *err = 0;
1258  pci_device_id_t *i;
1259 
1260  di = vlib_pci_get_device_info (vm, addr, &err);
1261 
1262  if (err)
1263  return err;
1264  for (i = ids; i->vendor_id != 0; i++)
1265  if (i->vendor_id == di->vendor_id && i->device_id == di->device_id)
1266  break;
1267 
1268  if (i->vendor_id == 0)
1269  return clib_error_return (0, "Wrong vendor or device id");
1270 
1271  pool_get (lpm->linux_pci_devices, p);
1272  p->handle = p - lpm->linux_pci_devices;
1273  p->addr.as_u32 = di->addr.as_u32;
1274  p->intx_irq.fd = -1;
1275  p->intx_irq.clib_file_index = -1;
1276  p->numa_node = di->numa_node;
1277  /*
1278  * pci io bar read/write fd
1279  */
1280  p->io_fd = -1;
1281 
1282  pci_log_debug (vm, p, "open vid:0x%04x did:0x%04x driver:%s iommu_group:%d",
1283  di->vendor_id, di->device_id, di->driver_name,
1284  di->iommu_group);
1285 
1286  if (clib_strncmp ("vfio-pci", (char *) di->driver_name, 8) == 0)
1287  err = add_device_vfio (vm, p, di, 0);
1288  else if (clib_strncmp ("uio_pci_generic", (char *) di->driver_name, 8) == 0)
1289  err = add_device_uio (vm, p, di, 0);
1290  else
1291  err = clib_error_create ("device not bound to 'vfio-pci' or "
1292  "'uio_pci_generic' kernel module");
1293  if (err)
1294  goto error;
1295 
1296  *handle = p->handle;
1297 
1298 error:
1300  if (err)
1301  {
1302  pci_log_err (vm, p, "%U", format_clib_error, err);
1303  clib_memset (p, 0, sizeof (linux_pci_device_t));
1304  pool_put (lpm->linux_pci_devices, p);
1305  }
1306 
1307  return err;
1308 }
1309 
1310 void
1312 {
1315  linux_pci_irq_t *irq;
1316  linux_pci_region_t *res;
1317  clib_error_t *err = 0;
1318 
1319  if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
1320  {
1321  irq = &p->intx_irq;
1322  if (irq->clib_file_index != -1)
1324  close (p->config_fd);
1325  if (p->io_fd != -1)
1326  close (p->io_fd);
1327  }
1328  else if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
1329  {
1330  irq = &p->intx_irq;
1331  /* close INTx irqs */
1332  if (irq->fd != -1)
1333  {
1334  err = vfio_set_irqs (vm, p, VFIO_PCI_INTX_IRQ_INDEX, 0, 0,
1335  VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1336  clib_error_free (err);
1337  if (irq->clib_file_index != -1)
1339  close (irq->fd);
1340  }
1341 
1342  /* close MSI-X irqs */
1343  if (vec_len (p->msix_irqs))
1344  {
1345  err = vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, 0, 0,
1346  VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1347  clib_error_free (err);
1348  /* *INDENT-OFF* */
1349  vec_foreach (irq, p->msix_irqs)
1350  {
1351  if (irq->fd == -1)
1352  continue;
1354  close (irq->fd);
1355  }
1356  /* *INDENT-ON* */
1357  vec_free (p->msix_irqs);
1358  }
1359  }
1360 
1361  /* *INDENT-OFF* */
1362  vec_foreach (res, p->regions)
1363  {
1364  if (res->size == 0)
1365  continue;
1366  clib_mem_vm_unmap (res->addr);
1367  if (res->fd != -1)
1368  close (res->fd);
1369  }
1370  /* *INDENT-ON* */
1371  vec_free (p->regions);
1372 
1373  close (p->fd);
1374  clib_memset (p, 0, sizeof (linux_pci_device_t));
1375  pool_put (lpm->linux_pci_devices, p);
1376 }
1377 
1378 void
1380 {
1381  vlib_pci_main_t *pm = &pci_main;
1384  pci_device_id_t *i;
1385  clib_error_t *err = 0;
1386  linux_pci_device_t *p;
1387 
1388  pool_get (lpm->linux_pci_devices, p);
1389  p->handle = p - lpm->linux_pci_devices;
1390  p->intx_irq.fd = -1;
1391 
1393 
1394  while (r)
1395  {
1396  for (i = r->supported_devices; i->vendor_id != 0; i++)
1397  if (i->vendor_id == di->vendor_id && i->device_id == di->device_id)
1398  {
1399  if (di->iommu_group != -1)
1400  err = add_device_vfio (vm, p, di, r);
1401  else
1402  err = add_device_uio (vm, p, di, r);
1403 
1404  if (err)
1405  clib_error_report (err);
1406  else
1407  return;
1408  }
1409  r = r->next_registration;
1410  }
1411 
1412  /* No driver, close the PCI config-space FD */
1413  clib_memset (p, 0, sizeof (linux_pci_device_t));
1414  pool_put (lpm->linux_pci_devices, p);
1415 }
1416 
1417 static clib_error_t *
1418 scan_pci_addr (void *arg, u8 * dev_dir_name, u8 * ignored)
1419 {
1420  vlib_pci_addr_t addr, **addrv = arg;
1421  unformat_input_t input;
1422  clib_error_t *err = 0;
1423 
1424  unformat_init_string (&input, (char *) dev_dir_name,
1425  vec_len (dev_dir_name));
1426 
1427  if (!unformat (&input, "/sys/bus/pci/devices/%U",
1429  err = clib_error_return (0, "unformat error `%v`", dev_dir_name);
1430 
1431  unformat_free (&input);
1432 
1433  if (err)
1434  return err;
1435 
1436  vec_add1 (*addrv, addr);
1437  return 0;
1438 }
1439 
1440 static int
1441 pci_addr_cmp (void *v1, void *v2)
1442 {
1443  vlib_pci_addr_t *a1 = v1;
1444  vlib_pci_addr_t *a2 = v2;
1445 
1446  if (a1->domain > a2->domain)
1447  return 1;
1448  if (a1->domain < a2->domain)
1449  return -1;
1450  if (a1->bus > a2->bus)
1451  return 1;
1452  if (a1->bus < a2->bus)
1453  return -1;
1454  if (a1->slot > a2->slot)
1455  return 1;
1456  if (a1->slot < a2->slot)
1457  return -1;
1458  if (a1->function > a2->function)
1459  return 1;
1460  if (a1->function < a2->function)
1461  return -1;
1462  return 0;
1463 }
1464 
1465 vlib_pci_addr_t *
1467 {
1468  vlib_pci_addr_t *addrs = 0;
1469  clib_error_t *err;
1471  &addrs, /* scan_dirs */ 0);
1472  if (err)
1473  {
1474  vec_free (addrs);
1475  return 0;
1476  }
1477 
1479 
1480  return addrs;
1481 }
1482 
1483 clib_error_t *
1485 {
1486  vlib_pci_main_t *pm = &pci_main;
1487  vlib_pci_addr_t *addr = 0, *addrs;
1488 
1489  pm->vlib_main = vm;
1490 
1491  ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32));
1492 
1493  addrs = vlib_pci_get_all_dev_addrs ();
1494  /* *INDENT-OFF* */
1495  vec_foreach (addr, addrs)
1496  {
1498  if ((d = vlib_pci_get_device_info (vm, addr, 0)))
1499  {
1502  }
1503  }
1504  /* *INDENT-ON* */
1505 
1506  return 0;
1507 }
1508 
1509 /* *INDENT-OFF* */
1511 {
1512  .runs_after = VLIB_INITS("unix_input_init"),
1513 };
1514 /* *INDENT-ON* */
1515 
1516 /*
1517  * fd.io coding-style-patch-verification: ON
1518  *
1519  * Local Variables:
1520  * eval: (c-set-style "gnu")
1521  * End:
1522  */
vlib_physmem_map_t::base
void * base
Definition: physmem.h:49
vec_reset_length
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
Definition: vec_bootstrap.h:194
pci_bus_init
clib_error_t * pci_bus_init(vlib_main_t *vm)
Definition: pci.c:251
vlib.h
uio_drv_name
static const char uio_drv_name[]
Definition: vmbus.c:39
clib_file::file_descriptor
u32 file_descriptor
Definition: file.h:54
vlib_pci_set_private_data
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
tmp
u32 * tmp
Definition: interface_output.c:1078
clib_strncpy
#define clib_strncpy(d, s, n)
Definition: string.h:992
vlib_pci_get_all_dev_addrs
vlib_pci_addr_t * vlib_pci_get_all_dev_addrs()
Definition: pci.c:1466
linux_pci_irq_t::msix_handler
pci_msix_handler_function_t * msix_handler
Definition: pci.c:84
file_main
clib_file_main_t file_main
Definition: main.c:63
linux_pci_uio_read_ready
static clib_error_t * linux_pci_uio_read_ready(clib_file_t *uf)
Definition: pci.c:649
clib_sysfs_link_to_name
__clib_export u8 * clib_sysfs_link_to_name(char *link)
Definition: sysfs.c:91
format_vfio_region_info
format_function_t format_vfio_region_info
Definition: vfio.h:52
vlib_pci_region
static clib_error_t * vlib_pci_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, int *fd, u64 *size, u64 *offset)
Definition: pci.c:1055
foreach_directory_file
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
Definition: util.c:49
pci_config_header_t::vendor_id
u16 vendor_id
Definition: pci_config.h:177
vlib_pci_disable_msix_irq
clib_error_t * vlib_pci_disable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:928
format_vlib_pci_addr
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
Definition: pci.c:138
linux_pci_device_t::io_offset
u64 io_offset
Definition: pci.c:114
vlib_physmem_map_t
Definition: physmem.h:45
LINUX_PCI_DEVICE_TYPE_UIO
@ LINUX_PCI_DEVICE_TYPE_UIO
Definition: pci.c:91
f
vlib_frame_t * f
Definition: interface_output.c:1080
vlib_pci_get_numa_node
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
pool_elt_at_index
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:553
linux_pci_region_t::size
size_t size
Definition: pci.c:74
clib_bitmap_t
uword clib_bitmap_t
Definition: bitmap.h:50
clib_mem_free
static void clib_mem_free(void *p)
Definition: mem.h:311
scan_uio_dir
static clib_error_t * scan_uio_dir(void *arg, u8 *path_name, u8 *file_name)
Definition: pci.c:578
linux_pci_device_t::numa_node
u32 numa_node
Definition: pci.c:100
vlib_pci_main_t::pci_device_registrations
pci_device_registration_t * pci_device_registrations
Definition: pci.h:156
directory_exists
static int directory_exists(char *path)
Definition: pci.c:387
clib_mem_vm_map_shared
__clib_export void * clib_mem_vm_map_shared(void *base, uword size, int fd, uword offset, char *fmt,...)
Definition: mem.c:59
vlib_buffer_pool_t
Definition: buffer.h:450
path
vl_api_fib_path_t path
Definition: mfib_types.api:44
clib_mem_vm_unmap
__clib_export int clib_mem_vm_unmap(void *base)
Definition: mem.c:510
vlib_pci_main_t::log_default
vlib_log_class_t log_default
Definition: pci.h:158
vlib_buffer_pool_t::physmem_map_index
u32 physmem_map_index
Definition: buffer.h:458
clib_error_return
#define clib_error_return(e, args...)
Definition: error.h:99
clib_file::read_function
clib_file_function_t * read_function
Definition: file.h:67
vlib_pci_dev_handle_t
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
vlib_pci_enable_msix_irq
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:895
u16
unsigned short u16
Definition: types.h:57
sysfs_pci_drv_path
static const char * sysfs_pci_drv_path
Definition: pci.c:59
linux_pci_device_t
Definition: pci.c:95
clib_file::private_data
u64 private_data
Definition: file.h:64
linux_pci_irq_t::clib_file_index
u32 clib_file_index
Definition: pci.c:80
pci_log_err
#define pci_log_err(vm, dev, f,...)
Definition: pci.c:66
pool_put
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:305
vlib_main_t::buffer_main
vlib_buffer_main_t * buffer_main
Definition: main.h:165
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
linux_pci_main_t::vlib_main
vlib_main_t * vlib_main
Definition: pci.c:133
vlib_pci_register_msix_handler
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:838
vlib_physmem_get_map
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
Definition: physmem.c:86
clib_file::description
u8 * description
Definition: file.h:70
vfio_set_irqs
static clib_error_t * vfio_set_irqs(vlib_main_t *vm, linux_pci_device_t *p, u32 index, u32 start, u32 count, u32 flags, int *efds)
Definition: pci.c:593
vlib_pci_bind_to_uio
clib_error_t * vlib_pci_bind_to_uio(vlib_main_t *vm, vlib_pci_addr_t *addr, char *uio_drv_name)
Definition: pci.c:397
clib_error_report
#define clib_error_report(e)
Definition: error.h:113
unformat_input_t
struct _unformat_input_t unformat_input_t
addr
vhost_vring_addr_t addr
Definition: vhost_user.h:130
CLIB_MEM_VM_MAP_FAILED
#define CLIB_MEM_VM_MAP_FAILED
Definition: mem.h:54
r
vnet_hw_if_output_node_runtime_t * r
Definition: interface_output.c:1071
clib_memcpy_fast
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
Definition: string.h:92
pci_addr_cmp
static int pci_addr_cmp(void *v1, void *v2)
Definition: pci.c:1441
vlib_pci_map_region
clib_error_t * vlib_pci_map_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, void **result)
Definition: pci.c:1182
vlib_physmem_map_t::log2_page_size
u32 log2_page_size
Definition: physmem.h:52
h
h
Definition: flowhash_template.h:372
clib_unix_warning
#define clib_unix_warning(format, args...)
Definition: error.h:68
error
Definition: cJSON.c:88
pci_device_registration_t
struct _pci_device_registration pci_device_registration_t
sysfs.h
vlib_pci_intr_enable
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:239
vlib_pci_map_dma
clib_error_t * vlib_pci_map_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h, void *ptr)
Definition: pci.c:1232
clib_sysfs_write
__clib_export clib_error_t * clib_sysfs_write(char *file_name, char *fmt,...)
Definition: sysfs.c:27
linux_pci_irq_t
Definition: pci.c:77
format_clib_error
__clib_export u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
unformat
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
vlib_pci_device_info
Definition: pci.h:60
vfio_map_physmem_page
clib_error_t * vfio_map_physmem_page(vlib_main_t *vm, void *addr)
Definition: vfio.c:38
linux_pci_device_type_t
linux_pci_device_type_t
Definition: pci.c:88
clib_sysfs_read
__clib_export clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
Definition: sysfs.c:51
linux_pci_device_t::io_fd
int io_fd
Definition: pci.c:113
init_device_from_registered
void init_device_from_registered(vlib_main_t *vm, vlib_pci_device_info_t *di)
Definition: pci.c:1379
sysfs_mod_vfio_noiommu
static char * sysfs_mod_vfio_noiommu
Definition: pci.c:60
pci_intx_handler_function_t
void() pci_intx_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle)
Definition: pci.h:130
sysfs_pci_dev_path
static const char * sysfs_pci_dev_path
Definition: pci.c:58
vlib_pci_get_private_data
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
count
u8 count
Definition: dhcp.api:208
clib_error_create
#define clib_error_create(args...)
Definition: error.h:96
vec_len
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
Definition: vec_bootstrap.h:142
unformat_free
static void unformat_free(unformat_input_t *i)
Definition: format.h:155
linux_pci_irq_t::fd
int fd
Definition: pci.c:79
len
u8 len
Definition: ip_types.api:103
vlib_physmem_map_t::n_pages
u32 n_pages
Definition: physmem.h:50
vec_add1
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:606
unformat_vlib_pci_addr
uword unformat_vlib_pci_addr(unformat_input_t *input, va_list *args)
Definition: pci.c:121
linux_pci_device_t::uio_minor
u32 uio_minor
Definition: pci.c:117
linux_pci_device_t::fd
int fd
Definition: pci.c:110
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
clib_file
Definition: file.h:51
linux_pci_device_t::regions
linux_pci_region_t * regions
Definition: pci.c:103
vlib_pci_io_region
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
Definition: pci.c:1196
offset
struct clib_bihash_value offset
template key/value backing page structure
clib_bitmap_count_set_bits
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:468
linux_pci_device_t::config_offset
u64 config_offset
Definition: pci.c:107
linux_pci_device_t::type
linux_pci_device_type_t type
Definition: pci.c:97
uword
u64 uword
Definition: types.h:112
linux_pci_vfio_error_ready
static clib_error_t * linux_pci_vfio_error_ready(clib_file_t *uf)
Definition: pci.c:723
linux_pci_region_t::fd
int fd
Definition: pci.c:72
linux_pci_irq_t::intx_handler
pci_intx_handler_function_t * intx_handler
Definition: pci.c:83
PCI_COMMAND_MEMORY
#define PCI_COMMAND_MEMORY
Definition: pci_config.h:182
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
vlib_read_or_write_t
vlib_read_or_write_t
Definition: defs.h:62
pool_get
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:255
vec_validate
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment)
Definition: vec.h:523
address
manual_print typedef address
Definition: ip_types.api:96
linux_pci_main_t
Definition: pci.c:131
vlib_pci_supports_virtual_addr_dma
int vlib_pci_supports_virtual_addr_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1243
vlib_pci_get_device_info
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_main_t *vm, vlib_pci_addr_t *addr, clib_error_t **error)
Definition: pci.c:202
vlib_pci_register_intx_handler
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
Definition: pci.c:787
vlib_pci_read_write_config
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
Definition: pci.c:1034
LINUX_PCI_DEVICE_TYPE_UNKNOWN
@ LINUX_PCI_DEVICE_TYPE_UNKNOWN
Definition: pci.c:90
vlib_pci_get_num_msix_interrupts
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:177
linux_pci_device_t::addr
vlib_pci_addr_t addr
Definition: pci.c:99
vlib_pci_map_region_fixed
clib_error_t * vlib_pci_map_region_fixed(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, u8 *addr, void **result)
Definition: pci.c:1189
linux_pci_device_t::supports_va_dma
u8 supports_va_dma
Definition: pci.c:126
scan_pci_addr
static clib_error_t * scan_pci_addr(void *arg, u8 *dev_dir_name, u8 *ignored)
Definition: pci.c:1418
linux_pci_uio_error_ready
static clib_error_t * linux_pci_uio_error_ready(clib_file_t *uf)
Definition: pci.c:676
data
u8 data[128]
Definition: ipsec_types.api:92
VLIB_READ
@ VLIB_READ
Definition: defs.h:64
linux_pci_device_t::intx_irq
linux_pci_irq_t intx_irq
Definition: pci.c:120
vec_free
#define vec_free(V)
Free vector's memory (no header).
Definition: vec.h:395
size
u32 size
Definition: vhost_user.h:125
index
u32 index
Definition: flow_types.api:221
clib_bihash_value
template key/value backing page structure
Definition: bihash_doc.h:44
vlib_buffer_main_t::buffer_pools
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:482
unformat_init_string
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1029
u64
unsigned long u64
Definition: types.h:89
format
description fragment has unexpected format
Definition: map.api:433
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
add_device_uio
static clib_error_t * add_device_uio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
Definition: pci.c:731
data_len
u8 data_len
Definition: ikev2_types.api:24
vec_validate_init_empty
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header,...
Definition: vec.h:570
clib_strncmp
#define clib_strncmp(s1, s2, n)
Definition: string.h:877
buf
u64 buf
Definition: application.c:493
u32
unsigned int u32
Definition: types.h:88
VLIB_INIT_FUNCTION
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:172
pci_config_header_t
Definition: pci_config.h:175
linux_pci_vfio_intx_read_ready
static clib_error_t * linux_pci_vfio_intx_read_ready(clib_file_t *uf)
Definition: pci.c:703
vlib_pci_main_t::vlib_main
vlib_main_t * vlib_main
Definition: pci.h:155
n_bytes
u32 n_bytes
Definition: interface_output.c:401
linux_pci_main
linux_pci_main_t linux_pci_main
Definition: pci.c:199
linux_pci_init
clib_error_t * linux_pci_init(vlib_main_t *vm)
Definition: pci.c:1484
vec_foreach
#define vec_foreach(var, vec)
Vector iterator.
Definition: vec_bootstrap.h:213
linux_pci_device_t::private_data
uword private_data
Definition: pci.c:124
pci.h
clib_error_return_unix
#define clib_error_return_unix(e, args...)
Definition: error.h:102
linux_pci_device_t::handle
vlib_pci_dev_handle_t handle
Definition: pci.c:98
linux_pci_get_device
static linux_pci_device_t * linux_pci_get_device(vlib_pci_dev_handle_t h)
Definition: pci.c:141
clib_file_add
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
vlib_log
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:131
LINUX_PCI_DEVICE_TYPE_VFIO
@ LINUX_PCI_DEVICE_TYPE_VFIO
Definition: pci.c:92
linux_pci_vfio_unmask_intx
static clib_error_t * linux_pci_vfio_unmask_intx(vlib_main_t *vm, linux_pci_device_t *d)
Definition: pci.c:669
pci_msix_handler_function_t
void() pci_msix_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle, u16 line)
Definition: pci.h:132
linux_pci_device_t::config_fd
int config_fd
Definition: pci.c:106
linux_pci_vfio_msix_read_ready
static clib_error_t * linux_pci_vfio_msix_read_ready(clib_file_t *uf)
Definition: pci.c:684
VLIB_PCI_DEVICE_INFO_F_NOIOMMU
#define VLIB_PCI_DEVICE_INFO_F_NOIOMMU
Definition: pci.h:63
vec_sort_with_function
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
Definition: vec.h:1097
length
char const int length
Definition: cJSON.h:163
clib_memset
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vlib_main_t
Definition: main.h:102
VLIB_INITS
#define VLIB_INITS(...)
Definition: init.h:352
vlib_get_main
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:38
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
vfio.h
unix.h
vlib_init_function_t
clib_error_t *() vlib_init_function_t(struct vlib_main_t *vm)
Definition: init.h:51
vlib_pci_read_write_io
clib_error_t * vlib_pci_read_write_io(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword offset, void *data, u32 length)
Definition: pci.c:1212
vlib_pci_get_msix_file_index
uword vlib_pci_get_msix_file_index(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 index)
Definition: pci.c:917
vlib_pci_get_addr
vlib_pci_addr_t * vlib_pci_get_addr(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:163
clib_file_del_by_index
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
Definition: file.h:119
vlib_pci_device_open
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1251
clib_error_free
#define clib_error_free(e)
Definition: error.h:86
rv
int __clib_unused rv
Definition: application.c:491
linux_pci_region_t::addr
void * addr
Definition: pci.c:73
vlib_pci_map_region_int
static clib_error_t * vlib_pci_map_region_int(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, u8 *addr, void **result)
Definition: pci.c:1130
pci_config_type1_little_to_host
static void pci_config_type1_little_to_host(pci_config_type1_regs_t *r)
Definition: pci_config.h:353
clib_file::error_function
clib_file_function_t * error_function
Definition: file.h:67
vlib_pci_device_close
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1311
vlib_pci_main_t
Definition: pci.h:153
vlib_pci_free_device_info
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
Definition: pci.h:114
di
void di(unformat_input_t *i)
Definition: unformat.c:163
pci_main
vlib_pci_main_t pci_main
Definition: pci.c:53
linux_pci_region_t
Definition: pci.c:70
linux_pci_main_t::linux_pci_devices
linux_pci_device_t * linux_pci_devices
Definition: pci.c:134
add_device_vfio
static clib_error_t * add_device_vfio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
Definition: pci.c:946
pci_config_type0_little_to_host
static void pci_config_type0_little_to_host(pci_config_type0_regs_t *r)
Definition: pci_config.h:279
clib_mem_alloc
static void * clib_mem_alloc(uword size)
Definition: mem.h:253
unformat_bitmap_list
__clib_export uword unformat_bitmap_list(unformat_input_t *input, va_list *va)
unformat a list of bit ranges into a bitmap (eg "0-3,5-7,11" )
Definition: bitmap.c:55
pci_log_debug
#define pci_log_debug(vm, dev, f,...)
Definition: pci.c:63
linux_vfio_group_get_device_fd
clib_error_t * linux_vfio_group_get_device_fd(vlib_pci_addr_t *addr, int *fdp, int *is_noiommu)
Definition: vfio.c:177
linux_pci_device_t::msix_irqs
linux_pci_irq_t * msix_irqs
Definition: pci.c:121
pci_device_id_t
Definition: pci.h:125
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105