47 #include <sys/types.h> 51 #include <sys/ioctl.h> 53 #include <linux/ethtool.h> 54 #include <linux/sockios.h> 56 #include <sys/eventfd.h> 61 "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode";
63 #define pci_log_debug(vm, dev, f, ...) \ 64 vlib_log(VLIB_LOG_LEVEL_DEBUG, pci_main.log_default, "%U: " f, \ 65 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__) 66 #define pci_log_err(vm, dev, f, ...) \ 67 vlib_log(VLIB_LOG_LEVEL_ERR, pci_main.log_default, "%U: " f, \ 68 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__) 196 di->
addr.as_u32 = addr->as_u32;
201 f =
format (0,
"%v/config%c", dev_dir_name, 0);
202 fd = open ((
char *) f, O_RDWR);
206 fd = open ((
char *) f, O_RDONLY);
229 if (!memcmp (&di->
config0.
header, &all_ones, sizeof (all_ones)))
244 f =
format (f,
"%v/numa_node%c", dev_dir_name, 0);
253 f =
format (f,
"%v/class%c", dev_dir_name, 0);
260 f =
format (f,
"%v/vendor%c", dev_dir_name, 0);
267 f =
format (f,
"%v/device%c", dev_dir_name, 0);
274 f =
format (f,
"%v/driver%c", dev_dir_name, 0);
282 f =
format (f,
"%v/iommu_group%c", dev_dir_name, 0);
290 f =
format (f,
"%v/iommu_group/name%c", dev_dir_name, 0);
294 if (strncmp ((
char *) tmpstr,
"vfio-noiommu", 12) == 0)
305 f =
format (f,
"%v/vpd%c", dev_dir_name, 0);
306 fd = open ((
char *) f, O_RDONLY);
315 if (read (fd, &tag, 3) != 3)
318 if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91)
321 len = (tag[2] << 8) | tag[1];
324 if (read (fd, data, len) != len)
331 else if (tag[0] == 0x90)
333 else if (tag[0] == 0x91)
360 struct stat s = { 0 };
361 if (stat (path, &s) == -1)
364 return S_ISDIR (s.st_mode);
372 u8 *s = 0, *driver_name = 0;
376 int fd, clear_driver_override = 0;
385 if (strncmp (
"auto", uio_drv_name, 5) == 0)
387 int vfio_pci_loaded = 0;
395 if (!vfio_pci_loaded)
398 "is bound to IOMMU group and " 399 "vfio-pci driver is not loaded",
404 uio_drv_name =
"vfio-pci";
409 if (vfio_pci_loaded &&
411 uio_drv_name =
"vfio-pci";
413 uio_drv_name =
"uio_pci_generic";
415 uio_drv_name =
"igb_uio";
420 "kernel VFIO or UIO driver",
428 s =
format (s,
"%v/driver%c", dev_dir_name, 0);
433 ((strcmp (
"vfio-pci", (
char *) driver_name) == 0) ||
434 (strcmp (
"uio_pci_generic", (
char *) driver_name) == 0) ||
435 (strcmp (
"igb_uio", (
char *) driver_name) == 0)))
440 dir = opendir (
"/sys/class/net");
446 "read /sys/class/net",
451 fd = socket (PF_INET, SOCK_DGRAM, 0);
458 while ((e = readdir (dir)))
461 struct ethtool_drvinfo drvinfo;
463 if (e->d_name[0] ==
'.')
468 ifr.ifr_data = (
char *) &drvinfo;
469 strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name));
470 ifr.ifr_name[
ARRAY_LEN (ifr.ifr_name) - 1] =
'\0';
471 drvinfo.cmd = ETHTOOL_GDRVINFO;
472 if (ioctl (fd, SIOCETHTOOL, &ifr) < 0)
475 if ((errno != ENOTSUP) && (errno != ENODEV))
481 if (strcmp ((
char *) s, drvinfo.bus_info))
485 strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name));
486 ifr.ifr_name[
ARRAY_LEN (ifr.ifr_name) - 1] =
'\0';
487 if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
495 if (ifr.ifr_flags & IFF_UP)
498 "interface %s is up",
508 s =
format (s,
"%v/driver/unbind%c", dev_dir_name, 0);
512 s =
format (s,
"%v/driver_override%c", dev_dir_name, 0);
513 if (access ((
char *) s, F_OK) == 0)
516 clear_driver_override = 1;
531 if (clear_driver_override)
533 s =
format (s,
"%v/driver_override%c", dev_dir_name, 0);
566 int data_len = efds ? count *
sizeof (int) : 0;
567 u8 buf[
sizeof (
struct vfio_irq_set) + data_len];
568 struct vfio_irq_info ii = { 0 };
569 struct vfio_irq_set *irq_set = (
struct vfio_irq_set *) buf;
572 ii.argsz =
sizeof (
struct vfio_irq_info);
575 if (ioctl (p->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
579 pci_log_debug (vm, p,
"%s index:%u count:%u flags: %s%s%s%s(0x%x)",
580 __func__, ii.index, ii.count,
581 ii.flags & VFIO_IRQ_INFO_EVENTFD ?
"eventfd " :
"",
582 ii.flags & VFIO_IRQ_INFO_MASKABLE ?
"maskable " :
"",
583 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ?
"automasked " :
"",
584 ii.flags & VFIO_IRQ_INFO_NORESIZE ?
"noresize " :
"",
587 if (ii.count < start + count)
594 flags |= VFIO_IRQ_SET_DATA_EVENTFD;
598 flags |= VFIO_IRQ_SET_DATA_NONE;
600 ASSERT ((flags & (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD)) !=
601 (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD));
603 irq_set->argsz =
sizeof (
struct vfio_irq_set) + data_len;
604 irq_set->index = index;
605 irq_set->start = start;
606 irq_set->count =
count;
607 irq_set->flags =
flags;
609 if (ioctl (p->
fd, VFIO_DEVICE_SET_IRQS, irq_set) < 0)
611 "[index = %u, start = %u, count = %u, " 614 index, start, count, flags);
622 int __attribute__ ((unused)) rv;
642 VFIO_IRQ_SET_ACTION_UNMASK, 0);
657 int __attribute__ ((unused)) rv;
676 int __attribute__ ((unused)) rv;
714 p->
config_fd = open ((
char *) s, O_RDWR);
731 p->
fd = open ((
char *) s, O_RDWR);
738 if (r && r->interrupt_handler)
741 if (r && r->init_function)
767 struct vfio_irq_info ii = { 0 };
768 ii.argsz =
sizeof (
struct vfio_irq_info);
769 ii.index = VFIO_PCI_INTX_IRQ_INDEX;
770 if (ioctl (p->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
773 pci_log_debug (vm, p,
"%s index:%u count:%u flags: %s%s%s%s(0x%x)",
774 __func__, ii.index, ii.count,
775 ii.flags & VFIO_IRQ_INFO_EVENTFD ?
"eventfd " :
"",
776 ii.flags & VFIO_IRQ_INFO_MASKABLE ?
"maskable " :
"",
777 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ?
"automasked " :
"",
778 ii.flags & VFIO_IRQ_INFO_NORESIZE ?
"noresize " :
"",
784 irq->
fd = eventfd (0, EFD_NONBLOCK);
825 for (i = start; i < start +
count; i++)
831 irq->
fd = eventfd (0, EFD_NONBLOCK);
876 for (i = start; i < start +
count; i++)
882 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
883 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
897 for (i = start; i < start +
count; i++)
900 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
901 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
909 struct vfio_device_info device_info = { 0 };
910 struct vfio_region_info reg = { 0 };
923 device_info.argsz =
sizeof (device_info);
924 if (ioctl (p->
fd, VFIO_DEVICE_GET_INFO, &device_info) < 0)
931 reg.argsz =
sizeof (
struct vfio_region_info);
932 reg.index = VFIO_PCI_CONFIG_REGION_INDEX;
933 if (ioctl (p->
fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
940 pci_log_debug (vm, p,
"%s region_info index:%u size:0x%lx offset:0x%lx " 941 "flags: %s%s%s(0x%x)", __func__,
942 reg.index, reg.size, reg.offset,
943 reg.flags & VFIO_REGION_INFO_FLAG_READ ?
"rd " :
"",
944 reg.flags & VFIO_REGION_INFO_FLAG_WRITE ?
"wr " :
"",
945 reg.flags & VFIO_REGION_INFO_FLAG_MMAP ?
"mmap " :
"",
952 if (device_info.flags & VFIO_DEVICE_FLAGS_RESET)
953 if (ioctl (p->
fd, VFIO_DEVICE_RESET) < 0)
960 if (r && r->interrupt_handler)
975 for (i = 0; i < pm->
n_pages; i++)
981 if (r && r->init_function)
1014 ?
"read" :
"write");
1026 u64 _size = 0, _offset = 0;
1035 struct stat stat_buf;
1039 _fd = open ((
char *) file_name, O_RDWR);
1047 if (fstat (_fd, &stat_buf) < 0)
1056 _size = stat_buf.st_size;
1061 struct vfio_region_info reg = { 0 };
1062 reg.argsz =
sizeof (
struct vfio_region_info);
1064 if (ioctl (p->
fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
1070 _offset = reg.offset;
1071 pci_log_debug (vm, p,
"%s region_info index:%u size:0x%lx offset:0x%lx " 1072 "flags: %s%s%s(0x%x)", __func__,
1073 reg.index, reg.size, reg.offset,
1074 reg.flags & VFIO_REGION_INFO_FLAG_READ ?
"rd " :
"",
1075 reg.flags & VFIO_REGION_INFO_FLAG_WRITE ?
"wr " :
"",
1076 reg.flags & VFIO_REGION_INFO_FLAG_MMAP ?
"mmap " :
"",
1096 int flags = MAP_SHARED;
1107 *result = mmap (addr, size, PROT_READ | PROT_WRITE, flags, fd,
offset);
1108 if (*result == (
void *) -1)
1173 ?
"read" :
"write");
1227 pci_log_debug (vm, p,
"open vid:0x%04x did:0x%04x driver:%s iommu_group:%d",
1231 if (strncmp (
"vfio-pci", (
char *) di->
driver_name, 8) == 0)
1233 else if (strncmp (
"uio_pci_generic", (
char *) di->
driver_name, 8) == 0)
1237 "'uio_pci_generic' kernel module");
1279 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1289 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1339 for (i = r->supported_devices; i->
vendor_id != 0; i++)
1352 r = r->next_registration;
1363 vlib_pci_addr_t
addr, **addrv = arg;
1370 if (!
unformat (&input,
"/sys/bus/pci/devices/%U",
1386 vlib_pci_addr_t *a1 = v1;
1387 vlib_pci_addr_t *a2 = v2;
1389 if (a1->domain > a2->domain)
1391 if (a1->domain < a2->domain)
1393 if (a1->bus > a2->bus)
1395 if (a1->bus < a2->bus)
1397 if (a1->slot > a2->slot)
1399 if (a1->slot < a2->slot)
1401 if (a1->function > a2->function)
1403 if (a1->function < a2->function)
1411 vlib_pci_addr_t *addrs = 0;
1430 vlib_pci_addr_t *
addr = 0, *addrs;
1438 ASSERT (
sizeof (vlib_pci_addr_t) ==
sizeof (
u32));
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
u8 * format_clib_error(u8 *s, va_list *va)
linux_pci_device_type_t type
uword unformat_vlib_pci_addr(unformat_input_t *input, va_list *args)
static clib_error_t * vlib_pci_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, int *fd, u64 *size, u64 *offset)
pci_device_registration_t * pci_device_registrations
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define pci_log_debug(vm, dev, f,...)
clib_error_t * vlib_pci_read_write_io(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword offset, void *data, u32 length)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static const char uio_drv_name[]
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
static clib_error_t * linux_pci_uio_error_ready(clib_file_t *uf)
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
vlib_pci_addr_t * vlib_pci_get_all_dev_addrs()
static clib_error_t * vfio_set_irqs(vlib_main_t *vm, linux_pci_device_t *p, u32 index, u32 start, u32 count, u32 flags, int *efds)
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
clib_error_t * clib_sysfs_write(char *file_name, char *fmt,...)
static char * sysfs_mod_vfio_noiommu
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
linux_pci_main_t linux_pci_main
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
static void pci_config_type1_little_to_host(pci_config_type1_regs_t *r)
#define VLIB_INIT_FUNCTION(x)
static clib_error_t * linux_pci_vfio_intx_read_ready(clib_file_t *uf)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
clib_file_main_t file_main
struct _pci_device_registration pci_device_registration_t
linux_pci_irq_t * msix_irqs
#define clib_error_create(args...)
clib_error_t * linux_vfio_group_get_device_fd(vlib_pci_addr_t *addr, int *fdp, int *is_noiommu)
#define vlib_call_init_function(vm, x)
pci_config_type1_regs_t config1
vlib_pci_dev_handle_t handle
#define pci_log_err(vm, dev, f,...)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
clib_error_t * linux_pci_init(vlib_main_t *vm)
static clib_error_t * scan_pci_addr(void *arg, u8 *dev_dir_name, u8 *ignored)
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
u32 vlib_pci_dev_handle_t
#define clib_error_return_unix(e, args...)
#define pool_put(P, E)
Free an object E in pool P.
void() pci_intx_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle)
vlib_buffer_pool_t * buffer_pools
static const char * sysfs_pci_drv_path
static clib_error_t * linux_pci_vfio_error_ready(clib_file_t *uf)
static clib_error_t * vlib_pci_map_region_int(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, u8 *addr, void **result)
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
int vlib_pci_supports_virtual_addr_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static clib_error_t * linux_pci_uio_read_ready(clib_file_t *uf)
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
static void pci_config_type0_little_to_host(pci_config_type0_regs_t *r)
static clib_error_t * add_device_vfio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
clib_error_t * vlib_pci_map_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, void **result)
#define vec_free(V)
Free vector's memory (no header).
clib_error_t * pci_bus_init(vlib_main_t *vm)
clib_error_t * vlib_pci_disable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
static linux_pci_device_t * linux_pci_get_device(vlib_pci_dev_handle_t h)
static clib_error_t * linux_pci_vfio_msix_read_ready(clib_file_t *uf)
pci_msix_handler_function_t * msix_handler
#define VLIB_PCI_DEVICE_INFO_F_NOIOMMU
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_main_t *vm, vlib_pci_addr_t *addr, clib_error_t **error)
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
pci_config_type0_regs_t config0
vlib_pci_addr_t * vlib_pci_get_addr(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define clib_error_report(e)
clib_error_t * vfio_map_physmem_page(vlib_main_t *vm, void *addr)
void() pci_msix_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle, u16 line)
static void * clib_mem_alloc(uword size)
static vlib_main_t * vlib_get_main(void)
template key/value backing page structure
static clib_error_t * linux_pci_vfio_unmask_intx(vlib_main_t *vm, linux_pci_device_t *d)
static int pci_addr_cmp(void *v1, void *v2)
clib_error_t * vlib_pci_map_region_fixed(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, u8 *addr, void **result)
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static int directory_exists(char *path)
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
clib_error_t * vlib_pci_map_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h, void *ptr)
#define clib_unix_warning(format, args...)
#define clib_error_free(e)
clib_file_function_t * error_function
linux_pci_region_t * regions
pci_intx_handler_function_t * intx_handler
struct clib_bihash_value offset
template key/value backing page structure
static clib_error_t * add_device_uio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
vlib_buffer_main_t buffer_main
#define vec_foreach(var, vec)
Vector iterator.
static const char * sysfs_pci_dev_path
clib_error_t * vlib_pci_bind_to_uio(vlib_main_t *vm, vlib_pci_addr_t *addr, char *uio_drv_name)
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
void init_device_from_registered(vlib_main_t *vm, vlib_pci_device_info_t *di)
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
linux_pci_device_t * linux_pci_devices
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
pci_config_header_t header
static clib_error_t * scan_uio_dir(void *arg, u8 *path_name, u8 *file_name)
u8 * clib_sysfs_link_to_name(char *link)
linux_vfio_main_t vfio_main
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)