47 #include <sys/types.h> 51 #include <sys/ioctl.h> 53 #include <linux/ethtool.h> 54 #include <linux/sockios.h> 56 #include <sys/eventfd.h> 61 "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode";
63 #define pci_log_debug(vm, dev, f, ...) \ 64 vlib_log(VLIB_LOG_LEVEL_DEBUG, pci_main.log_default, "%U: " f, \ 65 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__) 66 #define pci_log_err(vm, dev, f, ...) \ 67 vlib_log(VLIB_LOG_LEVEL_ERR, pci_main.log_default, "%U: " f, \ 68 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__) 183 struct vfio_irq_info ii = { 0 };
185 ii.argsz =
sizeof (
struct vfio_irq_info);
186 ii.index = VFIO_PCI_MSIX_IRQ_INDEX;
187 if (ioctl (d->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
214 di->
addr.as_u32 = addr->as_u32;
219 f =
format (0,
"%v/config%c", dev_dir_name, 0);
220 fd = open ((
char *) f, O_RDWR);
224 fd = open ((
char *) f, O_RDONLY);
247 if (!memcmp (&di->
config0.
header, &all_ones, sizeof (all_ones)))
262 f =
format (f,
"%v/numa_node%c", dev_dir_name, 0);
271 f =
format (f,
"%v/class%c", dev_dir_name, 0);
278 f =
format (f,
"%v/vendor%c", dev_dir_name, 0);
285 f =
format (f,
"%v/device%c", dev_dir_name, 0);
292 f =
format (f,
"%v/driver%c", dev_dir_name, 0);
299 f =
format (f,
"%v/iommu_group%c", dev_dir_name, 0);
308 f =
format (f,
"%v/iommu_group/name%c", dev_dir_name, 0);
312 if (strncmp ((
char *) tmpstr,
"vfio-noiommu", 12) == 0)
322 f =
format (f,
"%v/vpd%c", dev_dir_name, 0);
323 fd = open ((
char *) f, O_RDONLY);
332 if (read (fd, &tag, 3) != 3)
335 if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91)
338 len = (tag[2] << 8) | tag[1];
341 if (read (fd, data, len) != len)
348 else if (tag[0] == 0x90)
350 else if (tag[0] == 0x91)
377 struct stat s = { 0 };
378 if (stat (path, &s) == -1)
381 return S_ISDIR (s.st_mode);
389 u8 *s = 0, *driver_name = 0;
393 int fd, clear_driver_override = 0;
402 if (strncmp (
"auto", uio_drv_name, 5) == 0)
404 int vfio_pci_loaded = 0;
412 if (!vfio_pci_loaded)
415 "is bound to IOMMU group and " 416 "vfio-pci driver is not loaded",
421 uio_drv_name =
"vfio-pci";
426 if (vfio_pci_loaded &&
428 uio_drv_name =
"vfio-pci";
430 uio_drv_name =
"uio_pci_generic";
432 uio_drv_name =
"igb_uio";
437 "kernel VFIO or UIO driver",
445 s =
format (s,
"%v/driver%c", dev_dir_name, 0);
450 ((strcmp (
"vfio-pci", (
char *) driver_name) == 0) ||
451 (strcmp (
"uio_pci_generic", (
char *) driver_name) == 0) ||
452 (strcmp (
"igb_uio", (
char *) driver_name) == 0)))
457 dir = opendir (
"/sys/class/net");
463 "read /sys/class/net",
468 fd = socket (PF_INET, SOCK_DGRAM, 0);
475 while ((e = readdir (dir)))
478 struct ethtool_drvinfo drvinfo;
480 if (e->d_name[0] ==
'.')
485 ifr.ifr_data = (
char *) &drvinfo;
486 clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
488 drvinfo.cmd = ETHTOOL_GDRVINFO;
489 if (ioctl (fd, SIOCETHTOOL, &ifr) < 0)
492 if ((errno != ENOTSUP) && (errno != ENODEV))
498 if (strcmp ((
char *) s, drvinfo.bus_info))
502 clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
504 if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
512 if (ifr.ifr_flags & IFF_UP)
515 "interface %s is up",
525 s =
format (s,
"%v/driver/unbind%c", dev_dir_name, 0);
529 s =
format (s,
"%v/driver_override%c", dev_dir_name, 0);
530 if (access ((
char *) s, F_OK) == 0)
533 clear_driver_override = 1;
548 if (clear_driver_override)
550 s =
format (s,
"%v/driver_override%c", dev_dir_name, 0);
583 int data_len = efds ? count *
sizeof (int) : 0;
584 u8 buf[
sizeof (
struct vfio_irq_set) + data_len];
585 struct vfio_irq_info ii = { 0 };
586 struct vfio_irq_set *irq_set = (
struct vfio_irq_set *) buf;
589 ii.argsz =
sizeof (
struct vfio_irq_info);
592 if (ioctl (p->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
596 pci_log_debug (vm, p,
"%s index:%u count:%u flags: %s%s%s%s(0x%x)",
597 __func__, ii.index, ii.count,
598 ii.flags & VFIO_IRQ_INFO_EVENTFD ?
"eventfd " :
"",
599 ii.flags & VFIO_IRQ_INFO_MASKABLE ?
"maskable " :
"",
600 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ?
"automasked " :
"",
601 ii.flags & VFIO_IRQ_INFO_NORESIZE ?
"noresize " :
"",
604 if (ii.count < start + count)
611 flags |= VFIO_IRQ_SET_DATA_EVENTFD;
615 flags |= VFIO_IRQ_SET_DATA_NONE;
617 ASSERT ((flags & (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD)) !=
618 (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD));
620 irq_set->argsz =
sizeof (
struct vfio_irq_set) + data_len;
621 irq_set->index = index;
622 irq_set->start = start;
623 irq_set->count =
count;
624 irq_set->flags =
flags;
626 if (ioctl (p->
fd, VFIO_DEVICE_SET_IRQS, irq_set) < 0)
628 "[index = %u, start = %u, count = %u, " 631 index, start, count, flags);
639 int __attribute__ ((unused)) rv;
659 VFIO_IRQ_SET_ACTION_UNMASK, 0);
674 int __attribute__ ((unused)) rv;
693 int __attribute__ ((unused)) rv;
731 p->
config_fd = open ((
char *) s, O_RDWR);
748 p->
fd = open ((
char *) s, O_RDWR);
755 if (r && r->interrupt_handler)
758 if (r && r->init_function)
784 struct vfio_irq_info ii = { 0 };
785 ii.argsz =
sizeof (
struct vfio_irq_info);
786 ii.index = VFIO_PCI_INTX_IRQ_INDEX;
787 if (ioctl (p->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
790 pci_log_debug (vm, p,
"%s index:%u count:%u flags: %s%s%s%s(0x%x)",
791 __func__, ii.index, ii.count,
792 ii.flags & VFIO_IRQ_INFO_EVENTFD ?
"eventfd " :
"",
793 ii.flags & VFIO_IRQ_INFO_MASKABLE ?
"maskable " :
"",
794 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ?
"automasked " :
"",
795 ii.flags & VFIO_IRQ_INFO_NORESIZE ?
"noresize " :
"",
801 irq->
fd = eventfd (0, EFD_NONBLOCK);
842 for (i = start; i < start +
count; i++)
848 irq->
fd = eventfd (0, EFD_NONBLOCK);
893 for (i = start; i < start +
count; i++)
899 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
900 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
914 for (i = start; i < start +
count; i++)
917 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
918 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
926 struct vfio_device_info device_info = { 0 };
927 struct vfio_region_info reg = { 0 };
940 device_info.argsz =
sizeof (device_info);
941 if (ioctl (p->
fd, VFIO_DEVICE_GET_INFO, &device_info) < 0)
948 reg.argsz =
sizeof (
struct vfio_region_info);
949 reg.index = VFIO_PCI_CONFIG_REGION_INDEX;
950 if (ioctl (p->
fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
957 pci_log_debug (vm, p,
"%s region_info index:%u size:0x%lx offset:0x%lx " 958 "flags: %s%s%s(0x%x)", __func__,
959 reg.index, reg.size, reg.offset,
960 reg.flags & VFIO_REGION_INFO_FLAG_READ ?
"rd " :
"",
961 reg.flags & VFIO_REGION_INFO_FLAG_WRITE ?
"wr " :
"",
962 reg.flags & VFIO_REGION_INFO_FLAG_MMAP ?
"mmap " :
"",
969 if (device_info.flags & VFIO_DEVICE_FLAGS_RESET)
970 if (ioctl (p->
fd, VFIO_DEVICE_RESET) < 0)
977 if (r && r->interrupt_handler)
992 for (i = 0; i < pm->
n_pages; i++)
998 if (r && r->init_function)
1031 ?
"read" :
"write");
1043 u64 _size = 0, _offset = 0;
1052 struct stat stat_buf;
1056 _fd = open ((
char *) file_name, O_RDWR);
1064 if (fstat (_fd, &stat_buf) < 0)
1073 _size = stat_buf.st_size;
1078 struct vfio_region_info reg = { 0 };
1079 reg.argsz =
sizeof (
struct vfio_region_info);
1081 if (ioctl (p->
fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
1087 _offset = reg.offset;
1088 pci_log_debug (vm, p,
"%s region_info index:%u size:0x%lx offset:0x%lx " 1089 "flags: %s%s%s(0x%x)", __func__,
1090 reg.index, reg.size, reg.offset,
1091 reg.flags & VFIO_REGION_INFO_FLAG_READ ?
"rd " :
"",
1092 reg.flags & VFIO_REGION_INFO_FLAG_WRITE ?
"wr " :
"",
1093 reg.flags & VFIO_REGION_INFO_FLAG_MMAP ?
"mmap " :
"",
1113 int flags = MAP_SHARED;
1124 *result = mmap (addr, size, PROT_READ | PROT_WRITE, flags, fd,
offset);
1125 if (*result == (
void *) -1)
1190 ?
"read" :
"write");
1245 pci_log_debug (vm, p,
"open vid:0x%04x did:0x%04x driver:%s iommu_group:%d",
1255 "'uio_pci_generic' kernel module");
1298 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1309 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1359 for (i = r->supported_devices; i->
vendor_id != 0; i++)
1372 r = r->next_registration;
1383 vlib_pci_addr_t
addr, **addrv = arg;
1390 if (!
unformat (&input,
"/sys/bus/pci/devices/%U",
1406 vlib_pci_addr_t *a1 = v1;
1407 vlib_pci_addr_t *a2 = v2;
1409 if (a1->domain > a2->domain)
1411 if (a1->domain < a2->domain)
1413 if (a1->bus > a2->bus)
1415 if (a1->bus < a2->bus)
1417 if (a1->slot > a2->slot)
1419 if (a1->slot < a2->slot)
1421 if (a1->function > a2->function)
1423 if (a1->function < a2->function)
1431 vlib_pci_addr_t *addrs = 0;
1450 vlib_pci_addr_t *
addr = 0, *addrs;
1454 ASSERT (
sizeof (vlib_pci_addr_t) ==
sizeof (
u32));
static clib_error_t * linux_pci_vfio_unmask_intx(vlib_main_t *vm, linux_pci_device_t *d)
clib_error_t * pci_bus_init(vlib_main_t *vm)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
clib_error_t * vlib_pci_map_region_fixed(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, u8 *addr, void **result)
u8 * format_clib_error(u8 *s, va_list *va)
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define clib_strncmp(s1, s2, n)
static const char * sysfs_pci_dev_path
linux_pci_device_type_t type
pci_device_registration_t * pci_device_registrations
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static int directory_exists(char *path)
static clib_error_t * scan_pci_addr(void *arg, u8 *dev_dir_name, u8 *ignored)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static const char uio_drv_name[]
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
static int pci_addr_cmp(void *v1, void *v2)
vlib_buffer_main_t * buffer_main
static clib_error_t * linux_pci_uio_read_ready(clib_file_t *uf)
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
void() pci_msix_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle, u16 line)
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
#define pci_log_err(vm, dev, f,...)
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
clib_error_t * clib_sysfs_write(char *file_name, char *fmt,...)
static clib_error_t * linux_pci_vfio_error_ready(clib_file_t *uf)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
static clib_error_t * linux_pci_vfio_intx_read_ready(clib_file_t *uf)
static clib_error_t * vlib_pci_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, int *fd, u64 *size, u64 *offset)
static void pci_config_type1_little_to_host(pci_config_type1_regs_t *r)
clib_error_t * vlib_pci_map_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h, void *ptr)
#define VLIB_INIT_FUNCTION(x)
static clib_error_t * linux_pci_vfio_msix_read_ready(clib_file_t *uf)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
vlib_pci_addr_t * vlib_pci_get_all_dev_addrs()
clib_file_main_t file_main
static char * sysfs_mod_vfio_noiommu
linux_pci_irq_t * msix_irqs
#define clib_error_create(args...)
clib_error_t * linux_vfio_group_get_device_fd(vlib_pci_addr_t *addr, int *fdp, int *is_noiommu)
pci_config_type1_regs_t config1
vlib_pci_dev_handle_t handle
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_main_t *vm, vlib_pci_addr_t *addr, clib_error_t **error)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static const char * sysfs_pci_drv_path
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
clib_error_t * vlib_pci_bind_to_uio(vlib_main_t *vm, vlib_pci_addr_t *addr, char *uio_drv_name)
static clib_error_t * scan_uio_dir(void *arg, u8 *path_name, u8 *file_name)
#define clib_error_return_unix(e, args...)
#define pool_put(P, E)
Free an object E in pool P.
vlib_buffer_pool_t * buffer_pools
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
linux_pci_main_t linux_pci_main
clib_error_t * linux_pci_init(vlib_main_t *vm)
void init_device_from_registered(vlib_main_t *vm, vlib_pci_device_info_t *di)
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
u32 vlib_pci_dev_handle_t
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
static void pci_config_type0_little_to_host(pci_config_type0_regs_t *r)
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define clib_strncpy(d, s, n)
#define VLIB_PCI_DEVICE_INFO_F_NOIOMMU
#define vec_free(V)
Free vector's memory (no header).
#define pci_log_debug(vm, dev, f,...)
clib_error_t * vlib_pci_read_write_io(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword offset, void *data, u32 length)
void() pci_intx_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle)
clib_error_t * vlib_pci_disable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
pci_msix_handler_function_t * msix_handler
vlib_pci_addr_t * vlib_pci_get_addr(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static clib_error_t * vfio_set_irqs(vlib_main_t *vm, linux_pci_device_t *p, u32 index, u32 start, u32 count, u32 flags, int *efds)
manual_print typedef address
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
static clib_error_t * linux_pci_uio_error_ready(clib_file_t *uf)
pci_config_type0_regs_t config0
#define clib_error_report(e)
clib_error_t * vfio_map_physmem_page(vlib_main_t *vm, void *addr)
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
static void * clib_mem_alloc(uword size)
static vlib_main_t * vlib_get_main(void)
template key/value backing page structure
static clib_error_t * vlib_pci_map_region_int(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, u8 *addr, void **result)
static clib_error_t * add_device_vfio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
uword unformat_vlib_pci_addr(unformat_input_t *input, va_list *args)
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
#define clib_unix_warning(format, args...)
#define clib_error_free(e)
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
clib_error_t * vlib_pci_map_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, void **result)
clib_file_function_t * error_function
linux_pci_region_t * regions
pci_intx_handler_function_t * intx_handler
struct clib_bihash_value offset
template key/value backing page structure
int vlib_pci_supports_virtual_addr_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h)
struct _pci_device_registration pci_device_registration_t
#define vec_foreach(var, vec)
Vector iterator.
static clib_error_t * add_device_uio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
linux_pci_device_t * linux_pci_devices
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
pci_config_header_t header
u8 * clib_sysfs_link_to_name(char *link)
static linux_pci_device_t * linux_pci_get_device(vlib_pci_dev_handle_t h)