47 #include <sys/types.h> 51 #include <sys/ioctl.h> 53 #include <linux/ethtool.h> 54 #include <linux/sockios.h> 56 #include <sys/eventfd.h> 61 "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode";
63 #define pci_log_debug(vm, dev, f, ...) \ 64 vlib_log(VLIB_LOG_LEVEL_DEBUG, pci_main.log_default, "%U: " f, \ 65 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__) 66 #define pci_log_err(vm, dev, f, ...) \ 67 vlib_log(VLIB_LOG_LEVEL_ERR, pci_main.log_default, "%U: " f, \ 68 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__) 183 struct vfio_irq_info ii = { 0 };
185 ii.argsz =
sizeof (
struct vfio_irq_info);
186 ii.index = VFIO_PCI_MSIX_IRQ_INDEX;
187 if (ioctl (d->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
215 di->
addr.as_u32 = addr->as_u32;
220 f =
format (0,
"%v/config%c", dev_dir_name, 0);
221 fd = open ((
char *) f, O_RDWR);
225 fd = open ((
char *) f, O_RDONLY);
248 if (!memcmp (&di->
config0.
header, &all_ones, sizeof (all_ones)))
263 f =
format (f,
"%v/numa_node%c", dev_dir_name, 0);
275 unformat_bitmap_list, &bmp)))
282 f =
format (f,
"%v/class%c", dev_dir_name, 0);
289 f =
format (f,
"%v/vendor%c", dev_dir_name, 0);
296 f =
format (f,
"%v/device%c", dev_dir_name, 0);
303 f =
format (f,
"%v/driver%c", dev_dir_name, 0);
310 f =
format (f,
"%v/iommu_group%c", dev_dir_name, 0);
319 f =
format (f,
"%v/iommu_group/name%c", dev_dir_name, 0);
323 if (strncmp ((
char *) tmpstr,
"vfio-noiommu", 12) == 0)
333 f =
format (f,
"%v/vpd%c", dev_dir_name, 0);
334 fd = open ((
char *) f, O_RDONLY);
343 if (read (fd, &tag, 3) != 3)
346 if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91)
349 len = (tag[2] << 8) | tag[1];
352 if (read (fd, data, len) != len)
359 else if (tag[0] == 0x90)
361 else if (tag[0] == 0x91)
389 struct stat s = { 0 };
390 if (stat (path, &s) == -1)
393 return S_ISDIR (s.st_mode);
401 u8 *s = 0, *driver_name = 0;
405 int fd, clear_driver_override = 0;
414 if (strncmp (
"auto", uio_drv_name, 5) == 0)
416 int vfio_pci_loaded = 0;
424 if (!vfio_pci_loaded)
427 "is bound to IOMMU group and " 428 "vfio-pci driver is not loaded",
433 uio_drv_name =
"vfio-pci";
438 if (vfio_pci_loaded &&
440 uio_drv_name =
"vfio-pci";
442 uio_drv_name =
"uio_pci_generic";
444 uio_drv_name =
"igb_uio";
449 "kernel VFIO or UIO driver",
457 s =
format (s,
"%v/driver%c", dev_dir_name, 0);
462 ((strcmp (
"vfio-pci", (
char *) driver_name) == 0) ||
463 (strcmp (
"uio_pci_generic", (
char *) driver_name) == 0) ||
464 (strcmp (
"igb_uio", (
char *) driver_name) == 0)))
469 dir = opendir (
"/sys/class/net");
475 "read /sys/class/net",
480 fd = socket (PF_INET, SOCK_DGRAM, 0);
487 while ((e = readdir (dir)))
490 struct ethtool_drvinfo drvinfo;
492 if (e->d_name[0] ==
'.')
497 ifr.ifr_data = (
char *) &drvinfo;
498 clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
500 drvinfo.cmd = ETHTOOL_GDRVINFO;
501 if (ioctl (fd, SIOCETHTOOL, &ifr) < 0)
504 if ((errno != ENOTSUP) && (errno != ENODEV))
510 if (strcmp ((
char *) s, drvinfo.bus_info))
514 clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
516 if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
524 if (ifr.ifr_flags & IFF_UP)
527 "interface %s is up",
537 s =
format (s,
"%v/driver/unbind%c", dev_dir_name, 0);
541 s =
format (s,
"%v/driver_override%c", dev_dir_name, 0);
542 if (access ((
char *) s, F_OK) == 0)
545 clear_driver_override = 1;
560 if (clear_driver_override)
562 s =
format (s,
"%v/driver_override%c", dev_dir_name, 0);
595 int data_len = efds ? count *
sizeof (int) : 0;
596 u8 buf[
sizeof (
struct vfio_irq_set) + data_len];
597 struct vfio_irq_info ii = { 0 };
598 struct vfio_irq_set *irq_set = (
struct vfio_irq_set *) buf;
601 ii.argsz =
sizeof (
struct vfio_irq_info);
604 if (ioctl (p->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
608 pci_log_debug (vm, p,
"%s index:%u count:%u flags: %s%s%s%s(0x%x)",
609 __func__, ii.index, ii.count,
610 ii.flags & VFIO_IRQ_INFO_EVENTFD ?
"eventfd " :
"",
611 ii.flags & VFIO_IRQ_INFO_MASKABLE ?
"maskable " :
"",
612 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ?
"automasked " :
"",
613 ii.flags & VFIO_IRQ_INFO_NORESIZE ?
"noresize " :
"",
616 if (ii.count < start + count)
623 flags |= VFIO_IRQ_SET_DATA_EVENTFD;
627 flags |= VFIO_IRQ_SET_DATA_NONE;
629 ASSERT ((flags & (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD)) !=
630 (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD));
632 irq_set->argsz =
sizeof (
struct vfio_irq_set) + data_len;
633 irq_set->index = index;
634 irq_set->start = start;
635 irq_set->count =
count;
636 irq_set->flags =
flags;
638 if (ioctl (p->
fd, VFIO_DEVICE_SET_IRQS, irq_set) < 0)
640 "[index = %u, start = %u, count = %u, " 643 index, start, count, flags);
651 int __attribute__ ((unused)) rv;
671 VFIO_IRQ_SET_ACTION_UNMASK, 0);
686 int __attribute__ ((unused)) rv;
705 int __attribute__ ((unused)) rv;
743 p->
config_fd = open ((
char *) s, O_RDWR);
760 p->
fd = open ((
char *) s, O_RDWR);
767 if (r && r->interrupt_handler)
770 if (r && r->init_function)
796 struct vfio_irq_info ii = { 0 };
797 ii.argsz =
sizeof (
struct vfio_irq_info);
798 ii.index = VFIO_PCI_INTX_IRQ_INDEX;
799 if (ioctl (p->
fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
802 pci_log_debug (vm, p,
"%s index:%u count:%u flags: %s%s%s%s(0x%x)",
803 __func__, ii.index, ii.count,
804 ii.flags & VFIO_IRQ_INFO_EVENTFD ?
"eventfd " :
"",
805 ii.flags & VFIO_IRQ_INFO_MASKABLE ?
"maskable " :
"",
806 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ?
"automasked " :
"",
807 ii.flags & VFIO_IRQ_INFO_NORESIZE ?
"noresize " :
"",
813 irq->
fd = eventfd (0, EFD_NONBLOCK);
854 for (i = start; i < start +
count; i++)
860 irq->
fd = eventfd (0, EFD_NONBLOCK);
905 for (i = start; i < start +
count; i++)
911 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
912 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
926 for (i = start; i < start +
count; i++)
929 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
930 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
938 struct vfio_device_info device_info = { 0 };
939 struct vfio_region_info reg = { 0 };
952 device_info.argsz =
sizeof (device_info);
953 if (ioctl (p->
fd, VFIO_DEVICE_GET_INFO, &device_info) < 0)
960 reg.argsz =
sizeof (
struct vfio_region_info);
961 reg.index = VFIO_PCI_CONFIG_REGION_INDEX;
962 if (ioctl (p->
fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
969 pci_log_debug (vm, p,
"%s region_info index:%u size:0x%lx offset:0x%lx " 970 "flags: %s%s%s(0x%x)", __func__,
971 reg.index, reg.size, reg.offset,
972 reg.flags & VFIO_REGION_INFO_FLAG_READ ?
"rd " :
"",
973 reg.flags & VFIO_REGION_INFO_FLAG_WRITE ?
"wr " :
"",
974 reg.flags & VFIO_REGION_INFO_FLAG_MMAP ?
"mmap " :
"",
981 if (device_info.flags & VFIO_DEVICE_FLAGS_RESET)
982 if (ioctl (p->
fd, VFIO_DEVICE_RESET) < 0)
989 if (r && r->interrupt_handler)
1004 for (i = 0; i < pm->
n_pages; i++)
1010 if (r && r->init_function)
1043 ?
"read" :
"write");
1055 u64 _size = 0, _offset = 0;
1064 struct stat stat_buf;
1068 _fd = open ((
char *) file_name, O_RDWR);
1076 if (fstat (_fd, &stat_buf) < 0)
1085 _size = stat_buf.st_size;
1090 struct vfio_region_info reg = { 0 };
1091 reg.argsz =
sizeof (
struct vfio_region_info);
1093 if (ioctl (p->
fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
1099 _offset = reg.offset;
1100 pci_log_debug (vm, p,
"%s region_info index:%u size:0x%lx offset:0x%lx " 1101 "flags: %s%s%s(0x%x)", __func__,
1102 reg.index, reg.size, reg.offset,
1103 reg.flags & VFIO_REGION_INFO_FLAG_READ ?
"rd " :
"",
1104 reg.flags & VFIO_REGION_INFO_FLAG_WRITE ?
"wr " :
"",
1105 reg.flags & VFIO_REGION_INFO_FLAG_MMAP ?
"mmap " :
"",
1125 int flags = MAP_SHARED;
1136 *result = mmap (addr, size, PROT_READ | PROT_WRITE, flags, fd,
offset);
1137 if (*result == (
void *) -1)
1202 ?
"read" :
"write");
1257 pci_log_debug (vm, p,
"open vid:0x%04x did:0x%04x driver:%s iommu_group:%d",
1267 "'uio_pci_generic' kernel module");
1310 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1321 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1371 for (i = r->supported_devices; i->
vendor_id != 0; i++)
1384 r = r->next_registration;
1395 vlib_pci_addr_t
addr, **addrv = arg;
1402 if (!
unformat (&input,
"/sys/bus/pci/devices/%U",
1418 vlib_pci_addr_t *a1 = v1;
1419 vlib_pci_addr_t *a2 = v2;
1421 if (a1->domain > a2->domain)
1423 if (a1->domain < a2->domain)
1425 if (a1->bus > a2->bus)
1427 if (a1->bus < a2->bus)
1429 if (a1->slot > a2->slot)
1431 if (a1->slot < a2->slot)
1433 if (a1->function > a2->function)
1435 if (a1->function < a2->function)
1443 vlib_pci_addr_t *addrs = 0;
1462 vlib_pci_addr_t *
addr = 0, *addrs;
1466 ASSERT (
sizeof (vlib_pci_addr_t) ==
sizeof (
u32));
static clib_error_t * linux_pci_vfio_unmask_intx(vlib_main_t *vm, linux_pci_device_t *d)
clib_error_t * pci_bus_init(vlib_main_t *vm)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
clib_error_t * vlib_pci_map_region_fixed(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, u8 *addr, void **result)
u8 * format_clib_error(u8 *s, va_list *va)
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define clib_strncmp(s1, s2, n)
static const char * sysfs_pci_dev_path
linux_pci_device_type_t type
pci_device_registration_t * pci_device_registrations
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static int directory_exists(char *path)
static clib_error_t * scan_pci_addr(void *arg, u8 *dev_dir_name, u8 *ignored)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static const char uio_drv_name[]
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
static int pci_addr_cmp(void *v1, void *v2)
vlib_buffer_main_t * buffer_main
static clib_error_t * linux_pci_uio_read_ready(clib_file_t *uf)
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
void() pci_msix_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle, u16 line)
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
#define pci_log_err(vm, dev, f,...)
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
clib_error_t * clib_sysfs_write(char *file_name, char *fmt,...)
static clib_error_t * linux_pci_vfio_error_ready(clib_file_t *uf)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
static clib_error_t * linux_pci_vfio_intx_read_ready(clib_file_t *uf)
static clib_error_t * vlib_pci_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, int *fd, u64 *size, u64 *offset)
static void pci_config_type1_little_to_host(pci_config_type1_regs_t *r)
clib_error_t * vlib_pci_map_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h, void *ptr)
#define VLIB_INIT_FUNCTION(x)
static clib_error_t * linux_pci_vfio_msix_read_ready(clib_file_t *uf)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
vlib_pci_addr_t * vlib_pci_get_all_dev_addrs()
clib_file_main_t file_main
static char * sysfs_mod_vfio_noiommu
linux_pci_irq_t * msix_irqs
#define clib_error_create(args...)
clib_error_t * linux_vfio_group_get_device_fd(vlib_pci_addr_t *addr, int *fdp, int *is_noiommu)
pci_config_type1_regs_t config1
vlib_pci_dev_handle_t handle
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_main_t *vm, vlib_pci_addr_t *addr, clib_error_t **error)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static const char * sysfs_pci_drv_path
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
clib_error_t * vlib_pci_bind_to_uio(vlib_main_t *vm, vlib_pci_addr_t *addr, char *uio_drv_name)
static clib_error_t * scan_uio_dir(void *arg, u8 *path_name, u8 *file_name)
#define clib_error_return_unix(e, args...)
#define pool_put(P, E)
Free an object E in pool P.
vlib_buffer_pool_t * buffer_pools
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
linux_pci_main_t linux_pci_main
clib_error_t * linux_pci_init(vlib_main_t *vm)
void init_device_from_registered(vlib_main_t *vm, vlib_pci_device_info_t *di)
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
u32 vlib_pci_dev_handle_t
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
static void pci_config_type0_little_to_host(pci_config_type0_regs_t *r)
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define clib_strncpy(d, s, n)
#define VLIB_PCI_DEVICE_INFO_F_NOIOMMU
sll srl srl sll sra u16x4 i
#define vec_free(V)
Free vector's memory (no header).
#define pci_log_debug(vm, dev, f,...)
clib_error_t * vlib_pci_read_write_io(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword offset, void *data, u32 length)
void() pci_intx_handler_function_t(vlib_main_t *vm, vlib_pci_dev_handle_t handle)
clib_error_t * vlib_pci_disable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
pci_msix_handler_function_t * msix_handler
vlib_pci_addr_t * vlib_pci_get_addr(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static clib_error_t * vfio_set_irqs(vlib_main_t *vm, linux_pci_device_t *p, u32 index, u32 start, u32 count, u32 flags, int *efds)
manual_print typedef address
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
static clib_error_t * linux_pci_uio_error_ready(clib_file_t *uf)
pci_config_type0_regs_t config0
#define clib_error_report(e)
clib_error_t * vfio_map_physmem_page(vlib_main_t *vm, void *addr)
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
static void * clib_mem_alloc(uword size)
static vlib_main_t * vlib_get_main(void)
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
template key/value backing page structure
static clib_error_t * vlib_pci_map_region_int(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 bar, u8 *addr, void **result)
static clib_error_t * add_device_vfio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
uword unformat_vlib_pci_addr(unformat_input_t *input, va_list *args)
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
#define clib_unix_warning(format, args...)
#define clib_error_free(e)
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
clib_error_t * vlib_pci_map_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, void **result)
clib_file_function_t * error_function
linux_pci_region_t * regions
pci_intx_handler_function_t * intx_handler
struct clib_bihash_value offset
template key/value backing page structure
int vlib_pci_supports_virtual_addr_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h)
struct _pci_device_registration pci_device_registration_t
#define vec_foreach(var, vec)
Vector iterator.
static clib_error_t * add_device_uio(vlib_main_t *vm, linux_pci_device_t *p, vlib_pci_device_info_t *di, pci_device_registration_t *r)
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
linux_pci_device_t * linux_pci_devices
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
pci_config_header_t header
u8 * clib_sysfs_link_to_name(char *link)
static linux_pci_device_t * linux_pci_get_device(vlib_pci_dev_handle_t h)