17 #include <sys/ioctl.h> 28 #define PCI_VENDOR_ID_VIRTIO 0x1af4 29 #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000 31 #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041 33 #define PCI_CAPABILITY_LIST 0x34 34 #define PCI_CAP_ID_VNDR 0x09 35 #define PCI_CAP_ID_MSIX 0x11 37 #define PCI_MSIX_ENABLE 0x8000 39 #define PCI_CONFIG_SIZE(vif) ((vif->msix_enabled == VIRTIO_MSIX_ENABLED) ? \ 76 dst = (
u8 *) dst + size;
106 src = (
u8 *) src + size;
135 if ((features >> 32) != 0)
137 clib_warning (
"only 32 bit features are allowed for legacy virtio!");
139 u32 feature = 0, guest_features = (
u32) features;
158 if (status != VIRTIO_CONFIG_STATUS_RESET)
192 u16 queue_id,
void *p)
202 if ((
u32) addr == addr2)
262 u16 max_queue_pairs = 1;
269 max_virtqueue_pairs));
274 if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
276 " should be in range [1, 0x8000]",
341 vif->
flags |= VIRTIO_IF_FLAG_ADMIN_UP;
347 vif->
flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
370 for (; line < vif->
num_rxqs; line++)
386 struct status_struct *status_entry;
387 static struct status_struct status_array[] = {
388 #define _(s,b) { .str = #s, .bit = b, }, 396 status_entry = (
struct status_struct *) &status_array;
397 while (status_entry->str)
399 if (vif->
status & status_entry->bit)
464 for (; i < 64; i += 4)
474 struct virtio_net_ctrl_hdr ctrl;
484 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
488 u16 used, next, avail;
494 avail = vring->
avail->idx;
495 struct vring_desc *d = &vring->
desc[next];
500 return VIRTIO_NET_ERR;
508 d->flags = VRING_DESC_F_NEXT;
510 d->len =
sizeof (
struct virtio_net_ctrl_hdr);
511 vring->
avail->ring[avail & mask] = next;
513 next = (next + 1) & mask;
517 d = &vring->
desc[next];
518 d->flags = VRING_DESC_F_NEXT;
522 next = (next + 1) & mask;
526 d = &vring->
desc[next];
527 d->flags = VRING_DESC_F_WRITE;
530 d->len =
sizeof (data->
status);
531 next = (next + 1) & mask;
535 vring->
avail->idx = avail;
549 struct vring_used_elem *e = &vring->
used->ring[last & mask];
553 while (d->flags & VRING_DESC_F_NEXT)
579 virtio_net_ctrl_ack
status = VIRTIO_NET_ERR;
583 offload_hdr.
status = VIRTIO_NET_ERR;
599 virtio_net_ctrl_ack
status = VIRTIO_NET_ERR;
603 csum_offload_hdr.
status = VIRTIO_NET_ERR;
620 virtio_net_ctrl_ack
status = VIRTIO_NET_ERR;
624 gso_hdr.
status = VIRTIO_NET_ERR;
639 int csum_offload_enabled)
663 else if (csum_offload_enabled
703 virtio_net_ctrl_ack
status = VIRTIO_NET_ERR;
707 mq_hdr.
status = VIRTIO_NET_ERR;
718 if (qsz < 64 || qsz > 4096)
743 if (queue_size > 32768)
760 vring->
desc = vr.desc;
761 vring->
avail = vr.avail;
762 vring->
used = vr.used;
768 vring->
size = queue_size;
794 if (queue_size > 32768)
822 vring->
desc = vr.desc;
823 vring->
avail = vr.avail;
824 vring->
used = vr.used;
841 vring->
size = queue_size;
877 if (req_features == 0)
879 req_features = supported_features;
930 ((status & VIRTIO_CONFIG_STATUS_ACK)
931 && (status & VIRTIO_CONFIG_STATUS_DRIVER)))
942 struct virtio_pci_cap cap;
943 u8 pos, common_cfg = 0, notify_base = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
958 "error in reading the capability at", pos);
960 "error in reading the capability at [%2x]", pos);
965 u16 flags, table_size, table_size_mask = 0x07FF;
971 "error in reading the capability at [%2x]",
974 table_size = flags & table_size_mask;
976 "msix interrupt vector table-size", table_size);
995 "skipping non VNDR cap id:", cap.cap_vndr);
1000 "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
1001 pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
1002 switch (cap.cfg_type)
1025 if (common_cfg == 0 || notify_base == 0 || dev_cfg == 0 || isr == 0)
1048 args->
rv = VNET_API_ERROR_UNSUPPORTED;
1055 args->
rv = VNET_API_ERROR_INIT_FAILED;
1067 "support VIRTIO_RING_F_INDIRECT_DESC features");
1072 "support VIRTIO_NET_F_MRG_RXBUF features");
1081 if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
1083 args->
rv = VNET_API_ERROR_UNSUPPORTED;
1085 "error encountered: Device doesn't support requested features");
1097 rnd = (
u32) (now * 1e6);
1100 memcpy (vif->
mac_addr + 2, &rnd, sizeof (rnd));
1113 args->
rv = VNET_API_ERROR_EXCEEDED_NUMBER_OF_RANGES_CAPACITY;
1122 "error MSIX lines (%u) <= Number of RXQs (%u)",
1125 "error MSIX lines (%u) <= Number of RXQs (%u)",
1135 args->
rv = VNET_API_ERROR_INIT_FAILED;
1158 "no VPP worker thread is available");
1164 args->
rv = VNET_API_ERROR_INIT_FAILED;
1200 VIRTIO_MSI_NO_VECTOR)
1211 VIRTIO_MSI_NO_VECTOR)
1219 RX_QUEUE (i),
"msix vector is set at", j);
1241 u32 interrupt_count = 0;
1245 if (vif->pci_addr.as_u32 == args->addr)
1247 args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
1249 clib_error_return (error,
"PCI address in use");
1250 vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default,
"%U: %s",
1251 format_vlib_pci_addr, &args->addr,
1252 " PCI address in use");
1259 vif->dev_instance = vif - vim->interfaces;
1260 vif->per_interface_next_index = ~0;
1261 vif->pci_addr.as_u32 = args->addr;
1265 virtio_pci_device_ids, &
h)))
1267 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1271 vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default,
"%U: %s",
1273 "error encountered on pci device open");
1277 vif->pci_dev_handle =
h;
1280 vif->type = VIRTIO_IF_TYPE_PCI;
1295 if (interrupt_count > 1)
1300 args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1302 "error encountered on pci register msix handler 0");
1310 args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1312 "error encountered on pci register msix handler 1");
1321 vif->support_int_mode = 1;
1324 else if (interrupt_count == 1)
1333 "error encountered on pci register interrupt handler");
1336 vif->support_int_mode = 1;
1345 vif->support_int_mode = 0;
1363 vif->dev_instance, vif->mac_addr,
1369 args->rv = VNET_API_ERROR_INVALID_REGISTRATION;
1371 "error encountered on ethernet register interface");
1394 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1402 args->checksum_offload_enabled);
1415 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1416 args->error = error;
1426 if (vif->
type != VIRTIO_IF_TYPE_PCI)
1427 return VNET_API_ERROR_INVALID_INTERFACE;
1500 memset (vif, 0,
sizeof (*vif));
1509 int checksum_offload_enabled,
1510 int offloads_disabled)
1512 if (vif->
type != VIRTIO_IF_TYPE_PCI)
1513 return VNET_API_ERROR_INVALID_INTERFACE;
1517 else if (checksum_offload_enabled)
1519 else if (offloads_disabled)
static u8 virtio_pci_legacy_get_isr(vlib_main_t *vm, virtio_if_t *vif)
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
#define vec_foreach_index(var, v)
Iterate over vector indices.
static int virtio_pci_disable_offload(vlib_main_t *vm, virtio_if_t *vif)
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
void virtio_set_net_hdr_size(virtio_if_t *vif)
clib_error_t * virtio_pci_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
static int virtio_pci_legacy_setup_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id, void *p)
static u32 virtio_pci_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
static void virtio_pci_legacy_del_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
#define VIRTIO_PCI_CAP_ISR_CFG
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_main_t * vnet_get_main(void)
static u16 virtio_pci_legacy_get_queue_num(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
#define VIRTIO_MSI_QUEUE_VECTOR
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define VIRTIO_PCI_STATUS
static void virtio_pci_irq_queue_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
#define CLIB_MEMORY_STORE_BARRIER()
#define VIRTIO_PCI_QUEUE_PFN
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static u8 virtio_pci_legacy_get_status(vlib_main_t *vm, virtio_if_t *vif)
static f64 vlib_time_now(vlib_main_t *vm)
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
static clib_error_t * vlib_pci_intr_disable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t virtio_device_class
static clib_error_t * virtio_pci_device_init(vlib_main_t *vm, virtio_if_t *vif, virtio_pci_create_if_args_t *args)
static heap_elt_t * last(heap_header_t *h)
void virtio_vring_set_numa_node(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
#define STRUCT_OFFSET_OF(t, f)
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
#define VIRTIO_MSI_CONFIG_VECTOR
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
#define VIRTIO_PCI_HOST_FEATURES
static int virtio_pci_enable_gso(vlib_main_t *vm, virtio_if_t *vif)
static u64 virtio_pci_legacy_get_host_features(vlib_main_t *vm, virtio_if_t *vif)
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS
static void virtio_pci_set_mac(vlib_main_t *vm, virtio_if_t *vif)
static int virtio_pci_send_ctrl_msg(vlib_main_t *vm, virtio_if_t *vif, struct virtio_ctrl_msg *data, u32 len)
void virtio_pci_create_if(vlib_main_t *vm, virtio_pci_create_if_args_t *args)
static void clib_spinlock_free(clib_spinlock_t *p)
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define clib_memcpy(d, s, n)
static void virtio_pci_irq_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h)
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
static u32 virtio_pci_legacy_set_guest_features(vlib_main_t *vm, virtio_if_t *vif, u64 features)
#define VIRTIO_PCI_CAP_COMMON_CFG
struct vring_avail * avail
int virtio_pci_reset_device(vlib_main_t *vm, virtio_if_t *vif)
u8 * format_ethernet_address(u8 *s, va_list *args)
static u8 virtio_pci_queue_size_valid(u16 qsz)
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
static u16 virtio_pci_legacy_set_queue_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec, u16 queue_id)
#define virtio_log_error(vif, f,...)
#define PCI_DEVICE_ID_VIRTIO_NIC
#define TX_QUEUE_ACCESS(X)
void device_status(vlib_main_t *vm, virtio_if_t *vif)
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
static void virtio_negotiate_features(vlib_main_t *vm, virtio_if_t *vif, u64 req_features)
vnet_hw_interface_flags_t flags
void virtio_free_rx_buffers(vlib_main_t *vm, virtio_vring_t *vring)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
#define VIRTIO_PCI_CAP_NOTIFY_CFG
static void virtio_pci_legacy_read_config(vlib_main_t *vm, virtio_if_t *vif, void *dst, int len, u32 addr)
#define virtio_log_warning(vif, f,...)
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
static void clib_spinlock_init(clib_spinlock_t *p)
static void virtio_pci_legacy_set_status(vlib_main_t *vm, virtio_if_t *vif, u8 status)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
#define VIRTIO_PCI_GUEST_FEATURES
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
#define VIRTIO_PCI_ISR_CONFIG
struct virtio_net_ctrl_hdr ctrl
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
#define pool_put(P, E)
Free an object E in pool P.
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
u32 vlib_pci_dev_handle_t
static u16 virtio_pci_legacy_set_config_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec)
void virtio_pci_read_device_feature(vlib_main_t *vm, virtio_if_t *vif)
static u32 virtio_pci_legacy_get_guest_features(vlib_main_t *vm, virtio_if_t *vif)
clib_error_t * virtio_pci_read_caps(vlib_main_t *vm, virtio_if_t *vif)
static clib_error_t * virtio_pci_get_max_virtqueue_pairs(vlib_main_t *vm, virtio_if_t *vif)
#define PCI_CAPABILITY_LIST
int virtio_pci_enable_disable_offloads(vlib_main_t *vm, virtio_if_t *vif, int gso_enabled, int checksum_offload_enabled, int offloads_disabled)
#define virtio_log_debug(vif, f,...)
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
#define VIRTIO_PCI_VRING_ALIGN
virtio_vring_t * rxq_vrings
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
#define VIRTIO_PCI_QUEUE_NOTIFY
#define VIRTIO_PCI_ISR_INTR
static void vlib_physmem_free(vlib_main_t *vm, void *p)
#define VIRTIO_RING_FLAG_MASK_INT
sll srl srl sll sra u16x4 i
#define vec_free(V)
Free vector's memory (no header).
#define clib_warning(format, args...)
static u16 virtio_pci_is_link_up(vlib_main_t *vm, virtio_if_t *vif)
void debug_device_config_space(vlib_main_t *vm, virtio_if_t *vif)
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
static void virtio_pci_legacy_write_config(vlib_main_t *vm, virtio_if_t *vif, void *src, int len, u32 addr)
clib_error_t * virtio_pci_control_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
static uword round_pow2(uword x, uword pow2)
void virtio_pci_legacy_notify_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
int virtio_pci_delete_if(vlib_main_t *vm, virtio_if_t *vif)
void virtio_free_used_desc(vlib_main_t *vm, virtio_vring_t *vring)
#define PCI_DEVICE_ID_VIRTIO_NIC_MODERN
#define PCI_CONFIG_SIZE(vif)
static int virtio_pci_offloads(vlib_main_t *vm, virtio_if_t *vif, int gso_enabled, int csum_offload_enabled)
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
#define VIRTIO_FEATURE(X)
static int virtio_pci_enable_checksum_offload(vlib_main_t *vm, virtio_if_t *vif)
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
virtio_main_t virtio_main
static uword is_pow2(uword x)
#define VIRTIO_PCI_QUEUE_SEL
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
#define RX_QUEUE_ACCESS(X)
virtio_net_ctrl_ack status
#define VIRTIO_PCI_CAP_DEVICE_CFG
#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
VLIB buffer representation.
static void virtio_pci_irq_config_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
#define clib_error_free(e)
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
static u8 virtio_pci_legacy_reset(vlib_main_t *vm, virtio_if_t *vif)
virtio_vring_t * cxq_vring
static u32 random_u32(u32 *seed)
32-bit random number generator
static vlib_thread_main_t * vlib_get_thread_main()
#define CLIB_MEMORY_BARRIER()
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
#define VIRTIO_PCI_CAP_PCI_CFG
#define CLIB_CACHE_LINE_BYTES
#define PCI_VENDOR_ID_VIRTIO
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
static u32 virtio_pci_get_mac(vlib_main_t *vm, virtio_if_t *vif)
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
#define VIRTIO_NET_CTRL_MQ
#define VIRTIO_PCI_QUEUE_NUM
static int virtio_pci_enable_multiqueue(vlib_main_t *vm, virtio_if_t *vif, u16 num_queues)
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
virtio_vring_t * txq_vrings
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
static_always_inline void clib_memset_u32(void *p, u32 val, uword count)