22 #include <sys/ioctl.h> 23 #include <sys/socket.h> 26 #include <sys/types.h> 28 #include <netinet/in.h> 31 #include <linux/if_arp.h> 32 #include <linux/if_tun.h> 60 #define VHOST_USER_TX_COPY_THRESHOLD (VHOST_USER_COPY_ARRAY_N - 200) 64 #define foreach_vhost_user_tx_func_error \ 66 _(NOT_READY, "vhost vring not ready") \ 67 _(DOWN, "vhost interface is down") \ 68 _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \ 69 _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \ 70 _(MMAP_FAIL, "mmap failure") \ 71 _(INDIRECT_OVERFLOW, "indirect descriptor table overflow") 75 #define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f, 87 static __clib_unused
u8 *
91 u32 show_dev_instance = ~0;
97 if (show_dev_instance != ~0)
98 i = show_dev_instance;
100 s =
format (s,
"VirtualEthernet0/0/%d", i);
104 static __clib_unused
int 118 vu_log_debug (vui,
"renumbered vhost-user interface dev_instance %d to %d",
168 hdr_desc = &rxvq->
desc[desc_current];
179 if (!(rxvq->
desc[desc_current].
flags & VIRTQ_DESC_F_NEXT) &&
180 !(rxvq->
desc[desc_current].
flags & VIRTQ_DESC_F_INDIRECT))
190 u16 copy_len,
u32 * map_hint)
192 void *dst0, *dst1, *dst2, *dst3;
237 virtio_net_hdr_t * hdr)
240 int is_ip4 = b->
flags & VNET_BUFFER_F_IS_IP4;
243 ASSERT (!(is_ip4 && is_ip6));
245 if (b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
255 if (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
257 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
264 else if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
266 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
275 if (b->
flags & VNET_BUFFER_F_GSO)
277 if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
280 (vui->
features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_TSO4)))
283 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
286 (vui->
features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_TSO6)))
289 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
292 else if ((vui->
features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_UFO)) &&
293 (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
296 hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
303 u16 * n_descs_processed,
u8 chained,
307 vring_packed_desc_t *desc_table = rxvq->
packed_desc;
314 flags = desc_table[last_used_idx & rxvq->
qsz_mask].flags |
317 flags = desc_table[last_used_idx & rxvq->
qsz_mask].flags &
322 for (desc_idx = 1; desc_idx < *n_descs_processed; desc_idx++)
335 *n_descs_processed = 0;
339 vring_packed_desc_t *desc_table = rxvq->
packed_desc;
368 u32 desc_current = last_avail_idx & rxvq->
qsz_mask;
369 vring_packed_desc_t *hdr_desc = 0;
388 if (!(rxvq->
packed_desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
389 !(rxvq->
packed_desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT))
416 vring_packed_desc_t *desc_table;
418 u16 desc_head, desc_index, desc_len;
419 u16 n_descs_processed;
420 u8 indirect, chained;
424 rxvq = &vui->vrings[qid];
427 error = VHOST_USER_TX_FUNC_ERROR_NONE;
430 n_descs_processed = 0;
435 uword buffer_map_addr;
438 u32 total_desc_len = 0;
459 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
470 sizeof (vring_packed_desc_t)))
472 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
475 n_entries = desc_table[desc_head].len >> 4;
480 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
488 desc_len = vui->virtio_net_hdr_sz;
489 buffer_map_addr = desc_table[desc_index].addr;
490 buffer_len = desc_table[desc_index].len;
493 virtio_net_hdr_mrg_rxbuf_t *hdr = &cpu->
tx_headers[tx_headers_len];
496 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
497 hdr->num_buffers = 1;
499 or_flags = (b0->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
500 (b0->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
501 (b0->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
505 (vui->features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_CSUM)))
512 cpy->
len = vui->virtio_net_hdr_sz;
513 cpy->
dst = buffer_map_addr;
516 buffer_map_addr += vui->virtio_net_hdr_sz;
517 buffer_len -= vui->virtio_net_hdr_sz;
540 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
546 buffer_map_addr = desc_table[desc_index].addr;
547 buffer_len = desc_table[desc_index].len;
548 total_desc_len += desc_len;
564 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
567 total_desc_len += desc_len;
568 desc_index = (desc_index + 1) & rxvq->
qsz_mask;
569 buffer_map_addr = desc_table[desc_index].addr;
570 buffer_len = desc_table[desc_index].len;
573 else if (vui->virtio_net_hdr_sz == 12)
579 virtio_net_hdr_mrg_rxbuf_t *hdr =
582 desc_table[desc_index].len = desc_len;
584 desc_head = desc_index =
596 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
600 buffer_map_addr = desc_table[desc_index].addr;
601 buffer_len = desc_table[desc_index].len;
605 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
613 cpy->
len = bytes_left;
614 cpy->
len = (cpy->
len > buffer_len) ? buffer_len : cpy->
len;
615 cpy->
dst = buffer_map_addr;
619 bytes_left -= cpy->
len;
620 buffer_len -= cpy->
len;
621 buffer_map_addr += cpy->
len;
622 desc_len += cpy->
len;
630 (current_b0->
flags & VLIB_BUFFER_NEXT_PRESENT))
644 total_desc_len += desc_len;
664 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
669 chained, frame, n_left);
681 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
699 if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry)
707 if (
PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
742 error = VHOST_USER_TX_FUNC_ERROR_DOWN;
748 error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
754 rxvq = &vui->vrings[qid];
757 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
768 error = VHOST_USER_TX_FUNC_ERROR_NONE;
774 u16 desc_head, desc_index, desc_len;
776 uword buffer_map_addr;
794 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
798 desc_table = rxvq->
desc;
799 desc_head = desc_index =
809 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
817 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
823 desc_len = vui->virtio_net_hdr_sz;
824 buffer_map_addr = desc_table[desc_index].
addr;
825 buffer_len = desc_table[desc_index].
len;
829 virtio_net_hdr_mrg_rxbuf_t *hdr = &cpu->
tx_headers[tx_headers_len];
832 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
833 hdr->num_buffers = 1;
835 or_flags = (b0->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
836 (b0->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
837 (b0->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
841 && (vui->features & (1ULL << FEAT_VIRTIO_NET_F_GUEST_CSUM)))
848 cpy->
len = vui->virtio_net_hdr_sz;
849 cpy->
dst = buffer_map_addr;
853 buffer_map_addr += vui->virtio_net_hdr_sz;
854 buffer_len -= vui->virtio_net_hdr_sz;
864 desc_index = desc_table[desc_index].
next;
865 buffer_map_addr = desc_table[desc_index].
addr;
866 buffer_len = desc_table[desc_index].
len;
868 else if (vui->virtio_net_hdr_sz == 12)
870 virtio_net_hdr_mrg_rxbuf_t *hdr =
893 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
897 desc_table = rxvq->
desc;
898 desc_head = desc_index =
901 (rxvq->
desc[desc_head].
flags & VIRTQ_DESC_F_INDIRECT))
908 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
917 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
922 buffer_map_addr = desc_table[desc_index].
addr;
923 buffer_len = desc_table[desc_index].
len;
927 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
936 cpy->
len = bytes_left;
937 cpy->
len = (cpy->
len > buffer_len) ? buffer_len : cpy->
len;
938 cpy->
dst = buffer_map_addr;
942 bytes_left -= cpy->
len;
943 buffer_len -= cpy->
len;
944 buffer_map_addr += cpy->
len;
945 desc_len += cpy->
len;
954 (current_b0->
flags & VLIB_BUFFER_NEXT_PRESENT))
992 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
1010 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
1029 if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry)
1048 if (
PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
1054 thread_index, vui->sw_if_index, n_left);
1058 return frame->n_vectors;
1116 vu_log_err (vui,
"unhandled mode %d changed for if %d queue %d", mode,
1132 u8 link_old, link_new;
1140 if (link_old != link_new)
1149 .name =
"vhost-user",
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static __clib_unused u8 * format_vhost_user_interface_name(u8 *s, va_list *args)
static __clib_unused int vhost_user_name_renumber(vnet_hw_interface_t *hi, u32 new_dev_instance)
static __clib_unused clib_error_t * vhost_user_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
static_always_inline int vhost_user_vring_try_lock(vhost_user_intf_t *vui, u32 qid)
Try once to lock the vring.
u32 virtio_ring_flags
The device index.
virtio_net_hdr_mrg_rxbuf_t hdr
Length of the first data descriptor.
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
u8 runtime_data[0]
Function dependent node-runtime data.
vhost_user_tx_func_error_t
#define foreach_vhost_user_tx_func_error
vnet_interface_main_t interface_main
static __clib_unused char * vhost_user_tx_func_error_strings[]
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
#define VHOST_USER_TX_COPY_THRESHOLD
#define clib_memcpy_fast(a, b, c)
vring_used_elem_t ring[0]
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vring_packed_desc_t * packed_desc
vhost_copy_t copy[VHOST_USER_COPY_ARRAY_N]
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u16 current_length
Nbytes between current data and the end of this buffer.
#define VHOST_USER_EVENT_START_TIMER
#define VRING_AVAIL_F_NO_INTERRUPT
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
static_always_inline void vhost_user_advance_last_avail_idx(vhost_user_vring_t *vring)
static_always_inline void vhost_user_vring_lock(vhost_user_intf_t *vui, u32 qid)
Spin until the vring is successfully locked.
struct _vnet_device_class vnet_device_class_t
struct _tcp_header tcp_header_t
vhost_trace_t * current_trace
static_always_inline void vhost_user_dequeue_descs(vhost_user_vring_t *rxvq, virtio_net_hdr_mrg_rxbuf_t *hdr, u16 *n_descs_processed)
#define vu_log_debug(dev, f,...)
vnet_hw_interface_rx_mode
#define static_always_inline
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
static_always_inline u32 vhost_user_tx_copy(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
#define VRING_EVENT_F_DISABLE
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
#define VHOST_VRING_IDX_TX(qid)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VRING_USED_F_NO_NOTIFY
#define clib_error_return(e, args...)
static_always_inline u8 * format_vhost_trace(u8 *s, va_list *va)
#define VNET_DEVICE_CLASS_TX_FN(devclass)
#define clib_atomic_test_and_set(a)
VNET_DEVICE_CLASS(vhost_user_device_class)
#define VHOST_USER_COPY_ARRAY_N
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
#define clib_atomic_release(a)
static_always_inline void vhost_user_tx_trace_packed(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *rxvq)
static_always_inline void vhost_user_dequeue_chained_descs(vhost_user_vring_t *rxvq, u16 *n_descs_processed)
#define VIRTQ_DESC_F_INDIRECT
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
vhost_user_main_t vhost_user_main
static_always_inline uword vhost_user_device_class_packed(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
vlib_simple_counter_main_t * sw_if_counters
u32 node_index
Node index.
static_always_inline void vhost_user_vring_unlock(vhost_user_intf_t *vui, u32 qid)
Unlock the vring lock.
static_always_inline u8 vui_is_link_up(vhost_user_intf_t *vui)
vl_api_tunnel_mode_t mode
static_always_inline void vhost_user_mark_desc_available(vlib_main_t *vm, vhost_user_vring_t *rxvq, u16 *n_descs_processed, u8 chained, vlib_frame_t *frame, u32 n_left)
#define VHOST_VRING_IDX_RX(qid)
vnet_device_class_t vhost_user_device_class
u32 * show_dev_instance_by_real_dev_instance
u16 device_index
The interface queue index (Not the virtio vring idx)
vhost_user_intf_t * vhost_user_interfaces
#define CLIB_PREFETCH(addr, size, type)
sll srl srl sll sra u16x4 i
static __clib_unused clib_error_t * vhost_user_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
static_always_inline u64 vhost_user_is_packed_ring_supported(vhost_user_intf_t *vui)
#define VHOST_USER_EVENT_STOP_TIMER
u16 first_desc_len
Runtime queue flags.
static_always_inline u8 vhost_user_packed_desc_available(vhost_user_vring_t *vring, u16 idx)
vlib_main_t vlib_node_runtime_t * node
static_always_inline void vhost_user_advance_last_avail_table_idx(vhost_user_intf_t *vui, vhost_user_vring_t *vring, u8 chained)
#define VIRTQ_DESC_F_USED
#define vu_log_err(dev, f,...)
#define VIRTQ_DESC_F_AVAIL
#define VIRTQ_DESC_F_NEXT
static_always_inline void vhost_user_send_call(vlib_main_t *vm, vhost_user_vring_t *vq)
volatile u32 * vring_locks[VHOST_VRING_MAX_N]
vring_desc_event_t * avail_event
vlib_node_registration_t vhost_user_send_interrupt_node
(constructor) VLIB_REGISTER_NODE (vhost_user_send_interrupt_node)
static_always_inline void vhost_user_tx_trace(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *rxvq)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
static_always_inline void vnet_generic_header_offset_parser(vlib_buffer_t *b0, generic_header_offset_t *gho, int is_l2, int is_ip4, int is_ip6)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
VLIB buffer representation.
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
static_always_inline void vhost_user_log_dirty_pages_2(vhost_user_intf_t *vui, u64 addr, u64 len, u8 is_host_address)
#define vhost_user_log_dirty_ring(vui, vq, member)
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
#define CLIB_MEMORY_BARRIER()
static_always_inline void vhost_user_advance_last_used_idx(vhost_user_vring_t *vring)
virtio_net_hdr_mrg_rxbuf_t tx_headers[VLIB_FRAME_SIZE]
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
#define CLIB_CACHE_LINE_BYTES
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static u16 ip4_header_checksum(ip4_header_t *i)
static_always_inline void vhost_user_handle_tx_offload(vhost_user_intf_t *vui, vlib_buffer_t *b, virtio_net_hdr_t *hdr)