22 #include <sys/ioctl.h> 23 #include <sys/socket.h> 26 #include <sys/types.h> 28 #include <netinet/in.h> 31 #include <linux/if_arp.h> 32 #include <linux/if_tun.h> 59 #define VHOST_USER_TX_COPY_THRESHOLD (VHOST_USER_COPY_ARRAY_N - 200) 63 #define foreach_vhost_user_tx_func_error \ 65 _(NOT_READY, "vhost vring not ready") \ 66 _(DOWN, "vhost interface is down") \ 67 _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \ 68 _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \ 69 _(MMAP_FAIL, "mmap failure") \ 70 _(INDIRECT_OVERFLOW, "indirect descriptor table overflow") 74 #define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f, 86 static __clib_unused
u8 *
90 u32 show_dev_instance = ~0;
96 if (show_dev_instance != ~0)
97 i = show_dev_instance;
99 s =
format (s,
"VirtualEthernet0/0/%d", i);
103 static __clib_unused
int 117 vu_log_debug (vui,
"renumbered vhost-user interface dev_instance %d to %d",
167 hdr_desc = &rxvq->
desc[desc_current];
178 if (!(rxvq->
desc[desc_current].
flags & VRING_DESC_F_NEXT) &&
179 !(rxvq->
desc[desc_current].
flags & VRING_DESC_F_INDIRECT))
189 u16 copy_len,
u32 * map_hint)
191 void *dst0, *dst1, *dst2, *dst3;
236 virtio_net_hdr_t * hdr)
239 int is_ip4 = b->
flags & VNET_BUFFER_F_IS_IP4;
242 ASSERT (!(is_ip4 && is_ip6));
244 if (b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
254 if (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
260 else if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
268 if (b->
flags & VNET_BUFFER_F_GSO)
270 if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
286 (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
296 u16 * n_descs_processed,
u8 chained,
300 vring_packed_desc_t *desc_table = rxvq->
packed_desc;
307 flags = desc_table[last_used_idx & rxvq->
qsz_mask].flags |
310 flags = desc_table[last_used_idx & rxvq->
qsz_mask].flags &
315 for (desc_idx = 1; desc_idx < *n_descs_processed; desc_idx++)
328 *n_descs_processed = 0;
332 vring_packed_desc_t *desc_table = rxvq->
packed_desc;
361 u32 desc_current = last_avail_idx & rxvq->
qsz_mask;
362 vring_packed_desc_t *hdr_desc = 0;
381 if (!(rxvq->
packed_desc[desc_current].flags & VRING_DESC_F_NEXT) &&
382 !(rxvq->
packed_desc[desc_current].flags & VRING_DESC_F_INDIRECT))
409 vring_packed_desc_t *desc_table;
411 u16 desc_head, desc_index, desc_len;
412 u16 n_descs_processed;
413 u8 indirect, chained;
417 rxvq = &vui->vrings[qid];
420 error = VHOST_USER_TX_FUNC_ERROR_NONE;
423 n_descs_processed = 0;
428 uword buffer_map_addr;
431 u32 total_desc_len = 0;
452 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
463 sizeof (vring_packed_desc_t)))
465 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
468 n_entries = desc_table[desc_head].len >> 4;
473 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
481 desc_len = vui->virtio_net_hdr_sz;
482 buffer_map_addr = desc_table[desc_index].addr;
483 buffer_len = desc_table[desc_index].len;
486 virtio_net_hdr_mrg_rxbuf_t *hdr = &cpu->
tx_headers[tx_headers_len];
490 hdr->num_buffers = 1;
492 or_flags = (b0->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
493 (b0->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
494 (b0->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
505 cpy->
len = vui->virtio_net_hdr_sz;
506 cpy->
dst = buffer_map_addr;
509 buffer_map_addr += vui->virtio_net_hdr_sz;
510 buffer_len -= vui->virtio_net_hdr_sz;
533 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
539 buffer_map_addr = desc_table[desc_index].addr;
540 buffer_len = desc_table[desc_index].len;
541 total_desc_len += desc_len;
557 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
560 total_desc_len += desc_len;
561 desc_index = (desc_index + 1) & rxvq->
qsz_mask;
562 buffer_map_addr = desc_table[desc_index].addr;
563 buffer_len = desc_table[desc_index].len;
566 else if (vui->virtio_net_hdr_sz == 12)
572 virtio_net_hdr_mrg_rxbuf_t *hdr =
575 desc_table[desc_index].len = desc_len;
577 desc_head = desc_index =
589 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
593 buffer_map_addr = desc_table[desc_index].addr;
594 buffer_len = desc_table[desc_index].len;
598 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
606 cpy->
len = bytes_left;
607 cpy->
len = (cpy->
len > buffer_len) ? buffer_len : cpy->
len;
608 cpy->
dst = buffer_map_addr;
612 bytes_left -= cpy->
len;
613 buffer_len -= cpy->
len;
614 buffer_map_addr += cpy->
len;
615 desc_len += cpy->
len;
623 (current_b0->
flags & VLIB_BUFFER_NEXT_PRESENT))
637 total_desc_len += desc_len;
657 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
662 chained, frame, n_left);
674 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
692 if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry)
700 if (
PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
735 error = VHOST_USER_TX_FUNC_ERROR_DOWN;
741 error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
747 rxvq = &vui->vrings[qid];
750 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
761 error = VHOST_USER_TX_FUNC_ERROR_NONE;
767 u16 desc_head, desc_index, desc_len;
769 uword buffer_map_addr;
787 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
791 desc_table = rxvq->
desc;
792 desc_head = desc_index =
802 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
810 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
816 desc_len = vui->virtio_net_hdr_sz;
817 buffer_map_addr = desc_table[desc_index].
addr;
818 buffer_len = desc_table[desc_index].
len;
822 virtio_net_hdr_mrg_rxbuf_t *hdr = &cpu->
tx_headers[tx_headers_len];
826 hdr->num_buffers = 1;
828 or_flags = (b0->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
829 (b0->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
830 (b0->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
841 cpy->
len = vui->virtio_net_hdr_sz;
842 cpy->
dst = buffer_map_addr;
846 buffer_map_addr += vui->virtio_net_hdr_sz;
847 buffer_len -= vui->virtio_net_hdr_sz;
857 desc_index = desc_table[desc_index].
next;
858 buffer_map_addr = desc_table[desc_index].
addr;
859 buffer_len = desc_table[desc_index].
len;
861 else if (vui->virtio_net_hdr_sz == 12)
863 virtio_net_hdr_mrg_rxbuf_t *hdr =
886 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
890 desc_table = rxvq->
desc;
891 desc_head = desc_index =
894 (rxvq->
desc[desc_head].
flags & VRING_DESC_F_INDIRECT))
901 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
910 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
915 buffer_map_addr = desc_table[desc_index].
addr;
916 buffer_len = desc_table[desc_index].
len;
920 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
929 cpy->
len = bytes_left;
930 cpy->
len = (cpy->
len > buffer_len) ? buffer_len : cpy->
len;
931 cpy->
dst = buffer_map_addr;
935 bytes_left -= cpy->
len;
936 buffer_len -= cpy->
len;
937 buffer_map_addr += cpy->
len;
938 desc_len += cpy->
len;
947 (current_b0->
flags & VLIB_BUFFER_NEXT_PRESENT))
985 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
1003 VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
1022 if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry)
1041 if (
PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
1047 thread_index, vui->sw_if_index, n_left);
1051 return frame->n_vectors;
1109 vu_log_err (vui,
"unhandled mode %d changed for if %d queue %d", mode,
1125 u8 link_old, link_new;
1133 if (link_old != link_new)
1142 .name =
"vhost-user",
#define VRING_EVENT_F_DISABLE
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static __clib_unused u8 * format_vhost_user_interface_name(u8 *s, va_list *args)
static __clib_unused int vhost_user_name_renumber(vnet_hw_interface_t *hi, u32 new_dev_instance)
static __clib_unused clib_error_t * vhost_user_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
static_always_inline int vhost_user_vring_try_lock(vhost_user_intf_t *vui, u32 qid)
Try once to lock the vring.
u32 virtio_ring_flags
The device index.
virtio_net_hdr_mrg_rxbuf_t hdr
Length of the first data descriptor.
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
vl_api_wireguard_peer_flags_t flags
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
u8 runtime_data[0]
Function dependent node-runtime data.
#define VIRTIO_NET_HDR_F_NEEDS_CSUM
vhost_user_tx_func_error_t
#define foreach_vhost_user_tx_func_error
vnet_interface_main_t interface_main
static __clib_unused char * vhost_user_tx_func_error_strings[]
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
#define VHOST_USER_TX_COPY_THRESHOLD
#define clib_memcpy_fast(a, b, c)
vring_used_elem_t ring[0]
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vring_packed_desc_t * packed_desc
vhost_copy_t copy[VHOST_USER_COPY_ARRAY_N]
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u16 current_length
Nbytes between current data and the end of this buffer.
#define VHOST_USER_EVENT_START_TIMER
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
static_always_inline void vhost_user_advance_last_avail_idx(vhost_user_vring_t *vring)
static_always_inline void vhost_user_vring_lock(vhost_user_intf_t *vui, u32 qid)
Spin until the vring is successfully locked.
struct _vnet_device_class vnet_device_class_t
struct _tcp_header tcp_header_t
vhost_trace_t * current_trace
static_always_inline void vhost_user_dequeue_descs(vhost_user_vring_t *rxvq, virtio_net_hdr_mrg_rxbuf_t *hdr, u16 *n_descs_processed)
#define VIRTIO_FEATURE(X)
#define vu_log_debug(dev, f,...)
vnet_hw_interface_rx_mode
#define VRING_AVAIL_F_NO_INTERRUPT
#define static_always_inline
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
static_always_inline u32 vhost_user_tx_copy(vhost_user_intf_t *vui, vhost_copy_t *cpy, u16 copy_len, u32 *map_hint)
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
#define VHOST_VRING_IDX_TX(qid)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
static_always_inline u8 * format_vhost_trace(u8 *s, va_list *va)
#define VNET_DEVICE_CLASS_TX_FN(devclass)
#define clib_atomic_test_and_set(a)
VNET_DEVICE_CLASS(vhost_user_device_class)
#define VRING_USED_F_NO_NOTIFY
#define VHOST_USER_COPY_ARRAY_N
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
#define clib_atomic_release(a)
static_always_inline void vhost_user_tx_trace_packed(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *rxvq)
static_always_inline void vhost_user_dequeue_chained_descs(vhost_user_vring_t *rxvq, u16 *n_descs_processed)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
vhost_user_main_t vhost_user_main
static_always_inline uword vhost_user_device_class_packed(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
vlib_simple_counter_main_t * sw_if_counters
u32 node_index
Node index.
static_always_inline void vhost_user_vring_unlock(vhost_user_intf_t *vui, u32 qid)
Unlock the vring lock.
static_always_inline u8 vui_is_link_up(vhost_user_intf_t *vui)
#define VIRTIO_NET_HDR_GSO_TCPV4
vl_api_tunnel_mode_t mode
static_always_inline void vhost_user_mark_desc_available(vlib_main_t *vm, vhost_user_vring_t *rxvq, u16 *n_descs_processed, u8 chained, vlib_frame_t *frame, u32 n_left)
#define VHOST_VRING_IDX_RX(qid)
vnet_device_class_t vhost_user_device_class
u32 * show_dev_instance_by_real_dev_instance
u16 device_index
The interface queue index (Not the virtio vring idx)
vhost_user_intf_t * vhost_user_interfaces
#define CLIB_PREFETCH(addr, size, type)
sll srl srl sll sra u16x4 i
static __clib_unused clib_error_t * vhost_user_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
static_always_inline u64 vhost_user_is_packed_ring_supported(vhost_user_intf_t *vui)
#define VHOST_USER_EVENT_STOP_TIMER
u16 first_desc_len
Runtime queue flags.
static_always_inline u8 vhost_user_packed_desc_available(vhost_user_vring_t *vring, u16 idx)
vlib_main_t vlib_node_runtime_t * node
static_always_inline void vhost_user_advance_last_avail_table_idx(vhost_user_intf_t *vui, vhost_user_vring_t *vring, u8 chained)
#define vu_log_err(dev, f,...)
static_always_inline void vhost_user_send_call(vlib_main_t *vm, vhost_user_vring_t *vq)
volatile u32 * vring_locks[VHOST_VRING_MAX_N]
vring_desc_event_t * avail_event
vlib_node_registration_t vhost_user_send_interrupt_node
(constructor) VLIB_REGISTER_NODE (vhost_user_send_interrupt_node)
#define VRING_DESC_F_NEXT
static_always_inline void vhost_user_tx_trace(vhost_trace_t *t, vhost_user_intf_t *vui, u16 qid, vlib_buffer_t *b, vhost_user_vring_t *rxvq)
static_always_inline void vnet_generic_header_offset_parser(vlib_buffer_t *b0, generic_header_offset_t *gho, int is_l2, int is_ip4, int is_ip6)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
#define VIRTIO_NET_HDR_GSO_NONE
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
VLIB buffer representation.
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
static_always_inline void vhost_user_log_dirty_pages_2(vhost_user_intf_t *vui, u64 addr, u64 len, u8 is_host_address)
#define vhost_user_log_dirty_ring(vui, vq, member)
#define VRING_DESC_F_USED
#define CLIB_MEMORY_BARRIER()
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
static_always_inline void vhost_user_advance_last_used_idx(vhost_user_vring_t *vring)
virtio_net_hdr_mrg_rxbuf_t tx_headers[VLIB_FRAME_SIZE]
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
#define CLIB_CACHE_LINE_BYTES
#define VIRTIO_NET_HDR_GSO_UDP
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static u16 ip4_header_checksum(ip4_header_t *i)
#define VRING_DESC_F_INDIRECT
#define VIRTIO_NET_HDR_GSO_TCPV6
static_always_inline void vhost_user_handle_tx_offload(vhost_user_intf_t *vui, vlib_buffer_t *b, virtio_net_hdr_t *hdr)
#define VRING_DESC_F_AVAIL