18 #include <sys/types.h> 33 #define foreach_virtio_tx_func_error \ 34 _(NO_FREE_SLOTS, "no free tx slots") \ 35 _(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \ 36 _(PENDING_MSGS, "pending msgs in tx ring") \ 37 _(INDIRECT_DESC_ALLOC_FAILED, "indirect descriptor allocation failed - packet drop") \ 38 _(OUT_OF_ORDER, "out-of-order buffers in used ring") \ 39 _(GSO_PACKET_DROP, "gso disabled on itf -- gso packet drop") \ 40 _(CSUM_OFFLOAD_PACKET_DROP, "checksum offload disabled on itf -- csum offload packet drop") 44 #define _(f,s) VIRTIO_TX_ERROR_##f, 59 u32 dev_instance = va_arg (*args,
u32);
60 int verbose = va_arg (*args,
int);
63 s =
format (s,
"VIRTIO interface");
75 s =
format (s,
"Unimplemented...");
91 ASSERT (n_buffers <= ring_size);
113 u16 out_of_order_count = 0;
120 struct vring_used_elem *e = &vring->
used->ring[last & mask];
122 slot = n_buffers = e->id;
124 while (e->id == (n_buffers & mask))
129 struct vring_desc *d = &vring->
desc[e->id];
131 while (d->flags & VRING_DESC_F_NEXT)
135 d = &vring->
desc[next];
139 e = &vring->
used->ring[last & mask];
142 sz, (n_buffers - slot));
144 used -= (n_buffers -
slot);
153 out_of_order_count++;
162 if (out_of_order_count)
174 if (b->
flags & VNET_BUFFER_F_IS_IP4)
180 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
182 if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
186 else if (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
197 if (b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
200 else if (b->
flags & VNET_BUFFER_F_IS_IP6)
205 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
207 if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
211 else if (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
221 if (b->
flags & VNET_BUFFER_F_IS_IP4)
227 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
229 hdr->hdr_len = gho.
hdr_sz;
230 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
239 if (b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
242 else if (b->
flags & VNET_BUFFER_F_IS_IP6)
247 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
249 hdr->hdr_len = gho.
hdr_sz;
250 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
259 u16 avail,
u16 next,
u16 mask,
int do_gso,
260 int csum_offload,
uword node_index)
264 struct vring_desc *d;
265 d = &vring->
desc[next];
268 int is_l2 = (vif->
type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI));
272 if (b->
flags & VNET_BUFFER_F_GSO)
279 VIRTIO_TX_ERROR_GSO_PACKET_DROP);
283 else if (b->
flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
284 VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
291 VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP);
315 u32 indirect_buffer = 0;
319 VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED);
325 indirect_desc->
flags |= VLIB_BUFFER_NEXT_PRESENT;
327 bi = indirect_buffer;
329 struct vring_desc *
id =
332 if (vif->
type == VIRTIO_IF_TYPE_PCI)
348 id->flags = VRING_DESC_F_NEXT;
355 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
357 id->flags = VRING_DESC_F_NEXT;
373 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
375 id->flags = VRING_DESC_F_NEXT;
386 d->len = count *
sizeof (
struct vring_desc);
387 d->flags = VRING_DESC_F_INDIRECT;
389 else if (vif->
type == VIRTIO_IF_TYPE_PCI)
393 u16 n_buffers_in_chain = 1;
400 while (b_temp->
flags & VLIB_BUFFER_NEXT_PRESENT)
402 n_buffers_in_chain++;
406 if (n_buffers_in_chain > free_desc_count)
407 return n_buffers_in_chain;
412 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
414 d->flags = VRING_DESC_F_NEXT;
417 ~(VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID);
421 count = (count + 1) & mask;
430 vring->
avail->ring[avail & mask] = next;
439 vring->
avail->ring[avail & mask] = next;
446 u16 req,
u16 next,
u32 * first_free_desc_index,
447 u16 * free_desc_count)
453 if (vring->
buffers[next & mask] == ~0)
455 if (*first_free_desc_index == ~0)
457 *first_free_desc_index = (next & mask);
459 (*free_desc_count)++;
466 if (start + *free_desc_count ==
i)
468 (*free_desc_count)++;
483 int do_gso,
int csum_offload)
489 u16 used, next, avail;
507 avail = vring->
avail->idx;
509 u16 free_desc_count = 0;
513 u32 first_free_desc_index = ~0;
516 &first_free_desc_index, &free_desc_count);
519 next = first_free_desc_index;
522 free_desc_count = sz - used;
524 while (n_left && free_desc_count)
529 avail, next, mask, do_gso, csum_offload,
542 next = (next + n_added) & mask;
546 free_desc_count -= n_added;
552 vring->
avail->idx = avail;
565 VIRTIO_TX_ERROR_NO_FREE_SLOTS);
603 if (node_index == ~0)
652 vif->
flags |= VIRTIO_IF_FLAG_ADMIN_UP;
658 vif->
flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
u32 per_interface_next_index
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
VNET_DEVICE_CLASS_TX_FN() virtio_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
static clib_error_t * virtio_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
vnet_main_t * vnet_get_main(void)
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
#define CLIB_MEMORY_STORE_BARRIER()
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u16 current_length
Nbytes between current data and the end of this buffer.
static heap_elt_t * last(heap_header_t *h)
#define STRUCT_OFFSET_OF(t, f)
struct _tcp_header tcp_header_t
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
vnet_hw_interface_rx_mode
struct vring_avail * avail
#define static_always_inline
static clib_error_t * virtio_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
vnet_hw_interface_flags_t flags
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
#define VNET_DEVICE_CLASS_TX_FN(devclass)
#define foreach_virtio_tx_func_error
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static_always_inline void virtio_find_free_desc(virtio_vring_t *vring, u16 size, u16 mask, u16 req, u16 next, u32 *first_free_desc_index, u16 *free_desc_count)
static u8 * format_virtio_tx_trace(u8 *s, va_list *args)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
static u8 * format_virtio_device(u8 *s, va_list *args)
u32 node_index
Node index.
vl_api_tunnel_mode_t mode
static void virtio_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
static_always_inline void virtio_interface_drop_inline(vlib_main_t *vm, uword node_index, u32 *buffers, u16 n, virtio_tx_func_error_t error)
virtio_vring_t * rxq_vrings
format_function_t format_virtio_device_name
#define VIRTIO_RING_FLAG_MASK_INT
sll srl srl sll sra u16x4 i
static_always_inline uword virtio_interface_tx_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, virtio_if_t *vif, int do_gso, int csum_offload)
static_always_inline void set_gso_offsets(vlib_buffer_t *b, struct virtio_net_hdr_v1 *hdr, int is_l2)
vlib_main_t vlib_node_runtime_t * node
VNET_DEVICE_CLASS(avf_device_class,)
#define VIRTIO_FEATURE(X)
static uword pointer_to_uword(const void *p)
virtio_main_t virtio_main
static vlib_main_t * vlib_get_main(void)
#define VRING_TX_OUT_OF_ORDER
static_always_inline void vnet_generic_header_offset_parser(vlib_buffer_t *b0, generic_header_offset_t *gho, int is_l2, int is_ip4, int is_ip6)
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
u32 next_buffer
Next buffer for this linked-list of buffers.
static void virtio_clear_hw_interface_counters(u32 instance)
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
VLIB buffer representation.
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
static_always_inline void set_checksum_offsets(vlib_buffer_t *b, struct virtio_net_hdr_v1 *hdr, int is_l2)
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
static_always_inline u16 add_buffer_to_slot(vlib_main_t *vm, virtio_if_t *vif, virtio_vring_t *vring, u32 bi, u16 free_desc_count, u16 avail, u16 next, u16 mask, int do_gso, int csum_offload, uword node_index)
static_always_inline void virtio_free_used_device_desc(vlib_main_t *vm, virtio_vring_t *vring, uword node_index)
static clib_error_t * virtio_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static u16 ip4_header_checksum(ip4_header_t *i)
static_always_inline void virtio_memset_ring_u32(u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
static char * virtio_tx_func_error_strings[]
virtio_vring_t * txq_vrings
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
static_always_inline void clib_memset_u32(void *p, u32 val, uword count)