27 #define foreach_dpdk_tx_func_error \ 28 _(BAD_RETVAL, "DPDK tx function returned an error") \ 29 _(RING_FULL, "Tx packet drops (ring full)") \ 30 _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \ 31 _(REPL_FAIL, "Tx packet drops (replication failure)") 35 #define _(f,s) DPDK_TX_FUNC_ERROR_##f, 41 #ifndef CLIB_MULTIARCH_VARIANT 55 error = rte_eth_dev_default_mac_addr_set (xd->
device_index,
56 (
struct ether_addr *) address);
71 static struct rte_mbuf *
75 struct rte_mbuf **mbufs = 0, *s, *d;
77 unsigned socket_id = rte_socket_id ();
85 if (rte_pktmbuf_alloc_bulk (dm->
pktmbuf_pools[socket_id], mbufs, nb_segs))
92 d->nb_segs = s->nb_segs;
93 d->data_len = s->data_len;
94 d->pkt_len = s->pkt_len;
95 d->data_off = s->data_off;
96 clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
98 for (i = 1; i < nb_segs; i++)
103 d->data_len = s->data_len;
105 RTE_PKTMBUF_HEADROOM + s->data_len);
131 sizeof (buffer[0]) -
sizeof (buffer->
pre_data));
141 struct rte_mbuf *mb, *first_mb, *last_mb;
149 rte_pktmbuf_reset (mb);
154 rte_pktmbuf_reset (mb);
159 first_mb->nb_segs = 1;
195 struct rte_mbuf **tx_vector)
206 ring =
vec_header (tx_vector,
sizeof (*ring));
225 ASSERT (n_packets < xd->nb_tx_desc);
243 while (__sync_lock_test_and_set (xd->
lockp[queue_id], 1))
245 queue_id = (queue_id + 1) % xd->
tx_q_used;
257 &tx_vector[tx_tail], tx_head - tx_tail);
258 rv = rte_ring_sp_enqueue_burst (hqos->
swq,
259 (
void **) &tx_vector[tx_tail],
260 (uint16_t) (tx_head - tx_tail), 0);
268 (uint16_t) (tx_head - tx_tail));
277 *xd->
lockp[queue_id] = 0;
295 n_packets -= (uint16_t) rv;
297 while (rv && n_packets && (n_retry > 0));
319 struct rte_mbuf *mb_new;
328 DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
341 u32 ip_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
342 u32 tcp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
343 u32 udp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
344 int is_ip4 = b->
flags & VNET_BUFFER_F_IS_IP4;
348 if (
PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum) == 0))
354 mb->outer_l3_len = 0;
355 mb->outer_l2_len = 0;
356 ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
357 ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
358 ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
359 ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
360 mb->ol_flags |= ol_flags;
365 rte_net_intel_cksum_flags_prepare (mb, ol_flags);
382 u32 n_packets = f->n_vectors;
385 struct rte_mbuf **tx_vector;
387 u16 nb_tx_desc = xd->nb_tx_desc;
398 tx_vector = xd->tx_vectors[queue_id];
399 ring =
vec_header (tx_vector,
sizeof (*ring));
417 u32 bi0 = from[n_packets];
420 rte_pktmbuf_free (mb0);
442 i = ring->
tx_head % nb_tx_desc;
446 u32 bi0, bi1, bi2, bi3;
447 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
496 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
497 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
498 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
516 tx_vector[i++ % nb_tx_desc] = mb0;
518 tx_vector[i++ % nb_tx_desc] = mb1;
520 tx_vector[i++ % nb_tx_desc] = mb2;
522 tx_vector[i++ % nb_tx_desc] = mb3;
528 tx_vector[i++ % nb_tx_desc] = mb0;
529 tx_vector[i++ % nb_tx_desc] = mb1;
530 tx_vector[i++ % nb_tx_desc] = mb2;
531 tx_vector[i++ % nb_tx_desc] = mb3;
536 tx_vector[i++] = mb0;
537 tx_vector[i++] = mb1;
538 tx_vector[i++] = mb2;
539 tx_vector[i++] = mb3;
561 struct rte_mbuf *mb0;
582 tx_vector[i % nb_tx_desc] = mb0;
599 tx_pkts = n_on_ring - n_packets;
618 rte_pktmbuf_free (tx_vector[ring->
tx_tail + n_packets]);
631 _vec_len (dm->
recycle[my_cpu]) = 0;
639 #ifndef CLIB_MULTIARCH_VARIANT 705 if (node_index == ~0)
738 if ((xd->
pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->
pmd != VNET_DPDK_PMD_I40EVF))
751 vlan_offload = rte_eth_dev_get_vlan_offload (xd->
device_index);
752 vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
754 if ((r = rte_eth_dev_set_vlan_offload (xd->
device_index, vlan_offload)))
802 static void __clib_constructor
813 #define UP_DOWN_FLAG_EVENT 1 815 #ifndef CLIB_MULTIARCH_VARIANT 822 uword *event_data = 0;
840 sw_if_index = event_data[0];
841 flags = event_data[1];
863 .name =
"admin-up-down-process",
864 .process_log2_n_stack_bytes = 17,
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
uword( vlib_node_function_t)(struct vlib_main_t *vm, struct vlib_node_runtime_t *node, struct vlib_frame_t *frame)
sll srl srl sll sra u16x4 i
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
#define DPDK_DEVICE_FLAG_PMD_INIT_FAIL
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
#define VLIB_BUFFER_RECYCLE
vnet_main_t * vnet_get_main(void)
vnet_interface_main_t interface_main
vnet_device_class_t dpdk_device_class
#define DPDK_DEVICE_FLAG_TX_OFFLOAD
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
static f64 vlib_time_now(vlib_main_t *vm)
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **tx_vector)
static char * dpdk_tx_func_error_strings[]
#define VLIB_BUFFER_PRE_DATA_SIZE
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, u32 buffer_index, vlib_buffer_t *buffer)
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
#define DPDK_DEVICE_FLAG_HQOS
u32 per_interface_next_index
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
#define VNET_HW_INTERFACE_FLAG_LINK_UP
struct rte_eth_xstat * last_cleared_xstats
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
format_function_t format_dpdk_tx_dma_trace
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
VNET_DEVICE_CLASS(af_packet_device_class)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define DPDK_DEVICE_FLAG_PMD
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
#define DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM
struct rte_eth_stats stats
#define VLIB_BUFFER_NEXT_PRESENT
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
#define static_always_inline
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
#define VLIB_BUFFER_REPL_FAIL
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
uword CLIB_MULTIARCH_FN() dpdk_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static struct rte_mbuf * dpdk_replicate_packet_mb(vlib_buffer_t *b)
static vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
A collection of simple counters.
dpdk_device_hqos_per_worker_thread_t * hqos_wt
static void __clib_constructor dpdk_interface_tx_multiarch_select(void)
vnet_hw_interface_t * hw_interfaces
#define rte_mbuf_from_vlib_buffer(x)
void dpdk_device_start(dpdk_device_t *xd)
u16 current_length
Nbytes between current data and the end of this buffer.
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
struct vnet_sub_interface_t::@180::@181::@183 flags
static void pcap_add_buffer(pcap_main_t *pm, vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
vlib_simple_counter_main_t * sw_if_counters
u32 node_index
Node index.
#define DPDK_DEVICE_FLAG_ADMIN_UP
struct vnet_sub_interface_t::@180 eth
#define CLIB_PREFETCH(addr, size, type)
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
#define vec_free(V)
Free vector's memory (no header).
vlib_node_function_t __clib_weak dpdk_interface_tx_avx512
#define clib_warning(format, args...)
#define VLIB_BUFFER_IS_TRACED
#define clib_memcpy(a, b, c)
format_function_t format_dpdk_device
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
void dpdk_device_stop(dpdk_device_t *xd)
struct rte_eth_xstat * xstats
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
format_function_t format_dpdk_device_name
uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
u32 next_buffer
Next buffer for this linked-list of buffers.
struct rte_mempool ** pktmbuf_pools
#define DPDK_DEVICE_FLAG_HAVE_SUBIF
#define clib_error_report(e)
#define VLIB_NODE_FLAG_TRACE
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
dpdk_portid_t device_index
u8 n_add_refs
Number of additional references to this buffer.
static vlib_main_t * vlib_get_main(void)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
static void dpdk_clear_hw_interface_counters(u32 instance)
vlib_node_function_t __clib_weak dpdk_interface_tx_avx2
#define foreach_dpdk_tx_func_error
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u8 admin_up_down_in_progress
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, char *address)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
#define UP_DOWN_FLAG_EVENT
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
#define VLIB_BUFFER_EXT_HDR_VALID
static_always_inline void dpdk_prefetch_buffer_by_index(vlib_main_t *vm, u32 bi)
#define VLIB_REGISTER_NODE(x,...)
static void * vec_header(void *v, uword header_bytes)
Find a user vector header.
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
struct rte_eth_stats last_cleared_stats
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
static_always_inline void dpdk_buffer_recycle(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b, u32 bi, struct rte_mbuf **mbp)
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
#define CLIB_MULTIARCH_FN(fn)