26 #define foreach_dpdk_tx_func_error \ 27 _(BAD_RETVAL, "DPDK tx function returned an error") \ 28 _(PKT_DROP, "Tx packet drops (dpdk tx failure)") 32 #define _(f,s) DPDK_TX_FUNC_ERROR_##f, 53 error = rte_eth_dev_mac_addr_add (xd->
port_id,
54 (
struct rte_ether_addr *) address, 0);
56 error = rte_eth_dev_mac_addr_remove (xd->
port_id,
57 (
struct rte_ether_addr *) address);
75 error = rte_eth_dev_default_mac_addr_set (xd->
port_id, (
void *) address);
106 sizeof (buffer[0]) -
sizeof (buffer->
pre_data));
117 struct rte_mbuf *mb, *first_mb, *last_mb;
123 rte_pktmbuf_reset (mb);
125 first_mb->nb_segs = 1;
131 while (maybe_multiseg && (b->
flags & VLIB_BUFFER_NEXT_PRESENT))
136 rte_pktmbuf_reset (mb);
158 struct rte_mbuf **mb,
u32 n_left)
177 n_sent = rte_eth_tx_burst (xd->
port_id, queue_id, mb, n_left);
204 while (n_sent && n_left && (n_retry > 0));
221 u32 ip_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
222 u32 tcp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
223 u32 udp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
224 int is_ip4 = b->
flags & VNET_BUFFER_F_IS_IP4;
225 u32 tso = b->
flags & VNET_BUFFER_F_GSO;
229 if (
PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
235 mb->outer_l3_len = 0;
236 mb->outer_l2_len = 0;
237 ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
238 ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
239 ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
240 ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
241 ol_flags |= tso ? (tcp_cksum ? PKT_TX_TCP_SEG : PKT_TX_UDP_SEG) : 0;
249 mb->ol_flags |= ol_flags;
253 if (xd->
flags & DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM)
254 rte_net_intel_cksum_flags_prepare (mb, ol_flags);
269 u32 n_packets = f->n_vectors;
272 int queue_id = thread_index;
273 u32 tx_pkts = 0, all_or_flags = 0;
276 struct rte_mbuf **mb;
283 (
void **) ptd->
mbufs, n_packets,
284 -(
i32) sizeof (
struct rte_mbuf));
289 #if (CLIB_N_PREFETCHES >= 8) 305 all_or_flags |= or_flags;
312 if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
327 if (
PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
329 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
330 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
331 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
341 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
343 if (b[1]->
flags & VLIB_BUFFER_IS_TRACED)
345 if (b[2]->
flags & VLIB_BUFFER_IS_TRACED)
347 if (b[3]->
flags & VLIB_BUFFER_IS_TRACED)
354 #elif (CLIB_N_PREFETCHES >= 4) 371 all_or_flags |= or_flags;
376 if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
387 if (
PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
389 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
390 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
391 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
399 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
401 if (b[1]->
flags & VLIB_BUFFER_IS_TRACED)
413 all_or_flags |= b[0]->
flags;
420 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
428 tx_pkts = n_packets = mb - ptd->
mbufs;
449 rte_pktmbuf_free (ptd->
mbufs[n_packets - n_left - 1]);
462 rte_eth_stats_reset (xd->
port_id);
463 rte_eth_xstats_reset (xd->
port_id);
474 if (xd->
flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
479 if ((xd->
flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
481 xd->
flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
489 if ((xd->
flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
491 xd->
flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
510 if (node_index == ~0)
539 if ((xd->
flags & DPDK_DEVICE_FLAG_PMD) == 0)
543 if ((xd->
pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->
pmd != VNET_DPDK_PMD_I40EVF))
556 vlan_offload = rte_eth_dev_get_vlan_offload (xd->
port_id);
557 vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
559 if ((r = rte_eth_dev_set_vlan_offload (xd->
port_id, vlan_offload)))
569 rte_eth_dev_vlan_filter (xd->
port_id,
580 xd->
flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
582 xd->
flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
597 struct rte_eth_rss_reta_entry64 *reta_conf = NULL;
598 struct rte_eth_dev_info dev_info;
600 u16 *valid_queue = NULL;
601 u16 valid_queue_count = 0;
605 rte_eth_dev_info_get (xd->
port_id, &dev_info);
628 clib_memset (reta, 0, dev_info.reta_size * sizeof (*reta));
630 valid_queue_count = 0;
633 if (i >= dev_info.nb_rx_queues)
635 err = clib_error_return (0,
"illegal queue number");
638 reta[valid_queue_count++] = i;
643 if (valid_queue_count == 0)
650 for (i = valid_queue_count, j = 0; i < dev_info.reta_size; i++, j++)
652 j = j % valid_queue_count;
653 reta[
i] = valid_queue[j];
658 (
struct rte_eth_rss_reta_entry64 *)
clib_mem_alloc (dev_info.reta_size /
659 RTE_RETA_GROUP_SIZE *
660 sizeof (*reta_conf));
661 if (reta_conf == NULL)
668 dev_info.reta_size / RTE_RETA_GROUP_SIZE *
669 sizeof (*reta_conf));
671 for (i = 0; i < dev_info.reta_size; i++)
673 uint32_t reta_id = i / RTE_RETA_GROUP_SIZE;
674 uint32_t reta_pos = i % RTE_RETA_GROUP_SIZE;
676 reta_conf[reta_id].mask = UINT64_MAX;
677 reta_conf[reta_id].reta[reta_pos] = reta[
i];
681 rte_eth_dev_rss_reta_update (xd->
port_id, reta_conf, dev_info.reta_size);
717 #define UP_DOWN_FLAG_EVENT 1 725 uword *event_data = 0;
743 sw_if_index = event_data[0];
744 flags = event_data[1];
766 .name =
"admin-up-down-process",
767 .process_log2_n_stack_bytes = 17,
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
format_function_t format_dpdk_tx_trace
format_function_t format_dpdk_flow
#define vlib_buffer_from_rte_mbuf(x)
vl_api_wireguard_peer_flags_t flags
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
vnet_main_t * vnet_get_main(void)
vnet_interface_main_t interface_main
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static f64 vlib_time_now(vlib_main_t *vm)
#define VLIB_BUFFER_PRE_DATA_SIZE
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u16 current_length
Nbytes between current data and the end of this buffer.
u32 per_interface_next_index
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
struct vnet_sub_interface_t::@356::@357::@359 flags
static clib_error_t * dpdk_add_del_mac_address(vnet_hw_interface_t *hi, const u8 *address, u8 is_add)
u8 buffer_pool_index
index of buffer pool this buffer belongs.
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
#define static_always_inline
VNET_DEVICE_CLASS(af_xdp_device_class)
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
#define VNET_DEVICE_CLASS_TX_FN(devclass)
A collection of simple counters.
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
vnet_hw_interface_t * hw_interfaces
void dpdk_device_start(dpdk_device_t *xd)
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
struct vnet_sub_interface_t::@356 eth
dpdk_per_thread_data_t * per_thread_data
#define rte_mbuf_from_vlib_buffer(x)
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
static void dpdk_clear_hw_interface_counters(u32 instance)
dpdk_tx_queue_t * tx_queues
vlib_simple_counter_main_t * sw_if_counters
#define VLIB_REGISTER_NODE(x,...)
#define CLIB_PREFETCH(addr, size, type)
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
static uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
sll srl srl sll sra u16x4 i
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **mb, u32 n_left)
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, vlib_buffer_t *buffer)
format_function_t format_dpdk_device
vlib_main_t vlib_node_runtime_t * node
void dpdk_device_stop(dpdk_device_t *xd)
static char * dpdk_tx_func_error_strings[]
format_function_t format_dpdk_device_name
manual_print typedef address
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static void clib_mem_free(void *p)
#define clib_error_report(e)
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
static void * clib_mem_alloc(uword size)
static vlib_main_t * vlib_get_main(void)
#define UP_DOWN_FLAG_EVENT
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
struct rte_mbuf * mbufs[DPDK_RX_BURST_SZ]
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
u8 admin_up_down_in_progress
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
struct rte_mempool ** dpdk_no_cache_mempool_by_buffer_pool_index
VLIB buffer representation.
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, const u8 *old_address, const u8 *address)
VNET_DEVICE_CLASS_TX_FN() dpdk_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
vnet_flow_dev_ops_function_t dpdk_flow_ops_fn
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, vnet_sw_interface_flags_t flags)
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
static_always_inline __clib_unused void dpdk_prefetch_buffer(vlib_main_t *vm, struct rte_mbuf *mb)
#define VLIB_NODE_FLAG_TRACE
#define CLIB_CACHE_LINE_BYTES
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
volatile u8 ref_count
Reference count for this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
vl_api_interface_index_t sw_if_index
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
static clib_error_t * dpdk_interface_set_rss_queues(struct vnet_main_t *vnm, struct vnet_hw_interface_t *hi, clib_bitmap_t *bitmap)
#define foreach_dpdk_tx_func_error