27 #define foreach_dpdk_tx_func_error \ 28 _(BAD_RETVAL, "DPDK tx function returned an error") \ 29 _(PKT_DROP, "Tx packet drops (dpdk tx failure)") 33 #define _(f,s) DPDK_TX_FUNC_ERROR_##f, 54 error = rte_eth_dev_mac_addr_add (xd->
port_id,
55 (
struct rte_ether_addr *) address, 0);
57 error = rte_eth_dev_mac_addr_remove (xd->
port_id,
58 (
struct rte_ether_addr *) address);
76 error = rte_eth_dev_default_mac_addr_set (xd->
port_id, (
void *) address);
107 sizeof (buffer[0]) -
sizeof (buffer->
pre_data));
118 struct rte_mbuf *mb, *first_mb, *last_mb;
124 rte_pktmbuf_reset (mb);
126 first_mb->nb_segs = 1;
132 while (maybe_multiseg && (b->
flags & VLIB_BUFFER_NEXT_PRESENT))
137 rte_pktmbuf_reset (mb);
159 struct rte_mbuf **mb,
u32 n_left)
180 queue_id = (queue_id + 1) % xd->
tx_q_used;
193 n_sent = rte_ring_sp_enqueue_burst (hqos->
swq, (
void **) mb,
201 n_sent = rte_eth_tx_burst (xd->
port_id, queue_id, mb, n_left);
231 while (n_sent && n_left && (n_retry > 0));
248 u32 ip_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
249 u32 tcp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
250 u32 udp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
252 u32 tso = b->
flags & VNET_BUFFER_F_GSO;
256 if (
PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
262 mb->outer_l3_len = 0;
263 mb->outer_l2_len = 0;
264 ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
265 ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
266 ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
267 ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
268 ol_flags |= tso ? (tcp_cksum ? PKT_TX_TCP_SEG : PKT_TX_UDP_SEG) : 0;
276 mb->ol_flags |= ol_flags;
280 if (xd->
flags & DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM)
281 rte_net_intel_cksum_flags_prepare (mb, ol_flags);
296 u32 n_packets = f->n_vectors;
299 int queue_id = thread_index;
300 u32 tx_pkts = 0, all_or_flags = 0;
303 struct rte_mbuf **mb;
310 (
void **) ptd->
mbufs, n_packets,
311 -(
i32) sizeof (
struct rte_mbuf));
316 #if (CLIB_N_PREFETCHES >= 8) 332 all_or_flags |= or_flags;
339 if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
354 if (
PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
356 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
357 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
358 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
368 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
370 if (b[1]->
flags & VLIB_BUFFER_IS_TRACED)
372 if (b[2]->
flags & VLIB_BUFFER_IS_TRACED)
374 if (b[3]->
flags & VLIB_BUFFER_IS_TRACED)
381 #elif (CLIB_N_PREFETCHES >= 4) 398 all_or_flags |= or_flags;
403 if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
414 if (
PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
416 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
417 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
418 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
426 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
428 if (b[1]->
flags & VLIB_BUFFER_IS_TRACED)
440 all_or_flags |= b[0]->
flags;
447 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
455 tx_pkts = n_packets = mb - ptd->
mbufs;
476 rte_pktmbuf_free (ptd->
mbufs[n_packets - n_left - 1]);
489 rte_eth_stats_reset (xd->
port_id);
490 rte_eth_xstats_reset (xd->
port_id);
501 if (xd->
flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
506 if ((xd->
flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
508 xd->
flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
516 if ((xd->
flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
518 xd->
flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
537 if (node_index == ~0)
566 if ((xd->
flags & DPDK_DEVICE_FLAG_PMD) == 0)
570 if ((xd->
pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->
pmd != VNET_DPDK_PMD_I40EVF))
583 vlan_offload = rte_eth_dev_get_vlan_offload (xd->
port_id);
584 vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
586 if ((r = rte_eth_dev_set_vlan_offload (xd->
port_id, vlan_offload)))
596 rte_eth_dev_vlan_filter (xd->
port_id,
607 xd->
flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
609 xd->
flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
633 #define UP_DOWN_FLAG_EVENT 1 641 uword *event_data = 0;
659 sw_if_index = event_data[0];
660 flags = event_data[1];
682 .name =
"admin-up-down-process",
683 .process_log2_n_stack_bytes = 17,
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
format_function_t format_dpdk_tx_trace
format_function_t format_dpdk_flow
#define vlib_buffer_from_rte_mbuf(x)
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
vnet_main_t * vnet_get_main(void)
vnet_interface_main_t interface_main
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
#define clib_memcpy_fast(a, b, c)
static f64 vlib_time_now(vlib_main_t *vm)
static char * dpdk_tx_func_error_strings[]
#define VLIB_BUFFER_PRE_DATA_SIZE
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u16 current_length
Nbytes between current data and the end of this buffer.
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, const u8 *old_address, const u8 *address)
u32 per_interface_next_index
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, vlib_buffer_t *buffer)
u8 buffer_pool_index
index of buffer pool this buffer belongs.
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
#define static_always_inline
vl_api_interface_index_t sw_if_index
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
#define VNET_DEVICE_CLASS_TX_FN(devclass)
vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
A collection of simple counters.
dpdk_device_hqos_per_worker_thread_t * hqos_wt
#define clib_atomic_test_and_set(a)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
struct vnet_sub_interface_t::@286::@287::@289 flags
vnet_hw_interface_t * hw_interfaces
void dpdk_device_start(dpdk_device_t *xd)
#define clib_atomic_release(a)
dpdk_per_thread_data_t * per_thread_data
#define rte_mbuf_from_vlib_buffer(x)
vlib_simple_counter_main_t * sw_if_counters
struct vnet_sub_interface_t::@286 eth
#define VLIB_REGISTER_NODE(x,...)
#define CLIB_PREFETCH(addr, size, type)
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
#define clib_warning(format, args...)
static_always_inline void dpdk_prefetch_buffer(vlib_main_t *vm, struct rte_mbuf *mb)
format_function_t format_dpdk_device
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
vlib_main_t vlib_node_runtime_t * node
void dpdk_device_stop(dpdk_device_t *xd)
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **mb, u32 n_left)
format_function_t format_dpdk_device_name
manual_print typedef address
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static clib_error_t * dpdk_add_del_mac_address(vnet_hw_interface_t *hi, const u8 *address, u8 is_add)
#define clib_error_report(e)
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
dpdk_portid_t device_index
static vlib_main_t * vlib_get_main(void)
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
static void dpdk_clear_hw_interface_counters(u32 instance)
struct rte_mbuf * mbufs[DPDK_RX_BURST_SZ]
static uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
VNET_DEVICE_CLASS_TX_FN() dpdk_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
#define foreach_dpdk_tx_func_error
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
u8 admin_up_down_in_progress
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
struct rte_mempool ** dpdk_no_cache_mempool_by_buffer_pool_index
VLIB buffer representation.
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
#define UP_DOWN_FLAG_EVENT
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
vnet_flow_dev_ops_function_t dpdk_flow_ops_fn
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, vnet_sw_interface_flags_t flags)
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
#define VLIB_NODE_FLAG_TRACE
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
#define CLIB_CACHE_LINE_BYTES
volatile u8 ref_count
Reference count for this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
VNET_DEVICE_CLASS(avf_device_class,)