27 #define foreach_dpdk_tx_func_error \ 28 _(BAD_RETVAL, "DPDK tx function returned an error") \ 29 _(PKT_DROP, "Tx packet drops (dpdk tx failure)") 33 #define _(f,s) DPDK_TX_FUNC_ERROR_##f, 53 error = rte_eth_dev_default_mac_addr_set (xd->
port_id,
54 (
struct ether_addr *) address);
85 sizeof (buffer[0]) -
sizeof (buffer->
pre_data));
95 struct rte_mbuf *mb, *first_mb, *last_mb;
103 rte_pktmbuf_reset (mb);
104 while (maybe_multiseg && (b2->
flags & VLIB_BUFFER_NEXT_PRESENT))
108 rte_pktmbuf_reset (mb);
113 first_mb->nb_segs = 1;
119 while (maybe_multiseg && (b->
flags & VLIB_BUFFER_NEXT_PRESENT))
146 struct rte_mbuf **mb,
u32 n_left)
165 while (__sync_lock_test_and_set (xd->
lockp[queue_id], 1))
167 queue_id = (queue_id + 1) % xd->
tx_q_used;
179 n_sent = rte_ring_sp_enqueue_burst (hqos->
swq, (
void **) mb,
185 n_sent = rte_eth_tx_burst (xd->
port_id, queue_id, mb, n_left);
194 *xd->
lockp[queue_id] = 0;
214 while (n_sent && n_left && (n_retry > 0));
231 u32 ip_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
232 u32 tcp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
233 u32 udp_cksum = b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
234 int is_ip4 = b->
flags & VNET_BUFFER_F_IS_IP4;
238 if (
PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum) == 0))
244 mb->outer_l3_len = 0;
245 mb->outer_l2_len = 0;
246 ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
247 ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
248 ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
249 ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
250 mb->ol_flags |= ol_flags;
254 if (xd->
flags & DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM)
255 rte_net_intel_cksum_flags_prepare (mb, ol_flags);
270 u32 n_packets = f->n_vectors;
274 int queue_id = thread_index;
275 u32 tx_pkts = 0, all_or_flags = 0;
278 struct rte_mbuf **mb;
304 (
void **) ptd->
mbufs, n_packets,
305 -(
i32) sizeof (
struct rte_mbuf));
326 all_or_flags |= or_flags;
333 if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
348 if (
PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
350 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
351 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
352 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
362 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
364 if (b[1]->
flags & VLIB_BUFFER_IS_TRACED)
366 if (b[2]->
flags & VLIB_BUFFER_IS_TRACED)
368 if (b[3]->
flags & VLIB_BUFFER_IS_TRACED)
378 all_or_flags |= b[0]->
flags;
385 if (b[0]->
flags & VLIB_BUFFER_IS_TRACED)
393 tx_pkts = n_packets = mb - ptd->
mbufs;
414 rte_pktmbuf_free (ptd->
mbufs[n_packets - n_left - 1]);
448 if (xd->
flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
453 if ((xd->
flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
455 xd->
flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
463 if ((xd->
flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
465 xd->
flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
484 if (node_index == ~0)
513 if ((xd->
flags & DPDK_DEVICE_FLAG_PMD) == 0)
517 if ((xd->
pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->
pmd != VNET_DPDK_PMD_I40EVF))
530 vlan_offload = rte_eth_dev_get_vlan_offload (xd->
port_id);
531 vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
533 if ((r = rte_eth_dev_set_vlan_offload (xd->
port_id, vlan_offload)))
543 rte_eth_dev_vlan_filter (xd->
port_id,
554 xd->
flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
556 xd->
flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
579 #define UP_DOWN_FLAG_EVENT 1 587 uword *event_data = 0;
605 sw_if_index = event_data[0];
606 flags = event_data[1];
628 .name =
"admin-up-down-process",
629 .process_log2_n_stack_bytes = 17,
format_function_t format_dpdk_tx_trace
format_function_t format_dpdk_flow
#define vlib_buffer_from_rte_mbuf(x)
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
vnet_main_t * vnet_get_main(void)
vnet_interface_main_t interface_main
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
static f64 vlib_time_now(vlib_main_t *vm)
static char * dpdk_tx_func_error_strings[]
#define VLIB_BUFFER_PRE_DATA_SIZE
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, const u8 *old_address, const u8 *address)
u32 per_interface_next_index
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
struct rte_eth_xstat * last_cleared_xstats
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, vlib_buffer_t *buffer)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
struct vnet_sub_interface_t::@198::@199::@201 flags
struct rte_eth_stats stats
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
#define static_always_inline
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
#define VNET_DEVICE_CLASS_TX_FN(devclass)
vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
A collection of simple counters.
dpdk_device_hqos_per_worker_thread_t * hqos_wt
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
vnet_hw_interface_t * hw_interfaces
#define rte_mbuf_from_vlib_buffer(x)
void dpdk_device_start(dpdk_device_t *xd)
u16 current_length
Nbytes between current data and the end of this buffer.
dpdk_per_thread_data_t * per_thread_data
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
static void pcap_add_buffer(pcap_main_t *pm, vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
vlib_simple_counter_main_t * sw_if_counters
#define VLIB_REGISTER_NODE(x,...)
#define CLIB_PREFETCH(addr, size, type)
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
struct vnet_sub_interface_t::@198 eth
#define clib_warning(format, args...)
#define clib_memcpy(a, b, c)
static_always_inline void dpdk_prefetch_buffer(vlib_main_t *vm, struct rte_mbuf *mb)
format_function_t format_dpdk_device
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
void dpdk_device_stop(dpdk_device_t *xd)
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **mb, u32 n_left)
struct rte_eth_xstat * xstats
format_function_t format_dpdk_device_name
u32 next_buffer
Next buffer for this linked-list of buffers.
#define clib_error_report(e)
VNET_DEVICE_CLASS(bond_dev_class)
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
dpdk_portid_t device_index
u8 n_add_refs
Number of additional references to this buffer.
static vlib_main_t * vlib_get_main(void)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
static void dpdk_clear_hw_interface_counters(u32 instance)
struct rte_mbuf * mbufs[DPDK_RX_BURST_SZ]
static uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
VNET_DEVICE_CLASS_TX_FN() dpdk_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
#define foreach_dpdk_tx_func_error
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u8 admin_up_down_in_progress
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
dpdk_pcap_t pcap[VLIB_N_RX_TX]
#define UP_DOWN_FLAG_EVENT
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
vnet_flow_dev_ops_function_t dpdk_flow_ops_fn
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
struct rte_eth_stats last_cleared_stats
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
#define VLIB_NODE_FLAG_TRACE
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".