27 #define foreach_dpdk_tx_func_error \ 28 _(BAD_RETVAL, "DPDK tx function returned an error") \ 29 _(RING_FULL, "Tx packet drops (ring full)") \ 30 _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \ 31 _(REPL_FAIL, "Tx packet drops (replication failure)") 35 #define _(f,s) DPDK_TX_FUNC_ERROR_##f, 54 error = rte_eth_dev_default_mac_addr_set (xd->
device_index,
55 (
struct ether_addr *) address);
71 struct ether_addr mc_addr_vec[],
int naddr)
77 error = rte_eth_dev_set_mc_addr_list (xd->
device_index, mc_addr_vec, naddr);
93 struct rte_mbuf **mbufs = 0, *s, *d;
95 unsigned socket_id = rte_socket_id ();
100 nb_segs = s->nb_segs;
103 if (rte_pktmbuf_alloc_bulk (dm->
pktmbuf_pools[socket_id], mbufs, nb_segs))
110 d->nb_segs = s->nb_segs;
111 d->data_len = s->data_len;
112 d->pkt_len = s->pkt_len;
113 d->data_off = s->data_off;
114 clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
116 for (i = 1; i < nb_segs; i++)
121 d->data_len = s->data_len;
123 RTE_PKTMBUF_HEADROOM + s->data_len);
149 sizeof (buffer[0]) -
sizeof (buffer->
pre_data));
158 struct rte_mbuf *mb, *first_mb, *last_mb;
166 rte_pktmbuf_reset (mb);
171 rte_pktmbuf_reset (mb);
176 first_mb->nb_segs = 1;
212 struct rte_mbuf **tx_vector)
223 ring =
vec_header (tx_vector,
sizeof (*ring));
242 ASSERT (n_packets < xd->nb_tx_desc);
260 while (__sync_lock_test_and_set (xd->
lockp[queue_id], 1))
262 queue_id = (queue_id + 1) % xd->
tx_q_used;
274 &tx_vector[tx_tail], tx_head - tx_tail);
275 rv = rte_ring_sp_enqueue_burst (hqos->
swq,
276 (
void **) &tx_vector[tx_tail],
277 (uint16_t) (tx_head - tx_tail));
285 (uint16_t) (tx_head - tx_tail));
294 *xd->
lockp[queue_id] = 0;
312 n_packets -= (uint16_t) rv;
314 while (rv && n_packets && (n_retry > 0));
336 struct rte_mbuf *mb_new;
345 DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
370 struct rte_mbuf **tx_vector;
372 u16 nb_tx_desc = xd->nb_tx_desc;
383 tx_vector = xd->tx_vectors[queue_id];
384 ring =
vec_header (tx_vector,
sizeof (*ring));
402 u32 bi0 = from[n_packets];
405 rte_pktmbuf_free (mb0);
427 i = ring->
tx_head % nb_tx_desc;
431 u32 bi0, bi1, bi2, bi3;
432 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
484 tx_vector[i++ % nb_tx_desc] = mb0;
486 tx_vector[i++ % nb_tx_desc] = mb1;
488 tx_vector[i++ % nb_tx_desc] = mb2;
490 tx_vector[i++ % nb_tx_desc] = mb3;
496 tx_vector[i++ % nb_tx_desc] = mb0;
497 tx_vector[i++ % nb_tx_desc] = mb1;
498 tx_vector[i++ % nb_tx_desc] = mb2;
499 tx_vector[i++ % nb_tx_desc] = mb3;
504 tx_vector[i++] = mb0;
505 tx_vector[i++] = mb1;
506 tx_vector[i++] = mb2;
507 tx_vector[i++] = mb3;
529 struct rte_mbuf *mb0;
548 tx_vector[i % nb_tx_desc] = mb0;
565 tx_pkts = n_on_ring - n_packets;
584 rte_pktmbuf_free (tx_vector[ring->
tx_tail + n_packets]);
597 _vec_len (dm->
recycle[my_cpu]) = 0;
641 rv = rte_eth_dev_default_mac_addr_set (xd->
device_index,
642 (
struct ether_addr *)
665 if (xd->
pmd == VNET_DPDK_PMD_BOND)
668 int nlink = rte_eth_bond_slaves_get (xd->
device_index, slink, 16);
671 u8 dpdk_port = slink[--nlink];
672 rte_eth_dev_stop (dpdk_port);
678 clib_warning (
"rte_eth_dev_%s error: %d", is_up ?
"start" :
"stop", rv);
696 if (node_index == ~0)
729 if ((xd->
pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->
pmd != VNET_DPDK_PMD_I40EVF))
742 vlan_offload = rte_eth_dev_get_vlan_offload (xd->
device_index);
743 vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
745 if ((r = rte_eth_dev_set_vlan_offload (xd->
device_index, vlan_offload)))
792 #define UP_DOWN_FLAG_EVENT 1 800 uword *event_data = 0;
818 sw_if_index = event_data[0];
819 flags = event_data[1];
841 .name =
"admin-up-down-process",
842 .process_log2_n_stack_bytes = 17,
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
#define DPDK_DEVICE_FLAG_PROMISC
sll srl srl sll sra u16x4 i
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
u8 runtime_data[0]
Function dependent node-runtime data.
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
#define VLIB_BUFFER_RECYCLE
vnet_main_t * vnet_get_main(void)
vnet_interface_main_t interface_main
vnet_device_class_t dpdk_device_class
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
static f64 vlib_time_now(vlib_main_t *vm)
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **tx_vector)
static char * dpdk_tx_func_error_strings[]
#define VLIB_BUFFER_PRE_DATA_SIZE
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, u32 buffer_index, vlib_buffer_t *buffer)
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
#define DPDK_DEVICE_FLAG_HQOS
u32 per_interface_next_index
struct rte_eth_xstat * last_cleared_xstats
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
format_function_t format_dpdk_tx_dma_trace
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
VNET_DEVICE_CLASS(af_packet_device_class)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define DPDK_DEVICE_FLAG_PMD
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
struct rte_eth_stats stats
struct vnet_sub_interface_t::@117::@118::@120 flags
#define VLIB_BUFFER_NEXT_PRESENT
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
#define static_always_inline
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
#define VLIB_BUFFER_REPL_FAIL
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static uword dpdk_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
static vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
A collection of simple counters.
dpdk_device_hqos_per_worker_thread_t * hqos_wt
vnet_hw_interface_t * hw_interfaces
#define rte_mbuf_from_vlib_buffer(x)
u16 current_length
Nbytes between current data and the end of this buffer.
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
static void pcap_add_buffer(pcap_main_t *pm, vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
vlib_simple_counter_main_t * sw_if_counters
u32 node_index
Node index.
struct rte_mbuf * dpdk_replicate_packet_mb(vlib_buffer_t *b)
#define DPDK_DEVICE_FLAG_ADMIN_UP
#define CLIB_PREFETCH(addr, size, type)
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
#define vec_free(V)
Free vector's memory (no header).
#define clib_warning(format, args...)
#define VLIB_BUFFER_IS_TRACED
#define clib_memcpy(a, b, c)
struct vnet_sub_interface_t::@117 eth
format_function_t format_dpdk_device
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
struct rte_eth_xstat * xstats
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
format_function_t format_dpdk_device_name
uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
u32 next_buffer
Next buffer for this linked-list of buffers.
struct rte_mempool ** pktmbuf_pools
#define DPDK_DEVICE_FLAG_HAVE_SUBIF
#define clib_error_report(e)
#define VLIB_NODE_FLAG_TRACE
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
u8 n_add_refs
Number of additional references to this buffer.
static vlib_main_t * vlib_get_main(void)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
static void dpdk_clear_hw_interface_counters(u32 instance)
#define foreach_dpdk_tx_func_error
clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, char *address)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u8 admin_up_down_in_progress
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
#define UP_DOWN_FLAG_EVENT
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
#define VLIB_BUFFER_EXT_HDR_VALID
static_always_inline void dpdk_prefetch_buffer_by_index(vlib_main_t *vm, u32 bi)
#define VLIB_REGISTER_NODE(x,...)
static void * vec_header(void *v, uword header_bytes)
Find a user vector header.
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
u16 flags
Copy of main node flags.
struct rte_eth_stats last_cleared_stats
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
static_always_inline void dpdk_buffer_recycle(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b, u32 bi, struct rte_mbuf **mbp)
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 cpu_index, u32 index, u64 increment)
Increment a simple counter.
#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn)
clib_error_t * dpdk_set_mc_filter(vnet_hw_interface_t *hi, struct ether_addr mc_addr_vec[], int naddr)
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".