FD.io VPP  v17.04.2-2-ga8f93f8
Vector Packet Processing
dpdk.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __included_dpdk_h__
16 #define __included_dpdk_h__
17 
18 /* $$$$ We should rename always_inline -> clib_always_inline */
19 #undef always_inline
20 
21 #include <rte_config.h>
22 
23 #include <rte_common.h>
24 #include <rte_dev.h>
25 #include <rte_log.h>
26 #include <rte_memory.h>
27 #include <rte_memzone.h>
28 #include <rte_tailq.h>
29 #include <rte_eal.h>
30 #include <rte_per_lcore.h>
31 #include <rte_launch.h>
32 #include <rte_atomic.h>
33 #include <rte_cycles.h>
34 #include <rte_prefetch.h>
35 #include <rte_lcore.h>
36 #include <rte_per_lcore.h>
37 #include <rte_branch_prediction.h>
38 #include <rte_interrupts.h>
39 #include <rte_pci.h>
40 #include <rte_random.h>
41 #include <rte_debug.h>
42 #include <rte_ether.h>
43 #include <rte_ethdev.h>
44 #include <rte_ring.h>
45 #include <rte_mempool.h>
46 #include <rte_mbuf.h>
47 #include <rte_virtio_net.h>
48 #include <rte_version.h>
49 #include <rte_eth_bond.h>
50 #include <rte_sched.h>
51 
52 #include <vnet/unix/pcap.h>
53 #include <vnet/devices/devices.h>
54 
55 #if CLIB_DEBUG > 0
56 #define always_inline static inline
57 #else
58 #define always_inline static inline __attribute__ ((__always_inline__))
59 #endif
60 
61 #include <vlib/pci/pci.h>
62 
63 #define NB_MBUF (16<<10)
64 
68 
69 #if RTE_VERSION >= RTE_VERSION_NUM(17, 2, 0, 0)
70 #define foreach_dpdk_pmd \
71  _ ("net_thunderx", THUNDERX) \
72  _ ("net_e1000_em", E1000EM) \
73  _ ("net_e1000_igb", IGB) \
74  _ ("net_e1000_igb_vf", IGBVF) \
75  _ ("net_ixgbe", IXGBE) \
76  _ ("net_ixgbe_vf", IXGBEVF) \
77  _ ("net_i40e", I40E) \
78  _ ("net_i40e_vf", I40EVF) \
79  _ ("net_virtio", VIRTIO) \
80  _ ("net_enic", ENIC) \
81  _ ("net_vmxnet3", VMXNET3) \
82  _ ("AF_PACKET PMD", AF_PACKET) \
83  _ ("net_bonding", BOND) \
84  _ ("net_fm10k", FM10K) \
85  _ ("net_cxgbe", CXGBE) \
86  _ ("net_mlx5", MLX5) \
87  _ ("net_dpaa2", DPAA2)
88 #else
89 #define foreach_dpdk_pmd \
90  _ ("net_thunderx", THUNDERX) \
91  _ ("net_e1000_em", E1000EM) \
92  _ ("net_e1000_igb", IGB) \
93  _ ("net_e1000_igb_vf", IGBVF) \
94  _ ("net_ixgbe", IXGBE) \
95  _ ("net_ixgbe_vf", IXGBEVF) \
96  _ ("net_i40e", I40E) \
97  _ ("net_i40e_vf", I40EVF) \
98  _ ("net_virtio", VIRTIO) \
99  _ ("net_enic", ENIC) \
100  _ ("net_vmxnet3", VMXNET3) \
101  _ ("AF_PACKET PMD", AF_PACKET) \
102  _ ("rte_bond_pmd", BOND) \
103  _ ("net_fm10k", FM10K) \
104  _ ("net_cxgbe", CXGBE) \
105  _ ("net_mlx5", MLX5) \
106  _ ("net_dpaa2", DPAA2)
107 #endif
108 
109 typedef enum
110 {
112 #define _(s,f) VNET_DPDK_PMD_##f,
114 #undef _
115  VNET_DPDK_PMD_UNKNOWN, /* must be last */
116 } dpdk_pmd_t;
117 
118 typedef enum
119 {
129 
130 /*
131  * The header for the tx_vector in dpdk_device_t.
132  * Head and tail are indexes into the tx_vector and are of type
133  * u64 so they never overflow.
134  */
135 typedef struct
136 {
139 } tx_ring_hdr_t;
140 
141 typedef struct
142 {
143  struct rte_ring *swq;
144 
154  u32 hqos_tc_table[64];
156 
157 typedef struct
158 {
159  struct rte_ring **swq;
160  struct rte_mbuf **pkts_enq;
161  struct rte_mbuf **pkts_deq;
162  struct rte_sched_port *hqos;
169 
170 typedef struct
171 {
172  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
173  volatile u32 **lockp;
174 
175  /* Instance ID */
177 
180 
181  /* next node index if we decide to steal the rx graph arc */
183 
184  /* dpdk rte_mbuf rx and tx vectors, VLIB_FRAME_SIZE */
185  struct rte_mbuf ***tx_vectors; /* one per worker thread */
186  struct rte_mbuf ***rx_vectors;
187 
188  /* vector of traced contexts, per device */
190 
191  dpdk_pmd_t pmd:8;
193 
195 #define DPDK_DEVICE_FLAG_ADMIN_UP (1 << 0)
196 #define DPDK_DEVICE_FLAG_PROMISC (1 << 1)
197 #define DPDK_DEVICE_FLAG_PMD (1 << 2)
198 #define DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE (1 << 3)
199 #define DPDK_DEVICE_FLAG_MAYBE_MULTISEG (1 << 4)
200 #define DPDK_DEVICE_FLAG_HAVE_SUBIF (1 << 5)
201 #define DPDK_DEVICE_FLAG_HQOS (1 << 6)
202 
204  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
205 
207 
208  /* number of sub-interfaces */
210 
211  /* PMD related */
216  struct rte_eth_conf port_conf;
217  struct rte_eth_txconf tx_conf;
218 
219  /* HQoS related */
222 
223  /* af_packet or BondEthernet instance number */
225 
226  struct rte_eth_link link;
228 
229  struct rte_eth_stats stats;
230  struct rte_eth_stats last_stats;
231  struct rte_eth_stats last_cleared_stats;
232  struct rte_eth_xstat *xstats;
233  struct rte_eth_xstat *last_cleared_xstats;
236 
237  /* mac address */
239 } dpdk_device_t;
240 
241 #define DPDK_STATS_POLL_INTERVAL (10.0)
242 #define DPDK_MIN_STATS_POLL_INTERVAL (0.001) /* 1msec */
243 
244 #define DPDK_LINK_POLL_INTERVAL (3.0)
245 #define DPDK_MIN_LINK_POLL_INTERVAL (0.001) /* 1msec */
246 
247 typedef struct
248 {
252 
253 #ifndef DPDK_HQOS_DBG_BYPASS
254 #define DPDK_HQOS_DBG_BYPASS 0
255 #endif
256 
257 #ifndef HQOS_FLUSH_COUNT_THRESHOLD
258 #define HQOS_FLUSH_COUNT_THRESHOLD 100000
259 #endif
260 
262 {
265 
269 
277 
278  struct rte_sched_port_params port;
279  struct rte_sched_subport_params *subport;
280  struct rte_sched_pipe_params *pipe;
281  uint32_t *pipe_map;
283 
284 int dpdk_hqos_validate_mask (u64 mask, u32 n);
286  hqos, u32 pipe_profile_id);
291  struct rte_mbuf **pkts, u32 n_pkts);
292 
293 #define foreach_dpdk_device_config_item \
294  _ (num_rx_queues) \
295  _ (num_tx_queues) \
296  _ (num_rx_desc) \
297  _ (num_tx_desc) \
298  _ (rss_fn)
299 
300 typedef struct
301 {
302  vlib_pci_addr_t pci_addr;
305 #define DPDK_DEVICE_VLAN_STRIP_DEFAULT 0
306 #define DPDK_DEVICE_VLAN_STRIP_OFF 1
307 #define DPDK_DEVICE_VLAN_STRIP_ON 2
308 
309 #define _(x) uword x;
311 #undef _
316 
317 typedef struct
318 {
319 
320  /* Config stuff */
327 
328  /* Required config parameters */
334  u8 num_kni; /* while kni_init allows u32, port_id in callback fn is only u8 */
335 
336  /*
337  * format interface names ala xxxEthernet%d/%d/%d instead of
338  * xxxEthernet%x/%x/%x.
339  */
341 
342  /* per-device config */
346 
348 
350 
351 typedef struct
352 {
353 
354  /* Devices */
358 
359  /* per-thread recycle lists */
361 
362  /* per-thread buffer templates */
364 
365  /* buffer flags template, configurable to enable/disable tcp / udp cksum */
367 
368  /* vlib buffer free list, must be same size as an rte_mbuf */
370 
371  /* Ethernet input node index */
373 
374  /* pcap tracing [only works if (CLIB_DEBUG > 0)] */
380 
381  /* hashes */
386 
387  /*
388  * flag indicating that a posted admin up/down
389  * (via post_sw_interface_set_flags) is in progress
390  */
392 
394 
395  /* which cpus are running dpdk-input */
398 
399  /* which cpus are running I/O TX */
402 
403  /* control interval of dpdk link state and stat polling */
406 
407  /* Sleep for this many usec after each device poll */
409 
410  /* convenience */
414 
415  /* mempool */
416  struct rte_mempool **pktmbuf_pools;
417 
418  /* API message ID base */
420 } dpdk_main_t;
421 
423 
424 typedef struct
425 {
429  struct rte_mbuf mb;
430  /* Copy of VLIB buffer; packet data stored in pre_data. */
433 
434 typedef struct
435 {
439  struct rte_mbuf mb;
440  vlib_buffer_t buffer; /* Copy of VLIB buffer; pkt data stored in pre_data. */
441  u8 data[256]; /* First 256 data bytes, used for hexdump */
443 
445 
447 
449  struct ether_addr mc_addr_vec[], int naddr);
450 
452 
454 
455 u32 dpdk_interface_tx_vector (vlib_main_t * vm, u32 dev_instance);
456 
457 struct rte_mbuf *dpdk_replicate_packet_mb (vlib_buffer_t * b);
458 struct rte_mbuf *dpdk_zerocopy_replicate_packet_mb (vlib_buffer_t * b);
459 
460 #define foreach_dpdk_error \
461  _(NONE, "no error") \
462  _(RX_PACKET_ERROR, "Rx packet errors") \
463  _(RX_BAD_FCS, "Rx bad fcs") \
464  _(IP_CHECKSUM_ERROR, "Rx ip checksum errors") \
465  _(RX_ALLOC_FAIL, "rx buf alloc from free list failed") \
466  _(RX_ALLOC_NO_PHYSMEM, "rx buf alloc failed no physmem") \
467  _(RX_ALLOC_DROP_PKTS, "rx packets dropped due to alloc error")
468 
469 typedef enum
470 {
471 #define _(f,s) DPDK_ERROR_##f,
473 #undef _
475 } dpdk_error_t;
476 
477 int dpdk_set_stat_poll_interval (f64 interval);
479 void dpdk_update_link_state (dpdk_device_t * xd, f64 now);
482 
483 void dpdk_rx_trace (dpdk_main_t * dm,
484  vlib_node_runtime_t * node,
485  dpdk_device_t * xd,
486  u16 queue_id, u32 * buffers, uword n_buffers);
487 
488 #define EFD_OPERATION_LESS_THAN 0
489 #define EFD_OPERATION_GREATER_OR_EQUAL 1
490 
501 
502 uword
505 
506 #endif /* __included_dpdk_h__ */
507 
508 /*
509  * fd.io coding-style-patch-verification: ON
510  *
511  * Local Variables:
512  * eval: (c-set-style "gnu")
513  * End:
514  */
u32 ** d_trace_buffers
Definition: dpdk.h:189
u8 * default_mac_address
Definition: dpdk.h:238
void dpdk_device_lock_free(dpdk_device_t *xd)
Definition: init.c:469
f64 time_last_link_update
Definition: dpdk.h:227
struct dpdk_device_config_hqos_t dpdk_device_config_hqos_t
vmrglw vmrglh hi
uword( unformat_function_t)(unformat_input_t *input, va_list *args)
Definition: format.h:231
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:68
struct rte_mbuf * dpdk_replicate_packet_mb(vlib_buffer_t *b)
Definition: device.c:90
u8 * eal_init_args_str
Definition: dpdk.h:322
vlib_buffer_t buffer
Definition: dpdk.h:431
u8 interface_name_format_decimal
Definition: dpdk.h:340
dpdk_main_t dpdk_main
Definition: dpdk.h:422
int dpdk_hqos_validate_mask(u64 mask, u32 n)
Definition: hqos.c:171
u8 use_rss
Definition: dpdk.h:393
vnet_device_class_t dpdk_device_class
clib_error_t * dpdk_set_mc_filter(vnet_hw_interface_t *hi, struct ether_addr mc_addr_vec[], int naddr)
Definition: device.c:70
#define foreach_dpdk_error
Definition: dpdk.h:460
dpdk_error_t
Definition: dpdk.h:469
PCAP utility definitions.
u16 flags
Definition: dpdk.h:194
dpdk_device_and_queue_t ** devices_by_hqos_cpu
Definition: dpdk.h:357
struct _vlib_node_registration vlib_node_registration_t
struct rte_sched_port_params port
Definition: dpdk.h:278
u32 per_interface_next_index
Definition: dpdk.h:182
u8 enable_tcp_udp_checksum
Definition: dpdk.h:325
struct rte_mbuf ** pkts_enq
Definition: dpdk.h:160
struct rte_ring ** swq
Definition: dpdk.h:159
vlib_buffer_t * buffer_templates
Definition: dpdk.h:363
struct rte_sched_port * hqos
Definition: dpdk.h:162
u64 tx_tail
Definition: dpdk.h:138
u8 *( format_function_t)(u8 *s, va_list *args)
Definition: format.h:48
struct rte_eth_xstat * last_cleared_xstats
Definition: dpdk.h:233
format_function_t format_dpdk_rte_mbuf
Definition: dpdk.h:495
dpdk_device_config_hqos_t hqos
Definition: dpdk.h:314
u16 num_subifs
Definition: dpdk.h:209
struct _vnet_device_class vnet_device_class_t
u8 * pcap_filename
Definition: dpdk.h:377
format_function_t format_dpdk_tx_dma_trace
Definition: dpdk.h:493
u8 port_id
Definition: dpdk.h:224
clib_error_t * dpdk_port_setup(dpdk_main_t *dm, dpdk_device_t *xd)
Definition: init.c:322
struct rte_sched_pipe_params * pipe
Definition: dpdk.h:280
struct rte_mbuf *** tx_vectors
Definition: dpdk.h:185
foreach_dpdk_device_config_item clib_bitmap_t * workers
Definition: dpdk.h:312
PCAP main state data structure.
Definition: pcap.h:122
dpdk_config_main_t dpdk_config_main
Definition: dpdk.h:349
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:655
struct rte_mbuf ** pkts_deq
Definition: dpdk.h:161
dpdk_device_config_t default_devconf
Definition: dpdk.h:343
f64 stat_poll_interval
Definition: dpdk.h:405
u32 pcap_pkts_to_capture
Definition: dpdk.h:379
unformat_function_t unformat_socket_mem
Definition: dpdk.h:497
int input_cpu_first_index
Definition: dpdk.h:396
u32 dpdk_interface_tx_vector(vlib_main_t *vm, u32 dev_instance)
char i8
Definition: types.h:45
u16 rx_q_used
Definition: dpdk.h:213
unsigned long u64
Definition: types.h:89
format_function_t format_dpdk_rx_rte_mbuf
Definition: dpdk.h:496
clib_error_t * unformat_rss_fn(unformat_input_t *input, uword *rss_fn)
Definition: format.c:706
u32 device_index
Definition: dpdk.h:176
dpdk_device_hqos_per_worker_thread_t * hqos_wt
Definition: dpdk.h:220
vlib_node_registration_t handoff_dispatch_node
(constructor) VLIB_REGISTER_NODE (handoff_dispatch_node)
Definition: handoff.c:48
f64 time_last_stats_update
Definition: dpdk.h:234
u32 pcap_sw_if_index
Definition: dpdk.h:378
u32 vlib_sw_if_index
Definition: dpdk.h:179
vlib_pci_addr_t pci_addr
Definition: dpdk.h:302
format_function_t format_dpdk_rx_dma_trace
Definition: dpdk.h:494
dpdk_device_and_queue_t ** devices_by_cpu
Definition: dpdk.h:356
int dpdk_set_link_state_poll_interval(f64 interval)
Definition: init.c:1990
u8 ** eal_init_args
Definition: dpdk.h:321
void vnet_buffer_needs_dpdk_mb(vlib_buffer_t *b)
u32 vlib_hw_if_index
Definition: dpdk.h:178
pcap_main_t pcap_main
Definition: dpdk.h:376
struct _unformat_input_t unformat_input_t
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
Definition: hqos.c:644
#define foreach_dpdk_pmd
Definition: dpdk.h:70
dpdk_port_type_t port_type
Definition: dpdk.h:235
int input_cpu_count
Definition: dpdk.h:397
void dpdk_rx_trace(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, u32 *buffers, uword n_buffers)
Definition: node.c:118
u16 tx_q_used
Definition: dpdk.h:212
u16 nb_rx_desc
Definition: dpdk.h:214
void dpdk_thread_input(dpdk_main_t *dm, dpdk_device_t *xd)
u32 ** recycle
Definition: dpdk.h:360
dpdk_device_t * devices
Definition: dpdk.h:355
vlib_main_t * vm
Definition: buffer.c:276
u8 nchannels_set_manually
Definition: dpdk.h:330
u16 * cpu_socket_id_by_queue
Definition: dpdk.h:215
void dpdk_device_config_hqos_pipe_profile_default(dpdk_device_config_hqos_t *hqos, u32 pipe_profile_id)
Definition: hqos.c:200
volatile u32 ** lockp
Definition: dpdk.h:173
dpdk_device_config_t * dev_confs
Definition: dpdk.h:344
struct rte_mbuf *** rx_vectors
Definition: dpdk.h:186
int dpdk_set_stat_poll_interval(f64 interval)
Definition: init.c:1979
u8 coremask_set_manually
Definition: dpdk.h:329
dpdk_pmd_t
Definition: dpdk.h:109
uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:795
format_function_t format_dpdk_device
Definition: dpdk.h:492
struct rte_eth_xstat * xstats
Definition: dpdk.h:232
u8 * interface_name_suffix
Definition: dpdk.h:206
format_function_t format_dpdk_device_name
Definition: dpdk.h:491
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:208
unsigned int u32
Definition: types.h:88
int hqos_cpu_count
Definition: dpdk.h:401
u32 * vu_inactive_interfaces_device_index
Definition: dpdk.h:385
u64 tx_head
Definition: dpdk.h:137
u32 poll_sleep_usec
Definition: dpdk.h:408
void dpdk_device_lock_init(dpdk_device_t *xd)
Definition: init.c:456
struct rte_mempool ** pktmbuf_pools
Definition: dpdk.h:416
dpdk_device_hqos_per_hqos_thread_t * hqos_ht
Definition: dpdk.h:221
struct rte_mbuf * dpdk_zerocopy_replicate_packet_mb(vlib_buffer_t *b)
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1688
clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, char *address)
Definition: device.c:48
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:250
u64 uword
Definition: types.h:112
dpdk_port_type_t
Definition: dpdk.h:118
unsigned short u16
Definition: types.h:57
vlib_buffer_t buffer
Definition: dpdk.h:440
u32 ethernet_input_node_index
Definition: dpdk.h:372
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
u8 admin_up_down_in_progress
Definition: dpdk.h:391
uint32_t * pipe_map
Definition: dpdk.h:281
u32 buffer_flags_template
Definition: dpdk.h:366
u32 vlib_buffer_free_list_index
Definition: dpdk.h:369
int hqos_cpu_first_index
Definition: dpdk.h:400
struct rte_sched_subport_params * subport
Definition: dpdk.h:279
#define foreach_dpdk_device_config_item
Definition: dpdk.h:293
uword clib_bitmap_t
Definition: bitmap.h:50
i8 cpu_socket
Definition: dpdk.h:192
u16 msg_id_base
Definition: dpdk.h:419
u8 * uio_driver_name
Definition: dpdk.h:323
int tx_pcap_enable
Definition: dpdk.h:375
vnet_main_t * vnet_main
Definition: dpdk.h:412
u16 nb_tx_desc
Definition: dpdk.h:203
clib_error_t * unformat_hqos(unformat_input_t *input, dpdk_device_config_hqos_t *hqos)
Definition: format.c:729
uword * device_config_index_by_pci_addr
Definition: dpdk.h:345
uword * dpdk_device_by_kni_port_id
Definition: dpdk.h:382
uword * vu_sw_if_index_by_listener_fd
Definition: dpdk.h:383
f64 link_state_poll_interval
Definition: dpdk.h:404
dpdk_config_main_t * conf
Definition: dpdk.h:413
uword * vu_sw_if_index_by_sock_fd
Definition: dpdk.h:384
vlib_main_t * vlib_main
Definition: dpdk.h:411