FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
rdma.h
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #ifndef _RDMA_H_
19 #define _RDMA_H_
20 
21 #include <infiniband/verbs.h>
22 #include <vlib/log.h>
23 #include <vlib/pci/pci.h>
24 #include <vnet/interface.h>
26 #include <rdma/rdma_mlx5dv.h>
27 
28 #define foreach_rdma_device_flags \
29  _(0, ERROR, "error") \
30  _(1, ADMIN_UP, "admin-up") \
31  _(2, LINK_UP, "link-up") \
32  _(3, PROMISC, "promiscuous") \
33  _(4, MLX5DV, "mlx5dv")
34 
35 enum
36 {
37 #define _(a, b, c) RDMA_DEVICE_F_##b = (1 << a),
39 #undef _
40 };
41 
42 #ifndef MLX5_ETH_L2_INLINE_HEADER_SIZE
43 #define MLX5_ETH_L2_INLINE_HEADER_SIZE 18
44 #endif
45 
46 typedef struct
47 {
48  CLIB_ALIGN_MARK (align0, MLX5_SEND_WQE_BB);
49  union
50  {
51  struct mlx5_wqe_ctrl_seg ctrl;
52  struct
53  {
58  };
59  };
60  struct mlx5_wqe_eth_seg eseg;
61  struct mlx5_wqe_data_seg dseg;
63 #define RDMA_MLX5_WQE_SZ sizeof(rdma_mlx5_wqe_t)
64 #define RDMA_MLX5_WQE_DS (RDMA_MLX5_WQE_SZ/sizeof(struct mlx5_wqe_data_seg))
65 STATIC_ASSERT (RDMA_MLX5_WQE_SZ == MLX5_SEND_WQE_BB &&
66  RDMA_MLX5_WQE_SZ % sizeof (struct mlx5_wqe_data_seg) == 0,
67  "bad size");
68 
69 typedef struct
70 {
71  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
72  struct ibv_cq *cq;
73  struct ibv_wq *wq;
85  volatile u32 *wq_db;
86  volatile u32 *cq_db;
90 } rdma_rxq_t;
91 
92 typedef struct
93 {
94  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
95 
96  /* following fields are accessed in datapath */
98 
99  union
100  {
101  struct
102  {
103  /* ibverb datapath. Cache of cq, sq below */
104  struct ibv_cq *ibv_cq;
105  struct ibv_qp *ibv_qp;
106  };
107  struct
108  {
109  /* direct verbs datapath */
110  rdma_mlx5_wqe_t *dv_sq_wqes;
111  volatile u32 *dv_sq_dbrec;
112  volatile u64 *dv_sq_db;
113  struct mlx5_cqe64 *dv_cq_cqes;
114  volatile u32 *dv_cq_dbrec;
115  };
116  };
117 
118  u32 *bufs; /* vlib_buffer ring buffer */
121  u16 dv_cq_idx; /* monotonic CQE index (valid only for direct verbs) */
122  u8 bufs_log2sz; /* log2 vlib_buffer entries */
123  u8 dv_sq_log2sz:4; /* log2 SQ WQE entries (valid only for direct verbs) */
124  u8 dv_cq_log2sz:4; /* log2 CQ CQE entries (valid only for direct verbs) */
125  STRUCT_MARK (cacheline1);
126 
127  /* WQE template (valid only for direct verbs) */
128  u8 dv_wqe_tmpl[64];
129 
130  /* end of 2nd 64-bytes cacheline (or 1st 128-bytes cacheline) */
131  STRUCT_MARK (cacheline2);
132 
133  /* fields below are not accessed in datapath */
134  struct ibv_cq *cq;
135  struct ibv_qp *qp;
136 
137 } rdma_txq_t;
138 STATIC_ASSERT_OFFSET_OF (rdma_txq_t, cacheline1, 64);
139 STATIC_ASSERT_OFFSET_OF (rdma_txq_t, cacheline2, 128);
140 
141 #define RDMA_TXQ_DV_INVALID_ID 0xffffffff
142 
143 #define RDMA_TXQ_BUF_SZ(txq) (1U << (txq)->bufs_log2sz)
144 #define RDMA_TXQ_DV_SQ_SZ(txq) (1U << (txq)->dv_sq_log2sz)
145 #define RDMA_TXQ_DV_CQ_SZ(txq) (1U << (txq)->dv_cq_log2sz)
146 
147 #define RDMA_TXQ_USED_SZ(head, tail) ((u16)((u16)(tail) - (u16)(head)))
148 #define RDMA_TXQ_AVAIL_SZ(txq, head, tail) ((u16)(RDMA_TXQ_BUF_SZ (txq) - RDMA_TXQ_USED_SZ (head, tail)))
149 
150 typedef struct
151 {
152  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
153 
154  /* following fields are accessed in datapath */
161  u32 lkey; /* cache of mr->lkey */
162  u8 pool; /* buffer pool index */
163 
164  /* fields below are not accessed in datapath */
171 
172  struct ibv_context *ctx;
173  struct ibv_pd *pd;
174  struct ibv_mr *mr;
175  struct ibv_qp *rx_qp4;
176  struct ibv_qp *rx_qp6;
177  struct ibv_rwq_ind_table *rx_rwq_ind_tbl;
178  struct ibv_flow *flow_ucast4;
179  struct ibv_flow *flow_mcast4;
180  struct ibv_flow *flow_ucast6;
181  struct ibv_flow *flow_mcast6;
182 
184 } rdma_device_t;
185 
186 typedef struct
187 {
188  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
189  union
190  {
191  u16 cqe_flags[VLIB_FRAME_SIZE];
192  u16x8 cqe_flags8[VLIB_FRAME_SIZE / 8];
193  u16x16 cqe_flags16[VLIB_FRAME_SIZE / 16];
194  };
197 
198 typedef struct
199 {
204 } rdma_main_t;
205 
206 extern rdma_main_t rdma_main;
207 
208 typedef enum
209 {
213 } rdma_mode_t;
214 
215 typedef struct
216 {
222  rdma_mode_t mode;
223 
224  /* return */
225  int rv;
229 
232 
235 
241 
242 typedef struct
243 {
248 
249 #define foreach_rdma_tx_func_error \
250 _(SEGMENT_SIZE_EXCEEDED, "segment size exceeded") \
251 _(NO_FREE_SLOTS, "no free tx slots") \
252 _(SUBMISSION, "tx submission errors") \
253 _(COMPLETION, "tx completion errors")
254 
255 typedef enum
256 {
257 #define _(f,s) RDMA_TX_ERROR_##f,
259 #undef _
262 
263 #endif /* _RDMA_H_ */
264 
265 /*
266  * fd.io coding-style-patch-verification: ON
267  *
268  * Local Variables:
269  * eval: (c-set-style "gnu")
270  * End:
271  */
struct ibv_mr * mr
Definition: rdma.h:174
u16 head
Definition: rdma.h:119
volatile u32 * dv_sq_dbrec
Definition: rdma.h:111
struct mlx5_cqe64 * dv_cq_cqes
Definition: rdma.h:113
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
void rdma_delete_if(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:884
u8 * linux_ifname
Definition: rdma.h:167
rdma_mlx5_wqe_t * dv_sq_wqes
Definition: rdma.h:110
u32 wq_stride
Definition: rdma.h:89
struct ibv_flow * flow_mcast6
Definition: rdma.h:181
u16 tail
Definition: rdma.h:120
volatile u32 * cq_db
Definition: rdma.h:86
u32 cqn
Definition: rdma.h:87
format_function_t format_rdma_device
Definition: rdma.h:236
unsigned long u64
Definition: types.h:89
void rdma_create_if(vlib_main_t *vm, rdma_create_if_args_t *args)
Definition: device.c:726
u32 size
Definition: rdma.h:75
rdma_mode_t
Definition: rdma.h:208
vlib_pci_device_info_t * pci
Definition: rdma.h:165
u32 dev_instance
Definition: rdma.h:170
mlx5dv_rwq_t * wqes
Definition: rdma.h:84
struct ibv_wq * wq
Definition: rdma.h:73
volatile u32 * dv_cq_dbrec
Definition: rdma.h:114
u32 per_interface_next_index
Definition: rdma.h:158
format_function_t format_rdma_input_trace
Definition: rdma.h:238
#define STRUCT_MARK(mark)
Definition: clib.h:75
vlib_main_t * vm
Definition: in2out_ed.c:1582
rdma_main_t rdma_main
Definition: device.c:46
struct _vnet_device_class vnet_device_class_t
u32 tail
Definition: rdma.h:77
mac_address_t hwaddr
Definition: rdma.h:168
unsigned char u8
Definition: types.h:56
clib_spinlock_t lock
Definition: rdma.h:97
vlib_log_class_t log_class
Definition: rdma.h:202
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
struct ibv_flow * flow_ucast6
Definition: rdma.h:180
rdma_per_thread_data_t * per_thread_data
Definition: rdma.h:200
u32 vlib_log_class_t
Definition: vlib.h:51
u16 n_mini_cqes_left
Definition: rdma.h:81
struct ibv_flow * flow_mcast4
Definition: rdma.h:179
struct ibv_pd * pd
Definition: rdma.h:173
rdma_device_t * devices
Definition: rdma.h:201
struct ibv_cq * cq
Definition: rdma.h:134
unsigned int u32
Definition: types.h:88
u16 n_mini_cqes
Definition: rdma.h:80
unformat_function_t unformat_rdma_create_if_args
Definition: rdma.h:240
#define VLIB_FRAME_SIZE
Definition: node.h:377
u32 flags
Definition: rdma.h:157
u32 * bufs
Definition: rdma.h:74
format_function_t format_rdma_rxq
Definition: rdma.h:239
u8 wqe_index_hi
Definition: rdma.h:55
unsigned short u16
Definition: types.h:57
struct ibv_cq * cq
Definition: rdma.h:72
struct ibv_cq * ibv_cq
Definition: rdma.h:104
struct ibv_qp * qp
Definition: rdma.h:135
uword() unformat_function_t(unformat_input_t *input, va_list *args)
Definition: format.h:233
u8 wqe_index_lo
Definition: rdma.h:56
vlib_buffer_t buffer_template
Definition: rdma.h:195
u32 hw_if_index
Definition: rdma.h:160
u32 wqe_cnt
Definition: rdma.h:88
struct ibv_rwq_ind_table * rx_rwq_ind_tbl
Definition: rdma.h:177
clib_error_t * error
Definition: rdma.h:183
clib_error_t * error
Definition: rdma.h:227
rdma_mode_t mode
Definition: rdma.h:222
u16 dv_cq_idx
Definition: rdma.h:121
struct ibv_flow * flow_ucast4
Definition: rdma.h:178
STATIC_ASSERT(RDMA_MLX5_WQE_SZ==MLX5_SEND_WQE_BB &&RDMA_MLX5_WQE_SZ % sizeof(struct mlx5_wqe_data_seg)==0, "bad size")
u32 lkey
Definition: rdma.h:161
vlib_node_registration_t rdma_input_node
(constructor) VLIB_REGISTER_NODE (rdma_input_node)
Definition: input.c:655
u32 sw_if_index
Definition: rdma.h:159
STATIC_ASSERT_OFFSET_OF(rdma_txq_t, cacheline1, 64)
u16 log2_cq_size
Definition: rdma.h:79
u32 head
Definition: rdma.h:76
format_function_t format_rdma_device_name
Definition: rdma.h:237
u8 bufs_log2sz
Definition: rdma.h:122
u16 msg_id_base
Definition: rdma.h:203
u8 * name
Definition: rdma.h:166
volatile u64 * dv_sq_db
Definition: rdma.h:112
rdma_txq_t * txqs
Definition: rdma.h:156
u8 opc_mod
Definition: rdma.h:54
u8 dv_cq_log2sz
Definition: rdma.h:124
u8 dv_sq_log2sz
Definition: rdma.h:123
struct ibv_qp * rx_qp4
Definition: rdma.h:175
rdma_rxq_t * rxqs
Definition: rdma.h:155
vnet_device_class_t rdma_device_class
rdma_tx_func_error_t
Definition: rdma.h:255
struct _vlib_node_registration vlib_node_registration_t
mlx5dv_cqe_t * cqes
Definition: rdma.h:83
#define CLIB_ALIGN_MARK(name, alignment)
Definition: clib.h:90
u32 async_event_clib_file_index
Definition: rdma.h:169
VLIB buffer representation.
Definition: buffer.h:102
u32 * bufs
Definition: rdma.h:118
#define foreach_rdma_tx_func_error
Definition: rdma.h:249
volatile u32 * wq_db
Definition: rdma.h:85
#define foreach_rdma_device_flags
Definition: rdma.h:28
u32 cq_ci
Definition: rdma.h:78
u16 last_cqe_flags
Definition: rdma.h:82
epu16_epi64 u16x16
Definition: vector_avx2.h:123
struct ibv_context * ctx
Definition: rdma.h:172
#define RDMA_MLX5_WQE_SZ
Definition: rdma.h:63
struct ibv_qp * rx_qp6
Definition: rdma.h:176
struct ibv_qp * ibv_qp
Definition: rdma.h:105