FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vppinfra/ring.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/devices/devices.h>
24 
25 #include <rdma/rdma.h>
26 
29 {
30  struct ibv_wc wc[VLIB_FRAME_SIZE];
31  u32 tail, slot;
32  int n;
33 
34  n = ibv_poll_cq (txq->cq, VLIB_FRAME_SIZE, wc);
35  if (n <= 0)
36  return;
37 
38  tail = wc[n - 1].wr_id;
39  slot = txq->head & (txq->size - 1);
40  vlib_buffer_free_from_ring (vm, txq->bufs, slot, txq->size,
41  tail - txq->head);
42  txq->head = tail;
43 }
44 
47  rdma_txq_t * txq, u32 n_left_from, u32 * bi)
48 {
49  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
50  struct ibv_send_wr wr[VLIB_FRAME_SIZE], *w = wr;
51  struct ibv_sge sge[VLIB_FRAME_SIZE], *s = sge;
52  u32 n, slot = txq->tail & (txq->size - 1);
53  u32 *tx = &txq->bufs[slot];
54 
55  /* do not enqueue more packet than ring space */
56  n_left_from = clib_min (n_left_from, txq->size - (txq->tail - txq->head));
57  /* avoid wrap-around logic in core loop */
58  n = n_left_from = clib_min (n_left_from, txq->size - slot);
59 
60  /* if ring is full, do nothing */
61  if (PREDICT_FALSE (0 == n_left_from))
62  return 0;
63 
64  vlib_get_buffers (vm, bi, bufs, n_left_from);
65  memset (w, 0, n_left_from * sizeof (w[0]));
66 
67  while (n >= 4)
68  {
69  if (PREDICT_TRUE (n >= 8))
70  {
71  vlib_prefetch_buffer_header (b[4 + 0], LOAD);
72  vlib_prefetch_buffer_header (b[4 + 1], LOAD);
73  vlib_prefetch_buffer_header (b[4 + 2], LOAD);
74  vlib_prefetch_buffer_header (b[4 + 3], LOAD);
75 
76  CLIB_PREFETCH (&s[4 + 0], 4 * sizeof (s[0]), STORE);
77 
78  CLIB_PREFETCH (&w[4 + 0], CLIB_CACHE_LINE_BYTES, STORE);
79  CLIB_PREFETCH (&w[4 + 1], CLIB_CACHE_LINE_BYTES, STORE);
80  CLIB_PREFETCH (&w[4 + 2], CLIB_CACHE_LINE_BYTES, STORE);
81  CLIB_PREFETCH (&w[4 + 3], CLIB_CACHE_LINE_BYTES, STORE);
82  }
83 
84  vlib_buffer_copy_indices (tx, bi, 4);
85 
86  s[0].addr = vlib_buffer_get_current_va (b[0]);
87  s[0].length = b[0]->current_length;
88  s[0].lkey = rd->lkey;
89 
90  s[1].addr = vlib_buffer_get_current_va (b[1]);
91  s[1].length = b[1]->current_length;
92  s[1].lkey = rd->lkey;
93 
94  s[2].addr = vlib_buffer_get_current_va (b[2]);
95  s[2].length = b[2]->current_length;
96  s[2].lkey = rd->lkey;
97 
98  s[3].addr = vlib_buffer_get_current_va (b[3]);
99  s[3].length = b[3]->current_length;
100  s[3].lkey = rd->lkey;
101 
102  w[0].next = &w[0] + 1;
103  w[0].sg_list = &s[0];
104  w[0].num_sge = 1;
105  w[0].opcode = IBV_WR_SEND;
106 
107  w[1].next = &w[1] + 1;
108  w[1].sg_list = &s[1];
109  w[1].num_sge = 1;
110  w[1].opcode = IBV_WR_SEND;
111 
112  w[2].next = &w[2] + 1;
113  w[2].sg_list = &s[2];
114  w[2].num_sge = 1;
115  w[2].opcode = IBV_WR_SEND;
116 
117  w[3].next = &w[3] + 1;
118  w[3].sg_list = &s[3];
119  w[3].num_sge = 1;
120  w[3].opcode = IBV_WR_SEND;
121 
122  s += 4;
123  w += 4;
124  b += 4;
125  bi += 4;
126  tx += 4;
127  n -= 4;
128  }
129 
130  while (n >= 1)
131  {
132  vlib_buffer_copy_indices (tx, bi, 1);
133 
134  s[0].addr = vlib_buffer_get_current_va (b[0]);
135  s[0].length = b[0]->current_length;
136  s[0].lkey = rd->lkey;
137 
138  w[0].next = &w[0] + 1;
139  w[0].sg_list = &s[0];
140  w[0].num_sge = 1;
141  w[0].opcode = IBV_WR_SEND;
142 
143  s += 1;
144  w += 1;
145  b += 1;
146  bi += 1;
147  tx += 1;
148  n -= 1;
149  }
150 
151  w[-1].wr_id = txq->tail + n_left_from; /* register item to free */
152  w[-1].next = 0; /* fix next pointer in WR linked-list */
153  w[-1].send_flags = IBV_SEND_SIGNALED; /* generate a CQE so we can free buffers */
154 
155  w = wr;
156  if (PREDICT_FALSE (0 != ibv_post_send (txq->qp, w, &w)))
157  n_left_from = w - wr;
158 
159  txq->tail += n_left_from;
160  return n_left_from;
161 }
162 
164  vlib_node_runtime_t * node,
165  vlib_frame_t * frame)
166 {
167  rdma_main_t *rm = &rdma_main;
168  vnet_interface_output_runtime_t *ord = (void *) node->runtime_data;
170  u32 thread_index = vm->thread_index;
171  rdma_txq_t *txq =
172  vec_elt_at_index (rd->txqs, thread_index % vec_len (rd->txqs));
173  u32 *from;
174  u32 n_left_from;
175  int i;
176 
177  ASSERT (txq->size >= VLIB_FRAME_SIZE && is_pow2 (txq->size));
178  ASSERT (txq->tail - txq->head <= txq->size);
179 
180  from = vlib_frame_vector_args (frame);
181  n_left_from = frame->n_vectors;
182 
184 
185  for (i = 0; i < 5 && n_left_from > 0; i++)
186  {
187  u32 n_enq;
188  rdma_device_output_free (vm, txq);
189  n_enq = rmda_device_output_tx (vm, rd, txq, n_left_from, from);
190  n_left_from -= n_enq;
191  from += n_enq;
192  }
193 
195 
196  if (PREDICT_FALSE (n_left_from))
197  {
198  vlib_buffer_free (vm, from, n_left_from);
199  vlib_error_count (vm, node->node_index,
200  RDMA_TX_ERROR_NO_FREE_SLOTS, n_left_from);
201  }
202 
203  return frame->n_vectors - n_left_from;
204 }
205 
206 /*
207  * fd.io coding-style-patch-verification: ON
208  *
209  * Local Variables:
210  * eval: (c-set-style "gnu")
211  * End:
212  */
#define clib_min(x, y)
Definition: clib.h:302
static uword vlib_buffer_get_current_va(vlib_buffer_t *b)
Definition: buffer.h:237
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
#define PREDICT_TRUE(x)
Definition: clib.h:113
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
static_always_inline void rdma_device_output_free(vlib_main_t *vm, rdma_txq_t *txq)
Definition: output.c:28
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:110
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u32 size
Definition: rdma.h:55
int i
clib_spinlock_t lock
Definition: rdma.h:51
#define static_always_inline
Definition: clib.h:100
rdma_device_t * devices
Definition: rdma.h:95
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
struct ibv_cq * cq
Definition: rdma.h:52
u32 tail
Definition: rdma.h:57
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:298
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:378
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:912
u32 head
Definition: rdma.h:56
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
#define PREDICT_FALSE(x)
Definition: clib.h:112
static_always_inline u32 rmda_device_output_tx(vlib_main_t *vm, const rdma_device_t *rd, rdma_txq_t *txq, u32 n_left_from, u32 *bi)
Definition: output.c:46
struct ibv_qp * qp
Definition: rdma.h:53
rdma_main_t rdma_main
Definition: device.c:46
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
vlib_main_t * vm
Definition: buffer.c:323
VNET_DEVICE_CLASS_TX_FN() rdma_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:163
u32 lkey
Definition: rdma.h:71
#define ASSERT(truth)
static uword is_pow2(uword x)
Definition: clib.h:236
static_always_inline void vlib_buffer_copy_indices(u32 *dst, u32 *src, u32 n_indices)
Definition: buffer_funcs.h:102
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
VLIB buffer representation.
Definition: buffer.h:102
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
u32 * bufs
Definition: rdma.h:54
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:95