FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 
24 #include <vmxnet3/vmxnet3.h>
25 
28 {
29  vmxnet3_tx_comp_ring *comp_ring = &txq->tx_comp_ring;
30 
31  comp_ring->next++;
32  if (PREDICT_FALSE (comp_ring->next == txq->size))
33  {
34  comp_ring->next = 0;
35  comp_ring->gen ^= VMXNET3_TXCF_GEN;
36  }
37 }
38 
41 {
42  txq->tx_ring.produce++;
43  if (PREDICT_FALSE (txq->tx_ring.produce == txq->size))
44  {
45  txq->tx_ring.produce = 0;
46  txq->tx_ring.gen ^= VMXNET3_TXF_GEN;
47  }
48 }
49 
52 {
53  txq->tx_ring.consume++;
54  txq->tx_ring.consume &= txq->size - 1;
55 }
56 
59  vmxnet3_txq_t * txq)
60 {
61  vmxnet3_tx_comp *tx_comp;
62  vmxnet3_tx_comp_ring *comp_ring;
63 
64  comp_ring = &txq->tx_comp_ring;
65  tx_comp = &txq->tx_comp[comp_ring->next];
66 
67  while ((tx_comp->flags & VMXNET3_TXCF_GEN) == comp_ring->gen)
68  {
69  u16 eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
70  u32 bi0 = txq->tx_ring.bufs[txq->tx_ring.consume];
71 
72  vlib_buffer_free_one (vm, bi0);
73  while (txq->tx_ring.consume != eop_idx)
74  {
76  }
78 
80  tx_comp = &txq->tx_comp[comp_ring->next];
81  }
82 }
83 
86 {
87  u16 count;
88 
89  count = (txq->tx_ring.consume - txq->tx_ring.produce - 1);
90  /* Wrapped? */
91  if (txq->tx_ring.produce >= txq->tx_ring.consume)
92  count += txq->size;
93  return count;
94 }
95 
99 {
100  vmxnet3_main_t *vmxm = &vmxnet3_main;
101  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
103  u32 *buffers = vlib_frame_vector_args (frame);
104  u32 bi0;
105  vlib_buffer_t *b0;
106  vmxnet3_tx_desc *txd = 0;
107  u32 desc_idx, generation, first_idx;
108  u16 space_left;
109  u16 n_left = frame->n_vectors;
110  vmxnet3_txq_t *txq;
111  u16 qid = vm->thread_index % vd->num_tx_queues, produce;
112 
113  if (PREDICT_FALSE (!(vd->flags & VMXNET3_DEVICE_F_LINK_UP)))
114  {
115  vlib_buffer_free (vm, buffers, n_left);
116  vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_LINK_DOWN,
117  n_left);
118  return (0);
119  }
120 
121  txq = vec_elt_at_index (vd->txqs, qid);
123 
124  vmxnet3_txq_release (vm, vd, txq);
125 
126  produce = txq->tx_ring.produce;
127  while (PREDICT_TRUE (n_left))
128  {
129  u16 space_needed = 1, i;
130  u32 gso_size = 0;
131  vlib_buffer_t *b;
132  u32 hdr_len = 0;
133 
134  bi0 = buffers[0];
135  b0 = vlib_get_buffer (vm, bi0);
136  b = b0;
137 
138  space_left = vmxnet3_tx_ring_space_left (txq);
139  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
140  {
141  u32 next_buffer = b->next_buffer;
142 
143  b = vlib_get_buffer (vm, next_buffer);
144  space_needed++;
145  }
146  if (PREDICT_FALSE (space_left < space_needed))
147  {
148  vmxnet3_txq_release (vm, vd, txq);
149  space_left = vmxnet3_tx_ring_space_left (txq);
150 
151  if (PREDICT_FALSE (space_left < space_needed))
152  {
153  vlib_buffer_free_one (vm, bi0);
154  vlib_error_count (vm, node->node_index,
155  VMXNET3_TX_ERROR_NO_FREE_SLOTS, 1);
156  buffers++;
157  n_left--;
158  /*
159  * Drop this packet. But we may have enough room for the next
160  * packet
161  */
162  continue;
163  }
164  }
165 
166  /*
167  * Toggle the generation bit for SOP fragment to avoid device starts
168  * reading incomplete packet
169  */
170  generation = txq->tx_ring.gen ^ VMXNET3_TXF_GEN;
171  first_idx = txq->tx_ring.produce;
172  for (i = 0; i < space_needed; i++)
173  {
174  b0 = vlib_get_buffer (vm, bi0);
176 
177  desc_idx = txq->tx_ring.produce;
178 
180  txq->tx_ring.bufs[desc_idx] = bi0;
181 
182  txd = &txq->tx_desc[desc_idx];
183 
184  txd->address = vlib_buffer_get_current_pa (vm, b0);
185 
186  txd->flags[0] = generation | b0->current_length;
187  txd->flags[1] = 0;
188  if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
189  {
190  /*
191  * We should not be getting GSO outbound traffic unless it is
192  * lro is enable
193  */
194  ASSERT (vd->gso_enable == 1);
195  gso_size = vnet_buffer2 (b0)->gso_size;
196  hdr_len = vnet_buffer (b0)->l4_hdr_offset +
197  sizeof (ethernet_header_t);
198  }
199 
200  generation = txq->tx_ring.gen;
201  bi0 = b0->next_buffer;
202  }
203  if (PREDICT_FALSE (gso_size != 0))
204  {
205  txd->flags[1] = hdr_len;
206  txd->flags[1] |= VMXNET3_TXF_OM (VMXNET3_OM_TSO);
207  txd->flags[0] |= VMXNET3_TXF_MSSCOF (gso_size);
208  }
209  txd->flags[1] |= VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
210  asm volatile ("":::"memory");
211  /*
212  * Now toggle back the generation bit for the first segment.
213  * Device can start reading the packet
214  */
215  txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
216 
217  buffers++;
218  n_left--;
219  }
220 
221  if (PREDICT_TRUE (produce != txq->tx_ring.produce))
223 
225 
226  return (frame->n_vectors - n_left);
227 }
228 
229 /*
230  * fd.io coding-style-patch-verification: ON
231  *
232  * Local Variables:
233  * eval: (c-set-style "gnu")
234  * End:
235  */
static_always_inline void vmxnet3_tx_ring_advance_consume(vmxnet3_txq_t *txq)
Definition: output.c:51
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
u8 count
Definition: dhcp.api:208
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:463
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
#define vnet_buffer2(b)
Definition: buffer.h:482
#define PREDICT_TRUE(x)
Definition: clib.h:119
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
clib_spinlock_t lock
Definition: vmxnet3.h:546
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:110
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
#define VMXNET3_TXF_OM(x)
Definition: vmxnet3.h:149
VNET_DEVICE_CLASS_TX_FN() vmxnet3_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:96
#define static_always_inline
Definition: clib.h:106
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:305
unsigned int u32
Definition: types.h:88
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
#define VMXNET3_TXC_INDEX
Definition: vmxnet3.h:156
static_always_inline u16 vmxnet3_tx_ring_space_left(vmxnet3_txq_t *txq)
Definition: output.c:85
unsigned short u16
Definition: types.h:57
static_always_inline void vmxnet3_tx_ring_advance_produce(vmxnet3_txq_t *txq)
Definition: output.c:40
vmxnet3_tx_comp_ring tx_comp_ring
Definition: vmxnet3.h:551
vmxnet3_tx_comp * tx_comp
Definition: vmxnet3.h:549
#define PREDICT_FALSE(x)
Definition: clib.h:118
vlib_main_t * vm
Definition: in2out_ed.c:1599
#define VMXNET3_TXF_CQ
Definition: vmxnet3.h:152
#define VMXNET3_TXF_GEN
Definition: vmxnet3.h:146
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1599
#define ASSERT(truth)
static_always_inline void vmxnet3_txq_release(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_txq_t *txq)
Definition: output.c:58
#define VMXNET3_TXCF_GEN
Definition: vmxnet3.h:155
#define VMXNET3_TXF_EOP
Definition: vmxnet3.h:151
static_always_inline void vmxnet3_tx_comp_ring_advance_next(vmxnet3_txq_t *txq)
Definition: output.c:27
u32 reg_txprod
Definition: vmxnet3.h:545
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:492
VLIB buffer representation.
Definition: buffer.h:102
vmxnet3_tx_desc * tx_desc
Definition: vmxnet3.h:548
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
#define vnet_buffer(b)
Definition: buffer.h:417
vmxnet3_device_t * devices
Definition: vmxnet3.h:592
#define VMXNET3_TXF_MSSCOF(x)
Definition: vmxnet3.h:150
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1600
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:970
vmxnet3_tx_ring tx_ring
Definition: vmxnet3.h:550
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:95
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static_always_inline void vmxnet3_reg_write_inline(vmxnet3_device_t *vd, u8 bar, u32 addr, u32 val)
Definition: vmxnet3.h:646