FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
input.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 
24 #include <vmxnet3/vmxnet3.h>
25 
26 #define foreach_vmxnet3_input_error \
27  _(BUFFER_ALLOC, "buffer alloc error") \
28  _(RX_PACKET_NO_SOP, "Rx packet error - no SOP") \
29  _(RX_PACKET, "Rx packet error") \
30  _(RX_PACKET_EOP, "Rx packet error found on EOP") \
31  _(NO_BUFFER, "Rx no buffer error")
32 
33 typedef enum
34 {
35 #define _(f,s) VMXNET3_INPUT_ERROR_##f,
37 #undef _
40 
41 static __clib_unused char *vmxnet3_input_error_strings[] = {
42 #define _(n,s) s,
44 #undef _
45 };
46 
48 vmxnet3_find_rid (vmxnet3_device_t * vd, vmxnet3_rx_comp * rx_comp)
49 {
50  u32 rid;
51 
52  // rid is bits 16-25 (10 bits number)
53  rid = rx_comp->index & (0xffffffff >> 6);
54  rid >>= 16;
55  if ((rid >= vd->num_rx_queues) && (rid < (vd->num_rx_queues << 1)))
56  return 1;
57  else
58  return 0;
59 }
60 
63 {
64  vmxnet3_rx_comp_ring *comp_ring = &rxq->rx_comp_ring;
65 
66  comp_ring->next++;
67  if (PREDICT_FALSE (comp_ring->next == rxq->size))
68  {
69  comp_ring->next = 0;
70  comp_ring->gen ^= VMXNET3_RXCF_GEN;
71  }
72 }
73 
76  vlib_frame_t * frame, vmxnet3_device_t * vd,
77  u16 qid)
78 {
79  vnet_main_t *vnm = vnet_get_main ();
80  uword n_trace = vlib_get_trace_count (vm, node);
81  u32 n_rx_packets = 0, n_rx_bytes = 0;
82  vmxnet3_rx_comp *rx_comp;
83  u32 desc_idx;
84  vmxnet3_rxq_t *rxq;
85  u32 thread_index = vm->thread_index;
86  u32 buffer_indices[VLIB_FRAME_SIZE], *bi;
87  u16 nexts[VLIB_FRAME_SIZE], *next;
88  vmxnet3_rx_ring *ring;
89  vmxnet3_rx_comp_ring *comp_ring;
90  u16 rid;
91  vlib_buffer_t *prev_b0 = 0, *hb = 0;
93  u8 known_next = 0, got_packet = 0;
94  vmxnet3_rx_desc *rxd;
95  clib_error_t *error;
96 
97  rxq = vec_elt_at_index (vd->rxqs, qid);
98  comp_ring = &rxq->rx_comp_ring;
99  bi = buffer_indices;
100  next = nexts;
101  rx_comp = &rxq->rx_comp[comp_ring->next];
102 
103  while (PREDICT_TRUE (n_rx_packets < VLIB_FRAME_SIZE) &&
104  (comp_ring->gen == (rx_comp->flags & VMXNET3_RXCF_GEN)))
105  {
106  vlib_buffer_t *b0;
107  u32 bi0;
108 
109  rid = vmxnet3_find_rid (vd, rx_comp);
110  ring = &rxq->rx_ring[rid];
111 
112  if (PREDICT_TRUE (ring->fill >= 1))
113  ring->fill--;
114  else
115  {
116  vlib_error_count (vm, node->node_index,
117  VMXNET3_INPUT_ERROR_NO_BUFFER, 1);
118  if (hb)
119  {
121  hb = 0;
122  }
123  prev_b0 = 0;
124  break;
125  }
126 
127  desc_idx = rx_comp->index & VMXNET3_RXC_INDEX;
128  ring->consume = desc_idx;
129  rxd = &rxq->rx_desc[rid][desc_idx];
130 
131  bi0 = ring->bufs[desc_idx];
132  ring->bufs[desc_idx] = ~0;
133 
134  b0 = vlib_get_buffer (vm, bi0);
135  vnet_buffer (b0)->sw_if_index[VLIB_RX] = vd->sw_if_index;
136  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
137  vnet_buffer (b0)->feature_arc_index = 0;
138  b0->current_length = rx_comp->len & VMXNET3_RXCL_LEN_MASK;
139  b0->current_data = 0;
141  b0->next_buffer = 0;
142  b0->flags = 0;
143  b0->error = 0;
144  b0->current_config_index = 0;
145  ASSERT (b0->current_length != 0);
146 
147  if (PREDICT_FALSE ((rx_comp->index & VMXNET3_RXCI_EOP) &&
148  (rx_comp->len & VMXNET3_RXCL_ERROR)))
149  {
150  vlib_buffer_free_one (vm, bi0);
151  vlib_error_count (vm, node->node_index,
152  VMXNET3_INPUT_ERROR_RX_PACKET_EOP, 1);
153  if (hb && vlib_get_buffer_index (vm, hb) != bi0)
154  {
156  hb = 0;
157  }
158  prev_b0 = 0;
159  goto next;
160  }
161 
162  if (rx_comp->index & VMXNET3_RXCI_SOP)
163  {
164  ASSERT (!(rxd->flags & VMXNET3_RXF_BTYPE));
165  /* start segment */
166  hb = b0;
167  bi[0] = bi0;
168  if (!(rx_comp->index & VMXNET3_RXCI_EOP))
169  {
170  hb->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
171  prev_b0 = b0;
172  }
173  else
174  {
175  /*
176  * Both start and end of packet is set. It is a complete packet
177  */
178  prev_b0 = 0;
179  got_packet = 1;
180  }
181  }
182  else if (rx_comp->index & VMXNET3_RXCI_EOP)
183  {
184  /* end of segment */
185  if (prev_b0)
186  {
187  prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
188  prev_b0->next_buffer = bi0;
189  hb->total_length_not_including_first_buffer +=
190  b0->current_length;
191  prev_b0 = 0;
192  got_packet = 1;
193  }
194  else
195  {
196  /* EOP without SOP, error */
197  vlib_error_count (vm, node->node_index,
198  VMXNET3_INPUT_ERROR_RX_PACKET_NO_SOP, 1);
199  vlib_buffer_free_one (vm, bi0);
200  if (hb && vlib_get_buffer_index (vm, hb) != bi0)
201  {
203  hb = 0;
204  }
205  goto next;
206  }
207  }
208  else if (prev_b0) // !sop && !eop
209  {
210  /* mid chain */
211  ASSERT (rxd->flags & VMXNET3_RXF_BTYPE);
212  prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
213  prev_b0->next_buffer = bi0;
214  prev_b0 = b0;
216  }
217  else
218  {
219  vlib_error_count (vm, node->node_index,
220  VMXNET3_INPUT_ERROR_RX_PACKET, 1);
221  vlib_buffer_free_one (vm, bi0);
222  if (hb && vlib_get_buffer_index (vm, hb) != bi0)
223  {
225  hb = 0;
226  }
227  goto next;
228  }
229 
230  n_rx_bytes += b0->current_length;
231 
232  if (got_packet)
233  {
234  ethernet_header_t *e = (ethernet_header_t *) hb->data;
235 
236  if (PREDICT_FALSE (vd->per_interface_next_index != ~0))
237  {
238  next_index = vd->per_interface_next_index;
239  known_next = 1;
240  }
241 
242  if (PREDICT_FALSE
244  {
246  &next_index, hb);
247  known_next = 1;
248  }
249 
250  if (PREDICT_FALSE (known_next))
251  {
252  next[0] = next_index;
253  }
254  else
255  {
258  else
259  {
260  if (rx_comp->flags & VMXNET3_RXCF_IP4)
261  {
263  hb->flags |= VNET_BUFFER_F_IS_IP4;
266  [next[0]]);
267  }
268  else if (rx_comp->flags & VMXNET3_RXCF_IP6)
269  {
271  hb->flags |= VNET_BUFFER_F_IS_IP6;
274  [next[0]]);
275  }
276  else
277  {
279  }
280  }
281  }
282 
283  n_rx_packets++;
284  next++;
285  bi++;
286  hb = 0;
287  got_packet = 0;
288  }
289 
290  next:
292  rx_comp = &rxq->rx_comp[comp_ring->next];
293  }
294 
295  if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
296  {
297  u32 n_left = n_rx_packets;
298 
299  bi = buffer_indices;
300  next = nexts;
301  while (n_trace && n_left)
302  {
303  vlib_buffer_t *b;
305 
306  b = vlib_get_buffer (vm, bi[0]);
307  vlib_trace_buffer (vm, node, next[0], b, /* follow_chain */ 0);
308  tr = vlib_add_trace (vm, node, b, sizeof (*tr));
309  tr->next_index = next[0];
310  tr->hw_if_index = vd->hw_if_index;
311  tr->buffer = *b;
312 
313  n_trace--;
314  n_left--;
315  bi++;
316  next++;
317  }
318  vlib_set_trace_count (vm, node, n_trace);
319  }
320 
321  if (PREDICT_TRUE (n_rx_packets))
322  {
323  vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts,
324  n_rx_packets);
327  VNET_INTERFACE_COUNTER_RX, thread_index,
328  vd->hw_if_index, n_rx_packets, n_rx_bytes);
329  }
330 
331  error = vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
332  if (PREDICT_FALSE (error != 0))
333  {
334  vlib_error_count (vm, node->node_index,
335  VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
336  }
337  error = vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
338  if (PREDICT_FALSE (error != 0))
339  {
340  vlib_error_count (vm, node->node_index,
341  VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
342  }
343 
344  return n_rx_packets;
345 }
346 
348  vlib_node_runtime_t * node,
349  vlib_frame_t * frame)
350 {
351  u32 n_rx = 0;
352  vmxnet3_main_t *vmxm = &vmxnet3_main;
353  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
355 
357  {
358  vmxnet3_device_t *vd;
359  vd = vec_elt_at_index (vmxm->devices, dq->dev_instance);
360  if ((vd->flags & VMXNET3_DEVICE_F_ADMIN_UP) == 0)
361  continue;
362  n_rx += vmxnet3_device_input_inline (vm, node, frame, vd, dq->queue_id);
363  }
364  return n_rx;
365 }
366 
367 #ifndef CLIB_MARCH_VARIANT
368 /* *INDENT-OFF* */
370  .name = "vmxnet3-input",
371  .sibling_of = "device-input",
372  .format_trace = format_vmxnet3_input_trace,
373  .type = VLIB_NODE_TYPE_INPUT,
374  .state = VLIB_NODE_STATE_DISABLED,
375  .n_errors = VMXNET3_INPUT_N_ERROR,
376  .error_strings = vmxnet3_input_error_strings,
377 };
378 #endif
379 
380 /* *INDENT-ON* */
381 
382 /*
383  * fd.io coding-style-patch-verification: ON
384  *
385  * Local Variables:
386  * eval: (c-set-style "gnu")
387  * End:
388  */
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring0(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:591
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:156
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:204
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:135
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
vmxnet3_rx_desc * rx_desc[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:423
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define PREDICT_TRUE(x)
Definition: clib.h:108
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define VMXNET3_RXCF_IP4
Definition: vmxnet3.h:85
u32 thread_index
Definition: main.h:179
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
#define VLIB_NODE_FN(node)
Definition: node.h:187
unsigned char u8
Definition: types.h:56
#define VMXNET3_RXCF_GEN
Definition: vmxnet3.h:86
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:114
vmxnet3_rxq_t * rxqs
Definition: vmxnet3.h:475
static __clib_unused char * vmxnet3_input_error_strings[]
Definition: input.c:41
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:109
#define static_always_inline
Definition: clib.h:95
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:855
static_always_inline int vnet_device_input_have_features(u32 sw_if_index)
Definition: feature.h:252
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VMXNET3_RXC_INDEX
Definition: vmxnet3.h:87
#define VMXNET3_RXCL_LEN_MASK
Definition: vmxnet3.h:112
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:382
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:154
const u32 device_input_next_node_advance[((VNET_DEVICE_INPUT_N_NEXT_NODES/CLIB_CACHE_LINE_BYTES)+1)*CLIB_CACHE_LINE_BYTES]
Definition: devices.c:47
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static_always_inline void vmxnet3_rx_comp_ring_advance_next(vmxnet3_rxq_t *rxq)
Definition: input.c:62
unsigned short u16
Definition: types.h:57
vmxnet3_rx_comp_ring rx_comp_ring
Definition: vmxnet3.h:425
#define PREDICT_FALSE(x)
Definition: clib.h:107
u32 node_index
Node index.
Definition: node.h:494
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:138
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:155
vlib_main_t * vm
Definition: buffer.c:294
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
vlib_buffer_t buffer
Definition: vmxnet3.h:520
u32 per_interface_next_index
Definition: vmxnet3.h:465
#define ASSERT(truth)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:129
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:97
#define VMXNET3_RXCI_EOP
Definition: vmxnet3.h:114
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:218
#define foreach_vmxnet3_input_error
Definition: input.c:26
static_always_inline uword vmxnet3_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vmxnet3_device_t *vd, u16 qid)
Definition: input.c:75
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:57
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:155
vlib_node_registration_t vmxnet3_input_node
(constructor) VLIB_REGISTER_NODE (vmxnet3_input_node)
Definition: input.c:369
#define foreach_device_and_queue(var, vec)
Definition: devices.h:156
Definition: defs.h:47
#define VMXNET3_RXCL_ERROR
Definition: vmxnet3.h:113
u64 uword
Definition: types.h:112
#define VMXNET3_RXF_BTYPE
Definition: vmxnet3.h:82
vmxnet3_rx_comp * rx_comp
Definition: vmxnet3.h:424
vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:422
vmxnet3_input_error_t
Definition: input.c:33
format_function_t format_vmxnet3_input_trace
Definition: vmxnet3.h:533
#define vnet_buffer(b)
Definition: buffer.h:344
#define VMXNET3_RXCF_IP6
Definition: vmxnet3.h:84
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:259
vmxnet3_device_t * devices
Definition: vmxnet3.h:495
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring1(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:633
#define VMXNET3_RXCI_SOP
Definition: vmxnet3.h:115
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:588
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:172
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:116
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:58
Definition: defs.h:46
static_always_inline u16 vmxnet3_find_rid(vmxnet3_device_t *vd, vmxnet3_rx_comp *rx_comp)
Definition: input.c:48