FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
input.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #define _GNU_SOURCE
19 #include <stdint.h>
20 #include <net/if.h>
21 #include <sys/ioctl.h>
22 #include <sys/uio.h>
23 
24 #include <vlib/vlib.h>
25 #include <vlib/unix/unix.h>
26 #include <vnet/ethernet/ethernet.h>
27 #include <vnet/devices/devices.h>
28 
29 #include <marvell/pp2/pp2.h>
30 
31 #define foreach_mrvl_pp2_input_error \
32  _(PPIO_RECV, "pp2_ppio_recv error") \
33  _(BPOOL_GET_NUM_BUFFS, "pp2_bpool_get_num_buffs error") \
34  _(BPOOL_PUT_BUFFS, "pp2_bpool_put_buffs error") \
35  _(BUFFER_ALLOC, "buffer alloc error") \
36  _(MAC_CE, "MAC error (CRC error)") \
37  _(MAC_OR, "overrun error") \
38  _(MAC_RSVD, "unknown MAC error") \
39  _(MAC_RE, "resource error") \
40  _(IP_HDR, "ip4 header error")
41 
42 typedef enum
43 {
44 #define _(f,s) MRVL_PP2_INPUT_ERROR_##f,
46 #undef _
49 
50 static __clib_unused char *mrvl_pp2_input_error_strings[] = {
51 #define _(n,s) s,
53 #undef _
54 };
55 
58  vlib_buffer_t * b0, uword * n_trace,
59  mrvl_pp2_if_t * ppif, struct pp2_ppio_desc *d)
60 {
62  vlib_trace_buffer (vm, node, next0, b0,
63  /* follow_chain */ 0);
64  vlib_set_trace_count (vm, node, --(*n_trace));
65  tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
66  tr->next_index = next0;
67  tr->hw_if_index = ppif->hw_if_index;
68  clib_memcpy_fast (&tr->desc, d, sizeof (struct pp2_ppio_desc));
69 }
70 
72 mrvl_pp2_set_buf_data_len_flags (vlib_buffer_t * b, struct pp2_ppio_desc *d,
73  u32 add_flags)
74 {
75  u16 len;
76  len = pp2_ppio_inq_desc_get_pkt_len (d);
78  b->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | add_flags;
79 
80  if (add_flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
81  vnet_buffer (b)->l2_hdr_offset = 2;
82 
83  if (add_flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID)
84  {
85  u16 offset = DM_RXD_GET_L3_OFF (d);
86  vnet_buffer (b)->l3_hdr_offset = offset;
87  b->current_data = offset;
88  b->current_length = len - offset + 2;
89  }
90  else
91  {
92  b->current_data = 2;
93  b->current_length = len;
94  }
95 
96  if (add_flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID)
97  vnet_buffer (b)->l4_hdr_offset = vnet_buffer (b)->l3_hdr_offset +
98  DM_RXD_GET_IPHDR_LEN (d) * 4;
99 
100  return len;
101 }
102 
104 mrvl_pp2_next_from_desc (vlib_node_runtime_t * node, struct pp2_ppio_desc * d,
105  vlib_buffer_t * b, u32 * next)
106 {
107  u8 l3_info;
108  /* ES bit set means MAC error - drop and count */
109  if (PREDICT_FALSE (DM_RXD_GET_ES (d)))
110  {
112  u8 ec = DM_RXD_GET_EC (d);
113  if (ec == 0)
114  b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_CE];
115  else if (ec == 1)
116  b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_OR];
117  else if (ec == 2)
118  b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_RSVD];
119  else if (ec == 3)
120  b->error = node->errors[MRVL_PP2_INPUT_ERROR_MAC_RE];
121  return mrvl_pp2_set_buf_data_len_flags (b, d, 0);
122  }
123  l3_info = DM_RXD_GET_L3_PRS_INFO (d);
124 
125  /* ipv4 packet can be value 1, 2 or 3 */
126  if (PREDICT_TRUE ((l3_info - 1) < 3))
127  {
128  if (PREDICT_FALSE (DM_RXD_GET_L3_IP4_HDR_ERR (d) != 0))
129  {
131  b->error = node->errors[MRVL_PP2_INPUT_ERROR_IP_HDR];
132  return mrvl_pp2_set_buf_data_len_flags (b, d, 0);
133  }
136  (b, d,
137  VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
138  VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
139  VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4);
140  }
141 
142  /* ipv4 packet can be value 4 or 5 */
143  if (PREDICT_TRUE ((l3_info - 4) < 2))
144  {
147  (b, d,
148  VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
149  VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
150  VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6);
151  }
152 
154  return mrvl_pp2_set_buf_data_len_flags (b, d,
155  VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
156 }
157 
160  vlib_frame_t * frame, mrvl_pp2_if_t * ppif,
161  u16 qid)
162 {
163  vnet_main_t *vnm = vnet_get_main ();
165  u32 thread_index = vm->thread_index;
166  mrvl_pp2_inq_t *inq = vec_elt_at_index (ppif->inqs, qid);
167  uword n_trace = vlib_get_trace_count (vm, node);
169  vec_elt_at_index (ppm->per_thread_data, thread_index);
172  u32 n_rx_packets = 0;
173  u32 n_rx_bytes = 0;
174  u32 *to_next = 0;
175  struct pp2_ppio_desc *d;
176  u16 n_desc = VLIB_FRAME_SIZE;
177  u32 n_bufs;
178  u32 *buffers;
179  int i;
180 
182  if (PREDICT_FALSE (pp2_ppio_recv (ppif->ppio, 0, qid, ptd->descs, &n_desc)))
183  {
184  vlib_error_count (vm, node->node_index, MRVL_PP2_INPUT_ERROR_PPIO_RECV,
185  1);
186  n_desc = 0;
187  }
188  n_rx_packets = n_desc;
189 
190  for (i = 0; i < n_desc; i++)
191  ptd->buffers[i] = pp2_ppio_inq_desc_get_cookie (&ptd->descs[i]);
192 
193  d = ptd->descs;
194  buffers = ptd->buffers;
195  sw_if_index[VLIB_RX] = ppif->sw_if_index;
196  sw_if_index[VLIB_TX] = (u32) ~ 0;
197  while (n_desc)
198  {
199  u32 n_left_to_next;
200  vlib_buffer_t *b0, *b1;
201  u32 bi0, bi1;
202  u32 next0, next1;
203  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
204  while (n_desc >= 4 && n_left_to_next >= 2)
205  {
206  /* prefetch */
207  bi0 = buffers[0];
208  bi1 = buffers[1];
209  to_next[0] = bi0;
210  to_next[1] = bi1;
211  b0 = vlib_get_buffer (vm, bi0);
212  b1 = vlib_get_buffer (vm, bi1);
213 
214  if (PREDICT_TRUE (ppif->per_interface_next_index == ~0))
215  {
216  n_rx_bytes += mrvl_pp2_next_from_desc (node, d, b0, &next0);
217  n_rx_bytes += mrvl_pp2_next_from_desc (node, d + 1, b1, &next1);
219  &next1, b0, b1);
220  }
221  else
222  {
223  n_rx_bytes += mrvl_pp2_set_buf_data_len_flags (b0, d, 0);
224  n_rx_bytes += mrvl_pp2_set_buf_data_len_flags (b1, d + 1, 0);
225  next0 = next1 = ppif->per_interface_next_index;
226  }
227 
228  clib_memcpy_fast (vnet_buffer (b0)->sw_if_index, sw_if_index,
229  sizeof (sw_if_index));
230  clib_memcpy_fast (vnet_buffer (b1)->sw_if_index, sw_if_index,
231  sizeof (sw_if_index));
232 
235 
236  if (PREDICT_FALSE (n_trace > 0))
237  {
238  mrvl_pp2_input_trace (vm, node, next0, b0, &n_trace, ppif, d);
239  if (n_trace > 0)
240  mrvl_pp2_input_trace (vm, node, next1, b1, &n_trace, ppif,
241  d + 1);
242  }
243 
244  to_next += 2;
245  n_left_to_next -= 2;
246  d += 2;
247  buffers += 2;
248  n_desc -= 2;
249 
250  /* enqueue */
251  vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
252  n_left_to_next, bi0, bi1, next0,
253  next1);
254 
255  }
256  while (n_desc && n_left_to_next)
257  {
258  u32 bi0 = buffers[0];
259  to_next[0] = bi0;
260  b0 = vlib_get_buffer (vm, bi0);
261 
262  if (PREDICT_TRUE (ppif->per_interface_next_index == ~0))
263  {
264  n_rx_bytes += mrvl_pp2_next_from_desc (node, d, b0, &next0);
266  b0);
267  }
268  else
269  {
270  n_rx_bytes += mrvl_pp2_set_buf_data_len_flags (b0, d, 0);
271  next0 = ppif->per_interface_next_index;
272  }
273 
274  clib_memcpy_fast (vnet_buffer (b0)->sw_if_index, sw_if_index,
275  sizeof (sw_if_index));
276 
278 
279  if (PREDICT_FALSE (n_trace > 0))
280  mrvl_pp2_input_trace (vm, node, next0, b0, &n_trace, ppif, d);
281 
282  to_next += 1;
283  n_left_to_next--;
284  d++;
285  buffers++;
286  n_desc--;
287 
288  /* enqueue */
289  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
290  n_left_to_next, bi0, next0);
291  }
292  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
293  }
295  interface_main.combined_sw_if_counters +
296  VNET_INTERFACE_COUNTER_RX, thread_index,
297  ppif->hw_if_index, n_rx_packets,
298  n_rx_bytes);
299 
300  if (PREDICT_FALSE (pp2_bpool_get_num_buffs (inq->bpool, &n_bufs)))
301  {
302  vlib_error_count (vm, node->node_index,
303  MRVL_PP2_INPUT_ERROR_BPOOL_GET_NUM_BUFFS, 1);
304  goto done;
305  }
306 
307  n_bufs = inq->size - n_bufs;
308  while (n_bufs >= MRVL_PP2_BUFF_BATCH_SZ)
309  {
310  u16 n_alloc, i;
311  struct buff_release_entry *e = ptd->bre;
312  u32 *buffers = ptd->buffers;
313 
314  n_alloc = vlib_buffer_alloc (vm, ptd->buffers, MRVL_PP2_BUFF_BATCH_SZ);
315  i = n_alloc;
316 
317  if (PREDICT_FALSE (n_alloc == 0))
318  {
319  vlib_error_count (vm, node->node_index,
320  MRVL_PP2_INPUT_ERROR_BUFFER_ALLOC, 1);
321  goto done;
322  }
323 
324  while (i--)
325  {
326  u32 bi = buffers[0];
327  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
328  e->buff.addr = vlib_buffer_get_pa (vm, b) - 64;
329  e->buff.cookie = bi;
330  e->bpool = inq->bpool;
331  e++;
332  buffers++;
333  }
334 
335  i = n_alloc;
336  if (PREDICT_FALSE (pp2_bpool_put_buffs (ptd->hif, ptd->bre, &i)))
337  {
338  vlib_error_count (vm, node->node_index,
339  MRVL_PP2_INPUT_ERROR_BPOOL_PUT_BUFFS, 1);
340  vlib_buffer_free (vm, ptd->buffers, n_alloc);
341  goto done;
342  }
343 
344  if (PREDICT_FALSE (i != n_alloc))
345  vlib_buffer_free (vm, ptd->buffers + i, n_alloc - i);
346 
347  n_bufs -= i;
348  }
349 
350 done:
351  return n_rx_packets;
352 }
353 
354 uword
357 {
358  u32 n_rx = 0;
360  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
362 
364  {
365  mrvl_pp2_if_t *ppif;
366  ppif = vec_elt_at_index (ppm->interfaces, dq->dev_instance);
367  if (ppif->flags & MRVL_PP2_IF_F_ADMIN_UP)
368  n_rx += mrvl_pp2_device_input_inline (vm, node, frame, ppif,
369  dq->queue_id);
370  }
371  return n_rx;
372 }
373 
374 /* *INDENT-OFF* */
376  .function = mrvl_pp2_input_fn,
378  .name = "mrvl-pp2-input",
379  .sibling_of = "device-input",
380  .format_trace = format_mrvl_pp2_input_trace,
381  .type = VLIB_NODE_TYPE_INPUT,
382  .state = VLIB_NODE_STATE_POLLING,
383  .n_errors = MRVL_PP2_INPUT_N_ERROR,
384  .error_strings = mrvl_pp2_input_error_strings,
385 };
386 
387 /* *INDENT-ON* */
388 
389 
390 /*
391  * fd.io coding-style-patch-verification: ON
392  *
393  * Local Variables:
394  * eval: (c-set-style "gnu")
395  * End:
396  */
struct pp2_ppio * ppio
Definition: pp2.h:55
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
u32 dev_instance
Definition: pp2.h:61
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:187
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:525
struct buff_release_entry bre[MRVL_PP2_BUFF_BATCH_SZ]
Definition: pp2.h:73
format_function_t format_mrvl_pp2_input_trace
Definition: pp2.h:135
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
u32 buffers[VLIB_FRAME_SIZE]
Definition: pp2.h:74
struct pp2_ppio_desc desc
Definition: pp2.h:129
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:457
#define PREDICT_TRUE(x)
Definition: clib.h:119
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
struct pp2_ppio_desc * descs
Definition: pp2.h:72
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define VLIB_NODE_FLAG_TRACE_SUPPORTED
Definition: node.h:308
mrvl_pp2_input_error_t
Definition: input.c:42
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static_always_inline void mrvl_pp2_input_trace(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next0, vlib_buffer_t *b0, uword *n_trace, mrvl_pp2_if_t *ppif, struct pp2_ppio_desc *d)
Definition: input.c:57
u32 per_interface_next_index
Definition: pp2.h:56
mrvl_pp2_main_t mrvl_pp2_main
Definition: pp2.c:36
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:518
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:472
#define foreach_mrvl_pp2_input_error
Definition: input.c:31
unsigned char u8
Definition: types.h:56
u32 hw_if_index
Definition: pp2.h:63
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:130
struct pp2_bpool * bpool
Definition: pp2.h:38
#define static_always_inline
Definition: clib.h:106
vl_api_interface_index_t sw_if_index
Definition: gre.api:53
u32 sw_if_index
Definition: pp2.h:62
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
static_always_inline uword mrvl_pp2_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, mrvl_pp2_if_t *ppif, u16 qid)
Definition: input.c:159
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:380
struct pp2_hif * hif
Definition: pp2.h:71
#define MRVL_PP2_BUFF_BATCH_SZ
Definition: pp2.h:66
uword mrvl_pp2_input_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: input.c:355
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
unsigned short u16
Definition: types.h:57
static_always_inline u16 mrvl_pp2_set_buf_data_len_flags(vlib_buffer_t *b, struct pp2_ppio_desc *d, u32 add_flags)
Definition: input.c:72
u32 flags
Definition: pp2.h:53
#define PREDICT_FALSE(x)
Definition: clib.h:118
u32 node_index
Node index.
Definition: node.h:498
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:224
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:338
vlib_main_t * vm
Definition: in2out_ed.c:1599
static_always_inline void vnet_feature_start_device_input_x2(u32 sw_if_index, u32 *next0, u32 *next1, vlib_buffer_t *b0, vlib_buffer_t *b1)
Definition: feature.h:373
u8 len
Definition: ip_types.api:92
vlib_node_registration_t mrvl_pp2_input_node
(constructor) VLIB_REGISTER_NODE (mrvl_pp2_input_node)
Definition: input.c:375
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:483
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1599
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
template key/value backing page structure
Definition: bihash_doc.h:44
#define foreach_device_and_queue(var, vec)
Definition: devices.h:161
Definition: defs.h:47
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:492
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
struct clib_bihash_value offset
template key/value backing page structure
#define vnet_buffer(b)
Definition: buffer.h:417
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:343
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1600
mrvl_pp2_if_t * interfaces
Definition: pp2.h:79
mrvl_pp2_per_thread_data_t * per_thread_data
Definition: pp2.h:80
static_always_inline u16 mrvl_pp2_next_from_desc(vlib_node_runtime_t *node, struct pp2_ppio_desc *d, vlib_buffer_t *b, u32 *next)
Definition: input.c:104
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:203
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:677
mrvl_pp2_inq_t * inqs
Definition: pp2.h:58
static __clib_unused char * mrvl_pp2_input_error_strings[]
Definition: input.c:50
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
#define MRVL_PP2_IF_F_ADMIN_UP
Definition: pp2.h:54
Definition: defs.h:46
u16 size
Definition: pp2.h:37