FD.io VPP  v18.04-17-g3a0d853
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2016 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #define _GNU_SOURCE
19 #include <stdint.h>
20 #include <net/if.h>
21 #include <sys/ioctl.h>
22 #include <sys/uio.h>
23 
24 #include <vlib/vlib.h>
25 #include <vlib/unix/unix.h>
26 #include <vnet/ethernet/ethernet.h>
27 #include <vnet/devices/devices.h>
28 #include <vnet/feature/feature.h>
29 
30 #include <memif/memif.h>
31 #include <memif/private.h>
32 
33 #define foreach_memif_input_error \
34  _(BUFFER_ALLOC_FAIL, "buffer allocation failed") \
35  _(NOT_IP, "not ip packet")
36 
37 typedef enum
38 {
39 #define _(f,s) MEMIF_INPUT_ERROR_##f,
41 #undef _
44 
45 static __clib_unused char *memif_input_error_strings[] = {
46 #define _(n,s) s,
48 #undef _
49 };
50 
51 typedef struct
52 {
57 
58 static __clib_unused u8 *
59 format_memif_input_trace (u8 * s, va_list * args)
60 {
61  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
62  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
63  memif_input_trace_t *t = va_arg (*args, memif_input_trace_t *);
64  u32 indent = format_get_indent (s);
65 
66  s = format (s, "memif: hw_if_index %d next-index %d",
67  t->hw_if_index, t->next_index);
68  s = format (s, "\n%Uslot: ring %u", format_white_space, indent + 2,
69  t->ring);
70  return s;
71 }
72 
75 {
76  u8 *ptr = vlib_buffer_get_current (b);
77  u8 v = *ptr & 0xf0;
78 
79  if (PREDICT_TRUE (v == 0x40))
81  else if (PREDICT_TRUE (v == 0x60))
83 
84  b->error = node->errors[MEMIF_INPUT_ERROR_NOT_IP];
86 }
87 
90  memif_if_t * mif, vlib_buffer_t * b, u32 next, u16 qid,
91  uword * n_tracep)
92 {
94 
95  if (PREDICT_TRUE (b != 0))
96  {
98  vlib_trace_buffer (vm, node, next, b, /* follow_chain */ 0);
99  vlib_set_trace_count (vm, node, --(*n_tracep));
100  tr = vlib_add_trace (vm, node, b, sizeof (*tr));
101  tr->next_index = next;
102  tr->hw_if_index = mif->hw_if_index;
103  tr->ring = qid;
104  }
105 }
106 
109  u16 buffer_offset, u16 buffer_vec_index)
110 {
111  memif_copy_op_t *co;
113  co->data = data;
114  co->data_len = len;
115  co->buffer_offset = buffer_offset;
116  co->buffer_vec_index = buffer_vec_index;
117 }
118 
121  u32 buffer_size)
122 {
123  vlib_buffer_t *seg = b;
124  i32 bytes_left = b->current_length - buffer_size + b->current_data;
125 
126  if (PREDICT_TRUE (bytes_left <= 0))
127  return;
128 
129  b->current_length -= bytes_left;
131 
132  while (bytes_left)
133  {
134  seg->flags |= VLIB_BUFFER_NEXT_PRESENT;
135  seg->next_buffer = buffers[0];
136  seg = vlib_get_buffer (vm, buffers[0]);
137  buffers++;
138  seg->current_data = 0;
139  seg->current_length = clib_min (buffer_size, bytes_left);
140  bytes_left -= seg->current_length;
141  }
142 }
143 
146 {
147  u32 res = x - y;
148  res &= -(res <= x);
149  return res;
150 }
151 
152 /* branchless validation of the descriptor - uses saturated subtraction */
155 {
156  u32 rv;
157  u16 valid_flags = MEMIF_DESC_FLAG_NEXT;
158 
159  rv = d->flags & (~valid_flags);
160  rv |= sat_sub (d->region + 1, vec_len (mif->regions));
161  rv |= sat_sub (d->length, buffer_length);
162  rv |= sat_sub (d->offset + d->length, mif->regions[d->region].region_size);
163 
164  if (PREDICT_FALSE (rv))
165  {
166  mif->flags |= MEMIF_IF_FLAG_ERROR;
167  return 1;
168  }
169 
170  return 0;
171 }
172 
175  vlib_frame_t * frame, memif_if_t * mif,
176  memif_ring_type_t type, u16 qid,
178 {
179  vnet_main_t *vnm = vnet_get_main ();
180  memif_main_t *mm = &memif_main;
181  memif_ring_t *ring;
182  memif_queue_t *mq;
184  u32 next_index;
185  uword n_trace = vlib_get_trace_count (vm, node);
186  u32 n_rx_packets = 0, n_rx_bytes = 0;
187  u32 n_left, *to_next = 0;
188  u32 bi0, bi1, bi2, bi3;
189  vlib_buffer_t *b0, *b1, *b2, *b3;
190  u32 thread_index = vlib_get_thread_index ();
192  thread_index);
193  vlib_buffer_t *bt = &ptd->buffer_template;
194  u16 cur_slot, last_slot, ring_size, n_slots, mask;
195  i16 start_offset;
196  u16 n_buffers = 0, n_alloc;
197  memif_copy_op_t *co;
198  memif_packet_op_t *po;
199  memif_region_index_t last_region = ~0;
200  void *last_region_shm = 0;
201 
202  mq = vec_elt_at_index (mif->rx_queues, qid);
203  ring = mq->ring;
204  ring_size = 1 << mq->log2_ring_size;
205  mask = ring_size - 1;
206 
207  next_index = (mode == MEMIF_INTERFACE_MODE_IP) ?
209 
210  /* asume that somebody will want to add ethernet header on the packet
211  so start with IP header at offset 14 */
212  start_offset = (mode == MEMIF_INTERFACE_MODE_IP) ? 14 : 0;
213 
214  /* for S2M rings, we are consumers of packet buffers, and for M2S rings we
215  are producers of empty buffers */
216  cur_slot = (type == MEMIF_RING_S2M) ? mq->last_head : mq->last_tail;
217  last_slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
218  if (cur_slot == last_slot)
219  goto refill;
220  n_slots = last_slot - cur_slot;
221 
222  /* construct copy and packet vector out of ring slots */
223  while (n_slots && n_rx_packets < MEMIF_RX_VECTOR_SZ)
224  {
225  u32 dst_off, src_off, n_bytes_left;
226  u16 s0;
227  memif_desc_t *d0;
228  void *mb0;
229  po = ptd->packet_ops + n_rx_packets;
230  n_rx_packets++;
231  po->first_buffer_vec_index = n_buffers++;
232  po->packet_len = 0;
233  src_off = 0;
234  dst_off = start_offset;
235 
236  next_slot:
237  CLIB_PREFETCH (&ring->desc[(cur_slot + 8) & mask],
238  CLIB_CACHE_LINE_BYTES, LOAD);
239  s0 = cur_slot & mask;
240  d0 = &ring->desc[s0];
241  n_bytes_left = d0->length;
242 
243  /* slave resets buffer length,
244  * so it can produce full size buffer for master
245  */
246  if (type == MEMIF_RING_M2S)
247  d0->length = mif->run.buffer_size;
248 
249  po->packet_len += n_bytes_left;
250  if (PREDICT_FALSE (last_region != d0->region))
251  {
252  last_region_shm = mif->regions[d0->region].shm;
253  last_region = d0->region;
254  }
255  mb0 = last_region_shm + d0->offset;
256 
257  do
258  {
259  u32 dst_free = buffer_size - dst_off;
260  if (dst_free == 0)
261  {
262  dst_off = 0;
263  dst_free = buffer_size;
264  n_buffers++;
265  }
266  u32 bytes_to_copy = clib_min (dst_free, n_bytes_left);
267  memif_add_copy_op (ptd, mb0 + src_off, bytes_to_copy, dst_off,
268  n_buffers - 1);
269  n_bytes_left -= bytes_to_copy;
270  src_off += bytes_to_copy;
271  dst_off += bytes_to_copy;
272  }
273  while (PREDICT_FALSE (n_bytes_left));
274 
275  cur_slot++;
276  n_slots--;
277  if ((d0->flags & MEMIF_DESC_FLAG_NEXT) && n_slots)
278  {
279  src_off = 0;
280  goto next_slot;
281  }
282  }
283 
284  /* allocate free buffers */
285  vec_validate_aligned (ptd->buffers, n_buffers - 1, CLIB_CACHE_LINE_BYTES);
286  n_alloc = vlib_buffer_alloc (vm, ptd->buffers, n_buffers);
287  if (PREDICT_FALSE (n_alloc != n_buffers))
288  {
289  if (n_alloc)
290  vlib_buffer_free (vm, ptd->buffers, n_alloc);
291  vlib_error_count (vm, node->node_index,
292  MEMIF_INPUT_ERROR_BUFFER_ALLOC_FAIL, 1);
293  goto refill;
294  }
295 
296  /* copy data */
297  n_left = vec_len (ptd->copy_ops);
298  co = ptd->copy_ops;
299  while (n_left >= 8)
300  {
301  CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD);
302  CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD);
303  CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD);
304  CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD);
305 
306  b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
307  b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]);
308  b2 = vlib_get_buffer (vm, ptd->buffers[co[2].buffer_vec_index]);
309  b3 = vlib_get_buffer (vm, ptd->buffers[co[3].buffer_vec_index]);
310 
311  clib_memcpy (b0->data + co[0].buffer_offset, co[0].data,
312  co[0].data_len);
313  clib_memcpy (b1->data + co[1].buffer_offset, co[1].data,
314  co[1].data_len);
315  clib_memcpy (b2->data + co[2].buffer_offset, co[2].data,
316  co[2].data_len);
317  clib_memcpy (b3->data + co[3].buffer_offset, co[3].data,
318  co[3].data_len);
319 
320  co += 4;
321  n_left -= 4;
322  }
323  while (n_left)
324  {
325  b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
326  clib_memcpy (b0->data + co[0].buffer_offset, co[0].data,
327  co[0].data_len);
328  co += 1;
329  n_left -= 1;
330  }
331 
332  /* release slots from the ring */
333  if (type == MEMIF_RING_S2M)
334  {
336  ring->tail = mq->last_head = cur_slot;
337  }
338  else
339  {
340  mq->last_tail = cur_slot;
341  }
342 
343  u32 n_from = n_rx_packets;
344  po = ptd->packet_ops;
345 
346  vnet_buffer (bt)->sw_if_index[VLIB_RX] = mif->sw_if_index;
347  bt->current_data = start_offset;
348 
349  while (n_from)
350  {
351  u32 n_left_to_next;
352  u32 next0, next1, next2, next3;
353 
354  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
355  while (n_from >= 8 && n_left_to_next >= 4)
356  {
357  b0 = vlib_get_buffer (vm, po[4].first_buffer_vec_index);
358  b1 = vlib_get_buffer (vm, po[5].first_buffer_vec_index);
359  b2 = vlib_get_buffer (vm, po[6].first_buffer_vec_index);
360  b3 = vlib_get_buffer (vm, po[7].first_buffer_vec_index);
361  vlib_prefetch_buffer_header (b0, STORE);
362  vlib_prefetch_buffer_header (b1, STORE);
363  vlib_prefetch_buffer_header (b2, STORE);
364  vlib_prefetch_buffer_header (b3, STORE);
365 
366  /* enqueue buffer */
367  u32 fbvi0 = po[0].first_buffer_vec_index;
368  u32 fbvi1 = po[1].first_buffer_vec_index;
369  u32 fbvi2 = po[2].first_buffer_vec_index;
370  u32 fbvi3 = po[3].first_buffer_vec_index;
371  to_next[0] = bi0 = ptd->buffers[fbvi0];
372  to_next[1] = bi1 = ptd->buffers[fbvi1];
373  to_next[2] = bi2 = ptd->buffers[fbvi2];
374  to_next[3] = bi3 = ptd->buffers[fbvi3];
375  to_next += 4;
376  n_left_to_next -= 4;
377 
378  b0 = vlib_get_buffer (vm, bi0);
379  b1 = vlib_get_buffer (vm, bi1);
380  b2 = vlib_get_buffer (vm, bi2);
381  b3 = vlib_get_buffer (vm, bi3);
382 
383  clib_memcpy64_x4 (b0, b1, b2, b3, bt);
384 
385  b0->current_length = po[0].packet_len;
386  n_rx_bytes += b0->current_length;
387  b1->current_length = po[1].packet_len;
388  n_rx_bytes += b1->current_length;
389  b2->current_length = po[2].packet_len;
390  n_rx_bytes += b2->current_length;
391  b3->current_length = po[3].packet_len;
392  n_rx_bytes += b3->current_length;
393 
394  memif_add_to_chain (vm, b0, ptd->buffers + fbvi0 + 1, buffer_size);
395  memif_add_to_chain (vm, b1, ptd->buffers + fbvi1 + 1, buffer_size);
396  memif_add_to_chain (vm, b2, ptd->buffers + fbvi2 + 1, buffer_size);
397  memif_add_to_chain (vm, b3, ptd->buffers + fbvi3 + 1, buffer_size);
398 
399  if (mode == MEMIF_INTERFACE_MODE_IP)
400  {
401  next0 = memif_next_from_ip_hdr (node, b0);
402  next1 = memif_next_from_ip_hdr (node, b1);
403  next2 = memif_next_from_ip_hdr (node, b2);
404  next3 = memif_next_from_ip_hdr (node, b3);
405  }
406  else if (mode == MEMIF_INTERFACE_MODE_ETHERNET)
407  {
408  if (PREDICT_FALSE (mif->per_interface_next_index != ~0))
409  {
410  next0 = mif->per_interface_next_index;
411  next1 = mif->per_interface_next_index;
412  next2 = mif->per_interface_next_index;
413  next3 = mif->per_interface_next_index;
414  }
415  else
416  {
417  next0 = next1 = next2 = next3 = next_index;
418  /* redirect if feature path enabled */
420  &next0, b0);
422  &next1, b1);
424  &next2, b2);
426  &next3, b3);
427  }
428  }
429 
430  /* trace */
431  if (PREDICT_FALSE (n_trace > 0))
432  {
433  memif_trace_buffer (vm, node, mif, b0, next0, qid, &n_trace);
434  if (PREDICT_FALSE (n_trace > 0))
435  memif_trace_buffer (vm, node, mif, b1, next1, qid, &n_trace);
436  if (PREDICT_FALSE (n_trace > 0))
437  memif_trace_buffer (vm, node, mif, b2, next2, qid, &n_trace);
438  if (PREDICT_FALSE (n_trace > 0))
439  memif_trace_buffer (vm, node, mif, b3, next3, qid, &n_trace);
440  }
441 
442  /* enqueue */
443  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
444  n_left_to_next, bi0, bi1, bi2, bi3,
445  next0, next1, next2, next3);
446 
447  /* next */
448  n_from -= 4;
449  po += 4;
450  }
451  while (n_from && n_left_to_next)
452  {
453  /* enqueue buffer */
454  u32 fbvi0 = po->first_buffer_vec_index;
455  to_next[0] = bi0 = ptd->buffers[fbvi0];
456  to_next += 1;
457  n_left_to_next--;
458 
459  b0 = vlib_get_buffer (vm, bi0);
460  clib_memcpy (b0, bt, 64);
461  b0->current_length = po->packet_len;
462  n_rx_bytes += b0->current_length;
463 
464  memif_add_to_chain (vm, b0, ptd->buffers + fbvi0 + 1, buffer_size);
465 
466  if (mode == MEMIF_INTERFACE_MODE_IP)
467  {
468  next0 = memif_next_from_ip_hdr (node, b0);
469  }
470  else if (mode == MEMIF_INTERFACE_MODE_ETHERNET)
471  {
472  if (PREDICT_FALSE (mif->per_interface_next_index != ~0))
473  next0 = mif->per_interface_next_index;
474  else
475  {
476  next0 = next_index;
477  /* redirect if feature path enabled */
479  &next0, b0);
480  }
481 
482  }
483 
484  /* trace */
485  if (PREDICT_FALSE (n_trace > 0))
486  memif_trace_buffer (vm, node, mif, b0, next0, qid, &n_trace);
487 
488  /* enqueue */
489  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
490  n_left_to_next, bi0, next0);
491 
492  /* next */
493  n_from--;
494  po++;
495  }
496  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
497  }
498 
500  + VNET_INTERFACE_COUNTER_RX, thread_index,
501  mif->hw_if_index, n_rx_packets,
502  n_rx_bytes);
503 
504  /* refill ring with empty buffers */
505 refill:
506  vec_reset_length (ptd->buffers);
507  vec_reset_length (ptd->copy_ops);
508 
509  if (type == MEMIF_RING_M2S)
510  {
511  u16 head = ring->head;
512  n_slots = ring_size - head + mq->last_tail;
513 
514  while (n_slots--)
515  {
516  u16 s = head++ & mask;
517  memif_desc_t *d = &ring->desc[s];
518  d->length = mif->run.buffer_size;
519  }
520 
522  ring->head = head;
523  }
524 
525  return n_rx_packets;
526 }
527 
530  vlib_frame_t * frame, memif_if_t * mif,
531  u16 qid, memif_interface_mode_t mode)
532 {
533  vnet_main_t *vnm = vnet_get_main ();
534  memif_main_t *mm = &memif_main;
535  memif_ring_t *ring;
536  memif_queue_t *mq;
537  u32 next_index;
538  uword n_trace = vlib_get_trace_count (vm, node);
539  u32 n_rx_packets = 0, n_rx_bytes = 0;
540  u32 *to_next = 0, *buffers;
541  u32 bi0, bi1, bi2, bi3;
542  u16 s0, s1, s2, s3;
543  memif_desc_t *d0, *d1, *d2, *d3;
544  vlib_buffer_t *b0, *b1, *b2, *b3;
545  u32 thread_index = vlib_get_thread_index ();
547  thread_index);
548  u16 cur_slot, last_slot, ring_size, n_slots, mask, head;
549  i16 start_offset;
550  u32 buffer_length;
551  u16 n_alloc, n_from;
552 
553  mq = vec_elt_at_index (mif->rx_queues, qid);
554  ring = mq->ring;
555  ring_size = 1 << mq->log2_ring_size;
556  mask = ring_size - 1;
557 
558  next_index = (mode == MEMIF_INTERFACE_MODE_IP) ?
560 
561  /* asume that somebody will want to add ethernet header on the packet
562  so start with IP header at offset 14 */
563  start_offset = (mode == MEMIF_INTERFACE_MODE_IP) ? 14 : 0;
564  buffer_length = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES - start_offset;
565 
566  cur_slot = mq->last_tail;
567  last_slot = ring->tail;
568  if (cur_slot == last_slot)
569  goto refill;
570  n_slots = last_slot - cur_slot;
571 
572  /* process ring slots */
575  while (n_slots && n_rx_packets < MEMIF_RX_VECTOR_SZ)
576  {
577  vlib_buffer_t *hb;
578 
579  s0 = cur_slot & mask;
580  bi0 = mq->buffers[s0];
581  ptd->buffers[n_rx_packets++] = bi0;
582 
583  CLIB_PREFETCH (&ring->desc[(cur_slot + 8) & mask],
584  CLIB_CACHE_LINE_BYTES, LOAD);
585  d0 = &ring->desc[s0];
586  hb = b0 = vlib_get_buffer (vm, bi0);
587  b0->current_data = start_offset;
588  b0->current_length = start_offset + d0->length;
589  n_rx_bytes += d0->length;
590 
591  if (0 && memif_desc_is_invalid (mif, d0, buffer_length))
592  return 0;
593 
594  cur_slot++;
595  n_slots--;
596  if (PREDICT_FALSE ((d0->flags & MEMIF_DESC_FLAG_NEXT) && n_slots))
597  {
598  hb->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
599  next_slot:
600  s0 = cur_slot & mask;
601  d0 = &ring->desc[s0];
602  bi0 = mq->buffers[s0];
603 
604  /* previous buffer */
605  b0->next_buffer = bi0;
606  b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
607 
608  /* current buffer */
609  b0 = vlib_get_buffer (vm, bi0);
610  b0->current_data = start_offset;
611  b0->current_length = start_offset + d0->length;
613  n_rx_bytes += d0->length;
614 
615  cur_slot++;
616  n_slots--;
617  if ((d0->flags & MEMIF_DESC_FLAG_NEXT) && n_slots)
618  goto next_slot;
619  }
620  }
621 
622  /* release slots from the ring */
623  mq->last_tail = cur_slot;
624 
625  n_from = n_rx_packets;
626  buffers = ptd->buffers;
627 
628  while (n_from)
629  {
630  u32 n_left_to_next;
631  u32 next0, next1, next2, next3;
632 
633  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
634  while (n_from >= 8 && n_left_to_next >= 4)
635  {
636  b0 = vlib_get_buffer (vm, buffers[4]);
637  b1 = vlib_get_buffer (vm, buffers[5]);
638  b2 = vlib_get_buffer (vm, buffers[6]);
639  b3 = vlib_get_buffer (vm, buffers[7]);
640  vlib_prefetch_buffer_header (b0, STORE);
641  vlib_prefetch_buffer_header (b1, STORE);
642  vlib_prefetch_buffer_header (b2, STORE);
643  vlib_prefetch_buffer_header (b3, STORE);
644 
645  /* enqueue buffer */
646  to_next[0] = bi0 = buffers[0];
647  to_next[1] = bi1 = buffers[1];
648  to_next[2] = bi2 = buffers[2];
649  to_next[3] = bi3 = buffers[3];
650  to_next += 4;
651  n_left_to_next -= 4;
652  buffers += 4;
653 
654  b0 = vlib_get_buffer (vm, bi0);
655  b1 = vlib_get_buffer (vm, bi1);
656  b2 = vlib_get_buffer (vm, bi2);
657  b3 = vlib_get_buffer (vm, bi3);
658 
659  vnet_buffer (b0)->sw_if_index[VLIB_RX] = mif->sw_if_index;
660  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
661  vnet_buffer (b1)->sw_if_index[VLIB_RX] = mif->sw_if_index;
662  vnet_buffer (b1)->sw_if_index[VLIB_TX] = ~0;
663  vnet_buffer (b2)->sw_if_index[VLIB_RX] = mif->sw_if_index;
664  vnet_buffer (b2)->sw_if_index[VLIB_TX] = ~0;
665  vnet_buffer (b3)->sw_if_index[VLIB_RX] = mif->sw_if_index;
666  vnet_buffer (b3)->sw_if_index[VLIB_TX] = ~0;
667 
668  if (mode == MEMIF_INTERFACE_MODE_IP)
669  {
670  next0 = memif_next_from_ip_hdr (node, b0);
671  next1 = memif_next_from_ip_hdr (node, b1);
672  next2 = memif_next_from_ip_hdr (node, b2);
673  next3 = memif_next_from_ip_hdr (node, b3);
674  }
675  else if (mode == MEMIF_INTERFACE_MODE_ETHERNET)
676  {
677  if (PREDICT_FALSE (mif->per_interface_next_index != ~0))
678  {
679  next0 = mif->per_interface_next_index;
680  next1 = mif->per_interface_next_index;
681  next2 = mif->per_interface_next_index;
682  next3 = mif->per_interface_next_index;
683  }
684  else
685  {
686  next0 = next1 = next2 = next3 = next_index;
687  /* redirect if feature path enabled */
689  &next0, b0);
691  &next1, b1);
693  &next2, b2);
695  &next3, b3);
696  }
697  }
698 
699  /* trace */
700  if (PREDICT_FALSE (n_trace > 0))
701  {
702  memif_trace_buffer (vm, node, mif, b0, next0, qid, &n_trace);
703  if (PREDICT_FALSE (n_trace > 0))
704  memif_trace_buffer (vm, node, mif, b1, next1, qid, &n_trace);
705  if (PREDICT_FALSE (n_trace > 0))
706  memif_trace_buffer (vm, node, mif, b2, next2, qid, &n_trace);
707  if (PREDICT_FALSE (n_trace > 0))
708  memif_trace_buffer (vm, node, mif, b3, next3, qid, &n_trace);
709  }
710 
711  /* enqueue */
712  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
713  n_left_to_next, bi0, bi1, bi2, bi3,
714  next0, next1, next2, next3);
715 
716  /* next */
717  n_from -= 4;
718  }
719  while (n_from && n_left_to_next)
720  {
721  /* enqueue buffer */
722  to_next[0] = bi0 = buffers[0];
723  to_next += 1;
724  n_left_to_next--;
725  buffers += 1;
726 
727  b0 = vlib_get_buffer (vm, bi0);
728  vnet_buffer (b0)->sw_if_index[VLIB_RX] = mif->sw_if_index;
729  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
730 
731  if (mode == MEMIF_INTERFACE_MODE_IP)
732  {
733  next0 = memif_next_from_ip_hdr (node, b0);
734  }
735  else if (mode == MEMIF_INTERFACE_MODE_ETHERNET)
736  {
737  if (PREDICT_FALSE (mif->per_interface_next_index != ~0))
738  next0 = mif->per_interface_next_index;
739  else
740  {
741  next0 = next_index;
742  /* redirect if feature path enabled */
744  &next0, b0);
745  }
746  }
747 
748  /* trace */
749  if (PREDICT_FALSE (n_trace > 0))
750  memif_trace_buffer (vm, node, mif, b0, next0, qid, &n_trace);
751 
752  /* enqueue */
753  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
754  n_left_to_next, bi0, next0);
755 
756  /* next */
757  n_from--;
758  }
759  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
760  }
761 
763  + VNET_INTERFACE_COUNTER_RX, thread_index,
764  mif->hw_if_index, n_rx_packets,
765  n_rx_bytes);
766 
767  /* refill ring with empty buffers */
768 refill:
769  vec_reset_length (ptd->buffers);
770 
771  head = ring->head;
772  n_slots = ring_size - head + mq->last_tail;
773 
774  if (n_slots < 32)
775  goto done;
776 
777  memif_desc_t *dt = &ptd->desc_template;
778  memset (dt, 0, sizeof (memif_desc_t));
779  dt->length = buffer_length;
780 
781  n_alloc = vlib_buffer_alloc_to_ring (vm, mq->buffers, head & mask,
782  ring_size, n_slots);
783 
784  if (PREDICT_FALSE (n_alloc != n_slots))
785  {
786  vlib_error_count (vm, node->node_index,
787  MEMIF_INPUT_ERROR_BUFFER_ALLOC_FAIL, 1);
788  }
789 
790  while (n_alloc >= 32)
791  {
792  bi0 = mq->buffers[(head + 4) & mask];
793  vlib_prefetch_buffer_with_index (vm, bi0, LOAD);
794  bi1 = mq->buffers[(head + 5) & mask];
795  vlib_prefetch_buffer_with_index (vm, bi1, LOAD);
796  bi2 = mq->buffers[(head + 6) & mask];
797  vlib_prefetch_buffer_with_index (vm, bi2, LOAD);
798  bi3 = mq->buffers[(head + 7) & mask];
799  vlib_prefetch_buffer_with_index (vm, bi3, LOAD);
800 
801  s0 = head++ & mask;
802  s1 = head++ & mask;
803  s2 = head++ & mask;
804  s3 = head++ & mask;
805 
806  d0 = &ring->desc[s0];
807  d1 = &ring->desc[s1];
808  d2 = &ring->desc[s2];
809  d3 = &ring->desc[s3];
810 
811  clib_memcpy (d0, dt, sizeof (memif_desc_t));
812  clib_memcpy (d1, dt, sizeof (memif_desc_t));
813  clib_memcpy (d2, dt, sizeof (memif_desc_t));
814  clib_memcpy (d3, dt, sizeof (memif_desc_t));
815 
816  b0 = vlib_get_buffer (vm, mq->buffers[s0]);
817  b1 = vlib_get_buffer (vm, mq->buffers[s1]);
818  b2 = vlib_get_buffer (vm, mq->buffers[s2]);
819  b3 = vlib_get_buffer (vm, mq->buffers[s3]);
820 
821  d0->region = b0->buffer_pool_index + 1;
822  d1->region = b1->buffer_pool_index + 1;
823  d2->region = b2->buffer_pool_index + 1;
824  d3->region = b3->buffer_pool_index + 1;
825 
826  d0->offset =
827  (void *) b0->data - mif->regions[d0->region].shm + start_offset;
828  d1->offset =
829  (void *) b1->data - mif->regions[d1->region].shm + start_offset;
830  d2->offset =
831  (void *) b2->data - mif->regions[d2->region].shm + start_offset;
832  d3->offset =
833  (void *) b3->data - mif->regions[d3->region].shm + start_offset;
834 
835  n_alloc -= 4;
836  }
837  while (n_alloc)
838  {
839  s0 = head++ & mask;
840  d0 = &ring->desc[s0];
841  clib_memcpy (d0, dt, sizeof (memif_desc_t));
842  b0 = vlib_get_buffer (vm, mq->buffers[s0]);
843  d0->region = b0->buffer_pool_index + 1;
844  d0->offset =
845  (void *) b0->data - mif->regions[d0->region].shm + start_offset;
846 
847  n_alloc -= 1;
848  }
849 
851  ring->head = head;
852 
853 done:
854  return n_rx_packets;
855 }
856 
857 uword
859  vlib_node_runtime_t * node,
860  vlib_frame_t * frame)
861 {
862  u32 n_rx = 0;
863  memif_main_t *mm = &memif_main;
864  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
868 
870  {
871  memif_if_t *mif;
872  mif = vec_elt_at_index (mm->interfaces, dq->dev_instance);
873  if ((mif->flags & MEMIF_IF_FLAG_ADMIN_UP) &&
874  (mif->flags & MEMIF_IF_FLAG_CONNECTED))
875  {
876  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
877  {
878  if (mif->mode == MEMIF_INTERFACE_MODE_IP)
879  n_rx += memif_device_input_zc_inline (vm, node, frame, mif,
880  dq->queue_id, mode_ip);
881  else
882  n_rx += memif_device_input_zc_inline (vm, node, frame, mif,
883  dq->queue_id, mode_eth);
884  }
885  else if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
886  {
887  if (mif->mode == MEMIF_INTERFACE_MODE_IP)
888  n_rx += memif_device_input_inline (vm, node, frame, mif,
889  MEMIF_RING_M2S, dq->queue_id,
890  mode_ip);
891  else
892  n_rx += memif_device_input_inline (vm, node, frame, mif,
893  MEMIF_RING_M2S, dq->queue_id,
894  mode_eth);
895  }
896  else
897  {
898  if (mif->mode == MEMIF_INTERFACE_MODE_IP)
899  n_rx += memif_device_input_inline (vm, node, frame, mif,
900  MEMIF_RING_S2M, dq->queue_id,
901  mode_ip);
902  else
903  n_rx += memif_device_input_inline (vm, node, frame, mif,
904  MEMIF_RING_S2M, dq->queue_id,
905  mode_eth);
906  }
907  }
908  }
909 
910  return n_rx;
911 }
912 
913 #ifndef CLIB_MULTIARCH_VARIANT
914 /* *INDENT-OFF* */
916  .function = memif_input_fn,
917  .name = "memif-input",
918  .sibling_of = "device-input",
919  .format_trace = format_memif_input_trace,
920  .type = VLIB_NODE_TYPE_INPUT,
921  .state = VLIB_NODE_STATE_INTERRUPT,
922  .n_errors = MEMIF_INPUT_N_ERROR,
923  .error_strings = memif_input_error_strings,
924 };
925 
928 
929 #if __x86_64__
930 static void __clib_constructor
932 {
933  if (memif_input_fn_avx512 && clib_cpu_supports_avx512f ())
935  else if (memif_input_fn_avx2 && clib_cpu_supports_avx2 ())
937 }
938 #endif
939 #endif
940 
941 /* *INDENT-ON* */
942 
943 
944 /*
945  * fd.io coding-style-patch-verification: ON
946  *
947  * Local Variables:
948  * eval: (c-set-style "gnu")
949  * End:
950  */
memif_if_t * interfaces
Definition: private.h:221
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
#define clib_min(x, y)
Definition: clib.h:340
#define CLIB_UNUSED(x)
Definition: clib.h:79
static_always_inline uword memif_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, memif_if_t *mif, memif_ring_type_t type, u16 qid, memif_interface_mode_t mode)
Definition: node.c:174
adds_epu subs_epu i16x8 y
Definition: vector_sse42.h:227
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:143
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:211
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:391
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
vnet_interface_main_t interface_main
Definition: vnet.h:56
struct memif_if_t::@424 run
#define PREDICT_TRUE(x)
Definition: clib.h:106
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:112
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:571
memif_interface_mode_t
Definition: memif.h:53
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:143
#define vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, bi2, bi3, next0, next1, next2, next3)
Finish enqueueing four buffers forward in the graph.
Definition: buffer_node.h:138
uint16_t memif_region_index_t
Definition: memif.h:60
static u32 format_get_indent(u8 *s)
Definition: format.h:72
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
u16 first_buffer_vec_index
Definition: private.h:186
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:445
u32 * buffers
Definition: private.h:107
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:415
uint32_t length
Definition: memif.h:152
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:104
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:104
#define static_always_inline
Definition: clib.h:93
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:181
static_always_inline u32 memif_next_from_ip_hdr(vlib_node_runtime_t *node, vlib_buffer_t *b)
Definition: node.c:74
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:718
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
u16 buffer_size
Definition: private.h:167
int i32
Definition: types.h:81
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:191
uint16_t flags
Definition: memif.h:149
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u32 per_interface_next_index
Definition: private.h:145
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:447
static __clib_unused char * memif_input_error_strings[]
Definition: node.c:45
memif_packet_op_t packet_ops[MEMIF_RX_VECTOR_SZ]
Definition: private.h:204
memif_region_index_t region
Definition: memif.h:151
u16 last_head
Definition: private.h:105
memif_copy_op_t * copy_ops
Definition: private.h:205
vlib_node_registration_t memif_input_node
(constructor) VLIB_REGISTER_NODE (memif_input_node)
Definition: node.c:915
memif_desc_t desc[0]
Definition: memif.h:173
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
uword dev_instance
Definition: private.h:142
#define v
Definition: acl.c:495
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:209
void * data
Definition: private.h:191
#define PREDICT_FALSE(x)
Definition: clib.h:105
#define foreach_memif_input_error
Definition: node.c:33
u32 node_index
Node index.
Definition: node.h:437
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
uword( vlib_node_function_t)(struct vlib_main_t *vm, struct vlib_node_runtime_t *node, struct vlib_frame_t *frame)
Definition: node.h:54
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:130
vlib_node_function_t __clib_weak memif_input_fn_avx512
Definition: node.c:926
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:74
vlib_main_t * vm
Definition: buffer.c:294
static void __clib_constructor memif_input_multiarch_select(void)
Definition: node.c:931
u16 buffer_vec_index
Definition: private.h:194
static_always_inline void clib_memcpy64_x4(void *d0, void *d1, void *d2, void *d3, void *s)
Definition: string.h:89
#define clib_memcpy(a, b, c)
Definition: string.h:75
u16 last_tail
Definition: private.h:106
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
static_always_inline u32 sat_sub(u32 x, u32 y)
Definition: node.c:145
i16 buffer_offset
Definition: private.h:193
static_always_inline void memif_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *node, memif_if_t *mif, vlib_buffer_t *b, u32 next, u16 qid, uword *n_tracep)
Definition: node.c:89
static_always_inline uword memif_device_input_zc_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, memif_if_t *mif, u16 qid, memif_interface_mode_t mode)
Definition: node.c:529
memif_region_t * regions
Definition: private.h:153
memif_input_error_t
Definition: node.c:37
unsigned int u32
Definition: types.h:88
#define MEMIF_DESC_FLAG_NEXT
Definition: memif.h:150
u32 flags
Definition: private.h:138
memif_ring_t * ring
Definition: private.h:100
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
vlib_buffer_t buffer_template
Definition: private.h:209
u32 hw_if_index
Definition: private.h:140
uword CLIB_MULTIARCH_FN() memif_input_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: node.c:858
u64 uword
Definition: types.h:112
memif_region_offset_t offset
Definition: memif.h:153
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:159
#define foreach_device_and_queue(var, vec)
Definition: devices.h:156
Definition: defs.h:47
static u32 vlib_buffer_alloc_to_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Allocate buffers into ring.
Definition: buffer_funcs.h:364
unsigned short u16
Definition: types.h:57
static_always_inline void memif_add_to_chain(vlib_main_t *vm, vlib_buffer_t *b, u32 *buffers, u32 buffer_size)
Definition: node.c:120
vlib_node_function_t __clib_weak memif_input_fn_avx2
Definition: node.c:927
#define MEMIF_RX_VECTOR_SZ
Definition: private.h:197
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:553
static_always_inline void memif_add_copy_op(memif_per_thread_data_t *ptd, void *data, u32 len, u16 buffer_offset, u16 buffer_vec_index)
Definition: node.c:108
void * shm
Definition: private.h:85
static_always_inline u32 memif_desc_is_invalid(memif_if_t *mif, memif_desc_t *d, u32 buffer_length)
Definition: node.c:154
memif_log2_ring_size_t log2_ring_size
Definition: private.h:101
short i16
Definition: types.h:46
#define vnet_buffer(b)
Definition: buffer.h:372
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:234
memif_per_thread_data_t * per_thread_data
Definition: private.h:228
u8 data[0]
Packet data.
Definition: buffer.h:179
memif_ring_type_t
Definition: memif.h:47
volatile uint16_t head
Definition: memif.h:169
memif_queue_t * rx_queues
Definition: private.h:155
memif_desc_t desc_template
Definition: private.h:210
static __clib_unused u8 * format_memif_input_trace(u8 *s, va_list *args)
Definition: node.c:59
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:159
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
memif_main_t memif_main
Definition: memif.c:43
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:347
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
u32 sw_if_index
Definition: private.h:141
volatile uint16_t tail
Definition: memif.h:171
memif_interface_mode_t mode
Definition: private.h:143
Definition: defs.h:46
memif_region_size_t region_size
Definition: private.h:86
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:59