FD.io VPP  v18.01.2-1-g9b554f3
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/format.h>
18 #include <vlib/unix/cj.h>
19 #include <assert.h>
20 
21 #include <vnet/ethernet/ethernet.h>
22 #include <dpdk/device/dpdk.h>
23 
24 #include <dpdk/device/dpdk_priv.h>
25 #include <vppinfra/error.h>
26 
27 #define foreach_dpdk_tx_func_error \
28  _(BAD_RETVAL, "DPDK tx function returned an error") \
29  _(RING_FULL, "Tx packet drops (ring full)") \
30  _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \
31  _(REPL_FAIL, "Tx packet drops (replication failure)")
32 
33 typedef enum
34 {
35 #define _(f,s) DPDK_TX_FUNC_ERROR_##f,
37 #undef _
40 
41 #ifndef CLIB_MULTIARCH_VARIANT
42 static char *dpdk_tx_func_error_strings[] = {
43 #define _(n,s) s,
45 #undef _
46 };
47 
48 static clib_error_t *
50 {
51  int error;
52  dpdk_main_t *dm = &dpdk_main;
54 
55  error = rte_eth_dev_default_mac_addr_set (xd->device_index,
56  (struct ether_addr *) address);
57 
58  if (error)
59  {
60  return clib_error_return (0, "mac address set failed: %d", error);
61  }
62  else
63  {
65  vec_add (xd->default_mac_address, address, sizeof (address));
66  return NULL;
67  }
68 }
69 #endif
70 
71 static struct rte_mbuf *
73 {
74  dpdk_main_t *dm = &dpdk_main;
75  struct rte_mbuf **mbufs = 0, *s, *d;
76  u8 nb_segs;
77  unsigned socket_id = rte_socket_id ();
78  int i;
79 
80  ASSERT (dm->pktmbuf_pools[socket_id]);
82  nb_segs = s->nb_segs;
83  vec_validate (mbufs, nb_segs - 1);
84 
85  if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs))
86  {
87  vec_free (mbufs);
88  return 0;
89  }
90 
91  d = mbufs[0];
92  d->nb_segs = s->nb_segs;
93  d->data_len = s->data_len;
94  d->pkt_len = s->pkt_len;
95  d->data_off = s->data_off;
96  clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
97 
98  for (i = 1; i < nb_segs; i++)
99  {
100  d->next = mbufs[i];
101  d = mbufs[i];
102  s = s->next;
103  d->data_len = s->data_len;
104  clib_memcpy (d->buf_addr, s->buf_addr,
105  RTE_PKTMBUF_HEADROOM + s->data_len);
106  }
107 
108  d = mbufs[0];
109  vec_free (mbufs);
110  return d;
111 }
112 
113 static void
115  vlib_node_runtime_t * node,
116  dpdk_device_t * xd,
117  u16 queue_id, u32 buffer_index, vlib_buffer_t * buffer)
118 {
119  vlib_main_t *vm = vlib_get_main ();
121  struct rte_mbuf *mb;
122 
123  mb = rte_mbuf_from_vlib_buffer (buffer);
124 
125  t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
126  t0->queue_index = queue_id;
127  t0->device_index = xd->device_index;
128  t0->buffer_index = buffer_index;
129  clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
130  clib_memcpy (&t0->buffer, buffer,
131  sizeof (buffer[0]) - sizeof (buffer->pre_data));
132  clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data,
133  sizeof (t0->buffer.pre_data));
134  clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data));
135 }
136 
139  int maybe_multiseg)
140 {
141  struct rte_mbuf *mb, *first_mb, *last_mb;
142 
143  /* buffer is coming from non-dpdk source so we need to init
144  rte_mbuf header */
146  {
147  vlib_buffer_t *b2 = b;
148  last_mb = mb = rte_mbuf_from_vlib_buffer (b2);
149  rte_pktmbuf_reset (mb);
150  while (maybe_multiseg && (b2->flags & VLIB_BUFFER_NEXT_PRESENT))
151  {
152  b2 = vlib_get_buffer (vm, b2->next_buffer);
153  mb = rte_mbuf_from_vlib_buffer (b2);
154  rte_pktmbuf_reset (mb);
155  }
156  }
157 
158  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
159  first_mb->nb_segs = 1;
160  mb->data_len = b->current_length;
161  mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
162  b->current_length;
163  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
164 
165  while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
166  {
167  b = vlib_get_buffer (vm, b->next_buffer);
168  mb = rte_mbuf_from_vlib_buffer (b);
169  last_mb->next = mb;
170  last_mb = mb;
171  mb->data_len = b->current_length;
172  mb->pkt_len = b->current_length;
173  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
174  first_mb->nb_segs++;
175  if (PREDICT_FALSE (b->n_add_refs))
176  {
177  rte_mbuf_refcnt_update (mb, b->n_add_refs);
178  b->n_add_refs = 0;
179  }
180  }
181 }
182 
183 /*
184  * This function calls the dpdk's tx_burst function to transmit the packets
185  * on the tx_vector. It manages a lock per-device if the device does not
186  * support multiple queues. It returns the number of packets untransmitted
187  * on the tx_vector. If all packets are transmitted (the normal case), the
188  * function returns 0.
189  *
190  * The function assumes there is at least one packet on the tx_vector.
191  */
194  dpdk_device_t * xd,
195  struct rte_mbuf **tx_vector)
196 {
197  dpdk_main_t *dm = &dpdk_main;
198  u32 n_packets;
199  u32 tx_head;
200  u32 tx_tail;
201  u32 n_retry;
202  int rv;
203  int queue_id;
204  tx_ring_hdr_t *ring;
205 
206  ring = vec_header (tx_vector, sizeof (*ring));
207 
208  n_packets = ring->tx_head - ring->tx_tail;
209 
210  tx_head = ring->tx_head % xd->nb_tx_desc;
211 
212  /*
213  * Ensure rte_eth_tx_burst is not called with 0 packets, which can lead to
214  * unpredictable results.
215  */
216  ASSERT (n_packets > 0);
217 
218  /*
219  * Check for tx_vector overflow. If this fails it is a system configuration
220  * error. The ring should be sized big enough to handle the largest un-flowed
221  * off burst from a traffic manager. A larger size also helps performance
222  * a bit because it decreases the probability of having to issue two tx_burst
223  * calls due to a ring wrap.
224  */
225  ASSERT (n_packets < xd->nb_tx_desc);
226  ASSERT (ring->tx_tail == 0);
227 
228  n_retry = 16;
229  queue_id = vm->thread_index;
230 
231  do
232  {
233  /* start the burst at the tail */
234  tx_tail = ring->tx_tail % xd->nb_tx_desc;
235 
236  /*
237  * This device only supports one TX queue,
238  * and we're running multi-threaded...
239  */
240  if (PREDICT_FALSE (xd->lockp != 0))
241  {
242  queue_id = queue_id % xd->tx_q_used;
243  while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
244  /* zzzz */
245  queue_id = (queue_id + 1) % xd->tx_q_used;
246  }
247 
248  if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */
249  {
250  /* no wrap, transmit in one burst */
252  &xd->hqos_wt[vm->thread_index];
253 
254  ASSERT (hqos->swq != NULL);
255 
257  &tx_vector[tx_tail], tx_head - tx_tail);
258  rv = rte_ring_sp_enqueue_burst (hqos->swq,
259  (void **) &tx_vector[tx_tail],
260  (uint16_t) (tx_head - tx_tail), 0);
261  }
262  else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
263  {
264  /* no wrap, transmit in one burst */
265  rv = rte_eth_tx_burst (xd->device_index,
266  (uint16_t) queue_id,
267  &tx_vector[tx_tail],
268  (uint16_t) (tx_head - tx_tail));
269  }
270  else
271  {
272  ASSERT (0);
273  rv = 0;
274  }
275 
276  if (PREDICT_FALSE (xd->lockp != 0))
277  *xd->lockp[queue_id] = 0;
278 
279  if (PREDICT_FALSE (rv < 0))
280  {
281  // emit non-fatal message, bump counter
282  vnet_main_t *vnm = dm->vnet_main;
284  u32 node_index;
285 
286  node_index = vec_elt_at_index (im->hw_interfaces,
287  xd->hw_if_index)->tx_node_index;
288 
289  vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1);
290  clib_warning ("rte_eth_tx_burst[%d]: error %d", xd->device_index,
291  rv);
292  return n_packets; // untransmitted packets
293  }
294  ring->tx_tail += (u16) rv;
295  n_packets -= (uint16_t) rv;
296  }
297  while (rv && n_packets && (n_retry > 0));
298 
299  return n_packets;
300 }
301 
304 {
305  vlib_buffer_t *b;
306  struct rte_mbuf *mb;
307  b = vlib_get_buffer (vm, bi);
308  mb = rte_mbuf_from_vlib_buffer (b);
309  CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, STORE);
311 }
312 
315  vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp)
316 {
317  dpdk_main_t *dm = &dpdk_main;
318  u32 my_cpu = vm->thread_index;
319  struct rte_mbuf *mb_new;
320 
321  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0)
322  return;
323 
324  mb_new = dpdk_replicate_packet_mb (b);
325  if (PREDICT_FALSE (mb_new == 0))
326  {
327  vlib_error_count (vm, node->node_index,
328  DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
330  }
331  else
332  *mbp = mb_new;
333 
334  vec_add1 (dm->recycle[my_cpu], bi);
335 }
336 
339  struct rte_mbuf *mb)
340 {
341  u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
342  u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
343  u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
344  int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
345  u64 ol_flags;
346 
347  /* Is there any work for us? */
348  if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum) == 0))
349  return;
350 
351  mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
352  mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
353  vnet_buffer (b)->l3_hdr_offset;
354  mb->outer_l3_len = 0;
355  mb->outer_l2_len = 0;
356  ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
357  ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
358  ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
359  ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
360  mb->ol_flags |= ol_flags;
361 
362  /* we are trying to help compiler here by using local ol_flags with known
363  state of all flags */
365  rte_net_intel_cksum_flags_prepare (mb, ol_flags);
366 }
367 
368 /*
369  * Transmits the packets on the frame to the interface associated with the
370  * node. It first copies packets on the frame to a tx_vector containing the
371  * rte_mbuf pointers. It then passes this vector to tx_burst_vector_internal
372  * which calls the dpdk tx_burst function.
373  */
374 uword
376  vlib_node_runtime_t * node,
377  vlib_frame_t * f)
378 {
379  dpdk_main_t *dm = &dpdk_main;
380  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
382  u32 n_packets = f->n_vectors;
383  u32 n_left;
384  u32 *from;
385  struct rte_mbuf **tx_vector;
386  u16 i;
387  u16 nb_tx_desc = xd->nb_tx_desc;
388  int queue_id;
389  u32 my_cpu;
390  u32 tx_pkts = 0;
391  tx_ring_hdr_t *ring;
392  u32 n_on_ring;
393 
394  my_cpu = vm->thread_index;
395 
396  queue_id = my_cpu;
397 
398  tx_vector = xd->tx_vectors[queue_id];
399  ring = vec_header (tx_vector, sizeof (*ring));
400 
401  n_on_ring = ring->tx_head - ring->tx_tail;
402  from = vlib_frame_vector_args (f);
403 
404  ASSERT (n_packets <= VLIB_FRAME_SIZE);
405 
406  if (PREDICT_FALSE (n_on_ring + n_packets > nb_tx_desc))
407  {
408  /*
409  * Overflowing the ring should never happen.
410  * If it does then drop the whole frame.
411  */
412  vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_RING_FULL,
413  n_packets);
414 
415  while (n_packets--)
416  {
417  u32 bi0 = from[n_packets];
418  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
419  struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer (b0);
420  rte_pktmbuf_free (mb0);
421  }
422  return n_on_ring;
423  }
424 
425  if (PREDICT_FALSE (dm->tx_pcap_enable))
426  {
427  n_left = n_packets;
428  while (n_left > 0)
429  {
430  u32 bi0 = from[0];
431  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
432  if (dm->pcap_sw_if_index == 0 ||
433  dm->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_TX])
434  pcap_add_buffer (&dm->pcap_main, vm, bi0, 512);
435  from++;
436  n_left--;
437  }
438  }
439 
440  from = vlib_frame_vector_args (f);
441  n_left = n_packets;
442  i = ring->tx_head % nb_tx_desc;
443 
444  while (n_left >= 8)
445  {
446  u32 bi0, bi1, bi2, bi3;
447  struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
448  vlib_buffer_t *b0, *b1, *b2, *b3;
449  u32 or_flags;
450 
451  dpdk_prefetch_buffer_by_index (vm, from[4]);
452  dpdk_prefetch_buffer_by_index (vm, from[5]);
453  dpdk_prefetch_buffer_by_index (vm, from[6]);
454  dpdk_prefetch_buffer_by_index (vm, from[7]);
455 
456  bi0 = from[0];
457  bi1 = from[1];
458  bi2 = from[2];
459  bi3 = from[3];
460  from += 4;
461 
462  b0 = vlib_get_buffer (vm, bi0);
463  b1 = vlib_get_buffer (vm, bi1);
464  b2 = vlib_get_buffer (vm, bi2);
465  b3 = vlib_get_buffer (vm, bi3);
466 
467  or_flags = b0->flags | b1->flags | b2->flags | b3->flags;
468 
473 
474  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
475  {
476  dpdk_validate_rte_mbuf (vm, b0, 1);
477  dpdk_validate_rte_mbuf (vm, b1, 1);
478  dpdk_validate_rte_mbuf (vm, b2, 1);
479  dpdk_validate_rte_mbuf (vm, b3, 1);
480  }
481  else
482  {
483  dpdk_validate_rte_mbuf (vm, b0, 0);
484  dpdk_validate_rte_mbuf (vm, b1, 0);
485  dpdk_validate_rte_mbuf (vm, b2, 0);
486  dpdk_validate_rte_mbuf (vm, b3, 0);
487  }
488 
489  mb0 = rte_mbuf_from_vlib_buffer (b0);
490  mb1 = rte_mbuf_from_vlib_buffer (b1);
491  mb2 = rte_mbuf_from_vlib_buffer (b2);
492  mb3 = rte_mbuf_from_vlib_buffer (b3);
493 
494  if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
495  (or_flags &
496  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
497  | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
498  | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
499  {
500  dpdk_buffer_tx_offload (xd, b0, mb0);
501  dpdk_buffer_tx_offload (xd, b1, mb1);
502  dpdk_buffer_tx_offload (xd, b2, mb2);
503  dpdk_buffer_tx_offload (xd, b3, mb3);
504  }
505 
506  if (PREDICT_FALSE (or_flags & VLIB_BUFFER_RECYCLE))
507  {
508  dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
509  dpdk_buffer_recycle (vm, node, b1, bi1, &mb1);
510  dpdk_buffer_recycle (vm, node, b2, bi2, &mb2);
511  dpdk_buffer_recycle (vm, node, b3, bi3, &mb3);
512 
513  /* dont enqueue packets if replication failed as they must
514  be sent back to recycle */
515  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
516  tx_vector[i++ % nb_tx_desc] = mb0;
517  if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_REPL_FAIL) == 0))
518  tx_vector[i++ % nb_tx_desc] = mb1;
519  if (PREDICT_TRUE ((b2->flags & VLIB_BUFFER_REPL_FAIL) == 0))
520  tx_vector[i++ % nb_tx_desc] = mb2;
521  if (PREDICT_TRUE ((b3->flags & VLIB_BUFFER_REPL_FAIL) == 0))
522  tx_vector[i++ % nb_tx_desc] = mb3;
523  }
524  else
525  {
526  if (PREDICT_FALSE (i + 3 >= nb_tx_desc))
527  {
528  tx_vector[i++ % nb_tx_desc] = mb0;
529  tx_vector[i++ % nb_tx_desc] = mb1;
530  tx_vector[i++ % nb_tx_desc] = mb2;
531  tx_vector[i++ % nb_tx_desc] = mb3;
532  i %= nb_tx_desc;
533  }
534  else
535  {
536  tx_vector[i++] = mb0;
537  tx_vector[i++] = mb1;
538  tx_vector[i++] = mb2;
539  tx_vector[i++] = mb3;
540  }
541  }
542 
543 
544  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
545  {
546  if (b0->flags & VLIB_BUFFER_IS_TRACED)
547  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
548  if (b1->flags & VLIB_BUFFER_IS_TRACED)
549  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi1, b1);
550  if (b2->flags & VLIB_BUFFER_IS_TRACED)
551  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi2, b2);
552  if (b3->flags & VLIB_BUFFER_IS_TRACED)
553  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi3, b3);
554  }
555 
556  n_left -= 4;
557  }
558  while (n_left > 0)
559  {
560  u32 bi0;
561  struct rte_mbuf *mb0;
562  vlib_buffer_t *b0;
563 
564  bi0 = from[0];
565  from++;
566 
567  b0 = vlib_get_buffer (vm, bi0);
569 
570  dpdk_validate_rte_mbuf (vm, b0, 1);
571 
572  mb0 = rte_mbuf_from_vlib_buffer (b0);
573  dpdk_buffer_tx_offload (xd, b0, mb0);
574  dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
575 
576  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
577  if (b0->flags & VLIB_BUFFER_IS_TRACED)
578  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
579 
580  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
581  {
582  tx_vector[i % nb_tx_desc] = mb0;
583  i++;
584  }
585  n_left--;
586  }
587 
588  /* account for additional packets in the ring */
589  ring->tx_head += n_packets;
590  n_on_ring = ring->tx_head - ring->tx_tail;
591 
592  /* transmit as many packets as possible */
593  n_packets = tx_burst_vector_internal (vm, xd, tx_vector);
594 
595  /*
596  * tx_pkts is the number of packets successfully transmitted
597  * This is the number originally on ring minus the number remaining on ring
598  */
599  tx_pkts = n_on_ring - n_packets;
600 
601  {
602  /* If there is no callback then drop any non-transmitted packets */
603  if (PREDICT_FALSE (n_packets))
604  {
606  vnet_main_t *vnm = vnet_get_main ();
607 
610 
611  vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
612  n_packets);
613 
614  vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
615  n_packets);
616 
617  while (n_packets--)
618  rte_pktmbuf_free (tx_vector[ring->tx_tail + n_packets]);
619  }
620 
621  /* Reset head/tail to avoid unnecessary wrap */
622  ring->tx_head = 0;
623  ring->tx_tail = 0;
624  }
625 
626  /* Recycle replicated buffers */
627  if (PREDICT_FALSE (vec_len (dm->recycle[my_cpu])))
628  {
629  vlib_buffer_free (vm, dm->recycle[my_cpu],
630  vec_len (dm->recycle[my_cpu]));
631  _vec_len (dm->recycle[my_cpu]) = 0;
632  }
633 
634  ASSERT (ring->tx_head >= ring->tx_tail);
635 
636  return tx_pkts;
637 }
638 
639 #ifndef CLIB_MULTIARCH_VARIANT
640 static void
642 {
643  dpdk_main_t *dm = &dpdk_main;
644  dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
645 
646  /*
647  * Set the "last_cleared_stats" to the current stats, so that
648  * things appear to clear from a display perspective.
649  */
651 
652  clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats));
655  sizeof (xd->last_cleared_xstats[0]));
656 
657 }
658 
659 static clib_error_t *
661 {
662  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
663  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
664  dpdk_main_t *dm = &dpdk_main;
666 
668  return clib_error_return (0, "Interface not initialized");
669 
670  if (is_up)
671  {
674  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
675  dpdk_device_start (xd);
677  f64 now = vlib_time_now (dm->vlib_main);
678  dpdk_update_counters (xd, now);
679  dpdk_update_link_state (xd, now);
680  }
681  else
682  {
684  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
685  dpdk_device_stop (xd);
687  }
688 
689  return /* no error */ 0;
690 }
691 
692 /*
693  * Dynamically redirect all pkts from a specific interface
694  * to the specified node
695  */
696 static void
698  u32 node_index)
699 {
700  dpdk_main_t *xm = &dpdk_main;
701  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
703 
704  /* Shut off redirection */
705  if (node_index == ~0)
706  {
707  xd->per_interface_next_index = node_index;
708  return;
709  }
710 
712  vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index);
713 }
714 
715 
716 static clib_error_t *
718  u32 hw_if_index,
719  struct vnet_sw_interface_t *st, int is_add)
720 {
721  dpdk_main_t *xm = &dpdk_main;
722  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
725  int r, vlan_offload;
726  u32 prev_subifs = xd->num_subifs;
727  clib_error_t *err = 0;
728 
729  if (is_add)
730  xd->num_subifs++;
731  else if (xd->num_subifs)
732  xd->num_subifs--;
733 
734  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
735  goto done;
736 
737  /* currently we program VLANS only for IXGBE VF and I40E VF */
738  if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
739  goto done;
740 
741  if (t->sub.eth.flags.no_tags == 1)
742  goto done;
743 
744  if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
745  {
746  xd->num_subifs = prev_subifs;
747  err = clib_error_return (0, "unsupported VLAN setup");
748  goto done;
749  }
750 
751  vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index);
752  vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
753 
754  if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload)))
755  {
756  xd->num_subifs = prev_subifs;
757  err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
758  xd->device_index, r);
759  goto done;
760  }
761 
762 
763  if ((r =
764  rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id,
765  is_add)))
766  {
767  xd->num_subifs = prev_subifs;
768  err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
769  xd->device_index, r);
770  goto done;
771  }
772 
773 done:
774  if (xd->num_subifs)
776  else
778 
779  return err;
780 }
781 
782 /* *INDENT-OFF* */
784  .name = "dpdk",
785  .tx_function = dpdk_interface_tx,
786  .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR,
787  .tx_function_error_strings = dpdk_tx_func_error_strings,
788  .format_device_name = format_dpdk_device_name,
789  .format_device = format_dpdk_device,
790  .format_tx_trace = format_dpdk_tx_dma_trace,
791  .clear_counters = dpdk_clear_hw_interface_counters,
792  .admin_up_down_function = dpdk_interface_admin_up_down,
793  .subif_add_del_function = dpdk_subif_add_del_function,
794  .rx_redirect_to_node = dpdk_set_interface_next_node,
795  .mac_addr_change_function = dpdk_set_mac_address,
796 };
797 /* *INDENT-ON* */
798 
799 #if __x86_64__
802 static void __clib_constructor
804 {
805  if (dpdk_interface_tx_avx512 && clib_cpu_supports_avx512f ())
807  else if (dpdk_interface_tx_avx2 && clib_cpu_supports_avx2 ())
809 }
810 #endif
811 #endif
812 
813 #define UP_DOWN_FLAG_EVENT 1
814 
815 #ifndef CLIB_MULTIARCH_VARIANT
816 uword
819 {
820  clib_error_t *error = 0;
821  uword event_type;
822  uword *event_data = 0;
823  u32 sw_if_index;
824  u32 flags;
825 
826  while (1)
827  {
829 
830  event_type = vlib_process_get_events (vm, &event_data);
831 
833 
834  switch (event_type)
835  {
836  case UP_DOWN_FLAG_EVENT:
837  {
838  if (vec_len (event_data) == 2)
839  {
840  sw_if_index = event_data[0];
841  flags = event_data[1];
842  error =
844  flags);
845  clib_error_report (error);
846  }
847  }
848  break;
849  }
850 
851  vec_reset_length (event_data);
852 
854 
855  }
856  return 0; /* or not */
857 }
858 
859 /* *INDENT-OFF* */
861  .function = admin_up_down_process,
862  .type = VLIB_NODE_TYPE_PROCESS,
863  .name = "admin-up-down-process",
864  .process_log2_n_stack_bytes = 17, // 256KB
865 };
866 /* *INDENT-ON* */
867 #endif
868 
869 /*
870  * fd.io coding-style-patch-verification: ON
871  *
872  * Local Variables:
873  * eval: (c-set-style "gnu")
874  * End:
875  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:432
u8 * default_mac_address
Definition: dpdk.h:234
vmrglw vmrglh hi
uword( vlib_node_function_t)(struct vlib_main_t *vm, struct vlib_node_runtime_t *node, struct vlib_frame_t *frame)
Definition: node.h:54
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:538
vlib_buffer_t buffer
Definition: dpdk.h:413
#define DPDK_DEVICE_FLAG_PMD_INIT_FAIL
Definition: dpdk.h:185
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:356
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:619
#define VLIB_BUFFER_RECYCLE
Definition: buffer.h:99
dpdk_main_t dpdk_main
Definition: init.c:39
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
vnet_interface_main_t interface_main
Definition: vnet.h:56
vnet_device_class_t dpdk_device_class
#define PREDICT_TRUE(x)
Definition: clib.h:106
#define DPDK_DEVICE_FLAG_TX_OFFLOAD
Definition: dpdk.h:191
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define NULL
Definition: clib.h:55
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:224
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **tx_vector)
Definition: device.c:193
static char * dpdk_tx_func_error_strings[]
Definition: device.c:42
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
u16 flags
Definition: dpdk.h:181
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, u32 buffer_index, vlib_buffer_t *buffer)
Definition: device.c:114
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:173
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:518
#define DPDK_DEVICE_FLAG_HQOS
Definition: dpdk.h:188
u32 per_interface_next_index
Definition: dpdk.h:169
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:78
u64 tx_tail
Definition: dpdk.h:119
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:394
struct rte_eth_xstat * last_cleared_xstats
Definition: dpdk.h:229
u16 num_subifs
Definition: dpdk.h:200
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:107
format_function_t format_dpdk_tx_dma_trace
Definition: dpdk.h:457
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1108
VNET_DEVICE_CLASS(af_packet_device_class)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define DPDK_DEVICE_FLAG_PMD
Definition: dpdk.h:184
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:595
#define DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM
Definition: dpdk.h:192
struct rte_eth_stats stats
Definition: dpdk.h:225
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:95
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:68
#define static_always_inline
Definition: clib.h:93
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:542
#define VLIB_BUFFER_REPL_FAIL
Definition: buffer.h:98
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:619
uword CLIB_MULTIARCH_FN() dpdk_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
Definition: device.c:375
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:697
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:151
unsigned long u64
Definition: types.h:89
static struct rte_mbuf * dpdk_replicate_packet_mb(vlib_buffer_t *b)
Definition: device.c:72
static vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
Definition: device.c:860
A collection of simple counters.
Definition: counter.h:58
dpdk_device_hqos_per_worker_thread_t * hqos_wt
Definition: dpdk.h:212
u32 pcap_sw_if_index
Definition: dpdk.h:372
static void __clib_constructor dpdk_interface_tx_multiarch_select(void)
Definition: device.c:803
vnet_hw_interface_t * hw_interfaces
Definition: interface.h:649
#define rte_mbuf_from_vlib_buffer(x)
Definition: dpdk_priv.h:16
void dpdk_device_start(dpdk_device_t *xd)
Definition: common.c:128
vnet_sub_interface_t sub
Definition: interface.h:612
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:72
pcap_main_t pcap_main
Definition: dpdk.h:370
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
Definition: hqos.c:641
struct rte_mbuf mb
Definition: dpdk.h:411
struct vnet_sub_interface_t::@180::@181::@183 flags
static void pcap_add_buffer(pcap_main_t *pm, vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
Definition: pcap.h:202
#define PREDICT_FALSE(x)
Definition: clib.h:105
#define VLIB_FRAME_SIZE
Definition: node.h:328
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:671
u16 tx_q_used
Definition: dpdk.h:203
u32 node_index
Node index.
Definition: node.h:437
u32 hw_if_index
Definition: dpdk.h:165
dpdk_tx_func_error_t
Definition: device.c:33
#define DPDK_DEVICE_FLAG_ADMIN_UP
Definition: dpdk.h:182
u32 ** recycle
Definition: dpdk.h:357
struct vnet_sub_interface_t::@180 eth
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
dpdk_device_t * devices
Definition: dpdk.h:353
vlib_main_t * vm
Definition: buffer.c:283
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:86
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
volatile u32 ** lockp
Definition: dpdk.h:160
vlib_node_function_t __clib_weak dpdk_interface_tx_avx512
Definition: device.c:800
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:93
#define clib_memcpy(a, b, c)
Definition: string.h:75
dpdk_pmd_t pmd
Definition: dpdk.h:178
format_function_t format_dpdk_device
Definition: dpdk.h:455
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
Definition: device.c:138
void dpdk_device_stop(dpdk_device_t *xd)
Definition: common.c:172
struct rte_eth_xstat * xstats
Definition: dpdk.h:228
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
Definition: interface.h:576
#define ASSERT(truth)
format_function_t format_dpdk_device_name
Definition: dpdk.h:454
unsigned int u32
Definition: types.h:88
u64 tx_head
Definition: dpdk.h:118
uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:817
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
struct rte_mempool ** pktmbuf_pools
Definition: dpdk.h:398
#define DPDK_DEVICE_FLAG_HAVE_SUBIF
Definition: dpdk.h:187
#define clib_error_report(e)
Definition: error.h:113
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1294
dpdk_portid_t device_index
Definition: dpdk.h:163
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:124
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
static void dpdk_clear_hw_interface_counters(u32 instance)
Definition: device.c:641
vlib_node_function_t __clib_weak dpdk_interface_tx_avx2
Definition: device.c:801
Definition: defs.h:47
unsigned short u16
Definition: types.h:57
#define foreach_dpdk_tx_func_error
Definition: device.c:27
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
u8 admin_up_down_in_progress
Definition: dpdk.h:379
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, char *address)
Definition: device.c:49
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:534
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
u8 data[256]
Definition: dpdk.h:414
#define UP_DOWN_FLAG_EVENT
Definition: device.c:813
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:660
#define VLIB_BUFFER_EXT_HDR_VALID
Definition: buffer.h:101
#define vnet_buffer(b)
Definition: buffer.h:326
static_always_inline void dpdk_prefetch_buffer_by_index(vlib_main_t *vm, u32 bi)
Definition: device.c:303
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
static void * vec_header(void *v, uword header_bytes)
Find a user vector header.
Definition: vec_bootstrap.h:92
u8 data[0]
Packet data.
Definition: buffer.h:159
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: interface.c:546
struct rte_eth_stats last_cleared_stats
Definition: dpdk.h:227
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:717
static_always_inline void dpdk_buffer_recycle(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b, u32 bi, struct rte_mbuf **mbp)
Definition: device.c:314
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
Definition: device.c:338
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:75
int tx_pcap_enable
Definition: dpdk.h:369
vnet_main_t * vnet_main
Definition: dpdk.h:394
u16 nb_tx_desc
Definition: dpdk.h:194
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:59
vlib_main_t * vlib_main
Definition: dpdk.h:393