FD.io VPP  v18.07-rc0-415-g6c78436
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/format.h>
18 #include <vlib/unix/cj.h>
19 #include <assert.h>
20 
21 #include <vnet/ethernet/ethernet.h>
22 #include <dpdk/device/dpdk.h>
23 
24 #include <dpdk/device/dpdk_priv.h>
25 #include <vppinfra/error.h>
26 
27 #define foreach_dpdk_tx_func_error \
28  _(BAD_RETVAL, "DPDK tx function returned an error") \
29  _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \
30  _(REPL_FAIL, "Tx packet drops (replication failure)")
31 
32 typedef enum
33 {
34 #define _(f,s) DPDK_TX_FUNC_ERROR_##f,
36 #undef _
39 
40 #ifndef CLIB_MARCH_VARIANT
41 static char *dpdk_tx_func_error_strings[] = {
42 #define _(n,s) s,
44 #undef _
45 };
46 
47 static clib_error_t *
49 {
50  int error;
51  dpdk_main_t *dm = &dpdk_main;
53 
54  error = rte_eth_dev_default_mac_addr_set (xd->port_id,
55  (struct ether_addr *) address);
56 
57  if (error)
58  {
59  return clib_error_return (0, "mac address set failed: %d", error);
60  }
61  else
62  {
64  vec_add (xd->default_mac_address, address, sizeof (address));
65  return NULL;
66  }
67 }
68 #endif
69 
70 static struct rte_mbuf *
72 {
73  dpdk_main_t *dm = &dpdk_main;
74  struct rte_mbuf **mbufs = 0, *s, *d;
75  u8 nb_segs;
76  unsigned socket_id = rte_socket_id ();
77  int i;
78 
79  ASSERT (dm->pktmbuf_pools[socket_id]);
81  nb_segs = s->nb_segs;
82  vec_validate (mbufs, nb_segs - 1);
83 
84  if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs))
85  {
86  vec_free (mbufs);
87  return 0;
88  }
89 
90  d = mbufs[0];
91  d->nb_segs = s->nb_segs;
92  d->data_len = s->data_len;
93  d->pkt_len = s->pkt_len;
94  d->data_off = s->data_off;
95  clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
96 
97  for (i = 1; i < nb_segs; i++)
98  {
99  d->next = mbufs[i];
100  d = mbufs[i];
101  s = s->next;
102  d->data_len = s->data_len;
103  clib_memcpy (d->buf_addr, s->buf_addr,
104  RTE_PKTMBUF_HEADROOM + s->data_len);
105  }
106 
107  d = mbufs[0];
108  vec_free (mbufs);
109  return d;
110 }
111 
112 static void
114  dpdk_device_t * xd, u16 queue_id,
115  vlib_buffer_t * buffer)
116 {
117  vlib_main_t *vm = vlib_get_main ();
118  dpdk_tx_trace_t *t0;
119  struct rte_mbuf *mb;
120 
121  mb = rte_mbuf_from_vlib_buffer (buffer);
122 
123  t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
124  t0->queue_index = queue_id;
125  t0->device_index = xd->device_index;
126  t0->buffer_index = vlib_get_buffer_index (vm, buffer);
127  clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
128  clib_memcpy (&t0->buffer, buffer,
129  sizeof (buffer[0]) - sizeof (buffer->pre_data));
130  clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data,
131  sizeof (t0->buffer.pre_data));
132  clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data));
133 }
134 
137  int maybe_multiseg)
138 {
139  struct rte_mbuf *mb, *first_mb, *last_mb;
140 
141  /* buffer is coming from non-dpdk source so we need to init
142  rte_mbuf header */
143  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
144  {
145  vlib_buffer_t *b2 = b;
146  last_mb = mb = rte_mbuf_from_vlib_buffer (b2);
147  rte_pktmbuf_reset (mb);
148  while (maybe_multiseg && (b2->flags & VLIB_BUFFER_NEXT_PRESENT))
149  {
150  b2 = vlib_get_buffer (vm, b2->next_buffer);
151  mb = rte_mbuf_from_vlib_buffer (b2);
152  rte_pktmbuf_reset (mb);
153  }
154  }
155 
156  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
157  first_mb->nb_segs = 1;
158  mb->data_len = b->current_length;
159  mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
160  b->current_length;
161  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
162 
163  while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
164  {
165  b = vlib_get_buffer (vm, b->next_buffer);
166  mb = rte_mbuf_from_vlib_buffer (b);
167  last_mb->next = mb;
168  last_mb = mb;
169  mb->data_len = b->current_length;
170  mb->pkt_len = b->current_length;
171  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
172  first_mb->nb_segs++;
173  if (PREDICT_FALSE (b->n_add_refs))
174  {
175  rte_mbuf_refcnt_update (mb, b->n_add_refs);
176  b->n_add_refs = 0;
177  }
178  }
179 }
180 
181 /*
182  * This function calls the dpdk's tx_burst function to transmit the packets.
183  * It manages a lock per-device if the device does not
184  * support multiple queues. It returns the number of packets untransmitted
185  * If all packets are transmitted (the normal case), the function returns 0.
186  */
189  dpdk_device_t * xd,
190  struct rte_mbuf **mb, u32 n_left)
191 {
192  dpdk_main_t *dm = &dpdk_main;
193  u32 n_retry;
194  int n_sent = 0;
195  int queue_id;
196 
197  n_retry = 16;
198  queue_id = vm->thread_index;
199 
200  do
201  {
202  /*
203  * This device only supports one TX queue,
204  * and we're running multi-threaded...
205  */
206  if (PREDICT_FALSE (xd->lockp != 0))
207  {
208  queue_id = queue_id % xd->tx_q_used;
209  while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
210  /* zzzz */
211  queue_id = (queue_id + 1) % xd->tx_q_used;
212  }
213 
214  if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */
215  {
216  /* no wrap, transmit in one burst */
218  &xd->hqos_wt[vm->thread_index];
219 
220  ASSERT (hqos->swq != NULL);
221 
222  dpdk_hqos_metadata_set (hqos, mb, n_left);
223  n_sent = rte_ring_sp_enqueue_burst (hqos->swq, (void **) mb,
224  n_left, 0);
225  }
226  else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
227  {
228  /* no wrap, transmit in one burst */
229  n_sent = rte_eth_tx_burst (xd->port_id, queue_id, mb, n_left);
230  }
231  else
232  {
233  ASSERT (0);
234  n_sent = 0;
235  }
236 
237  if (PREDICT_FALSE (xd->lockp != 0))
238  *xd->lockp[queue_id] = 0;
239 
240  if (PREDICT_FALSE (n_sent < 0))
241  {
242  // emit non-fatal message, bump counter
243  vnet_main_t *vnm = dm->vnet_main;
245  u32 node_index;
246 
247  node_index = vec_elt_at_index (im->hw_interfaces,
248  xd->hw_if_index)->tx_node_index;
249 
250  vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1);
251  clib_warning ("rte_eth_tx_burst[%d]: error %d",
252  xd->port_id, n_sent);
253  return n_left; // untransmitted packets
254  }
255  n_left -= n_sent;
256  mb += n_sent;
257  }
258  while (n_sent && n_left && (n_retry > 0));
259 
260  return n_left;
261 }
262 
264 dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb)
265 {
267  CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, STORE);
269 }
270 
273  vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp)
274 {
275  dpdk_main_t *dm = &dpdk_main;
276  struct rte_mbuf *mb_new;
277 
278  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0)
279  return;
280 
281  mb_new = dpdk_replicate_packet_mb (b);
282  if (PREDICT_FALSE (mb_new == 0))
283  {
284  vlib_error_count (vm, node->node_index,
285  DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
286  b->flags |= VLIB_BUFFER_REPL_FAIL;
287  }
288  else
289  *mbp = mb_new;
290 
291  vec_add1 (dm->recycle[vm->thread_index], bi);
292 }
293 
296  struct rte_mbuf *mb)
297 {
298  u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
299  u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
300  u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
301  int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
302  u64 ol_flags;
303 
304  /* Is there any work for us? */
305  if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum) == 0))
306  return;
307 
308  mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
309  mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
310  vnet_buffer (b)->l3_hdr_offset;
311  mb->outer_l3_len = 0;
312  mb->outer_l2_len = 0;
313  ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
314  ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
315  ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
316  ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
317  mb->ol_flags |= ol_flags;
318 
319  /* we are trying to help compiler here by using local ol_flags with known
320  state of all flags */
321  if (xd->flags & DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM)
322  rte_net_intel_cksum_flags_prepare (mb, ol_flags);
323 }
324 
325 /*
326  * Transmits the packets on the frame to the interface associated with the
327  * node. It first copies packets on the frame to a per-thread arrays
328  * containing the rte_mbuf pointers.
329  */
330 uword
332  vlib_node_runtime_t * node,
333  vlib_frame_t * f)
334 {
335  dpdk_main_t *dm = &dpdk_main;
336  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
338  u32 n_packets = f->n_vectors;
339  u32 n_left;
340  u32 *from;
341  u32 thread_index = vm->thread_index;
342  int queue_id = thread_index;
343  u32 tx_pkts = 0, all_or_flags = 0;
345  thread_index);
346  struct rte_mbuf **mb;
347  vlib_buffer_t *b[4];
348 
349  from = vlib_frame_vector_args (f);
350 
351  ASSERT (n_packets <= VLIB_FRAME_SIZE);
352 
353  /* TX PCAP tracing */
355  {
356  n_left = n_packets;
357  while (n_left > 0)
358  {
359  u32 bi0 = from[0];
360  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
361  if (dm->pcap[VLIB_TX].pcap_sw_if_index == 0 ||
363  == vnet_buffer (b0)->sw_if_index[VLIB_TX])
364  pcap_add_buffer (&dm->pcap[VLIB_TX].pcap_main, vm, bi0, 512);
365  from++;
366  n_left--;
367  }
368  }
369 
370  /* calculate rte_mbuf pointers out of buffer indices */
372  (void **) ptd->mbufs, n_packets,
373  -(i32) sizeof (struct rte_mbuf));
374 
375  from = vlib_frame_vector_args (f);
376  n_left = n_packets;
377  mb = ptd->mbufs;
378 
379  while (n_left >= 8)
380  {
381  u32 or_flags;
382 
383  dpdk_prefetch_buffer (vm, mb[4]);
384  dpdk_prefetch_buffer (vm, mb[5]);
385  dpdk_prefetch_buffer (vm, mb[6]);
386  dpdk_prefetch_buffer (vm, mb[7]);
387 
388  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
389  b[1] = vlib_buffer_from_rte_mbuf (mb[1]);
390  b[2] = vlib_buffer_from_rte_mbuf (mb[2]);
391  b[3] = vlib_buffer_from_rte_mbuf (mb[3]);
392 
393  or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
394  all_or_flags |= or_flags;
395 
400 
401  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
402  {
403  dpdk_validate_rte_mbuf (vm, b[0], 1);
404  dpdk_validate_rte_mbuf (vm, b[1], 1);
405  dpdk_validate_rte_mbuf (vm, b[2], 1);
406  dpdk_validate_rte_mbuf (vm, b[3], 1);
407  }
408  else
409  {
410  dpdk_validate_rte_mbuf (vm, b[0], 0);
411  dpdk_validate_rte_mbuf (vm, b[1], 0);
412  dpdk_validate_rte_mbuf (vm, b[2], 0);
413  dpdk_validate_rte_mbuf (vm, b[3], 0);
414  }
415 
416  if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
417  (or_flags &
418  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
419  | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
420  | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
421  {
422  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
423  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
424  dpdk_buffer_tx_offload (xd, b[2], mb[2]);
425  dpdk_buffer_tx_offload (xd, b[3], mb[3]);
426  }
427 
428  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
429  {
430  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
431  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
432  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
433  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[1]);
434  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
435  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[2]);
436  if (b[3]->flags & VLIB_BUFFER_IS_TRACED)
437  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[3]);
438  }
439 
440  mb += 4;
441  n_left -= 4;
442  }
443  while (n_left > 0)
444  {
445  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
446  all_or_flags |= b[0]->flags;
448 
449  dpdk_validate_rte_mbuf (vm, b[0], 1);
450  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
451 
452  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
453  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
454  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
455 
456  mb++;
457  n_left--;
458  }
459 
460  /* run inly if we have buffers to recycle */
461  if (PREDICT_FALSE (all_or_flags & VLIB_BUFFER_RECYCLE))
462  {
463  struct rte_mbuf **mb_old;
464  from = vlib_frame_vector_args (f);
465  n_left = n_packets;
466  mb_old = mb = ptd->mbufs;
467  while (n_left > 0)
468  {
469  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
470  dpdk_buffer_recycle (vm, node, b[0], from[0], &mb_old[0]);
471 
472  /* in case of REPL_FAIL we need to shift data */
473  mb[0] = mb_old[0];
474 
475  if (PREDICT_TRUE ((b[0]->flags & VLIB_BUFFER_REPL_FAIL) == 0))
476  mb++;
477  mb_old++;
478  from++;
479  n_left--;
480  }
481  }
482 
483  /* transmit as many packets as possible */
484  tx_pkts = n_packets = mb - ptd->mbufs;
485  n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets);
486 
487  {
488  /* If there is no callback then drop any non-transmitted packets */
489  if (PREDICT_FALSE (n_left))
490  {
491  tx_pkts -= n_left;
493  vnet_main_t *vnm = vnet_get_main ();
494 
497 
498  vlib_increment_simple_counter (cm, thread_index, xd->sw_if_index,
499  n_left);
500 
501  vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
502  n_left);
503 
504  while (n_left--)
505  rte_pktmbuf_free (ptd->mbufs[n_packets - n_left - 1]);
506  }
507  }
508 
509  /* Recycle replicated buffers */
510  if (PREDICT_FALSE (vec_len (dm->recycle[thread_index])))
511  {
512  vlib_buffer_free (vm, dm->recycle[thread_index],
513  vec_len (dm->recycle[thread_index]));
514  _vec_len (dm->recycle[thread_index]) = 0;
515  }
516 
517  return tx_pkts;
518 }
519 
520 #ifndef CLIB_MARCH_VARIANT
521 static void
523 {
524  dpdk_main_t *dm = &dpdk_main;
525  dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
526 
527  /*
528  * Set the "last_cleared_stats" to the current stats, so that
529  * things appear to clear from a display perspective.
530  */
532 
533  clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats));
536  sizeof (xd->last_cleared_xstats[0]));
537 
538 }
539 
540 static clib_error_t *
542 {
543  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
544  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
545  dpdk_main_t *dm = &dpdk_main;
547 
548  if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
549  return clib_error_return (0, "Interface not initialized");
550 
551  if (is_up)
552  {
555  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
556  dpdk_device_start (xd);
557  xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
558  f64 now = vlib_time_now (dm->vlib_main);
559  dpdk_update_counters (xd, now);
560  dpdk_update_link_state (xd, now);
561  }
562  else
563  {
565  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
566  dpdk_device_stop (xd);
567  xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
568  }
569 
570  return /* no error */ 0;
571 }
572 
573 /*
574  * Dynamically redirect all pkts from a specific interface
575  * to the specified node
576  */
577 static void
579  u32 node_index)
580 {
581  dpdk_main_t *xm = &dpdk_main;
582  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
584 
585  /* Shut off redirection */
586  if (node_index == ~0)
587  {
588  xd->per_interface_next_index = node_index;
589  return;
590  }
591 
593  vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index);
594 }
595 
596 
597 static clib_error_t *
599  u32 hw_if_index,
600  struct vnet_sw_interface_t *st, int is_add)
601 {
602  dpdk_main_t *xm = &dpdk_main;
603  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
606  int r, vlan_offload;
607  u32 prev_subifs = xd->num_subifs;
608  clib_error_t *err = 0;
609 
610  if (is_add)
611  xd->num_subifs++;
612  else if (xd->num_subifs)
613  xd->num_subifs--;
614 
615  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
616  goto done;
617 
618  /* currently we program VLANS only for IXGBE VF and I40E VF */
619  if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
620  goto done;
621 
622  if (t->sub.eth.flags.no_tags == 1)
623  goto done;
624 
625  if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
626  {
627  xd->num_subifs = prev_subifs;
628  err = clib_error_return (0, "unsupported VLAN setup");
629  goto done;
630  }
631 
632  vlan_offload = rte_eth_dev_get_vlan_offload (xd->port_id);
633  vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
634 
635  if ((r = rte_eth_dev_set_vlan_offload (xd->port_id, vlan_offload)))
636  {
637  xd->num_subifs = prev_subifs;
638  err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
639  xd->port_id, r);
640  goto done;
641  }
642 
643 
644  if ((r =
645  rte_eth_dev_vlan_filter (xd->port_id,
646  t->sub.eth.outer_vlan_id, is_add)))
647  {
648  xd->num_subifs = prev_subifs;
649  err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
650  xd->port_id, r);
651  goto done;
652  }
653 
654 done:
655  if (xd->num_subifs)
656  xd->flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
657  else
658  xd->flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
659 
660  return err;
661 }
662 
663 /* *INDENT-OFF* */
665  .name = "dpdk",
666  .tx_function = dpdk_interface_tx,
667  .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR,
668  .tx_function_error_strings = dpdk_tx_func_error_strings,
669  .format_device_name = format_dpdk_device_name,
670  .format_device = format_dpdk_device,
671  .format_tx_trace = format_dpdk_tx_trace,
672  .clear_counters = dpdk_clear_hw_interface_counters,
673  .admin_up_down_function = dpdk_interface_admin_up_down,
674  .subif_add_del_function = dpdk_subif_add_del_function,
675  .rx_redirect_to_node = dpdk_set_interface_next_node,
676  .mac_addr_change_function = dpdk_set_mac_address,
677  .format_flow = format_dpdk_flow,
678  .flow_ops_function = dpdk_flow_ops_fn,
679 };
680 /* *INDENT-ON* */
681 
682 #if __x86_64__
685 static void __clib_constructor
687 {
688  if (dpdk_interface_tx_avx512 && clib_cpu_supports_avx512f ())
690  else if (dpdk_interface_tx_avx2 && clib_cpu_supports_avx2 ())
692 }
693 #endif
694 #endif
695 
696 #define UP_DOWN_FLAG_EVENT 1
697 
698 #ifndef CLIB_MARCH_VARIANT
699 uword
702 {
703  clib_error_t *error = 0;
704  uword event_type;
705  uword *event_data = 0;
706  u32 sw_if_index;
707  u32 flags;
708 
709  while (1)
710  {
712 
713  event_type = vlib_process_get_events (vm, &event_data);
714 
716 
717  switch (event_type)
718  {
719  case UP_DOWN_FLAG_EVENT:
720  {
721  if (vec_len (event_data) == 2)
722  {
723  sw_if_index = event_data[0];
724  flags = event_data[1];
725  error =
727  flags);
728  clib_error_report (error);
729  }
730  }
731  break;
732  }
733 
734  vec_reset_length (event_data);
735 
737 
738  }
739  return 0; /* or not */
740 }
741 
742 /* *INDENT-OFF* */
744  .function = admin_up_down_process,
745  .type = VLIB_NODE_TYPE_PROCESS,
746  .name = "admin-up-down-process",
747  .process_log2_n_stack_bytes = 17, // 256KB
748 };
749 /* *INDENT-ON* */
750 #endif
751 
752 /*
753  * fd.io coding-style-patch-verification: ON
754  *
755  * Local Variables:
756  * eval: (c-set-style "gnu")
757  * End:
758  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
u8 * default_mac_address
Definition: dpdk.h:262
format_function_t format_dpdk_tx_trace
Definition: dpdk.h:513
vmrglw vmrglh hi
format_function_t format_dpdk_flow
Definition: dpdk.h:517
typedef address
Definition: ip_types.api:34
#define vlib_buffer_from_rte_mbuf(x)
Definition: dpdk_priv.h:17
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:541
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:534
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:619
dpdk_main_t dpdk_main
Definition: init.c:42
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
vnet_interface_main_t interface_main
Definition: vnet.h:56
vnet_device_class_t dpdk_device_class
#define PREDICT_TRUE(x)
Definition: clib.h:106
unsigned long u64
Definition: types.h:89
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define NULL
Definition: clib.h:55
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:225
static char * dpdk_tx_func_error_strings[]
Definition: device.c:41
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
u16 flags
Definition: dpdk.h:211
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:176
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
int i
u32 per_interface_next_index
Definition: dpdk.h:206
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:75
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:79
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:458
struct rte_eth_xstat * last_cleared_xstats
Definition: dpdk.h:257
u16 num_subifs
Definition: dpdk.h:219
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:250
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1110
unsigned char u8
Definition: types.h:56
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, vlib_buffer_t *buffer)
Definition: device.c:113
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:600
struct vnet_sub_interface_t::@198::@199::@201 flags
struct rte_eth_stats stats
Definition: dpdk.h:253
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:104
#define static_always_inline
Definition: clib.h:93
dpdk_portid_t port_id
Definition: dpdk.h:200
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:542
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:649
uword CLIB_MULTIARCH_FN() dpdk_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
Definition: device.c:331
u32 buffer_index
Definition: dpdk.h:455
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:578
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:164
unsigned int u32
Definition: types.h:88
static struct rte_mbuf * dpdk_replicate_packet_mb(vlib_buffer_t *b)
Definition: device.c:71
static vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
Definition: device.c:743
A collection of simple counters.
Definition: counter.h:58
#define VLIB_FRAME_SIZE
Definition: node.h:364
dpdk_device_hqos_per_worker_thread_t * hqos_wt
Definition: dpdk.h:239
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:141
static void __clib_constructor dpdk_interface_tx_multiarch_select(void)
Definition: device.c:686
vnet_hw_interface_t * hw_interfaces
Definition: interface.h:787
#define rte_mbuf_from_vlib_buffer(x)
Definition: dpdk_priv.h:16
void dpdk_device_start(dpdk_device_t *xd)
Definition: common.c:143
vnet_sub_interface_t sub
Definition: interface.h:697
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
u32 pcap_sw_if_index
Definition: dpdk.h:395
dpdk_per_thread_data_t * per_thread_data
Definition: dpdk.h:405
unsigned short u16
Definition: types.h:57
u8 data[256]
Definition: dpdk.h:461
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
Definition: hqos.c:640
static void pcap_add_buffer(pcap_main_t *pm, vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
Definition: pcap.h:205
#define PREDICT_FALSE(x)
Definition: clib.h:105
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:809
u16 tx_q_used
Definition: dpdk.h:222
u32 node_index
Node index.
Definition: node.h:473
uword( vlib_node_function_t)(struct vlib_main_t *vm, struct vlib_node_runtime_t *node, struct vlib_frame_t *frame)
Definition: node.h:54
u32 hw_if_index
Definition: dpdk.h:202
dpdk_tx_func_error_t
Definition: device.c:32
u32 ** recycle
Definition: dpdk.h:408
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:77
dpdk_device_t * devices
Definition: dpdk.h:403
vlib_main_t * vm
Definition: buffer.c:294
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:87
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
volatile u32 ** lockp
Definition: dpdk.h:194
vlib_node_function_t __clib_weak dpdk_interface_tx_avx512
Definition: device.c:683
struct vnet_sub_interface_t::@198 eth
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
dpdk_pmd_t pmd
Definition: dpdk.h:208
static_always_inline void dpdk_prefetch_buffer(vlib_main_t *vm, struct rte_mbuf *mb)
Definition: device.c:264
format_function_t format_dpdk_device
Definition: dpdk.h:511
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
Definition: device.c:136
void dpdk_device_stop(dpdk_device_t *xd)
Definition: common.c:190
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **mb, u32 n_left)
Definition: device.c:188
struct rte_eth_xstat * xstats
Definition: dpdk.h:256
struct rte_mbuf mb
Definition: dpdk.h:458
signed int i32
Definition: types.h:81
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
Definition: interface.h:660
#define ASSERT(truth)
format_function_t format_dpdk_device_name
Definition: dpdk.h:510
uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:700
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
struct rte_mempool ** pktmbuf_pools
Definition: dpdk.h:442
#define clib_error_report(e)
Definition: error.h:113
VNET_DEVICE_CLASS(bond_dev_class)
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1424
dpdk_portid_t device_index
Definition: dpdk.h:197
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:138
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
vlib_buffer_t buffer
Definition: dpdk.h:460
static void dpdk_clear_hw_interface_counters(u32 instance)
Definition: device.c:522
vlib_node_function_t __clib_weak dpdk_interface_tx_avx2
Definition: device.c:684
struct rte_mbuf * mbufs[DPDK_RX_BURST_SZ]
Definition: dpdk.h:382
pcap_main_t pcap_main
Definition: dpdk.h:393
Definition: defs.h:47
#define foreach_dpdk_tx_func_error
Definition: device.c:27
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u8 admin_up_down_in_progress
Definition: dpdk.h:426
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, char *address)
Definition: device.c:48
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:546
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
dpdk_pcap_t pcap[VLIB_N_RX_TX]
Definition: dpdk.h:414
#define UP_DOWN_FLAG_EVENT
Definition: device.c:696
u16 device_index
Definition: dpdk.h:456
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:541
#define vnet_buffer(b)
Definition: buffer.h:360
vnet_flow_dev_ops_function_t dpdk_flow_ops_fn
Definition: dpdk.h:519
u8 data[0]
Packet data.
Definition: buffer.h:172
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: interface.c:549
struct rte_eth_stats last_cleared_stats
Definition: dpdk.h:255
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:598
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:295
static_always_inline void dpdk_buffer_recycle(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b, u32 bi, struct rte_mbuf **mbp)
Definition: device.c:272
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
Definition: device.c:295
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
u8 queue_index
Definition: dpdk.h:457
vnet_main_t * vnet_main
Definition: dpdk.h:438
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
int pcap_enable
Definition: dpdk.h:392
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:59
vlib_main_t * vlib_main
Definition: dpdk.h:437