FD.io VPP  v20.01-48-g3e0dafb74
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/format.h>
18 #include <vlib/unix/cj.h>
19 #include <assert.h>
20 
21 #include <vnet/ethernet/ethernet.h>
22 #include <dpdk/buffer.h>
23 #include <dpdk/device/dpdk.h>
24 #include <dpdk/device/dpdk_priv.h>
25 #include <vppinfra/error.h>
26 
27 #define foreach_dpdk_tx_func_error \
28  _(BAD_RETVAL, "DPDK tx function returned an error") \
29  _(PKT_DROP, "Tx packet drops (dpdk tx failure)")
30 
31 typedef enum
32 {
33 #define _(f,s) DPDK_TX_FUNC_ERROR_##f,
35 #undef _
38 
39 static char *dpdk_tx_func_error_strings[] = {
40 #define _(n,s) s,
42 #undef _
43 };
44 
45 static clib_error_t *
47  const u8 * address, u8 is_add)
48 {
49  int error;
50  dpdk_main_t *dm = &dpdk_main;
52 
53  if (is_add)
54  error = rte_eth_dev_mac_addr_add (xd->port_id,
55  (struct rte_ether_addr *) address, 0);
56  else
57  error = rte_eth_dev_mac_addr_remove (xd->port_id,
58  (struct rte_ether_addr *) address);
59 
60  if (error)
61  {
62  return clib_error_return (0, "mac address add/del failed: %d", error);
63  }
64 
65  return NULL;
66 }
67 
68 static clib_error_t *
70  const u8 * old_address, const u8 * address)
71 {
72  int error;
73  dpdk_main_t *dm = &dpdk_main;
75 
76  error = rte_eth_dev_default_mac_addr_set (xd->port_id, (void *) address);
77 
78  if (error)
79  {
80  return clib_error_return (0, "mac address set failed: %d", error);
81  }
82  else
83  {
85  vec_add (xd->default_mac_address, address, sizeof (address));
86  return NULL;
87  }
88 }
89 
90 static void
92  dpdk_device_t * xd, u16 queue_id,
93  vlib_buffer_t * buffer)
94 {
95  vlib_main_t *vm = vlib_get_main ();
96  dpdk_tx_trace_t *t0;
97  struct rte_mbuf *mb;
98 
99  mb = rte_mbuf_from_vlib_buffer (buffer);
100 
101  t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
102  t0->queue_index = queue_id;
103  t0->device_index = xd->device_index;
104  t0->buffer_index = vlib_get_buffer_index (vm, buffer);
105  clib_memcpy_fast (&t0->mb, mb, sizeof (t0->mb));
106  clib_memcpy_fast (&t0->buffer, buffer,
107  sizeof (buffer[0]) - sizeof (buffer->pre_data));
108  clib_memcpy_fast (t0->buffer.pre_data, buffer->data + buffer->current_data,
109  sizeof (t0->buffer.pre_data));
110  clib_memcpy_fast (&t0->data, mb->buf_addr + mb->data_off,
111  sizeof (t0->data));
112 }
113 
116  int maybe_multiseg)
117 {
118  struct rte_mbuf *mb, *first_mb, *last_mb;
119  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
120 
121  /* buffer is coming from non-dpdk source so we need to init
122  rte_mbuf header */
123  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
124  rte_pktmbuf_reset (mb);
125 
126  first_mb->nb_segs = 1;
127  mb->data_len = b->current_length;
128  mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
129  b->current_length;
130  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
131 
132  while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
133  {
134  b = vlib_get_buffer (vm, b->next_buffer);
135  mb = rte_mbuf_from_vlib_buffer (b);
136  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
137  rte_pktmbuf_reset (mb);
138  last_mb->next = mb;
139  last_mb = mb;
140  mb->data_len = b->current_length;
141  mb->pkt_len = b->current_length;
142  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
143  first_mb->nb_segs++;
144  if (PREDICT_FALSE (b->ref_count > 1))
145  mb->pool =
147  }
148 }
149 
150 /*
151  * This function calls the dpdk's tx_burst function to transmit the packets.
152  * It manages a lock per-device if the device does not
153  * support multiple queues. It returns the number of packets untransmitted
154  * If all packets are transmitted (the normal case), the function returns 0.
155  */
158  dpdk_device_t * xd,
159  struct rte_mbuf **mb, u32 n_left)
160 {
161  dpdk_main_t *dm = &dpdk_main;
162  u32 n_retry;
163  int n_sent = 0;
164  int queue_id;
165 
166  n_retry = 16;
167  queue_id = vm->thread_index;
168 
169  do
170  {
171  /*
172  * This device only supports one TX queue,
173  * and we're running multi-threaded...
174  */
175  if (PREDICT_FALSE (xd->lockp != 0))
176  {
177  queue_id = queue_id % xd->tx_q_used;
178  while (clib_atomic_test_and_set (xd->lockp[queue_id]))
179  /* zzzz */
180  queue_id = (queue_id + 1) % xd->tx_q_used;
181  }
182 
183 #if 0
184  if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */
185  {
186  /* no wrap, transmit in one burst */
188  &xd->hqos_wt[vm->thread_index];
189 
190  ASSERT (hqos->swq != NULL);
191 
192  dpdk_hqos_metadata_set (hqos, mb, n_left);
193  n_sent = rte_ring_sp_enqueue_burst (hqos->swq, (void **) mb,
194  n_left, 0);
195  }
196  else
197 #endif
198  if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
199  {
200  /* no wrap, transmit in one burst */
201  n_sent = rte_eth_tx_burst (xd->port_id, queue_id, mb, n_left);
202  n_retry--;
203  }
204  else
205  {
206  ASSERT (0);
207  n_sent = 0;
208  }
209 
210  if (PREDICT_FALSE (xd->lockp != 0))
211  clib_atomic_release (xd->lockp[queue_id]);
212 
213  if (PREDICT_FALSE (n_sent < 0))
214  {
215  // emit non-fatal message, bump counter
216  vnet_main_t *vnm = dm->vnet_main;
218  u32 node_index;
219 
220  node_index = vec_elt_at_index (im->hw_interfaces,
221  xd->hw_if_index)->tx_node_index;
222 
223  vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1);
224  clib_warning ("rte_eth_tx_burst[%d]: error %d",
225  xd->port_id, n_sent);
226  return n_left; // untransmitted packets
227  }
228  n_left -= n_sent;
229  mb += n_sent;
230  }
231  while (n_sent && n_left && (n_retry > 0));
232 
233  return n_left;
234 }
235 
237 dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb)
238 {
240  CLIB_PREFETCH (mb, sizeof (struct rte_mbuf), STORE);
242 }
243 
246  struct rte_mbuf *mb)
247 {
248  u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
249  u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
250  u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
251  int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
252  u32 tso = b->flags & VNET_BUFFER_F_GSO;
253  u64 ol_flags;
254 
255  /* Is there any work for us? */
256  if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
257  return;
258 
259  mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
260  mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
261  vnet_buffer (b)->l3_hdr_offset;
262  mb->outer_l3_len = 0;
263  mb->outer_l2_len = 0;
264  ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
265  ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
266  ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
267  ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
268  ol_flags |= tso ? (tcp_cksum ? PKT_TX_TCP_SEG : PKT_TX_UDP_SEG) : 0;
269 
270  if (tso)
271  {
272  mb->l4_len = vnet_buffer2 (b)->gso_l4_hdr_sz;
273  mb->tso_segsz = vnet_buffer2 (b)->gso_size;
274  }
275 
276  mb->ol_flags |= ol_flags;
277 
278  /* we are trying to help compiler here by using local ol_flags with known
279  state of all flags */
280  if (xd->flags & DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM)
281  rte_net_intel_cksum_flags_prepare (mb, ol_flags);
282 }
283 
284 /*
285  * Transmits the packets on the frame to the interface associated with the
286  * node. It first copies packets on the frame to a per-thread arrays
287  * containing the rte_mbuf pointers.
288  */
291  vlib_frame_t * f)
292 {
293  dpdk_main_t *dm = &dpdk_main;
294  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
296  u32 n_packets = f->n_vectors;
297  u32 n_left;
298  u32 thread_index = vm->thread_index;
299  int queue_id = thread_index;
300  u32 tx_pkts = 0, all_or_flags = 0;
302  thread_index);
303  struct rte_mbuf **mb;
304  vlib_buffer_t *b[4];
305 
306  ASSERT (n_packets <= VLIB_FRAME_SIZE);
307 
308  /* calculate rte_mbuf pointers out of buffer indices */
310  (void **) ptd->mbufs, n_packets,
311  -(i32) sizeof (struct rte_mbuf));
312 
313  n_left = n_packets;
314  mb = ptd->mbufs;
315 
316 #if (CLIB_N_PREFETCHES >= 8)
317  while (n_left >= 8)
318  {
319  u32 or_flags;
320 
321  dpdk_prefetch_buffer (vm, mb[4]);
322  dpdk_prefetch_buffer (vm, mb[5]);
323  dpdk_prefetch_buffer (vm, mb[6]);
324  dpdk_prefetch_buffer (vm, mb[7]);
325 
326  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
327  b[1] = vlib_buffer_from_rte_mbuf (mb[1]);
328  b[2] = vlib_buffer_from_rte_mbuf (mb[2]);
329  b[3] = vlib_buffer_from_rte_mbuf (mb[3]);
330 
331  or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
332  all_or_flags |= or_flags;
333 
338 
339  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
340  {
341  dpdk_validate_rte_mbuf (vm, b[0], 1);
342  dpdk_validate_rte_mbuf (vm, b[1], 1);
343  dpdk_validate_rte_mbuf (vm, b[2], 1);
344  dpdk_validate_rte_mbuf (vm, b[3], 1);
345  }
346  else
347  {
348  dpdk_validate_rte_mbuf (vm, b[0], 0);
349  dpdk_validate_rte_mbuf (vm, b[1], 0);
350  dpdk_validate_rte_mbuf (vm, b[2], 0);
351  dpdk_validate_rte_mbuf (vm, b[3], 0);
352  }
353 
354  if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
355  (or_flags &
356  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
357  | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
358  | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
359  {
360  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
361  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
362  dpdk_buffer_tx_offload (xd, b[2], mb[2]);
363  dpdk_buffer_tx_offload (xd, b[3], mb[3]);
364  }
365 
366  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
367  {
368  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
369  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
370  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
371  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[1]);
372  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
373  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[2]);
374  if (b[3]->flags & VLIB_BUFFER_IS_TRACED)
375  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[3]);
376  }
377 
378  mb += 4;
379  n_left -= 4;
380  }
381 #elif (CLIB_N_PREFETCHES >= 4)
382  while (n_left >= 4)
383  {
384  vlib_buffer_t *b2, *b3;
385  u32 or_flags;
386 
387  CLIB_PREFETCH (mb[2], CLIB_CACHE_LINE_BYTES, STORE);
388  CLIB_PREFETCH (mb[3], CLIB_CACHE_LINE_BYTES, STORE);
389  b2 = vlib_buffer_from_rte_mbuf (mb[2]);
391  b3 = vlib_buffer_from_rte_mbuf (mb[3]);
393 
394  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
395  b[1] = vlib_buffer_from_rte_mbuf (mb[1]);
396 
397  or_flags = b[0]->flags | b[1]->flags;
398  all_or_flags |= or_flags;
399 
402 
403  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
404  {
405  dpdk_validate_rte_mbuf (vm, b[0], 1);
406  dpdk_validate_rte_mbuf (vm, b[1], 1);
407  }
408  else
409  {
410  dpdk_validate_rte_mbuf (vm, b[0], 0);
411  dpdk_validate_rte_mbuf (vm, b[1], 0);
412  }
413 
414  if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
415  (or_flags &
416  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
417  | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
418  | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
419  {
420  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
421  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
422  }
423 
424  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
425  {
426  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
427  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
428  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
429  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[1]);
430  }
431 
432  mb += 2;
433  n_left -= 2;
434  }
435 #endif
436 
437  while (n_left > 0)
438  {
439  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
440  all_or_flags |= b[0]->flags;
442 
443  dpdk_validate_rte_mbuf (vm, b[0], 1);
444  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
445 
446  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
447  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
448  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
449 
450  mb++;
451  n_left--;
452  }
453 
454  /* transmit as many packets as possible */
455  tx_pkts = n_packets = mb - ptd->mbufs;
456  n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets);
457 
458  {
459  /* If there is no callback then drop any non-transmitted packets */
460  if (PREDICT_FALSE (n_left))
461  {
462  tx_pkts -= n_left;
464  vnet_main_t *vnm = vnet_get_main ();
465 
468 
469  vlib_increment_simple_counter (cm, thread_index, xd->sw_if_index,
470  n_left);
471 
472  vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
473  n_left);
474 
475  while (n_left--)
476  rte_pktmbuf_free (ptd->mbufs[n_packets - n_left - 1]);
477  }
478  }
479 
480  return tx_pkts;
481 }
482 
483 static void
485 {
486  dpdk_main_t *dm = &dpdk_main;
487  dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
488 
489  rte_eth_stats_reset (xd->port_id);
490  rte_eth_xstats_reset (xd->port_id);
491 }
492 
493 static clib_error_t *
495 {
496  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
497  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
498  dpdk_main_t *dm = &dpdk_main;
500 
501  if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
502  return clib_error_return (0, "Interface not initialized");
503 
504  if (is_up)
505  {
506  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
507  dpdk_device_start (xd);
508  xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
509  f64 now = vlib_time_now (dm->vlib_main);
510  dpdk_update_counters (xd, now);
511  dpdk_update_link_state (xd, now);
512  }
513  else
514  {
516  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
517  dpdk_device_stop (xd);
518  xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
519  }
520 
521  return /* no error */ 0;
522 }
523 
524 /*
525  * Dynamically redirect all pkts from a specific interface
526  * to the specified node
527  */
528 static void
530  u32 node_index)
531 {
532  dpdk_main_t *xm = &dpdk_main;
533  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
535 
536  /* Shut off redirection */
537  if (node_index == ~0)
538  {
539  xd->per_interface_next_index = node_index;
540  return;
541  }
542 
544  vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index);
545 }
546 
547 
548 static clib_error_t *
550  u32 hw_if_index,
551  struct vnet_sw_interface_t *st, int is_add)
552 {
553  dpdk_main_t *xm = &dpdk_main;
554  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
557  int r, vlan_offload;
558  u32 prev_subifs = xd->num_subifs;
559  clib_error_t *err = 0;
560 
561  if (is_add)
562  xd->num_subifs++;
563  else if (xd->num_subifs)
564  xd->num_subifs--;
565 
566  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
567  goto done;
568 
569  /* currently we program VLANS only for IXGBE VF and I40E VF */
570  if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
571  goto done;
572 
573  if (t->sub.eth.flags.no_tags == 1)
574  goto done;
575 
576  if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
577  {
578  xd->num_subifs = prev_subifs;
579  err = clib_error_return (0, "unsupported VLAN setup");
580  goto done;
581  }
582 
583  vlan_offload = rte_eth_dev_get_vlan_offload (xd->port_id);
584  vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
585 
586  if ((r = rte_eth_dev_set_vlan_offload (xd->port_id, vlan_offload)))
587  {
588  xd->num_subifs = prev_subifs;
589  err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
590  xd->port_id, r);
591  goto done;
592  }
593 
594 
595  if ((r =
596  rte_eth_dev_vlan_filter (xd->port_id,
597  t->sub.eth.outer_vlan_id, is_add)))
598  {
599  xd->num_subifs = prev_subifs;
600  err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
601  xd->port_id, r);
602  goto done;
603  }
604 
605 done:
606  if (xd->num_subifs)
607  xd->flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
608  else
609  xd->flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
610 
611  return err;
612 }
613 
614 /* *INDENT-OFF* */
616  .name = "dpdk",
617  .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR,
618  .tx_function_error_strings = dpdk_tx_func_error_strings,
619  .format_device_name = format_dpdk_device_name,
620  .format_device = format_dpdk_device,
621  .format_tx_trace = format_dpdk_tx_trace,
622  .clear_counters = dpdk_clear_hw_interface_counters,
623  .admin_up_down_function = dpdk_interface_admin_up_down,
624  .subif_add_del_function = dpdk_subif_add_del_function,
625  .rx_redirect_to_node = dpdk_set_interface_next_node,
626  .mac_addr_change_function = dpdk_set_mac_address,
627  .mac_addr_add_del_function = dpdk_add_del_mac_address,
628  .format_flow = format_dpdk_flow,
629  .flow_ops_function = dpdk_flow_ops_fn,
630 };
631 /* *INDENT-ON* */
632 
633 #define UP_DOWN_FLAG_EVENT 1
634 
635 static uword
638 {
639  clib_error_t *error = 0;
640  uword event_type;
641  uword *event_data = 0;
643  u32 flags;
644 
645  while (1)
646  {
648 
649  event_type = vlib_process_get_events (vm, &event_data);
650 
652 
653  switch (event_type)
654  {
655  case UP_DOWN_FLAG_EVENT:
656  {
657  if (vec_len (event_data) == 2)
658  {
659  sw_if_index = event_data[0];
660  flags = event_data[1];
661  error =
663  flags);
664  clib_error_report (error);
665  }
666  }
667  break;
668  }
669 
670  vec_reset_length (event_data);
671 
673 
674  }
675  return 0; /* or not */
676 }
677 
678 /* *INDENT-OFF* */
680  .function = admin_up_down_process,
681  .type = VLIB_NODE_TYPE_PROCESS,
682  .name = "admin-up-down-process",
683  .process_log2_n_stack_bytes = 17, // 256KB
684 };
685 /* *INDENT-ON* */
686 
687 /*
688  * fd.io coding-style-patch-verification: ON
689  *
690  * Local Variables:
691  * eval: (c-set-style "gnu")
692  * End:
693  */
u8 * default_mac_address
Definition: dpdk.h:255
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
format_function_t format_dpdk_tx_trace
Definition: dpdk.h:494
format_function_t format_dpdk_flow
Definition: dpdk.h:498
#define vlib_buffer_from_rte_mbuf(x)
Definition: buffer.h:20
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:593
dpdk_main_t dpdk_main
Definition: init.c:45
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
#define vnet_buffer2(b)
Definition: buffer.h:467
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define PREDICT_TRUE(x)
Definition: clib.h:112
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define NULL
Definition: clib.h:58
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:279
static char * dpdk_tx_func_error_strings[]
Definition: device.c:39
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
u16 flags
Definition: dpdk.h:210
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u8 data[0]
Packet data.
Definition: buffer.h:181
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, const u8 *old_address, const u8 *address)
Definition: device.c:69
u32 per_interface_next_index
Definition: dpdk.h:205
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:178
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:78
u16 num_subifs
Definition: dpdk.h:219
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:366
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, vlib_buffer_t *buffer)
Definition: device.c:91
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:599
#define static_always_inline
Definition: clib.h:99
vl_api_interface_index_t sw_if_index
Definition: gre.api:59
dpdk_portid_t port_id
Definition: dpdk.h:199
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:516
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:477
u32 buffer_index
Definition: dpdk.h:436
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:529
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:305
unsigned int u32
Definition: types.h:88
vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
Definition: device.c:679
A collection of simple counters.
Definition: counter.h:57
#define VLIB_FRAME_SIZE
Definition: node.h:378
dpdk_device_hqos_per_worker_thread_t * hqos_wt
Definition: dpdk.h:239
#define clib_atomic_test_and_set(a)
Definition: atomics.h:42
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:257
vnet_crypto_main_t * cm
Definition: quic_crypto.c:41
struct vnet_sub_interface_t::@286::@287::@289 flags
vnet_hw_interface_t * hw_interfaces
Definition: interface.h:840
void dpdk_device_start(dpdk_device_t *xd)
Definition: common.c:161
vnet_sub_interface_t sub
Definition: interface.h:740
#define clib_atomic_release(a)
Definition: atomics.h:43
dpdk_per_thread_data_t * per_thread_data
Definition: dpdk.h:401
unsigned short u16
Definition: types.h:57
#define rte_mbuf_from_vlib_buffer(x)
Definition: buffer.h:19
u8 data[256]
Definition: dpdk.h:442
#define PREDICT_FALSE(x)
Definition: clib.h:111
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:862
u16 tx_q_used
Definition: dpdk.h:222
struct vnet_sub_interface_t::@286 eth
vlib_main_t * vm
Definition: in2out_ed.c:1810
u32 hw_if_index
Definition: dpdk.h:201
dpdk_tx_func_error_t
Definition: device.c:31
u8 is_ip4
Definition: lisp_gpe.api:232
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
u32 flags
Definition: vhost_user.h:141
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
dpdk_device_t * devices
Definition: dpdk.h:399
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:77
volatile u32 ** lockp
Definition: dpdk.h:193
#define clib_warning(format, args...)
Definition: error.h:59
dpdk_pmd_t pmd
Definition: dpdk.h:207
static_always_inline void dpdk_prefetch_buffer(vlib_main_t *vm, struct rte_mbuf *mb)
Definition: device.c:237
format_function_t format_dpdk_device
Definition: dpdk.h:492
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
Definition: device.c:115
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1810
void dpdk_device_stop(dpdk_device_t *xd)
Definition: common.c:195
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **mb, u32 n_left)
Definition: device.c:157
struct rte_mbuf mb
Definition: dpdk.h:439
signed int i32
Definition: types.h:77
#define ASSERT(truth)
format_function_t format_dpdk_device_name
Definition: dpdk.h:491
manual_print typedef address
Definition: ip_types.api:84
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:178
static clib_error_t * dpdk_add_del_mac_address(vnet_hw_interface_t *hi, const u8 *address, u8 is_add)
Definition: device.c:46
#define clib_error_report(e)
Definition: error.h:113
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1544
dpdk_portid_t device_index
Definition: dpdk.h:196
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
vl_api_ip4_address_t hi
Definition: arp.api:37
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
Definition: hqos.c:640
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
vlib_buffer_t buffer
Definition: dpdk.h:441
static void dpdk_clear_hw_interface_counters(u32 instance)
Definition: device.c:484
struct rte_mbuf * mbufs[DPDK_RX_BURST_SZ]
Definition: dpdk.h:387
static uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:636
VNET_DEVICE_CLASS_TX_FN() dpdk_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
Definition: device.c:289
#define foreach_dpdk_tx_func_error
Definition: device.c:27
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
u8 admin_up_down_in_progress
Definition: dpdk.h:410
u32 instance
Definition: gre.api:57
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:492
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:492
struct rte_mempool ** dpdk_no_cache_mempool_by_buffer_pool_index
Definition: buffer.c:34
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
#define UP_DOWN_FLAG_EVENT
Definition: device.c:633
u16 device_index
Definition: dpdk.h:437
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:494
#define vnet_buffer(b)
Definition: buffer.h:408
vnet_flow_dev_ops_function_t dpdk_flow_ops_fn
Definition: dpdk.h:502
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, vnet_sw_interface_flags_t flags)
Definition: interface.c:501
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:549
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:302
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
Definition: device.c:245
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u8 queue_index
Definition: dpdk.h:438
vnet_main_t * vnet_main
Definition: dpdk.h:422
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
VNET_DEVICE_CLASS(avf_device_class,)
vlib_main_t * vlib_main
Definition: dpdk.h:421