FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/format.h>
18 #include <assert.h>
19 
20 #include <vnet/ethernet/ethernet.h>
21 #include <dpdk/buffer.h>
22 #include <dpdk/device/dpdk.h>
23 #include <dpdk/device/dpdk_priv.h>
24 #include <vppinfra/error.h>
25 
26 #define foreach_dpdk_tx_func_error \
27  _(BAD_RETVAL, "DPDK tx function returned an error") \
28  _(PKT_DROP, "Tx packet drops (dpdk tx failure)")
29 
30 typedef enum
31 {
32 #define _(f,s) DPDK_TX_FUNC_ERROR_##f,
34 #undef _
37 
38 static char *dpdk_tx_func_error_strings[] = {
39 #define _(n,s) s,
41 #undef _
42 };
43 
44 static clib_error_t *
46  const u8 * address, u8 is_add)
47 {
48  int error;
49  dpdk_main_t *dm = &dpdk_main;
51 
52  if (is_add)
53  error = rte_eth_dev_mac_addr_add (xd->port_id,
54  (struct rte_ether_addr *) address, 0);
55  else
56  error = rte_eth_dev_mac_addr_remove (xd->port_id,
57  (struct rte_ether_addr *) address);
58 
59  if (error)
60  {
61  return clib_error_return (0, "mac address add/del failed: %d", error);
62  }
63 
64  return NULL;
65 }
66 
67 static clib_error_t *
69  const u8 * old_address, const u8 * address)
70 {
71  int error;
72  dpdk_main_t *dm = &dpdk_main;
74 
75  error = rte_eth_dev_default_mac_addr_set (xd->port_id, (void *) address);
76 
77  if (error)
78  {
79  return clib_error_return (0, "mac address set failed: %d", error);
80  }
81  else
82  {
84  vec_add (xd->default_mac_address, address, sizeof (mac_address_t));
85  return NULL;
86  }
87 }
88 
89 static void
91  dpdk_device_t * xd, u16 queue_id,
92  vlib_buffer_t * buffer)
93 {
94  vlib_main_t *vm = vlib_get_main ();
95  dpdk_tx_trace_t *t0;
96  struct rte_mbuf *mb;
97 
98  mb = rte_mbuf_from_vlib_buffer (buffer);
99 
100  t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
101  t0->queue_index = queue_id;
102  t0->device_index = xd->device_index;
103  t0->buffer_index = vlib_get_buffer_index (vm, buffer);
104  clib_memcpy_fast (&t0->mb, mb, sizeof (t0->mb));
105  clib_memcpy_fast (&t0->buffer, buffer,
106  sizeof (buffer[0]) - sizeof (buffer->pre_data));
107  clib_memcpy_fast (t0->buffer.pre_data, buffer->data + buffer->current_data,
108  sizeof (t0->buffer.pre_data));
109  clib_memcpy_fast (&t0->data, mb->buf_addr + mb->data_off,
110  sizeof (t0->data));
111 }
112 
115  int maybe_multiseg)
116 {
117  struct rte_mbuf *mb, *first_mb, *last_mb;
118  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
119 
120  /* buffer is coming from non-dpdk source so we need to init
121  rte_mbuf header */
122  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
123  rte_pktmbuf_reset (mb);
124 
125  first_mb->nb_segs = 1;
126  mb->data_len = b->current_length;
127  mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
128  b->current_length;
129  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
130 
131  while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
132  {
133  b = vlib_get_buffer (vm, b->next_buffer);
134  mb = rte_mbuf_from_vlib_buffer (b);
135  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
136  rte_pktmbuf_reset (mb);
137  last_mb->next = mb;
138  last_mb = mb;
139  mb->data_len = b->current_length;
140  mb->pkt_len = b->current_length;
141  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
142  first_mb->nb_segs++;
143  if (PREDICT_FALSE (b->ref_count > 1))
144  mb->pool =
146  }
147 }
148 
149 /*
150  * This function calls the dpdk's tx_burst function to transmit the packets.
151  * It manages a lock per-device if the device does not
152  * support multiple queues. It returns the number of packets untransmitted
153  * If all packets are transmitted (the normal case), the function returns 0.
154  */
157  dpdk_device_t * xd,
158  struct rte_mbuf **mb, u32 n_left)
159 {
160  dpdk_main_t *dm = &dpdk_main;
161  dpdk_tx_queue_t *txq;
162  u32 n_retry;
163  int n_sent = 0;
164  int queue_id;
165 
166  n_retry = 16;
167  queue_id = vm->thread_index % xd->tx_q_used;
168  txq = vec_elt_at_index (xd->tx_queues, queue_id);
169 
170  do
171  {
173 
174  if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
175  {
176  /* no wrap, transmit in one burst */
177  n_sent = rte_eth_tx_burst (xd->port_id, queue_id, mb, n_left);
178  n_retry--;
179  }
180  else
181  {
182  ASSERT (0);
183  n_sent = 0;
184  }
185 
187 
188  if (PREDICT_FALSE (n_sent < 0))
189  {
190  // emit non-fatal message, bump counter
191  vnet_main_t *vnm = dm->vnet_main;
193  u32 node_index;
194 
195  node_index = vec_elt_at_index (im->hw_interfaces,
196  xd->hw_if_index)->tx_node_index;
197 
198  vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1);
199  return n_left; // untransmitted packets
200  }
201  n_left -= n_sent;
202  mb += n_sent;
203  }
204  while (n_sent && n_left && (n_retry > 0));
205 
206  return n_left;
207 }
208 
209 static_always_inline __clib_unused void
210 dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb)
211 {
213  CLIB_PREFETCH (mb, sizeof (struct rte_mbuf), STORE);
215 }
216 
219  struct rte_mbuf *mb)
220 {
221  u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
222  u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
223  u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
224  int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
225  u32 tso = b->flags & VNET_BUFFER_F_GSO;
226  u64 ol_flags;
227 
228  /* Is there any work for us? */
229  if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
230  return;
231 
232  mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
233  mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
234  vnet_buffer (b)->l3_hdr_offset;
235  mb->outer_l3_len = 0;
236  mb->outer_l2_len = 0;
237  ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
238  ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
239  ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
240  ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
241  ol_flags |= tso ? (tcp_cksum ? PKT_TX_TCP_SEG : PKT_TX_UDP_SEG) : 0;
242 
243  if (tso)
244  {
245  mb->l4_len = vnet_buffer2 (b)->gso_l4_hdr_sz;
246  mb->tso_segsz = vnet_buffer2 (b)->gso_size;
247  }
248 
249  mb->ol_flags |= ol_flags;
250 
251  /* we are trying to help compiler here by using local ol_flags with known
252  state of all flags */
253  if (xd->flags & DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM)
254  rte_net_intel_cksum_flags_prepare (mb, ol_flags);
255 }
256 
257 /*
258  * Transmits the packets on the frame to the interface associated with the
259  * node. It first copies packets on the frame to a per-thread arrays
260  * containing the rte_mbuf pointers.
261  */
264  vlib_frame_t * f)
265 {
266  dpdk_main_t *dm = &dpdk_main;
267  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
269  u32 n_packets = f->n_vectors;
270  u32 n_left;
271  u32 thread_index = vm->thread_index;
272  int queue_id = thread_index;
273  u32 tx_pkts = 0, all_or_flags = 0;
275  thread_index);
276  struct rte_mbuf **mb;
277  vlib_buffer_t *b[4];
278 
279  ASSERT (n_packets <= VLIB_FRAME_SIZE);
280 
281  /* calculate rte_mbuf pointers out of buffer indices */
283  (void **) ptd->mbufs, n_packets,
284  -(i32) sizeof (struct rte_mbuf));
285 
286  n_left = n_packets;
287  mb = ptd->mbufs;
288 
289 #if (CLIB_N_PREFETCHES >= 8)
290  while (n_left >= 8)
291  {
292  u32 or_flags;
293 
294  dpdk_prefetch_buffer (vm, mb[4]);
295  dpdk_prefetch_buffer (vm, mb[5]);
296  dpdk_prefetch_buffer (vm, mb[6]);
297  dpdk_prefetch_buffer (vm, mb[7]);
298 
299  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
300  b[1] = vlib_buffer_from_rte_mbuf (mb[1]);
301  b[2] = vlib_buffer_from_rte_mbuf (mb[2]);
302  b[3] = vlib_buffer_from_rte_mbuf (mb[3]);
303 
304  or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
305  all_or_flags |= or_flags;
306 
311 
312  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
313  {
314  dpdk_validate_rte_mbuf (vm, b[0], 1);
315  dpdk_validate_rte_mbuf (vm, b[1], 1);
316  dpdk_validate_rte_mbuf (vm, b[2], 1);
317  dpdk_validate_rte_mbuf (vm, b[3], 1);
318  }
319  else
320  {
321  dpdk_validate_rte_mbuf (vm, b[0], 0);
322  dpdk_validate_rte_mbuf (vm, b[1], 0);
323  dpdk_validate_rte_mbuf (vm, b[2], 0);
324  dpdk_validate_rte_mbuf (vm, b[3], 0);
325  }
326 
327  if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
328  (or_flags &
329  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
330  | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
331  | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
332  {
333  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
334  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
335  dpdk_buffer_tx_offload (xd, b[2], mb[2]);
336  dpdk_buffer_tx_offload (xd, b[3], mb[3]);
337  }
338 
339  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
340  {
341  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
342  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
343  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
344  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[1]);
345  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
346  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[2]);
347  if (b[3]->flags & VLIB_BUFFER_IS_TRACED)
348  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[3]);
349  }
350 
351  mb += 4;
352  n_left -= 4;
353  }
354 #elif (CLIB_N_PREFETCHES >= 4)
355  while (n_left >= 4)
356  {
357  vlib_buffer_t *b2, *b3;
358  u32 or_flags;
359 
360  CLIB_PREFETCH (mb[2], CLIB_CACHE_LINE_BYTES, STORE);
361  CLIB_PREFETCH (mb[3], CLIB_CACHE_LINE_BYTES, STORE);
362  b2 = vlib_buffer_from_rte_mbuf (mb[2]);
364  b3 = vlib_buffer_from_rte_mbuf (mb[3]);
366 
367  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
368  b[1] = vlib_buffer_from_rte_mbuf (mb[1]);
369 
370  or_flags = b[0]->flags | b[1]->flags;
371  all_or_flags |= or_flags;
372 
375 
376  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
377  {
378  dpdk_validate_rte_mbuf (vm, b[0], 1);
379  dpdk_validate_rte_mbuf (vm, b[1], 1);
380  }
381  else
382  {
383  dpdk_validate_rte_mbuf (vm, b[0], 0);
384  dpdk_validate_rte_mbuf (vm, b[1], 0);
385  }
386 
387  if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
388  (or_flags &
389  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
390  | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
391  | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
392  {
393  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
394  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
395  }
396 
397  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
398  {
399  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
400  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
401  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
402  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[1]);
403  }
404 
405  mb += 2;
406  n_left -= 2;
407  }
408 #endif
409 
410  while (n_left > 0)
411  {
412  b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
413  all_or_flags |= b[0]->flags;
415 
416  dpdk_validate_rte_mbuf (vm, b[0], 1);
417  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
418 
419  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
420  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
421  dpdk_tx_trace_buffer (dm, node, xd, queue_id, b[0]);
422 
423  mb++;
424  n_left--;
425  }
426 
427  /* transmit as many packets as possible */
428  tx_pkts = n_packets = mb - ptd->mbufs;
429  n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets);
430 
431  {
432  /* If there is no callback then drop any non-transmitted packets */
433  if (PREDICT_FALSE (n_left))
434  {
435  tx_pkts -= n_left;
437  vnet_main_t *vnm = vnet_get_main ();
438 
441 
442  vlib_increment_simple_counter (cm, thread_index, xd->sw_if_index,
443  n_left);
444 
445  vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
446  n_left);
447 
448  while (n_left--)
449  rte_pktmbuf_free (ptd->mbufs[n_packets - n_left - 1]);
450  }
451  }
452 
453  return tx_pkts;
454 }
455 
456 static void
458 {
459  dpdk_main_t *dm = &dpdk_main;
460  dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
461 
462  rte_eth_stats_reset (xd->port_id);
463  rte_eth_xstats_reset (xd->port_id);
464 }
465 
466 static clib_error_t *
468 {
469  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
470  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
471  dpdk_main_t *dm = &dpdk_main;
473 
474  if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)
475  return clib_error_return (0, "Interface not initialized");
476 
477  if (is_up)
478  {
479  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
480  dpdk_device_start (xd);
481  xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
482  f64 now = vlib_time_now (dm->vlib_main);
483  dpdk_update_counters (xd, now);
484  dpdk_update_link_state (xd, now);
485  }
486  else
487  {
489  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0)
490  dpdk_device_stop (xd);
491  xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
492  }
493 
494  return /* no error */ 0;
495 }
496 
497 /*
498  * Dynamically redirect all pkts from a specific interface
499  * to the specified node
500  */
501 static void
503  u32 node_index)
504 {
505  dpdk_main_t *xm = &dpdk_main;
506  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
508 
509  /* Shut off redirection */
510  if (node_index == ~0)
511  {
512  xd->per_interface_next_index = node_index;
513  return;
514  }
515 
517  vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index);
518 }
519 
520 
521 static clib_error_t *
523  u32 hw_if_index,
524  struct vnet_sw_interface_t *st, int is_add)
525 {
526  dpdk_main_t *xm = &dpdk_main;
527  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
530  int r, vlan_offload;
531  u32 prev_subifs = xd->num_subifs;
532  clib_error_t *err = 0;
533 
534  if (is_add)
535  xd->num_subifs++;
536  else if (xd->num_subifs)
537  xd->num_subifs--;
538 
539  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
540  goto done;
541 
542  /* currently we program VLANS only for IXGBE VF and I40E VF */
543  if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
544  goto done;
545 
546  if (t->sub.eth.flags.no_tags == 1)
547  goto done;
548 
549  if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
550  {
551  xd->num_subifs = prev_subifs;
552  err = clib_error_return (0, "unsupported VLAN setup");
553  goto done;
554  }
555 
556  vlan_offload = rte_eth_dev_get_vlan_offload (xd->port_id);
557  vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
558 
559  if ((r = rte_eth_dev_set_vlan_offload (xd->port_id, vlan_offload)))
560  {
561  xd->num_subifs = prev_subifs;
562  err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
563  xd->port_id, r);
564  goto done;
565  }
566 
567 
568  if ((r =
569  rte_eth_dev_vlan_filter (xd->port_id,
570  t->sub.eth.outer_vlan_id, is_add)))
571  {
572  xd->num_subifs = prev_subifs;
573  err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
574  xd->port_id, r);
575  goto done;
576  }
577 
578 done:
579  if (xd->num_subifs)
580  xd->flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
581  else
582  xd->flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
583 
584  return err;
585 }
586 
587 static clib_error_t *
589  struct vnet_hw_interface_t *hi,
590  clib_bitmap_t * bitmap)
591 {
592  dpdk_main_t *xm = &dpdk_main;
593  u32 hw_if_index = hi->hw_if_index;
594  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
596  clib_error_t *err = 0;
597  struct rte_eth_rss_reta_entry64 *reta_conf = NULL;
598  struct rte_eth_dev_info dev_info;
599  u16 *reta = NULL;
600  u16 *valid_queue = NULL;
601  u16 valid_queue_count = 0;
602  uint32_t i, j;
603  uint32_t ret;
604 
605  rte_eth_dev_info_get (xd->port_id, &dev_info);
606 
607  /* parameter check */
608  if (clib_bitmap_count_set_bits (bitmap) == 0)
609  {
610  err = clib_error_return (0, "must assign at least one valid rss queue");
611  goto done;
612  }
613 
614  if (clib_bitmap_count_set_bits (bitmap) > dev_info.nb_rx_queues)
615  {
616  err = clib_error_return (0, "too many rss queues");
617  goto done;
618  }
619 
620  /* new RETA */
621  reta = clib_mem_alloc (dev_info.reta_size * sizeof (*reta));
622  if (reta == NULL)
623  {
624  err = clib_error_return (0, "clib_mem_alloc failed");
625  goto done;
626  }
627 
628  clib_memset (reta, 0, dev_info.reta_size * sizeof (*reta));
629 
630  valid_queue_count = 0;
631  /* *INDENT-OFF* */
632  clib_bitmap_foreach (i, bitmap, ({
633  if (i >= dev_info.nb_rx_queues)
634  {
635  err = clib_error_return (0, "illegal queue number");
636  goto done;
637  }
638  reta[valid_queue_count++] = i;
639  }));
640  /* *INDENT-ON* */
641 
642  /* check valid_queue_count not zero, make coverity happy */
643  if (valid_queue_count == 0)
644  {
645  err = clib_error_return (0, "must assign at least one valid rss queue");
646  goto done;
647  }
648 
649  valid_queue = reta;
650  for (i = valid_queue_count, j = 0; i < dev_info.reta_size; i++, j++)
651  {
652  j = j % valid_queue_count;
653  reta[i] = valid_queue[j];
654  }
655 
656  /* update reta table */
657  reta_conf =
658  (struct rte_eth_rss_reta_entry64 *) clib_mem_alloc (dev_info.reta_size /
659  RTE_RETA_GROUP_SIZE *
660  sizeof (*reta_conf));
661  if (reta_conf == NULL)
662  {
663  err = clib_error_return (0, "clib_mem_alloc failed");
664  goto done;
665  }
666 
667  clib_memset (reta_conf, 0,
668  dev_info.reta_size / RTE_RETA_GROUP_SIZE *
669  sizeof (*reta_conf));
670 
671  for (i = 0; i < dev_info.reta_size; i++)
672  {
673  uint32_t reta_id = i / RTE_RETA_GROUP_SIZE;
674  uint32_t reta_pos = i % RTE_RETA_GROUP_SIZE;
675 
676  reta_conf[reta_id].mask = UINT64_MAX;
677  reta_conf[reta_id].reta[reta_pos] = reta[i];
678  }
679 
680  ret =
681  rte_eth_dev_rss_reta_update (xd->port_id, reta_conf, dev_info.reta_size);
682  if (ret)
683  {
684  err = clib_error_return (0, "rte_eth_dev_rss_reta_update err %d", ret);
685  goto done;
686  }
687 
688 done:
689  if (reta)
690  clib_mem_free (reta);
691  if (reta_conf)
692  clib_mem_free (reta_conf);
693 
694  return err;
695 }
696 
697 /* *INDENT-OFF* */
699  .name = "dpdk",
700  .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR,
701  .tx_function_error_strings = dpdk_tx_func_error_strings,
702  .format_device_name = format_dpdk_device_name,
703  .format_device = format_dpdk_device,
704  .format_tx_trace = format_dpdk_tx_trace,
705  .clear_counters = dpdk_clear_hw_interface_counters,
706  .admin_up_down_function = dpdk_interface_admin_up_down,
707  .subif_add_del_function = dpdk_subif_add_del_function,
708  .rx_redirect_to_node = dpdk_set_interface_next_node,
709  .mac_addr_change_function = dpdk_set_mac_address,
710  .mac_addr_add_del_function = dpdk_add_del_mac_address,
711  .format_flow = format_dpdk_flow,
712  .flow_ops_function = dpdk_flow_ops_fn,
713  .set_rss_queues_function = dpdk_interface_set_rss_queues,
714 };
715 /* *INDENT-ON* */
716 
717 #define UP_DOWN_FLAG_EVENT 1
718 
719 static uword
722 {
723  clib_error_t *error = 0;
724  uword event_type;
725  uword *event_data = 0;
727  u32 flags;
728 
729  while (1)
730  {
732 
733  event_type = vlib_process_get_events (vm, &event_data);
734 
736 
737  switch (event_type)
738  {
739  case UP_DOWN_FLAG_EVENT:
740  {
741  if (vec_len (event_data) == 2)
742  {
743  sw_if_index = event_data[0];
744  flags = event_data[1];
745  error =
747  flags);
748  clib_error_report (error);
749  }
750  }
751  break;
752  }
753 
754  vec_reset_length (event_data);
755 
757 
758  }
759  return 0; /* or not */
760 }
761 
762 /* *INDENT-OFF* */
764  .function = admin_up_down_process,
765  .type = VLIB_NODE_TYPE_PROCESS,
766  .name = "admin-up-down-process",
767  .process_log2_n_stack_bytes = 17, // 256KB
768 };
769 /* *INDENT-ON* */
770 
771 /*
772  * fd.io coding-style-patch-verification: ON
773  *
774  * Local Variables:
775  * eval: (c-set-style "gnu")
776  * End:
777  */
u8 * default_mac_address
Definition: dpdk.h:228
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
format_function_t format_dpdk_tx_trace
Definition: dpdk.h:439
format_function_t format_dpdk_flow
Definition: dpdk.h:443
#define vlib_buffer_from_rte_mbuf(x)
Definition: buffer.h:20
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:103
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:656
dpdk_main_t dpdk_main
Definition: init.c:46
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
#define vnet_buffer2(b)
Definition: buffer.h:482
vnet_interface_main_t interface_main
Definition: vnet.h:59
#define PREDICT_TRUE(x)
Definition: clib.h:121
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:333
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:127
u16 flags
Definition: dpdk.h:186
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:249
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u32 per_interface_next_index
Definition: dpdk.h:182
vlib_main_t * vm
Definition: in2out_ed.c:1582
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:214
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:78
u16 num_subifs
Definition: dpdk.h:201
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:402
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1173
unsigned char u8
Definition: types.h:56
struct vnet_sub_interface_t::@356::@357::@359 flags
static clib_error_t * dpdk_add_del_mac_address(vnet_hw_interface_t *hi, const u8 *address, u8 is_add)
Definition: device.c:45
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:668
#define static_always_inline
Definition: clib.h:108
VNET_DEVICE_CLASS(af_xdp_device_class)
dpdk_portid_t port_id
Definition: dpdk.h:189
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:579
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:478
u32 buffer_index
Definition: dpdk.h:355
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:467
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:522
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
Definition: device.c:763
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:316
unsigned int u32
Definition: types.h:88
A collection of simple counters.
Definition: counter.h:57
#define VLIB_FRAME_SIZE
Definition: node.h:377
static_always_inline void dpdk_buffer_tx_offload(dpdk_device_t *xd, vlib_buffer_t *b, struct rte_mbuf *mb)
Definition: device.c:218
u32 device_index
Definition: dpdk.h:176
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:293
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
Definition: device.c:114
vnet_crypto_main_t * cm
Definition: quic_crypto.c:53
vnet_hw_interface_t * hw_interfaces
Definition: interface.h:858
void dpdk_device_start(dpdk_device_t *xd)
Definition: common.c:166
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
vnet_sub_interface_t sub
Definition: interface.h:758
struct vnet_sub_interface_t::@356 eth
dpdk_per_thread_data_t * per_thread_data
Definition: dpdk.h:324
unsigned short u16
Definition: types.h:57
#define rte_mbuf_from_vlib_buffer(x)
Definition: buffer.h:19
u8 data[256]
Definition: dpdk.h:361
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:502
static void dpdk_clear_hw_interface_counters(u32 instance)
Definition: device.c:457
#define PREDICT_FALSE(x)
Definition: clib.h:120
dpdk_tx_queue_t * tx_queues
Definition: dpdk.h:173
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:880
u16 tx_q_used
Definition: dpdk.h:185
u32 hw_if_index
Definition: dpdk.h:178
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
dpdk_device_t * devices
Definition: dpdk.h:323
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:95
static uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:720
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **mb, u32 n_left)
Definition: device.c:156
u8 data[]
Packet data.
Definition: buffer.h:181
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, vlib_buffer_t *buffer)
Definition: device.c:90
dpdk_pmd_t pmd
Definition: dpdk.h:190
format_function_t format_dpdk_device
Definition: dpdk.h:437
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1582
void dpdk_device_stop(dpdk_device_t *xd)
Definition: common.c:200
struct rte_mbuf mb
Definition: dpdk.h:358
static char * dpdk_tx_func_error_strings[]
Definition: device.c:38
signed int i32
Definition: types.h:77
#define ASSERT(truth)
format_function_t format_dpdk_device_name
Definition: dpdk.h:436
manual_print typedef address
Definition: ip_types.api:85
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:178
static void clib_mem_free(void *p)
Definition: mem.h:215
#define clib_error_report(e)
Definition: error.h:113
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1532
static void * clib_mem_alloc(uword size)
Definition: mem.h:157
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
vl_api_ip4_address_t hi
Definition: arp.api:37
vlib_buffer_t buffer
Definition: dpdk.h:360
#define UP_DOWN_FLAG_EVENT
Definition: device.c:717
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
struct rte_mbuf * mbufs[DPDK_RX_BURST_SZ]
Definition: dpdk.h:311
dpdk_tx_func_error_t
Definition: device.c:30
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
u8 admin_up_down_in_progress
Definition: dpdk.h:333
u32 instance
Definition: gre.api:51
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:498
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:492
struct rte_mempool ** dpdk_no_cache_mempool_by_buffer_pool_index
Definition: buffer.c:34
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:297
static clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, const u8 *old_address, const u8 *address)
Definition: device.c:68
u16 device_index
Definition: dpdk.h:356
clib_spinlock_t lock
Definition: dpdk.h:165
VNET_DEVICE_CLASS_TX_FN() dpdk_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
Definition: device.c:262
#define vnet_buffer(b)
Definition: buffer.h:417
vnet_flow_dev_ops_function_t dpdk_flow_ops_fn
Definition: dpdk.h:447
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, vnet_sw_interface_flags_t flags)
Definition: interface.c:507
uword clib_bitmap_t
Definition: bitmap.h:50
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:577
static_always_inline __clib_unused void dpdk_prefetch_buffer(vlib_main_t *vm, struct rte_mbuf *mb)
Definition: device.c:210
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:301
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u8 queue_index
Definition: dpdk.h:357
vnet_main_t * vnet_main
Definition: dpdk.h:341
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:104
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
vl_api_interface_index_t sw_if_index
Definition: wireguard.api:33
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
static clib_error_t * dpdk_interface_set_rss_queues(struct vnet_main_t *vnm, struct vnet_hw_interface_t *hi, clib_bitmap_t *bitmap)
Definition: device.c:588
vlib_main_t * vlib_main
Definition: dpdk.h:340
#define foreach_dpdk_tx_func_error
Definition: device.c:26