FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
init.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/error.h>
18 #include <vppinfra/format.h>
19 #include <vppinfra/bitmap.h>
20 #include <vppinfra/linux/sysfs.h>
21 #include <vlib/unix/unix.h>
22 #include <vlib/log.h>
23 
24 #include <vnet/ethernet/ethernet.h>
25 #include <dpdk/device/dpdk.h>
26 #include <vlib/pci/pci.h>
27 
28 #include <rte_ring.h>
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <unistd.h>
33 #include <sys/stat.h>
34 #include <sys/mount.h>
35 #include <string.h>
36 #include <fcntl.h>
37 
38 #include <dpdk/device/dpdk_priv.h>
39 
40 #define ETHER_MAX_LEN 1518 /**< Maximum frame len, including CRC. */
41 
44 
45 #define LINK_STATE_ELOGS 0
46 
47 /* Port configuration, mildly modified Intel app values */
48 
49 static dpdk_port_type_t
50 port_type_from_speed_capa (struct rte_eth_dev_info *dev_info)
51 {
52 
53  if (dev_info->speed_capa & ETH_LINK_SPEED_100G)
55  else if (dev_info->speed_capa & ETH_LINK_SPEED_56G)
57  else if (dev_info->speed_capa & ETH_LINK_SPEED_50G)
59  else if (dev_info->speed_capa & ETH_LINK_SPEED_40G)
61  else if (dev_info->speed_capa & ETH_LINK_SPEED_25G)
63  else if (dev_info->speed_capa & ETH_LINK_SPEED_20G)
65  else if (dev_info->speed_capa & ETH_LINK_SPEED_10G)
67  else if (dev_info->speed_capa & ETH_LINK_SPEED_5G)
69  else if (dev_info->speed_capa & ETH_LINK_SPEED_2_5G)
71  else if (dev_info->speed_capa & ETH_LINK_SPEED_1G)
73 
75 }
76 
77 
78 static u32
80 {
81  dpdk_main_t *dm = &dpdk_main;
83  u32 old = 0;
84 
86  {
87  old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0;
88 
90  xd->flags |= DPDK_DEVICE_FLAG_PROMISC;
91  else
92  xd->flags &= ~DPDK_DEVICE_FLAG_PROMISC;
93 
94  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
95  {
96  if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
97  rte_eth_promiscuous_enable (xd->port_id);
98  else
99  rte_eth_promiscuous_disable (xd->port_id);
100  }
101  }
102  else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
103  {
104  xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes;
105  dpdk_device_setup (xd);
106  }
107  return old;
108 }
109 
110 static void
112 {
113  int q;
114  vec_validate (xd->lockp, xd->tx_q_used - 1);
115  for (q = 0; q < xd->tx_q_used; q++)
116  {
119  memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES);
120  }
121 }
122 
123 static struct rte_mempool_ops *
124 get_ops_by_name (char *ops_name)
125 {
126  u32 i;
127 
128  for (i = 0; i < rte_mempool_ops_table.num_ops; i++)
129  {
130  if (!strcmp (ops_name, rte_mempool_ops_table.ops[i].name))
131  return &rte_mempool_ops_table.ops[i];
132  }
133 
134  return 0;
135 }
136 
137 static int
138 dpdk_ring_alloc (struct rte_mempool *mp)
139 {
140  u32 rg_flags = 0, count;
141  i32 ret;
142  char rg_name[RTE_RING_NAMESIZE];
143  struct rte_ring *r;
144 
145  ret = snprintf (rg_name, sizeof (rg_name), RTE_MEMPOOL_MZ_FORMAT, mp->name);
146  if (ret < 0 || ret >= (i32) sizeof (rg_name))
147  return -ENAMETOOLONG;
148 
149  /* ring flags */
150  if (mp->flags & MEMPOOL_F_SP_PUT)
151  rg_flags |= RING_F_SP_ENQ;
152  if (mp->flags & MEMPOOL_F_SC_GET)
153  rg_flags |= RING_F_SC_DEQ;
154 
155  count = rte_align32pow2 (mp->size + 1);
156  /*
157  * Allocate the ring that will be used to store objects.
158  * Ring functions will return appropriate errors if we are
159  * running as a secondary process etc., so no checks made
160  * in this function for that condition.
161  */
162  /* XXX can we get memory from the right socket? */
163  r = clib_mem_alloc_aligned (rte_ring_get_memsize (count),
165 
166  /* XXX rte_ring_lookup will not work */
167 
168  ret = rte_ring_init (r, rg_name, count, rg_flags);
169  if (ret)
170  return ret;
171 
172  mp->pool_data = r;
173 
174  return 0;
175 }
176 
177 static int
179 {
180 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
181  if (xd->port_conf.rxmode.hw_strip_crc)
182 #else
183  if (xd->port_conf.rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP)
184 #endif
185  return 1;
186  return 0;
187 }
188 
189 static clib_error_t *
191 {
192  u32 nports;
193  u32 mtu, max_rx_frame;
194  u32 nb_desc = 0;
195  int i;
196  clib_error_t *error;
202  dpdk_device_t *xd;
203  vlib_pci_addr_t last_pci_addr;
204  u32 last_pci_addr_port = 0;
206  uword *p_hqos;
207 
208  u32 next_hqos_cpu = 0;
209  u8 af_packet_instance_num = 0;
210  u8 bond_ether_instance_num = 0;
211  last_pci_addr.as_u32 = ~0;
212 
213  dm->hqos_cpu_first_index = 0;
214  dm->hqos_cpu_count = 0;
215 
216  /* find out which cpus will be used for I/O TX */
217  p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads");
218  tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0;
219 
220  if (tr_hqos && tr_hqos->count > 0)
221  {
222  dm->hqos_cpu_first_index = tr_hqos->first_index;
223  dm->hqos_cpu_count = tr_hqos->count;
224  }
225 
228 
229  nports = rte_eth_dev_count_avail ();
230 
231  if (nports < 1)
232  {
233  dpdk_log_notice ("DPDK drivers found no ports...");
234  }
235 
236  if (CLIB_DEBUG > 0)
237  dpdk_log_notice ("DPDK drivers found %d ports...", nports);
238 
239  if (dm->conf->enable_tcp_udp_checksum)
240  dm->buffer_flags_template &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT
241  | VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
242 
243  /* vlib_buffer_t template */
246  for (i = 0; i < tm->n_vlib_mains; i++)
247  {
250  fl = vlib_buffer_get_free_list (vm,
254  vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
255  }
256 
257  /* *INDENT-OFF* */
258  RTE_ETH_FOREACH_DEV(i)
259  {
260  u8 addr[6];
261  u8 vlan_strip = 0;
262  struct rte_eth_dev_info dev_info;
263  struct rte_pci_device *pci_dev;
264  struct rte_eth_link l;
265  dpdk_device_config_t *devconf = 0;
266  vlib_pci_addr_t pci_addr;
267  uword *p = 0;
268 
269  if (!rte_eth_dev_is_valid_port(i))
270  continue;
271 
272  rte_eth_link_get_nowait (i, &l);
273  rte_eth_dev_info_get (i, &dev_info);
274 
275  if (dev_info.device == 0)
276  {
277  clib_warning ("DPDK bug: missing device info. Skipping %s device",
278  dev_info.driver_name);
279  continue;
280  }
281  pci_dev = RTE_DEV_TO_PCI (dev_info.device);
282 
283  if (pci_dev) /* bonded interface has no pci info */
284  {
285  pci_addr.domain = pci_dev->addr.domain;
286  pci_addr.bus = pci_dev->addr.bus;
287  pci_addr.slot = pci_dev->addr.devid;
288  pci_addr.function = pci_dev->addr.function;
290  pci_addr.as_u32);
291  }
292 
293  if (p)
294  devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
295  else
296  devconf = &dm->conf->default_devconf;
297 
298  /* Create vnet interface */
302  xd->cpu_socket = (i8) rte_eth_dev_socket_id (i);
303 
304  /* Handle interface naming for devices with multiple ports sharing same PCI ID */
305  if (pci_dev)
306  {
307  struct rte_eth_dev_info di = { 0 };
308  struct rte_pci_device *next_pci_dev;
309  rte_eth_dev_info_get (i + 1, &di);
310  next_pci_dev = di.device ? RTE_DEV_TO_PCI (di.device) : 0;
311  if (pci_dev && next_pci_dev &&
312  pci_addr.as_u32 != last_pci_addr.as_u32 &&
313  memcmp (&pci_dev->addr, &next_pci_dev->addr,
314  sizeof (struct rte_pci_addr)) == 0)
315  {
316  xd->interface_name_suffix = format (0, "0");
317  last_pci_addr.as_u32 = pci_addr.as_u32;
318  last_pci_addr_port = i;
319  }
320  else if (pci_addr.as_u32 == last_pci_addr.as_u32)
321  {
323  format (0, "%u", i - last_pci_addr_port);
324  }
325  else
326  {
327  last_pci_addr.as_u32 = ~0;
328  }
329  }
330  else
331  last_pci_addr.as_u32 = ~0;
332 
333  clib_memcpy (&xd->tx_conf, &dev_info.default_txconf,
334  sizeof (struct rte_eth_txconf));
335 
336  if (dm->conf->no_multi_seg)
337  {
338 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
339  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
340  xd->port_conf.rxmode.jumbo_frame = 0;
341  xd->port_conf.rxmode.enable_scatter = 0;
342 #else
343  xd->port_conf.txmode.offloads &= ~DEV_TX_OFFLOAD_MULTI_SEGS;
344  xd->port_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
345  xd->port_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_SCATTER;
346 #endif
347  }
348  else
349  {
350 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
351  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS;
352  xd->port_conf.rxmode.jumbo_frame = 1;
353  xd->port_conf.rxmode.enable_scatter = 1;
354 #else
355  xd->port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS;
356  xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
357  xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER;
358 #endif
359  xd->flags |= DPDK_DEVICE_FLAG_MAYBE_MULTISEG;
360  }
361 
362  xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains);
363 
364  if (devconf->num_tx_queues > 0
365  && devconf->num_tx_queues < xd->tx_q_used)
366  xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues);
367 
368  if (devconf->num_rx_queues > 1
369  && dev_info.max_rx_queues >= devconf->num_rx_queues)
370  {
371  xd->rx_q_used = devconf->num_rx_queues;
372  xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
373  if (devconf->rss_fn == 0)
374  xd->port_conf.rx_adv_conf.rss_conf.rss_hf =
375  ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
376  else
377  {
378  u64 unsupported_bits;
379  xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn;
380  unsupported_bits = xd->port_conf.rx_adv_conf.rss_conf.rss_hf;
381  unsupported_bits &= ~dev_info.flow_type_rss_offloads;
382  if (unsupported_bits)
383  dpdk_log_warn ("Unsupported RSS hash functions: %U",
384  format_dpdk_rss_hf_name, unsupported_bits);
385  }
386  xd->port_conf.rx_adv_conf.rss_conf.rss_hf &=
387  dev_info.flow_type_rss_offloads;
388  }
389  else
390  xd->rx_q_used = 1;
391 
392  xd->flags |= DPDK_DEVICE_FLAG_PMD;
393 
394  /* workaround for drivers not setting driver_name */
395  if ((!dev_info.driver_name) && (pci_dev))
396  dev_info.driver_name = pci_dev->driver->driver.name;
397 
398  ASSERT (dev_info.driver_name);
399 
400  if (!xd->pmd)
401  {
402 
403 
404 #define _(s,f) else if (dev_info.driver_name && \
405  !strcmp(dev_info.driver_name, s)) \
406  xd->pmd = VNET_DPDK_PMD_##f;
407  if (0)
408  ;
410 #undef _
411  else
413 
417 
418  switch (xd->pmd)
419  {
420  /* Drivers with valid speed_capa set */
421  case VNET_DPDK_PMD_E1000EM:
422  case VNET_DPDK_PMD_IGB:
423  case VNET_DPDK_PMD_IXGBE:
424  case VNET_DPDK_PMD_I40E:
425  xd->port_type = port_type_from_speed_capa (&dev_info);
426  xd->supported_flow_actions = VNET_FLOW_ACTION_MARK |
427  VNET_FLOW_ACTION_REDIRECT_TO_NODE |
428  VNET_FLOW_ACTION_BUFFER_ADVANCE |
429  VNET_FLOW_ACTION_COUNT | VNET_FLOW_ACTION_DROP;
430 
431  if (dm->conf->no_tx_checksum_offload == 0)
432  {
433 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
434  xd->tx_conf.txq_flags &= ~(ETH_TXQ_FLAGS_NOXSUMUDP |
435  ETH_TXQ_FLAGS_NOXSUMTCP);
436 #else
437  xd->port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM;
438  xd->port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM;
439 #endif
440  xd->flags |=
441  DPDK_DEVICE_FLAG_TX_OFFLOAD |
442  DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM;
443  }
444 
445 
446  break;
447  case VNET_DPDK_PMD_CXGBE:
448  case VNET_DPDK_PMD_MLX4:
449  case VNET_DPDK_PMD_MLX5:
450  case VNET_DPDK_PMD_QEDE:
451  xd->port_type = port_type_from_speed_capa (&dev_info);
452  break;
453 
454  /* SR-IOV VFs */
455  case VNET_DPDK_PMD_IGBVF:
456  case VNET_DPDK_PMD_IXGBEVF:
457  case VNET_DPDK_PMD_I40EVF:
459 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
460  xd->port_conf.rxmode.hw_strip_crc = 1;
461 #else
462  xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
463 #endif
464  break;
465 
466  case VNET_DPDK_PMD_THUNDERX:
468 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
469  xd->port_conf.rxmode.hw_strip_crc = 1;
470 #else
471  xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
472 #endif
473  break;
474 
475  case VNET_DPDK_PMD_ENA:
477 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
478  xd->port_conf.rxmode.enable_scatter = 0;
479 #else
480  xd->port_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_SCATTER;
481 #endif
482  break;
483 
484  case VNET_DPDK_PMD_DPAA2:
486  break;
487 
488  /* Cisco VIC */
489  case VNET_DPDK_PMD_ENIC:
490  if (l.link_speed == 40000)
492  else
494  break;
495 
496  /* Intel Red Rock Canyon */
497  case VNET_DPDK_PMD_FM10K:
499 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
500  xd->port_conf.rxmode.hw_strip_crc = 1;
501 #else
502  xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
503 #endif
504  break;
505 
506  /* virtio */
507  case VNET_DPDK_PMD_VIRTIO:
511  break;
512 
513  /* vmxnet3 */
514  case VNET_DPDK_PMD_VMXNET3:
516 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
517  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
518 #else
519  xd->port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS;
520 #endif
521  break;
522 
523  case VNET_DPDK_PMD_AF_PACKET:
525  xd->af_packet_instance_num = af_packet_instance_num++;
526  break;
527 
528  case VNET_DPDK_PMD_BOND:
530  xd->bond_instance_num = bond_ether_instance_num++;
531  break;
532 
533  case VNET_DPDK_PMD_VIRTIO_USER:
535  break;
536 
537  case VNET_DPDK_PMD_VHOST_ETHER:
539  break;
540 
541  case VNET_DPDK_PMD_LIOVF_ETHER:
543  break;
544 
545  case VNET_DPDK_PMD_FAILSAFE:
547  xd->port_conf.intr_conf.lsc = 1;
548  break;
549 
550  default:
552  }
553 
554  if (devconf->num_rx_desc)
555  xd->nb_rx_desc = devconf->num_rx_desc;
556 
557  if (devconf->num_tx_desc)
558  xd->nb_tx_desc = devconf->num_tx_desc;
559  }
560 
561  if (xd->pmd == VNET_DPDK_PMD_AF_PACKET)
562  {
563  f64 now = vlib_time_now (vm);
564  u32 rnd;
565  rnd = (u32) (now * 1e6);
566  rnd = random_u32 (&rnd);
567  clib_memcpy (addr + 2, &rnd, sizeof (rnd));
568  addr[0] = 2;
569  addr[1] = 0xfe;
570  }
571  else
572  rte_eth_macaddr_get (i, (struct ether_addr *) addr);
573 
574  if (xd->tx_q_used < tm->n_vlib_mains)
576 
577  xd->port_id = i;
578  xd->device_index = xd - dm->devices;
579  xd->per_interface_next_index = ~0;
580 
581  /* assign interface to input thread */
582  int q;
583 
584  if (devconf->hqos_enabled)
585  {
586  xd->flags |= DPDK_DEVICE_FLAG_HQOS;
587 
588  int cpu;
589  if (devconf->hqos.hqos_thread_valid)
590  {
591  if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count)
592  return clib_error_return (0, "invalid HQoS thread index");
593 
594  cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread;
595  }
596  else
597  {
598  if (dm->hqos_cpu_count == 0)
599  return clib_error_return (0, "no HQoS threads available");
600 
601  cpu = dm->hqos_cpu_first_index + next_hqos_cpu;
602 
603  next_hqos_cpu++;
604  if (next_hqos_cpu == dm->hqos_cpu_count)
605  next_hqos_cpu = 0;
606 
607  devconf->hqos.hqos_thread_valid = 1;
608  devconf->hqos.hqos_thread = cpu;
609  }
610 
612  vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
613  dq->device = xd->device_index;
614  dq->queue_id = 0;
615  }
616 
617  /* count the number of descriptors used for this device */
618  nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
619 
621  (dm->vnet_main, dpdk_device_class.index, xd->device_index,
622  /* ethernet address */ addr,
624  if (error)
625  return error;
626 
627  /*
628  * Ensure default mtu is not > the mtu read from the hardware.
629  * Otherwise rte_eth_dev_configure() will fail and the port will
630  * not be available.
631  * Calculate max_frame_size and mtu supported by NIC
632  */
633  if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen)
634  {
635  /*
636  * This device does not support the platforms's max frame
637  * size. Use it's advertised mru instead.
638  */
639  max_rx_frame = dev_info.max_rx_pktlen;
640  mtu = dev_info.max_rx_pktlen - sizeof (ethernet_header_t);
641  }
642  else
643  {
644  /* VPP treats MTU and max_rx_pktlen both equal to
645  * ETHERNET_MAX_PACKET_BYTES, if dev_info.max_rx_pktlen >=
646  * ETHERNET_MAX_PACKET_BYTES + sizeof(ethernet_header_t)
647  */
648  if (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES +
649  sizeof (ethernet_header_t)))
650  {
652  max_rx_frame = ETHERNET_MAX_PACKET_BYTES;
653 
654  /*
655  * Some platforms do not account for Ethernet FCS (4 bytes) in
656  * MTU calculations. To interop with them increase mru but only
657  * if the device's settings can support it.
658  */
659  if (dpdk_port_crc_strip_enabled (xd) &&
660  (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES +
661  sizeof (ethernet_header_t) +
662  4)))
663  {
664  max_rx_frame += 4;
665  }
666  }
667  else
668  {
669  max_rx_frame = ETHERNET_MAX_PACKET_BYTES;
671 
672  if (dpdk_port_crc_strip_enabled (xd) &&
673  (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)))
674  {
675  max_rx_frame += 4;
676  }
677  }
678  }
679 
680  if (xd->pmd == VNET_DPDK_PMD_FAILSAFE)
681  {
682  /* failsafe device numerables are reported with active device only,
683  * need to query the mtu for current device setup to overwrite
684  * reported value.
685  */
686  uint16_t dev_mtu;
687  if (!rte_eth_dev_get_mtu (i, &dev_mtu))
688  {
689  mtu = dev_mtu;
690  max_rx_frame = mtu + sizeof (ethernet_header_t);
691 
693  {
694  max_rx_frame += 4;
695  }
696  }
697  }
698 
699  /*Set port rxmode config */
700  xd->port_conf.rxmode.max_rx_pkt_len = max_rx_frame;
701 
703  xd->sw_if_index = sw->sw_if_index;
705  dpdk_input_node.index);
706 
707  if (devconf->workers)
708  {
709  int i;
710  q = 0;
711  clib_bitmap_foreach (i, devconf->workers, ({
712  vnet_hw_interface_assign_rx_thread (dm->vnet_main, xd->hw_if_index, q++,
713  vdm->first_worker_thread_index + i);
714  }));
715  }
716  else
717  for (q = 0; q < xd->rx_q_used; q++)
718  {
720  ~1);
721  }
722 
723  /*Get vnet hardware interface */
725 
726  /*Override default max_packet_bytes and max_supported_bytes set in
727  * ethernet_register_interface() above*/
728  if (hi)
729  {
730  hi->max_packet_bytes = mtu;
731  hi->max_supported_packet_bytes = max_rx_frame;
732  }
733 
734  if (dm->conf->no_tx_checksum_offload == 0)
735  if (xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD && hi != NULL)
737 
738  dpdk_device_setup (xd);
739 
740  if (vec_len (xd->errors))
741  dpdk_log_err ("setup failed for device %U. Errors:\n %U",
744 
745  if (devconf->hqos_enabled)
746  {
747  clib_error_t *rv;
748  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
749  if (rv)
750  return rv;
751  }
752 
753  /*
754  * For cisco VIC vNIC, set default to VLAN strip enabled, unless
755  * specified otherwise in the startup config.
756  * For other NICs default to VLAN strip disabled, unless specified
757  * otherwise in the startup config.
758  */
759  if (xd->pmd == VNET_DPDK_PMD_ENIC)
760  {
761  if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF)
762  vlan_strip = 1; /* remove vlan tag from VIC port by default */
763  else
764  dpdk_log_warn ("VLAN strip disabled for interface\n");
765  }
766  else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON)
767  vlan_strip = 1;
768 
769  if (vlan_strip)
770  {
771  int vlan_off;
772  vlan_off = rte_eth_dev_get_vlan_offload (xd->port_id);
773  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
774 #if RTE_VERSION < RTE_VERSION_NUM(18, 8, 0, 0)
775  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
776 #else
777  if (vlan_off)
778  xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
779  else
780  xd->port_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
781 #endif
782  if (rte_eth_dev_set_vlan_offload (xd->port_id, vlan_off) == 0)
783  dpdk_log_info ("VLAN strip enabled for interface\n");
784  else
785  dpdk_log_warn ("VLAN strip cannot be supported by interface\n");
786  }
787 
788  if (hi)
789  hi->max_packet_bytes = xd->port_conf.rxmode.max_rx_pkt_len
790  - sizeof (ethernet_header_t);
791  else
792  clib_warning ("hi NULL");
793 
794  if (dm->conf->no_multi_seg)
795  mtu = mtu > ETHER_MAX_LEN ? ETHER_MAX_LEN : mtu;
796 
797  rte_eth_dev_set_mtu (xd->port_id, mtu);
798  }
799  /* *INDENT-ON* */
800 
801  if (nb_desc > dm->conf->num_mbufs)
802  dpdk_log_err ("%d mbufs allocated but total rx/tx ring size is %d\n",
803  dm->conf->num_mbufs, nb_desc);
804 
805  return 0;
806 }
807 
808 static void
810 {
811  clib_error_t *error;
812  u8 *pci_addr = 0;
813  int num_whitelisted = vec_len (conf->dev_confs);
814  vlib_pci_device_info_t *d = 0;
815  vlib_pci_addr_t *addr = 0, *addrs;
816 
817  addrs = vlib_pci_get_all_dev_addrs ();
818  /* *INDENT-OFF* */
819  vec_foreach (addr, addrs)
820  {
821  dpdk_device_config_t * devconf = 0;
822  vec_reset_length (pci_addr);
823  pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, addr, 0);
824  if (d)
825  {
827  d = 0;
828  }
829  d = vlib_pci_get_device_info (addr, &error);
830  if (error)
831  {
832  clib_error_report (error);
833  continue;
834  }
835 
837  continue;
838 
839  if (num_whitelisted)
840  {
841  uword * p = hash_get (conf->device_config_index_by_pci_addr, addr->as_u32);
842 
843  if (!p)
844  continue;
845 
846  devconf = pool_elt_at_index (conf->dev_confs, p[0]);
847  }
848 
849  /* virtio */
850  if (d->vendor_id == 0x1af4 &&
853  ;
854  /* vmxnet3 */
855  else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0)
856  ;
857  /* all Intel network devices */
858  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_NETWORK_ETHERNET)
859  ;
860  /* all Intel QAT devices VFs */
861  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_PROCESSOR_CO &&
862  (d->device_id == 0x0443 || d->device_id == 0x37c9 || d->device_id == 0x19e3))
863  ;
864  /* Cisco VIC */
865  else if (d->vendor_id == 0x1137 && d->device_id == 0x0043)
866  ;
867  /* Chelsio T4/T5 */
868  else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000)
869  ;
870  /* Amazen Elastic Network Adapter */
871  else if (d->vendor_id == 0x1d0f && d->device_id >= 0xec20 && d->device_id <= 0xec21)
872  ;
873  /* Cavium Network Adapter */
874  else if (d->vendor_id == 0x177d && d->device_id == 0x9712)
875  ;
876  /* Cavium FastlinQ QL41000 Series */
877  else if (d->vendor_id == 0x1077 && d->device_id >= 0x8070 && d->device_id <= 0x8090)
878  ;
879  /* Mellanox mlx4 */
880  else if (d->vendor_id == 0x15b3 && d->device_id >= 0x1003 && d->device_id <= 0x1004)
881  {
882  continue;
883  }
884  /* Mellanox mlx5 */
885  else if (d->vendor_id == 0x15b3 && d->device_id >= 0x1013 && d->device_id <= 0x101a)
886  {
887  continue;
888  }
889  else
890  {
891  dpdk_log_warn ("Unsupported PCI device 0x%04x:0x%04x found "
892  "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id,
893  pci_addr);
894  continue;
895  }
896 
897  error = vlib_pci_bind_to_uio (addr, (char *) conf->uio_driver_name);
898 
899  if (error)
900  {
901  if (devconf == 0)
902  {
903  pool_get (conf->dev_confs, devconf);
904  hash_set (conf->device_config_index_by_pci_addr, addr->as_u32,
905  devconf - conf->dev_confs);
906  devconf->pci_addr.as_u32 = addr->as_u32;
907  }
908  devconf->is_blacklisted = 1;
909  clib_error_report (error);
910  }
911  }
912  /* *INDENT-ON* */
913  vec_free (pci_addr);
915 }
916 
917 static clib_error_t *
918 dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
919  unformat_input_t * input, u8 is_default)
920 {
921  clib_error_t *error = 0;
922  uword *p;
923  dpdk_device_config_t *devconf;
924  unformat_input_t sub_input;
925 
926  if (is_default)
927  {
928  devconf = &conf->default_devconf;
929  }
930  else
931  {
932  p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32);
933 
934  if (!p)
935  {
936  pool_get (conf->dev_confs, devconf);
937  hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32,
938  devconf - conf->dev_confs);
939  }
940  else
941  return clib_error_return (0,
942  "duplicate configuration for PCI address %U",
943  format_vlib_pci_addr, &pci_addr);
944  }
945 
946  devconf->pci_addr.as_u32 = pci_addr.as_u32;
947  devconf->hqos_enabled = 0;
949 
950  if (!input)
951  return 0;
952 
955  {
956  if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues))
957  ;
958  else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues))
959  ;
960  else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc))
961  ;
962  else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc))
963  ;
964  else if (unformat (input, "workers %U", unformat_bitmap_list,
965  &devconf->workers))
966  ;
967  else
968  if (unformat
969  (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input))
970  {
971  error = unformat_rss_fn (&sub_input, &devconf->rss_fn);
972  if (error)
973  break;
974  }
975  else if (unformat (input, "vlan-strip-offload off"))
977  else if (unformat (input, "vlan-strip-offload on"))
979  else
980  if (unformat
981  (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input))
982  {
983  devconf->hqos_enabled = 1;
984  error = unformat_hqos (&sub_input, &devconf->hqos);
985  if (error)
986  break;
987  }
988  else if (unformat (input, "hqos"))
989  {
990  devconf->hqos_enabled = 1;
991  }
992  else
993  {
994  error = clib_error_return (0, "unknown input `%U'",
995  format_unformat_error, input);
996  break;
997  }
998  }
999 
1000  if (error)
1001  return error;
1002 
1003  if (devconf->workers && devconf->num_rx_queues == 0)
1004  devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers);
1005  else if (devconf->workers &&
1006  clib_bitmap_count_set_bits (devconf->workers) !=
1007  devconf->num_rx_queues)
1008  error =
1009  clib_error_return (0,
1010  "%U: number of worker threads must be "
1011  "equal to number of rx queues", format_vlib_pci_addr,
1012  &pci_addr);
1013 
1014  return error;
1015 }
1016 
1017 static clib_error_t *
1019 {
1020  unformat_input_t input;
1021  u8 *line, *s = 0;
1022  int n, n_try;
1023 
1024  n = n_try = 4096;
1025  while (n == n_try)
1026  {
1027  uword len = vec_len (s);
1028  vec_resize (s, len + n_try);
1029 
1030  n = read (uf->file_descriptor, s + len, n_try);
1031  if (n < 0 && errno != EAGAIN)
1032  return clib_error_return_unix (0, "read");
1033  _vec_len (s) = len + (n < 0 ? 0 : n);
1034  }
1035 
1036  unformat_init_vector (&input, s);
1037 
1038  while (unformat_user (&input, unformat_line, &line))
1039  {
1040  dpdk_log_notice ("%v", line);
1041  vec_free (line);
1042  }
1043 
1044  unformat_free (&input);
1045  return 0;
1046 }
1047 
1048 static clib_error_t *
1050 {
1051  clib_error_t *error = 0;
1054  dpdk_device_config_t *devconf;
1055  vlib_pci_addr_t pci_addr;
1056  unformat_input_t sub_input;
1057  uword x;
1058  u8 *s, *tmp = 0;
1059  u32 log_level;
1060  int ret, i;
1061  int num_whitelisted = 0;
1062  u8 no_pci = 0;
1063  u8 no_huge = 0;
1064  u8 huge_dir = 0;
1065  u8 file_prefix = 0;
1066  u8 *socket_mem = 0;
1067  u8 *huge_dir_path = 0;
1068 
1069  huge_dir_path =
1070  format (0, "%s/hugepages%c", vlib_unix_get_runtime_dir (), 0);
1071 
1072  conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
1073  log_level = RTE_LOG_NOTICE;
1074 
1075  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1076  {
1077  /* Prime the pump */
1078  if (unformat (input, "no-hugetlb"))
1079  {
1080  vec_add1 (conf->eal_init_args, (u8 *) "--no-huge");
1081  no_huge = 1;
1082  }
1083 
1084  else if (unformat (input, "enable-tcp-udp-checksum"))
1085  conf->enable_tcp_udp_checksum = 1;
1086 
1087  else if (unformat (input, "no-tx-checksum-offload"))
1088  conf->no_tx_checksum_offload = 1;
1089 
1090  else if (unformat (input, "decimal-interface-names"))
1092 
1093  else if (unformat (input, "log-level %U", unformat_dpdk_log_level, &x))
1094  log_level = x;
1095 
1096  else if (unformat (input, "no-multi-seg"))
1097  conf->no_multi_seg = 1;
1098 
1099  else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
1100  &sub_input))
1101  {
1102  error =
1103  dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input,
1104  1);
1105 
1106  if (error)
1107  return error;
1108  }
1109  else
1110  if (unformat
1111  (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr,
1112  unformat_vlib_cli_sub_input, &sub_input))
1113  {
1114  error = dpdk_device_config (conf, pci_addr, &sub_input, 0);
1115 
1116  if (error)
1117  return error;
1118 
1119  num_whitelisted++;
1120  }
1121  else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr))
1122  {
1123  error = dpdk_device_config (conf, pci_addr, 0, 0);
1124 
1125  if (error)
1126  return error;
1127 
1128  num_whitelisted++;
1129  }
1130  else if (unformat (input, "num-mem-channels %d", &conf->nchannels))
1131  conf->nchannels_set_manually = 0;
1132  else if (unformat (input, "num-mbufs %d", &conf->num_mbufs))
1133  ;
1134  else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
1135  ;
1136  else if (unformat (input, "socket-mem %s", &socket_mem))
1137  ;
1138  else if (unformat (input, "no-pci"))
1139  {
1140  no_pci = 1;
1141  tmp = format (0, "--no-pci%c", 0);
1142  vec_add1 (conf->eal_init_args, tmp);
1143  }
1144 
1145 #define _(a) \
1146  else if (unformat(input, #a)) \
1147  { \
1148  tmp = format (0, "--%s%c", #a, 0); \
1149  vec_add1 (conf->eal_init_args, tmp); \
1150  }
1152 #undef _
1153 #define _(a) \
1154  else if (unformat(input, #a " %s", &s)) \
1155  { \
1156  if (!strncmp(#a, "huge-dir", 8)) \
1157  huge_dir = 1; \
1158  else if (!strncmp(#a, "file-prefix", 11)) \
1159  file_prefix = 1; \
1160  tmp = format (0, "--%s%c", #a, 0); \
1161  vec_add1 (conf->eal_init_args, tmp); \
1162  vec_add1 (s, 0); \
1163  if (!strncmp(#a, "vdev", 4)) \
1164  if (strstr((char*)s, "af_packet")) \
1165  clib_warning ("af_packet obsoleted. Use CLI 'create host-interface'."); \
1166  vec_add1 (conf->eal_init_args, s); \
1167  }
1169 #undef _
1170 #define _(a,b) \
1171  else if (unformat(input, #a " %s", &s)) \
1172  { \
1173  tmp = format (0, "-%s%c", #b, 0); \
1174  vec_add1 (conf->eal_init_args, tmp); \
1175  vec_add1 (s, 0); \
1176  vec_add1 (conf->eal_init_args, s); \
1177  }
1179 #undef _
1180 #define _(a,b) \
1181  else if (unformat(input, #a " %s", &s)) \
1182  { \
1183  tmp = format (0, "-%s%c", #b, 0); \
1184  vec_add1 (conf->eal_init_args, tmp); \
1185  vec_add1 (s, 0); \
1186  vec_add1 (conf->eal_init_args, s); \
1187  conf->a##_set_manually = 1; \
1188  }
1190 #undef _
1191  else if (unformat (input, "default"))
1192  ;
1193 
1194  else if (unformat_skip_white_space (input))
1195  ;
1196  else
1197  {
1198  error = clib_error_return (0, "unknown input `%U'",
1199  format_unformat_error, input);
1200  goto done;
1201  }
1202  }
1203 
1204  if (!conf->uio_driver_name)
1205  conf->uio_driver_name = format (0, "auto%c", 0);
1206 
1207  /*
1208  * Use 1G huge pages if available.
1209  */
1210  if (!no_huge && !huge_dir)
1211  {
1212  u32 x, *mem_by_socket = 0;
1213  uword c = 0;
1214  int rv;
1215 
1216  umount ((char *) huge_dir_path);
1217 
1218  /* Process "socket-mem" parameter value */
1219  if (vec_len (socket_mem))
1220  {
1221  unformat_input_t in;
1222  unformat_init_vector (&in, socket_mem);
1224  {
1225  if (unformat (&in, "%u,", &x))
1226  ;
1227  else if (unformat (&in, "%u", &x))
1228  ;
1229  else if (unformat (&in, ","))
1230  x = 0;
1231  else
1232  break;
1233 
1234  vec_add1 (mem_by_socket, x);
1235  }
1236  /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */
1237  unformat_free (&in);
1238  socket_mem = 0;
1239  }
1240  else
1241  {
1242  /* *INDENT-OFF* */
1244  {
1245  vec_validate(mem_by_socket, c);
1246  mem_by_socket[c] = 64; /* default per-socket mem */
1247  }
1248  ));
1249  /* *INDENT-ON* */
1250  }
1251 
1252  /* *INDENT-OFF* */
1254  {
1255  clib_error_t *e;
1256 
1257  vec_validate(mem_by_socket, c);
1258 
1259  e = clib_sysfs_prealloc_hugepages(c, 2 << 10, mem_by_socket[c] / 2);
1260  if (e)
1261  clib_error_report (e);
1262  }));
1263  /* *INDENT-ON* */
1264 
1265  if (mem_by_socket == 0)
1266  {
1267  error = clib_error_return (0, "mem_by_socket NULL");
1268  goto done;
1269  }
1270  _vec_len (mem_by_socket) = c + 1;
1271 
1272  /* regenerate socket_mem string */
1273  vec_foreach_index (x, mem_by_socket)
1274  socket_mem = format (socket_mem, "%s%u",
1275  socket_mem ? "," : "", mem_by_socket[x]);
1276  socket_mem = format (socket_mem, "%c", 0);
1277 
1278  vec_free (mem_by_socket);
1279 
1280  error = vlib_unix_recursive_mkdir ((char *) huge_dir_path);
1281  if (error)
1282  {
1283  goto done;
1284  }
1285 
1286  rv = mount ("none", (char *) huge_dir_path, "hugetlbfs", 0, NULL);
1287 
1288  if (rv)
1289  {
1290  error = clib_error_return (0, "mount failed %d", errno);
1291  goto done;
1292  }
1293 
1294  tmp = format (0, "--huge-dir%c", 0);
1295  vec_add1 (conf->eal_init_args, tmp);
1296  tmp = format (0, "%s%c", huge_dir_path, 0);
1297  vec_add1 (conf->eal_init_args, tmp);
1298  if (!file_prefix)
1299  {
1300  tmp = format (0, "--file-prefix%c", 0);
1301  vec_add1 (conf->eal_init_args, tmp);
1302  tmp = format (0, "vpp%c", 0);
1303  vec_add1 (conf->eal_init_args, tmp);
1304  }
1305  }
1306 
1307  if (error)
1308  return error;
1309 
1310  /* I'll bet that -c and -n must be the first and second args... */
1311  if (!conf->coremask_set_manually)
1312  {
1314  uword *coremask = 0;
1315  int i;
1316 
1317  /* main thread core */
1318  coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
1319 
1320  for (i = 0; i < vec_len (tm->registrations); i++)
1321  {
1322  tr = tm->registrations[i];
1323  coremask = clib_bitmap_or (coremask, tr->coremask);
1324  }
1325 
1326  vec_insert (conf->eal_init_args, 2, 1);
1327  conf->eal_init_args[1] = (u8 *) "-c";
1328  tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
1329  conf->eal_init_args[2] = tmp;
1330  clib_bitmap_free (coremask);
1331  }
1332 
1333  if (!conf->nchannels_set_manually)
1334  {
1335  vec_insert (conf->eal_init_args, 2, 3);
1336  conf->eal_init_args[3] = (u8 *) "-n";
1337  tmp = format (0, "%d", conf->nchannels);
1338  conf->eal_init_args[4] = tmp;
1339  }
1340 
1341  if (no_pci == 0 && geteuid () == 0)
1342  dpdk_bind_devices_to_uio (conf);
1343 
1344 #define _(x) \
1345  if (devconf->x == 0 && conf->default_devconf.x > 0) \
1346  devconf->x = conf->default_devconf.x ;
1347 
1348  /* *INDENT-OFF* */
1349  pool_foreach (devconf, conf->dev_confs, ({
1350 
1351  /* default per-device config items */
1352  foreach_dpdk_device_config_item
1353 
1354  /* add DPDK EAL whitelist/blacklist entry */
1355  if (num_whitelisted > 0 && devconf->is_blacklisted == 0)
1356  {
1357  tmp = format (0, "-w%c", 0);
1358  vec_add1 (conf->eal_init_args, tmp);
1359  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1360  vec_add1 (conf->eal_init_args, tmp);
1361  }
1362  else if (num_whitelisted == 0 && devconf->is_blacklisted != 0)
1363  {
1364  tmp = format (0, "-b%c", 0);
1365  vec_add1 (conf->eal_init_args, tmp);
1366  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1367  vec_add1 (conf->eal_init_args, tmp);
1368  }
1369  }));
1370  /* *INDENT-ON* */
1371 
1372 #undef _
1373 
1374  /* set master-lcore */
1375  tmp = format (0, "--master-lcore%c", 0);
1376  vec_add1 (conf->eal_init_args, tmp);
1377  tmp = format (0, "%u%c", tm->main_lcore, 0);
1378  vec_add1 (conf->eal_init_args, tmp);
1379 
1380  /* set socket-mem */
1381  if (!no_huge)
1382  {
1383  tmp = format (0, "--socket-mem%c", 0);
1384  vec_add1 (conf->eal_init_args, tmp);
1385  tmp = format (0, "%s%c", socket_mem, 0);
1386  vec_add1 (conf->eal_init_args, tmp);
1387  }
1388 
1389  /* NULL terminate the "argv" vector, in case of stupidity */
1390  vec_add1 (conf->eal_init_args, 0);
1391  _vec_len (conf->eal_init_args) -= 1;
1392 
1393  /* Set up DPDK eal and packet mbuf pool early. */
1394 
1395  rte_log_set_global_level (log_level);
1396  int log_fds[2] = { 0 };
1397  if (pipe (log_fds) == 0)
1398  {
1399  if (fcntl (log_fds[1], F_SETFL, O_NONBLOCK) == 0)
1400  {
1401  FILE *f = fdopen (log_fds[1], "a");
1402  if (f && rte_openlog_stream (f) == 0)
1403  {
1404  clib_file_t t = { 0 };
1406  t.file_descriptor = log_fds[0];
1407  t.description = format (0, "DPDK logging pipe");
1408  clib_file_add (&file_main, &t);
1409  }
1410  }
1411  else
1412  {
1413  close (log_fds[0]);
1414  close (log_fds[1]);
1415  }
1416  }
1417 
1418  vm = vlib_get_main ();
1419 
1420  /* make copy of args as rte_eal_init tends to mess up with arg array */
1421  for (i = 1; i < vec_len (conf->eal_init_args); i++)
1422  conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ",
1423  conf->eal_init_args[i]);
1424 
1425  dpdk_log_warn ("EAL init args: %s", conf->eal_init_args_str);
1426  ret = rte_eal_init (vec_len (conf->eal_init_args),
1427  (char **) conf->eal_init_args);
1428 
1429  /* lazy umount hugepages */
1430  umount2 ((char *) huge_dir_path, MNT_DETACH);
1431  rmdir ((char *) huge_dir_path);
1432  vec_free (huge_dir_path);
1433 
1434  if (ret < 0)
1435  return clib_error_return (0, "rte_eal_init returned %d", ret);
1436 
1437  /* set custom ring memory allocator */
1438  {
1439  struct rte_mempool_ops *ops = NULL;
1440 
1441  ops = get_ops_by_name ("ring_sp_sc");
1442  ops->alloc = dpdk_ring_alloc;
1443 
1444  ops = get_ops_by_name ("ring_mp_sc");
1445  ops->alloc = dpdk_ring_alloc;
1446 
1447  ops = get_ops_by_name ("ring_sp_mc");
1448  ops->alloc = dpdk_ring_alloc;
1449 
1450  ops = get_ops_by_name ("ring_mp_mc");
1451  ops->alloc = dpdk_ring_alloc;
1452  }
1453 
1454  /* main thread 1st */
1455  error = dpdk_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ());
1456  if (error)
1457  return error;
1458 
1459  for (i = 0; i < RTE_MAX_LCORE; i++)
1460  {
1461  error = dpdk_buffer_pool_create (vm, conf->num_mbufs,
1462  rte_lcore_to_socket_id (i));
1463  if (error)
1464  return error;
1465  }
1466 
1467 done:
1468  return error;
1469 }
1470 
1472 
1473 void
1475 {
1476  vnet_main_t *vnm = vnet_get_main ();
1477  struct rte_eth_link prev_link = xd->link;
1478  u32 hw_flags = 0;
1479  u8 hw_flags_chg = 0;
1480 
1481  /* only update link state for PMD interfaces */
1482  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
1483  return;
1484 
1485  xd->time_last_link_update = now ? now : xd->time_last_link_update;
1486  memset (&xd->link, 0, sizeof (xd->link));
1487  rte_eth_link_get_nowait (xd->port_id, &xd->link);
1488 
1489  if (LINK_STATE_ELOGS)
1490  {
1492  ELOG_TYPE_DECLARE (e) =
1493  {
1494  .format =
1495  "update-link-state: sw_if_index %d, admin_up %d,"
1496  "old link_state %d new link_state %d",.format_args = "i4i1i1i1",};
1497 
1498  struct
1499  {
1500  u32 sw_if_index;
1501  u8 admin_up;
1502  u8 old_link_state;
1503  u8 new_link_state;
1504  } *ed;
1505  ed = ELOG_DATA (&vm->elog_main, e);
1506  ed->sw_if_index = xd->sw_if_index;
1507  ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
1508  ed->old_link_state = (u8)
1510  ed->new_link_state = (u8) xd->link.link_status;
1511  }
1512 
1513  if ((xd->flags & (DPDK_DEVICE_FLAG_ADMIN_UP | DPDK_DEVICE_FLAG_BOND_SLAVE))
1514  && ((xd->link.link_status != 0) ^
1516  {
1517  hw_flags_chg = 1;
1518  hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
1519  }
1520 
1521  if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex))
1522  {
1523  hw_flags_chg = 1;
1524  switch (xd->link.link_duplex)
1525  {
1526  case ETH_LINK_HALF_DUPLEX:
1528  break;
1529  case ETH_LINK_FULL_DUPLEX:
1531  break;
1532  default:
1533  break;
1534  }
1535  }
1536  if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed))
1537  {
1538  hw_flags_chg = 1;
1539  switch (xd->link.link_speed)
1540  {
1541  case ETH_SPEED_NUM_10M:
1543  break;
1544  case ETH_SPEED_NUM_100M:
1546  break;
1547  case ETH_SPEED_NUM_1G:
1548  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G;
1549  break;
1550  case ETH_SPEED_NUM_2_5G:
1552  break;
1553  case ETH_SPEED_NUM_5G:
1554  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_5G;
1555  break;
1556  case ETH_SPEED_NUM_10G:
1558  break;
1559  case ETH_SPEED_NUM_20G:
1561  break;
1562  case ETH_SPEED_NUM_25G:
1564  break;
1565  case ETH_SPEED_NUM_40G:
1567  break;
1568  case ETH_SPEED_NUM_50G:
1570  break;
1571  case ETH_SPEED_NUM_56G:
1573  break;
1574  case ETH_SPEED_NUM_100G:
1576  break;
1577  case 0:
1578  break;
1579  default:
1580  dpdk_log_warn ("unknown link speed %d", xd->link.link_speed);
1581  break;
1582  }
1583  }
1584  if (hw_flags_chg)
1585  {
1586  if (LINK_STATE_ELOGS)
1587  {
1589 
1590  ELOG_TYPE_DECLARE (e) =
1591  {
1592  .format =
1593  "update-link-state: sw_if_index %d, new flags %d",.format_args
1594  = "i4i4",};
1595 
1596  struct
1597  {
1598  u32 sw_if_index;
1599  u32 flags;
1600  } *ed;
1601  ed = ELOG_DATA (&vm->elog_main, e);
1602  ed->sw_if_index = xd->sw_if_index;
1603  ed->flags = hw_flags;
1604  }
1605  vnet_hw_interface_set_flags (vnm, xd->hw_if_index, hw_flags);
1606  }
1607 }
1608 
1609 static uword
1611 {
1612  clib_error_t *error;
1613  vnet_main_t *vnm = vnet_get_main ();
1614  dpdk_main_t *dm = &dpdk_main;
1616  dpdk_device_t *xd;
1618  int i;
1619  int j;
1620 
1621  error = dpdk_lib_init (dm);
1622 
1623  if (error)
1624  clib_error_report (error);
1625 
1626  tm->worker_thread_release = 1;
1627 
1628  f64 now = vlib_time_now (vm);
1629  vec_foreach (xd, dm->devices)
1630  {
1631  dpdk_update_link_state (xd, now);
1632  }
1633 
1634  {
1635  /*
1636  * Extra set up for bond interfaces:
1637  * 1. Setup MACs for bond interfaces and their slave links which was set
1638  * in dpdk_device_setup() but needs to be done again here to take
1639  * effect.
1640  * 2. Set up info and register slave link state change callback handling.
1641  * 3. Set up info for bond interface related CLI support.
1642  */
1643  int nports = rte_eth_dev_count_avail ();
1644  if (nports > 0)
1645  {
1646  /* *INDENT-OFF* */
1647  RTE_ETH_FOREACH_DEV(i)
1648  {
1649  xd = NULL;
1650  for (j = 0; j < nports; j++)
1651  {
1652  if (dm->devices[j].port_id == i)
1653  {
1654  xd = &dm->devices[j];
1655  }
1656  }
1657  if (xd != NULL && xd->pmd == VNET_DPDK_PMD_BOND)
1658  {
1659  u8 addr[6];
1660  dpdk_portid_t slink[16];
1661  int nlink = rte_eth_bond_slaves_get (i, slink, 16);
1662  if (nlink > 0)
1663  {
1664  vnet_hw_interface_t *bhi;
1665  ethernet_interface_t *bei;
1666  int rv;
1667 
1668  /* Get MAC of 1st slave link */
1669  rte_eth_macaddr_get
1670  (slink[0], (struct ether_addr *) addr);
1671 
1672  /* Set MAC of bounded interface to that of 1st slave link */
1673  dpdk_log_info ("Set MAC for bond port %d BondEthernet%d",
1674  i, xd->bond_instance_num);
1675  rv = rte_eth_bond_mac_address_set
1676  (i, (struct ether_addr *) addr);
1677  if (rv)
1678  dpdk_log_warn ("Set MAC addr failure rv=%d", rv);
1679 
1680  /* Populate MAC of bonded interface in VPP hw tables */
1681  bhi = vnet_get_hw_interface
1682  (vnm, dm->devices[i].hw_if_index);
1683  bei = pool_elt_at_index
1684  (em->interfaces, bhi->hw_instance);
1685  clib_memcpy (bhi->hw_address, addr, 6);
1686  clib_memcpy (bei->address, addr, 6);
1687 
1688  /* Init l3 packet size allowed on bonded interface */
1690  while (nlink >= 1)
1691  { /* for all slave links */
1692  int slave = slink[--nlink];
1693  dpdk_device_t *sdev = &dm->devices[slave];
1694  vnet_hw_interface_t *shi;
1695  vnet_sw_interface_t *ssi;
1696  ethernet_interface_t *sei;
1697  /* Add MAC to all slave links except the first one */
1698  if (nlink)
1699  {
1700  dpdk_log_info ("Add MAC for slave port %d",
1701  slave);
1702  rv = rte_eth_dev_mac_addr_add
1703  (slave, (struct ether_addr *) addr, 0);
1704  if (rv)
1705  dpdk_log_warn ("Add MAC addr failure rv=%d",
1706  rv);
1707  }
1708  /* Setup slave link state change callback handling */
1709  rte_eth_dev_callback_register
1710  (slave, RTE_ETH_EVENT_INTR_LSC,
1712  dpdk_device_t *sxd = &dm->devices[slave];
1713  sxd->flags |= DPDK_DEVICE_FLAG_BOND_SLAVE;
1714  sxd->bond_port = i;
1715  /* Set slaves bitmap for bonded interface */
1716  bhi->bond_info = clib_bitmap_set
1717  (bhi->bond_info, sdev->hw_if_index, 1);
1718  /* Set MACs and slave link flags on slave interface */
1719  shi = vnet_get_hw_interface (vnm, sdev->hw_if_index);
1720  ssi = vnet_get_sw_interface (vnm, sdev->sw_if_index);
1721  sei = pool_elt_at_index
1722  (em->interfaces, shi->hw_instance);
1725  clib_memcpy (shi->hw_address, addr, 6);
1726  clib_memcpy (sei->address, addr, 6);
1727  /* Set l3 packet size allowed as the lowest of slave */
1728  if (bhi->max_packet_bytes > shi->max_packet_bytes)
1729  bhi->max_packet_bytes = shi->max_packet_bytes;
1730  }
1731  }
1732  }
1733  }
1734  /* *INDENT-ON* */
1735  }
1736  }
1737 
1738  while (1)
1739  {
1740  /*
1741  * check each time through the loop in case intervals are changed
1742  */
1743  f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ?
1745 
1746  vlib_process_wait_for_event_or_clock (vm, min_wait);
1747 
1748  if (dm->admin_up_down_in_progress)
1749  /* skip the poll if an admin up down is in progress (on any interface) */
1750  continue;
1751 
1752  vec_foreach (xd, dm->devices)
1753  {
1754  f64 now = vlib_time_now (vm);
1755  if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval)
1756  dpdk_update_counters (xd, now);
1757  if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval)
1758  dpdk_update_link_state (xd, now);
1759 
1760  }
1761  }
1762 
1763  return 0;
1764 }
1765 
1766 /* *INDENT-OFF* */
1768  .function = dpdk_process,
1769  .type = VLIB_NODE_TYPE_PROCESS,
1770  .name = "dpdk-process",
1771  .process_log2_n_stack_bytes = 17,
1772 };
1773 /* *INDENT-ON* */
1774 
1775 static clib_error_t *
1777 {
1778  dpdk_main_t *dm = &dpdk_main;
1779  clib_error_t *error = 0;
1780 
1781  /* verify that structs are cacheline aligned */
1782  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
1783  "Cache line marker must be 1st element in dpdk_device_t");
1784  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) ==
1786  "Data in cache line 0 is bigger than cache line size");
1787  STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0,
1788  "Cache line marker must be 1st element in frame_queue_trace_t");
1789  STATIC_ASSERT (RTE_CACHE_LINE_SIZE == 1 << CLIB_LOG2_CACHE_LINE_BYTES,
1790  "DPDK RTE CACHE LINE SIZE does not match with 1<<CLIB_LOG2_CACHE_LINE_BYTES");
1791 
1792  dm->vlib_main = vm;
1793  dm->vnet_main = vnet_get_main ();
1794  dm->conf = &dpdk_config_main;
1795 
1796  dm->conf->nchannels = 4;
1797  dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
1798  vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
1799 
1800  /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
1801  dm->buffer_flags_template =
1802  (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID
1803  | VNET_BUFFER_F_L4_CHECKSUM_COMPUTED |
1804  VNET_BUFFER_F_L4_CHECKSUM_CORRECT | VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
1805 
1808 
1809  /* init CLI */
1810  if ((error = vlib_call_init_function (vm, dpdk_cli_init)))
1811  return error;
1812 
1813  dm->log_default = vlib_log_register_class ("dpdk", 0);
1814 
1815  return error;
1816 }
1817 
1819 
1820 
1821 /*
1822  * fd.io coding-style-patch-verification: ON
1823  *
1824  * Local Variables:
1825  * eval: (c-set-style "gnu")
1826  * End:
1827  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:227
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
static void dpdk_bind_devices_to_uio(dpdk_config_main_t *conf)
Definition: init.c:809
f64 time_last_link_update
Definition: dpdk.h:253
vmrglw vmrglh hi
format_function_t format_vlib_pci_addr
Definition: pci.h:288
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:255
#define VIRTIO_PCI_MODERN_DEVICEID_NET
Definition: pci_config.h:169
#define clib_min(x, y)
Definition: clib.h:291
#define VNET_HW_INTERFACE_FLAG_SPEED_1G
Definition: interface.h:507
ethernet_main_t ethernet_main
Definition: init.c:45
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:469
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
#define VNET_HW_INTERFACE_FLAG_SPEED_50G
Definition: interface.h:514
u8 interface_name_format_decimal
Definition: dpdk.h:368
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
#define NB_MBUF
Definition: dpdk.h:59
vnet_device_class_t dpdk_device_class
unsigned long u64
Definition: types.h:89
u32 sw_if_index
Definition: dpdk.h:205
#define DPDK_DEVICE_VLAN_STRIP_OFF
Definition: dpdk.h:335
#define NULL
Definition: clib.h:57
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:227
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:574
static uword * clib_bitmap_or(uword *ai, uword *bi)
Logical operator across two bitmaps.
static u32 dpdk_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 flags)
Definition: init.c:79
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1474
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_pci_addr_t *addr, clib_error_t **error)
Definition: pci.c:162
u16 flags
Definition: dpdk.h:213
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define LINK_STATE_ELOGS
Definition: init.c:45
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
static struct rte_mempool_ops * get_ops_by_name(char *ops_name)
Definition: init.c:124
u32 file_descriptor
Definition: file.h:54
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
dpdk_device_and_queue_t ** devices_by_hqos_cpu
Definition: dpdk.h:404
#define DPDK_NB_RX_DESC_VIRTIO
Definition: dpdk_priv.h:21
clib_error_t * errors
Definition: dpdk.h:267
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
u32 per_interface_next_index
Definition: dpdk.h:208
u8 enable_tcp_udp_checksum
Definition: dpdk.h:354
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:983
#define DPDK_DEVICE_VLAN_STRIP_ON
Definition: dpdk.h:336
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
#define VNET_HW_INTERFACE_FLAG_SPEED_100G
Definition: interface.h:516
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define DPDK_NB_TX_DESC_DEFAULT
Definition: dpdk_priv.h:20
u32 supported_flow_actions
Definition: dpdk.h:233
#define foreach_eal_double_hyphen_predicate_arg
Definition: dpdk_priv.h:32
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:287
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:494
int dpdk_port_state_callback(dpdk_portid_t port_id, enum rte_eth_event_type type, void *param, void *ret_param)
Definition: common.c:340
u16 bond_instance_num
Definition: dpdk.h:246
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
dpdk_device_config_hqos_t hqos
Definition: dpdk.h:343
vlib_pci_addr_t * vlib_pci_get_all_dev_addrs()
Definition: pci.c:1242
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
vhost_vring_addr_t addr
Definition: vhost_user.h:121
unsigned char u8
Definition: types.h:56
dpdk_config_main_t dpdk_config_main
Definition: init.c:43
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
#define ETHER_MAX_LEN
Maximum frame len, including CRC.
Definition: init.c:40
double f64
Definition: types.h:142
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define VNET_HW_INTERFACE_FLAG_SPEED_2_5G
Definition: interface.h:508
memset(h->entries, 0, sizeof(h->entries[0])*entries)
foreach_dpdk_device_config_item clib_bitmap_t * workers
Definition: dpdk.h:341
#define dpdk_log_warn(...)
Definition: dpdk.h:498
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
static clib_error_t * dpdk_log_read_ready(clib_file_t *uf)
Definition: init.c:1018
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
dpdk_portid_t port_id
Definition: dpdk.h:202
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:646
u32 sw_if_index
Definition: vxlan_gbp.api:39
dpdk_device_config_t default_devconf
Definition: dpdk.h:371
f64 stat_poll_interval
Definition: dpdk.h:431
static char * vlib_unix_get_runtime_dir(void)
Definition: unix.h:141
static dpdk_port_type_t port_type_from_speed_capa(struct rte_eth_dev_info *dev_info)
Definition: init.c:50
clib_error_t * vlib_pci_bind_to_uio(vlib_pci_addr_t *addr, char *uio_drv_name)
Definition: pci.c:345
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
u16 rx_q_used
Definition: dpdk.h:225
#define VNET_HW_INTERFACE_FLAG_SPEED_25G
Definition: interface.h:512
clib_file_main_t file_main
Definition: main.c:63
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:240
unsigned int u32
Definition: types.h:88
void dpdk_device_setup(dpdk_device_t *xd)
Definition: common.c:40
#define vlib_call_init_function(vm, x)
Definition: init.h:260
clib_error_t * unformat_rss_fn(unformat_input_t *input, uword *rss_fn)
Definition: format.c:878
#define DPDK_NB_TX_DESC_VIRTIO
Definition: dpdk_priv.h:22
struct rte_eth_conf port_conf
Definition: dpdk.h:229
static clib_error_t * dpdk_init(vlib_main_t *vm)
Definition: init.c:1776
#define fl(x, y)
u32 max_supported_packet_bytes
Definition: interface.h:570
f64 time_last_stats_update
Definition: dpdk.h:260
struct rte_eth_txconf tx_conf
Definition: dpdk.h:230
u8 * description
Definition: file.h:70
#define hash_get(h, key)
Definition: hash.h:249
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
#define vec_insert(V, N, M)
Insert N vector elements starting at element M, initialize new elements to zero (no header...
Definition: vec.h:687
vlib_pci_addr_t pci_addr
Definition: dpdk.h:331
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
Definition: buffer.c:511
static clib_error_t * dpdk_config(vlib_main_t *vm, unformat_input_t *input)
Definition: init.c:1049
#define foreach_eal_double_hyphen_arg
Definition: dpdk_priv.h:48
dpdk_portid_t bond_port
Definition: dpdk.h:250
#define ETHERNET_INTERFACE_FLAG_CONFIG_MTU(flags)
Definition: ethernet.h:165
u8 ** eal_init_args
Definition: dpdk.h:350
#define VNET_HW_INTERFACE_FLAG_SPEED_10M
Definition: interface.h:505
dpdk_per_thread_data_t * per_thread_data
Definition: dpdk.h:405
#define VNET_HW_INTERFACE_FLAG_SPEED_56G
Definition: interface.h:515
#define foreach_eal_single_hyphen_mandatory_arg
Definition: dpdk_priv.h:38
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
#define VNET_HW_INTERFACE_FLAG_HALF_DUPLEX
Definition: interface.h:497
#define clib_error_return_unix(e, args...)
Definition: error.h:102
#define foreach_dpdk_pmd
Definition: dpdk.h:65
vlib_buffer_t buffer_template
Definition: dpdk.h:389
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:440
#define ELOG_DATA(em, f)
Definition: elog.h:481
#define VIRTIO_PCI_LEGACY_DEVICEID_NET
Definition: pci_config.h:168
dpdk_port_type_t port_type
Definition: dpdk.h:261
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:172
vnet_sw_interface_flags_t flags
Definition: interface.h:720
signed char i8
Definition: types.h:45
u16 tx_q_used
Definition: dpdk.h:224
u16 nb_rx_desc
Definition: dpdk.h:226
uint16_t dpdk_portid_t
Definition: dpdk.h:122
#define dpdk_log_info(...)
Definition: dpdk.h:502
u32 hw_if_index
Definition: dpdk.h:204
u32 flags
Definition: vhost_user.h:115
void unformat_init_vector(unformat_input_t *input, u8 *vector_string)
Definition: unformat.c:1031
#define VNET_HW_INTERFACE_BOND_INFO_SLAVE
Definition: interface.h:590
u16 af_packet_instance_num
Definition: dpdk.h:245
#define foreach_eal_single_hyphen_arg
Definition: dpdk_priv.h:42
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:155
#define DPDK_NB_RX_DESC_DEFAULT
Definition: dpdk_priv.h:19
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
static clib_error_t * dpdk_device_config(dpdk_config_main_t *conf, vlib_pci_addr_t pci_addr, unformat_input_t *input, u8 is_default)
Definition: init.c:918
dpdk_device_t * devices
Definition: dpdk.h:403
vlib_main_t * vm
Definition: buffer.c:294
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:87
u8 nchannels_set_manually
Definition: dpdk.h:359
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
#define dpdk_log_err(...)
Definition: dpdk.h:496
volatile u32 ** lockp
Definition: dpdk.h:196
dpdk_device_config_t * dev_confs
Definition: dpdk.h:372
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
dpdk_pmd_t pmd
Definition: dpdk.h:210
format_function_t format_dpdk_device_errors
Definition: dpdk.h:509
elog_main_t elog_main
Definition: main.h:157
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:159
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD
Definition: interface.h:535
u8 coremask_set_manually
Definition: dpdk.h:358
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
#define VNET_HW_INTERFACE_FLAG_SPEED_10G
Definition: interface.h:510
static void dpdk_device_lock_init(dpdk_device_t *xd)
Definition: init.c:111
format_function_t format_dpdk_rss_hf_name
Definition: dpdk.h:515
u8 * interface_name_suffix
Definition: dpdk.h:218
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define VNET_HW_INTERFACE_FLAG_FULL_DUPLEX
Definition: interface.h:498
#define ASSERT(truth)
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:205
format_function_t format_dpdk_device_name
Definition: dpdk.h:507
int hqos_cpu_count
Definition: dpdk.h:427
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:138
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
vlib_log_class_t log_default
Definition: dpdk.h:445
static uword dpdk_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: init.c:1610
Bitmaps built as vectors of machine words.
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:277
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
#define DPDK_LINK_POLL_INTERVAL
Definition: dpdk.h:273
size_t count
Definition: vapi.c:46
dpdk_main_t dpdk_main
Definition: init.c:42
uword * thread_registrations_by_name
Definition: threads.h:287
clib_error_t * dpdk_cli_init(vlib_main_t *vm)
Definition: cli.c:2074
dpdk_portid_t device_index
Definition: dpdk.h:199
struct rte_eth_link link
Definition: dpdk.h:252
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:247
dpdk_port_type_t
Definition: dpdk.h:100
static int dpdk_port_crc_strip_enabled(dpdk_device_t *xd)
Definition: init.c:178
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
Definition: defs.h:47
clib_error_t * vlib_unix_recursive_mkdir(char *path)
Definition: util.c:103
#define DPDK_STATS_POLL_INTERVAL
Definition: dpdk.h:270
#define VNET_HW_INTERFACE_FLAG_SPEED_100M
Definition: interface.h:506
unformat_function_t unformat_line
Definition: format.h:279
static vlib_node_registration_t dpdk_process_node
(constructor) VLIB_REGISTER_NODE (dpdk_process_node)
Definition: init.c:1767
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
uword unformat_vlib_cli_sub_input(unformat_input_t *i, va_list *args)
Definition: cli.c:154
u8 admin_up_down_in_progress
Definition: dpdk.h:423
#define STATIC_ASSERT(truth,...)
#define VNET_HW_INTERFACE_FLAG_SPEED_5G
Definition: interface.h:509
u8 no_tx_checksum_offload
Definition: dpdk.h:355
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
static clib_error_t * dpdk_lib_init(dpdk_main_t *dm)
Definition: init.c:190
#define hash_get_mem(h, key)
Definition: hash.h:269
u32 buffer_flags_template
Definition: dpdk.h:408
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
#define vnet_buffer(b)
Definition: buffer.h:344
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
#define VNET_HW_INTERFACE_FLAG_SPEED_40G
Definition: interface.h:513
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
int hqos_cpu_first_index
Definition: dpdk.h:426
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
#define ETHERNET_MAX_PACKET_BYTES
Definition: ethernet.h:152
#define vec_foreach(var, vec)
Vector iterator.
i8 cpu_socket
Definition: dpdk.h:211
#define ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC(flags)
Definition: ethernet.h:160
uword * cpu_socket_bitmap
Definition: threads.h:322
Definition: file.h:51
static int dpdk_ring_alloc(struct rte_mempool *mp)
Definition: init.c:138
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:675
u8 * uio_driver_name
Definition: dpdk.h:352
vlib_thread_registration_t ** registrations
Definition: threads.h:285
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
Definition: pci.h:107
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
unformat_function_t unformat_dpdk_log_level
Definition: dpdk.h:518
ethernet_interface_t * interfaces
Definition: ethernet.h:290
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:116
vnet_main_t * vnet_main
Definition: dpdk.h:435
u16 nb_tx_desc
Definition: dpdk.h:215
clib_error_t * unformat_hqos(unformat_input_t *input, dpdk_device_config_hqos_t *hqos)
Definition: format.c:915
u16 device_class
Definition: pci.h:72
#define VNET_HW_INTERFACE_FLAG_SPEED_20G
Definition: interface.h:511
uword * device_config_index_by_pci_addr
Definition: dpdk.h:373
uword unformat_skip_white_space(unformat_input_t *input)
Definition: unformat.c:815
static uword vnet_hw_interface_is_link_up(vnet_main_t *vnm, u32 hw_if_index)
volatile u32 worker_thread_release
Definition: threads.h:328
#define dpdk_log_notice(...)
Definition: dpdk.h:500
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
vnet_device_main_t vnet_device_main
Definition: devices.c:22
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
f64 link_state_poll_interval
Definition: dpdk.h:430
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
dpdk_config_main_t * conf
Definition: dpdk.h:436
vlib_main_t * vlib_main
Definition: dpdk.h:434