FD.io VPP  v18.07.1-19-g511ce25
Vector Packet Processing
init.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/error.h>
18 #include <vppinfra/format.h>
19 #include <vppinfra/bitmap.h>
20 #include <vppinfra/linux/sysfs.h>
21 #include <vlib/unix/unix.h>
22 #include <vlib/log.h>
23 
24 #include <vnet/ethernet/ethernet.h>
25 #include <dpdk/device/dpdk.h>
26 #include <vlib/pci/pci.h>
27 
28 #include <rte_ring.h>
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <unistd.h>
33 #include <sys/stat.h>
34 #include <sys/mount.h>
35 #include <string.h>
36 #include <fcntl.h>
37 
38 #include <dpdk/device/dpdk_priv.h>
39 
40 #define ETHER_MAX_LEN 1518 /**< Maximum frame len, including CRC. */
41 
44 
45 #define LINK_STATE_ELOGS 0
46 
47 /* Port configuration, mildly modified Intel app values */
48 
49 static struct rte_eth_conf port_conf_template = {
50  .rxmode = {
51  .split_hdr_size = 0,
52  .header_split = 0, /**< Header Split disabled */
53  .hw_ip_checksum = 0, /**< IP checksum offload disabled */
54  .hw_vlan_filter = 0, /**< VLAN filtering disabled */
55  .hw_strip_crc = 0, /**< CRC stripped by hardware */
56  },
57  .txmode = {
58  .mq_mode = ETH_MQ_TX_NONE,
59  },
60 };
61 
62 static dpdk_port_type_t
63 port_type_from_speed_capa (struct rte_eth_dev_info *dev_info)
64 {
65 
66  if (dev_info->speed_capa & ETH_LINK_SPEED_100G)
68  else if (dev_info->speed_capa & ETH_LINK_SPEED_56G)
70  else if (dev_info->speed_capa & ETH_LINK_SPEED_50G)
72  else if (dev_info->speed_capa & ETH_LINK_SPEED_40G)
74  else if (dev_info->speed_capa & ETH_LINK_SPEED_25G)
76  else if (dev_info->speed_capa & ETH_LINK_SPEED_20G)
78  else if (dev_info->speed_capa & ETH_LINK_SPEED_10G)
80  else if (dev_info->speed_capa & ETH_LINK_SPEED_5G)
82  else if (dev_info->speed_capa & ETH_LINK_SPEED_2_5G)
84  else if (dev_info->speed_capa & ETH_LINK_SPEED_1G)
86 
88 }
89 
90 
91 static u32
93 {
94  dpdk_main_t *dm = &dpdk_main;
96  u32 old = 0;
97 
99  {
100  old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0;
101 
103  xd->flags |= DPDK_DEVICE_FLAG_PROMISC;
104  else
105  xd->flags &= ~DPDK_DEVICE_FLAG_PROMISC;
106 
107  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
108  {
109  if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
110  rte_eth_promiscuous_enable (xd->port_id);
111  else
112  rte_eth_promiscuous_disable (xd->port_id);
113  }
114  }
115  else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
116  {
117  xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes;
118  dpdk_device_setup (xd);
119  }
120  return old;
121 }
122 
123 static void
125 {
126  int q;
127  vec_validate (xd->lockp, xd->tx_q_used - 1);
128  for (q = 0; q < xd->tx_q_used; q++)
129  {
132  memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES);
133  }
134 }
135 
136 static struct rte_mempool_ops *
137 get_ops_by_name (char *ops_name)
138 {
139  u32 i;
140 
141  for (i = 0; i < rte_mempool_ops_table.num_ops; i++)
142  {
143  if (!strcmp (ops_name, rte_mempool_ops_table.ops[i].name))
144  return &rte_mempool_ops_table.ops[i];
145  }
146 
147  return 0;
148 }
149 
150 static int
151 dpdk_ring_alloc (struct rte_mempool *mp)
152 {
153  u32 rg_flags = 0, count;
154  i32 ret;
155  char rg_name[RTE_RING_NAMESIZE];
156  struct rte_ring *r;
157 
158  ret = snprintf (rg_name, sizeof (rg_name), RTE_MEMPOOL_MZ_FORMAT, mp->name);
159  if (ret < 0 || ret >= (i32) sizeof (rg_name))
160  return -ENAMETOOLONG;
161 
162  /* ring flags */
163  if (mp->flags & MEMPOOL_F_SP_PUT)
164  rg_flags |= RING_F_SP_ENQ;
165  if (mp->flags & MEMPOOL_F_SC_GET)
166  rg_flags |= RING_F_SC_DEQ;
167 
168  count = rte_align32pow2 (mp->size + 1);
169  /*
170  * Allocate the ring that will be used to store objects.
171  * Ring functions will return appropriate errors if we are
172  * running as a secondary process etc., so no checks made
173  * in this function for that condition.
174  */
175  /* XXX can we get memory from the right socket? */
176  r = clib_mem_alloc_aligned (rte_ring_get_memsize (count),
178 
179  /* XXX rte_ring_lookup will not work */
180 
181  ret = rte_ring_init (r, rg_name, count, rg_flags);
182  if (ret)
183  return ret;
184 
185  mp->pool_data = r;
186 
187  return 0;
188 }
189 
190 static clib_error_t *
192 {
193  u32 nports;
194  u32 mtu, max_rx_frame;
195  u32 nb_desc = 0;
196  int i;
197  clib_error_t *error;
203  dpdk_device_t *xd;
204  vlib_pci_addr_t last_pci_addr;
205  u32 last_pci_addr_port = 0;
207  uword *p_hqos;
208 
209  u32 next_hqos_cpu = 0;
210  u8 af_packet_instance_num = 0;
211  u8 bond_ether_instance_num = 0;
212  last_pci_addr.as_u32 = ~0;
213 
214  dm->hqos_cpu_first_index = 0;
215  dm->hqos_cpu_count = 0;
216 
217  /* find out which cpus will be used for I/O TX */
218  p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads");
219  tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0;
220 
221  if (tr_hqos && tr_hqos->count > 0)
222  {
223  dm->hqos_cpu_first_index = tr_hqos->first_index;
224  dm->hqos_cpu_count = tr_hqos->count;
225  }
226 
229 
230 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
231  nports = rte_eth_dev_count ();
232 #else
233  nports = rte_eth_dev_count_avail ();
234 #endif
235 
236  if (nports < 1)
237  {
238  dpdk_log_notice ("DPDK drivers found no ports...");
239  }
240 
241  if (CLIB_DEBUG > 0)
242  dpdk_log_notice ("DPDK drivers found %d ports...", nports);
243 
244  if (dm->conf->enable_tcp_udp_checksum)
245  dm->buffer_flags_template &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT
246  | VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
247 
248  /* vlib_buffer_t template */
251  for (i = 0; i < tm->n_vlib_mains; i++)
252  {
255  fl = vlib_buffer_get_free_list (vm,
259  vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
260  }
261 
262  /* *INDENT-OFF* */
263  RTE_ETH_FOREACH_DEV(i)
264  {
265  u8 addr[6];
266  u8 vlan_strip = 0;
267  struct rte_eth_dev_info dev_info;
268  struct rte_pci_device *pci_dev;
269  struct rte_eth_link l;
270  dpdk_device_config_t *devconf = 0;
271  vlib_pci_addr_t pci_addr;
272  uword *p = 0;
273 
274  if (!rte_eth_dev_is_valid_port(i))
275  continue;
276 
277  rte_eth_link_get_nowait (i, &l);
278  rte_eth_dev_info_get (i, &dev_info);
279 
280 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
281  pci_dev = dev_info.pci_dev;
282 #else
283  if (dev_info.device == 0)
284  {
285  clib_warning ("DPDK bug: missing device info. Skipping %s device",
286  dev_info.driver_name);
287  continue;
288  }
289  pci_dev = RTE_DEV_TO_PCI (dev_info.device);
290 #endif
291 
292  if (pci_dev) /* bonded interface has no pci info */
293  {
294  pci_addr.domain = pci_dev->addr.domain;
295  pci_addr.bus = pci_dev->addr.bus;
296  pci_addr.slot = pci_dev->addr.devid;
297  pci_addr.function = pci_dev->addr.function;
298  p =
300  pci_addr.as_u32);
301  }
302 
303  if (p)
304  devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
305  else
306  devconf = &dm->conf->default_devconf;
307 
308  /* Create vnet interface */
312  xd->cpu_socket = (i8) rte_eth_dev_socket_id (i);
313 
314  /* Handle interface naming for devices with multiple ports sharing same PCI ID */
315  if (pci_dev)
316  {
317  struct rte_eth_dev_info di = { 0 };
318  struct rte_pci_device *next_pci_dev;
319  rte_eth_dev_info_get (i + 1, &di);
320 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
321  next_pci_dev = di.pci_dev;
322 #else
323  next_pci_dev = di.device ? RTE_DEV_TO_PCI (di.device) : 0;
324 #endif
325  if (pci_dev && next_pci_dev &&
326  pci_addr.as_u32 != last_pci_addr.as_u32 &&
327  memcmp (&pci_dev->addr, &next_pci_dev->addr,
328  sizeof (struct rte_pci_addr)) == 0)
329  {
330  xd->interface_name_suffix = format (0, "0");
331  last_pci_addr.as_u32 = pci_addr.as_u32;
332  last_pci_addr_port = i;
333  }
334  else if (pci_addr.as_u32 == last_pci_addr.as_u32)
335  {
337  format (0, "%u", i - last_pci_addr_port);
338  }
339  else
340  {
341  last_pci_addr.as_u32 = ~0;
342  }
343  }
344  else
345  last_pci_addr.as_u32 = ~0;
346 
347  clib_memcpy (&xd->tx_conf, &dev_info.default_txconf,
348  sizeof (struct rte_eth_txconf));
349 
350  if (dm->conf->no_multi_seg)
351  {
352  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
353  port_conf_template.rxmode.jumbo_frame = 0;
354  port_conf_template.rxmode.enable_scatter = 0;
355  }
356  else
357  {
358  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS;
359  port_conf_template.rxmode.jumbo_frame = 1;
360  port_conf_template.rxmode.enable_scatter = 1;
361  xd->flags |= DPDK_DEVICE_FLAG_MAYBE_MULTISEG;
362  }
363 
365  sizeof (struct rte_eth_conf));
366 
367  xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains);
368 
369  if (devconf->num_tx_queues > 0
370  && devconf->num_tx_queues < xd->tx_q_used)
371  xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues);
372 
373  if (devconf->num_rx_queues > 1
374  && dev_info.max_rx_queues >= devconf->num_rx_queues)
375  {
376  xd->rx_q_used = devconf->num_rx_queues;
377  xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
378  if (devconf->rss_fn == 0)
379  xd->port_conf.rx_adv_conf.rss_conf.rss_hf =
380  ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
381  else
382  xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn;
383  }
384  else
385  xd->rx_q_used = 1;
386 
387  xd->flags |= DPDK_DEVICE_FLAG_PMD;
388 
389  /* workaround for drivers not setting driver_name */
390  if ((!dev_info.driver_name) && (pci_dev))
391  dev_info.driver_name = pci_dev->driver->driver.name;
392 
393  ASSERT (dev_info.driver_name);
394 
395  if (!xd->pmd)
396  {
397 
398 
399 #define _(s,f) else if (dev_info.driver_name && \
400  !strcmp(dev_info.driver_name, s)) \
401  xd->pmd = VNET_DPDK_PMD_##f;
402  if (0)
403  ;
405 #undef _
406  else
408 
412 
413  switch (xd->pmd)
414  {
415  /* Drivers with valid speed_capa set */
416  case VNET_DPDK_PMD_E1000EM:
417  case VNET_DPDK_PMD_IGB:
418  case VNET_DPDK_PMD_IXGBE:
419  case VNET_DPDK_PMD_I40E:
420  xd->port_type = port_type_from_speed_capa (&dev_info);
421  xd->supported_flow_actions = VNET_FLOW_ACTION_MARK |
422  VNET_FLOW_ACTION_REDIRECT_TO_NODE |
423  VNET_FLOW_ACTION_BUFFER_ADVANCE |
424  VNET_FLOW_ACTION_COUNT | VNET_FLOW_ACTION_DROP;
425 
426  if (dm->conf->no_tx_checksum_offload == 0)
427  {
428  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOXSUMS;
429  xd->flags |=
430  DPDK_DEVICE_FLAG_TX_OFFLOAD |
431  DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM;
432  }
433 
434 
435  break;
436  case VNET_DPDK_PMD_CXGBE:
437  case VNET_DPDK_PMD_MLX4:
438  case VNET_DPDK_PMD_MLX5:
439  case VNET_DPDK_PMD_QEDE:
440  xd->port_type = port_type_from_speed_capa (&dev_info);
441  break;
442 
443  /* SR-IOV VFs */
444  case VNET_DPDK_PMD_IGBVF:
445  case VNET_DPDK_PMD_IXGBEVF:
446  case VNET_DPDK_PMD_I40EVF:
448  xd->port_conf.rxmode.hw_strip_crc = 1;
449  break;
450 
451  case VNET_DPDK_PMD_THUNDERX:
453  xd->port_conf.rxmode.hw_strip_crc = 1;
454  break;
455 
456  case VNET_DPDK_PMD_ENA:
458  xd->port_conf.rxmode.enable_scatter = 0;
459  break;
460 
461  case VNET_DPDK_PMD_DPAA2:
463  break;
464 
465  /* Cisco VIC */
466  case VNET_DPDK_PMD_ENIC:
467  if (l.link_speed == 40000)
469  else
471  break;
472 
473  /* Intel Red Rock Canyon */
474  case VNET_DPDK_PMD_FM10K:
476  xd->port_conf.rxmode.hw_strip_crc = 1;
477  break;
478 
479  /* virtio */
480  case VNET_DPDK_PMD_VIRTIO:
484  break;
485 
486  /* vmxnet3 */
487  case VNET_DPDK_PMD_VMXNET3:
489  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
490  break;
491 
492  case VNET_DPDK_PMD_AF_PACKET:
494  xd->af_packet_instance_num = af_packet_instance_num++;
495  break;
496 
497  case VNET_DPDK_PMD_BOND:
499  xd->bond_instance_num = bond_ether_instance_num++;
500  break;
501 
502  case VNET_DPDK_PMD_VIRTIO_USER:
504  break;
505 
506  case VNET_DPDK_PMD_VHOST_ETHER:
508  break;
509 
510  case VNET_DPDK_PMD_LIOVF_ETHER:
512  break;
513 
514  case VNET_DPDK_PMD_FAILSAFE:
516  xd->port_conf.intr_conf.lsc = 1;
517  break;
518 
519  default:
521  }
522 
523  if (devconf->num_rx_desc)
524  xd->nb_rx_desc = devconf->num_rx_desc;
525 
526  if (devconf->num_tx_desc)
527  xd->nb_tx_desc = devconf->num_tx_desc;
528  }
529 
530  if (xd->pmd == VNET_DPDK_PMD_AF_PACKET)
531  {
532  f64 now = vlib_time_now (vm);
533  u32 rnd;
534  rnd = (u32) (now * 1e6);
535  rnd = random_u32 (&rnd);
536  clib_memcpy (addr + 2, &rnd, sizeof (rnd));
537  addr[0] = 2;
538  addr[1] = 0xfe;
539  }
540  else
541  rte_eth_macaddr_get (i, (struct ether_addr *) addr);
542 
543  if (xd->tx_q_used < tm->n_vlib_mains)
545 
546  xd->port_id = i;
547  xd->device_index = xd - dm->devices;
548  xd->per_interface_next_index = ~0;
549 
550  /* assign interface to input thread */
551  int q;
552 
553  if (devconf->hqos_enabled)
554  {
555  xd->flags |= DPDK_DEVICE_FLAG_HQOS;
556 
557  int cpu;
558  if (devconf->hqos.hqos_thread_valid)
559  {
560  if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count)
561  return clib_error_return (0, "invalid HQoS thread index");
562 
563  cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread;
564  }
565  else
566  {
567  if (dm->hqos_cpu_count == 0)
568  return clib_error_return (0, "no HQoS threads available");
569 
570  cpu = dm->hqos_cpu_first_index + next_hqos_cpu;
571 
572  next_hqos_cpu++;
573  if (next_hqos_cpu == dm->hqos_cpu_count)
574  next_hqos_cpu = 0;
575 
576  devconf->hqos.hqos_thread_valid = 1;
577  devconf->hqos.hqos_thread = cpu;
578  }
579 
581  vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
582  dq->device = xd->device_index;
583  dq->queue_id = 0;
584  }
585 
586  /* count the number of descriptors used for this device */
587  nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
588 
590  (dm->vnet_main, dpdk_device_class.index, xd->device_index,
591  /* ethernet address */ addr,
593  if (error)
594  return error;
595 
596  /*
597  * Ensure default mtu is not > the mtu read from the hardware.
598  * Otherwise rte_eth_dev_configure() will fail and the port will
599  * not be available.
600  * Calculate max_frame_size and mtu supported by NIC
601  */
602  if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen)
603  {
604  /*
605  * This device does not support the platforms's max frame
606  * size. Use it's advertised mru instead.
607  */
608  max_rx_frame = dev_info.max_rx_pktlen;
609  mtu = dev_info.max_rx_pktlen - sizeof (ethernet_header_t);
610  }
611  else
612  {
613  /* VPP treats MTU and max_rx_pktlen both equal to
614  * ETHERNET_MAX_PACKET_BYTES, if dev_info.max_rx_pktlen >=
615  * ETHERNET_MAX_PACKET_BYTES + sizeof(ethernet_header_t)
616  */
617  if (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES +
618  sizeof (ethernet_header_t)))
619  {
621  max_rx_frame = ETHERNET_MAX_PACKET_BYTES;
622 
623  /*
624  * Some platforms do not account for Ethernet FCS (4 bytes) in
625  * MTU calculations. To interop with them increase mru but only
626  * if the device's settings can support it.
627  */
628  if (xd->port_conf.rxmode.hw_strip_crc &&
629  (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES +
630  sizeof (ethernet_header_t) +
631  4)))
632  {
633  max_rx_frame += 4;
634  }
635  }
636  else
637  {
638  max_rx_frame = ETHERNET_MAX_PACKET_BYTES;
640 
641  if (xd->port_conf.rxmode.hw_strip_crc &&
642  (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)))
643  {
644  max_rx_frame += 4;
645  }
646  }
647  }
648 
649  if (xd->pmd == VNET_DPDK_PMD_FAILSAFE)
650  {
651  /* failsafe device numerables are reported with active device only,
652  * need to query the mtu for current device setup to overwrite
653  * reported value.
654  */
655  uint16_t dev_mtu;
656  if (!rte_eth_dev_get_mtu (i, &dev_mtu))
657  {
658  mtu = dev_mtu;
659  max_rx_frame = mtu + sizeof (ethernet_header_t);
660 
661  if (xd->port_conf.rxmode.hw_strip_crc)
662  {
663  max_rx_frame += 4;
664  }
665  }
666  }
667 
668  /*Set port rxmode config */
669  xd->port_conf.rxmode.max_rx_pkt_len = max_rx_frame;
670 
672  xd->sw_if_index = sw->sw_if_index;
674  dpdk_input_node.index);
675 
676  if (devconf->workers)
677  {
678  int i;
679  q = 0;
680  clib_bitmap_foreach (i, devconf->workers, ({
681  vnet_hw_interface_assign_rx_thread (dm->vnet_main, xd->hw_if_index, q++,
682  vdm->first_worker_thread_index + i);
683  }));
684  }
685  else
686  for (q = 0; q < xd->rx_q_used; q++)
687  {
689  ~1);
690  }
691 
692  /*Get vnet hardware interface */
694 
695  /*Override default max_packet_bytes and max_supported_bytes set in
696  * ethernet_register_interface() above*/
697  if (hi)
698  {
699  hi->max_packet_bytes = mtu;
700  hi->max_supported_packet_bytes = max_rx_frame;
701  }
702 
703  if (dm->conf->no_tx_checksum_offload == 0)
704  if (xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD)
706 
707  dpdk_device_setup (xd);
708 
709  if (vec_len (xd->errors))
710  dpdk_log_err ("setup failed for device %U. Errors:\n %U",
713 
714  if (devconf->hqos_enabled)
715  {
716  clib_error_t *rv;
717  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
718  if (rv)
719  return rv;
720  }
721 
722  /*
723  * For cisco VIC vNIC, set default to VLAN strip enabled, unless
724  * specified otherwise in the startup config.
725  * For other NICs default to VLAN strip disabled, unless specified
726  * otherwis in the startup config.
727  */
728  if (xd->pmd == VNET_DPDK_PMD_ENIC)
729  {
730  if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF)
731  vlan_strip = 1; /* remove vlan tag from VIC port by default */
732  else
733  dpdk_log_warn ("VLAN strip disabled for interface\n");
734  }
735  else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON)
736  vlan_strip = 1;
737 
738  if (vlan_strip)
739  {
740  int vlan_off;
741  vlan_off = rte_eth_dev_get_vlan_offload (xd->port_id);
742  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
743  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
744  if (rte_eth_dev_set_vlan_offload (xd->port_id, vlan_off) == 0)
745  dpdk_log_info ("VLAN strip enabled for interface\n");
746  else
747  dpdk_log_warn ("VLAN strip cannot be supported by interface\n");
748  }
749 
750  if (hi)
751  hi->max_packet_bytes = xd->port_conf.rxmode.max_rx_pkt_len
752  - sizeof (ethernet_header_t);
753  else
754  clib_warning ("hi NULL");
755 
756  if (dm->conf->no_multi_seg)
757  mtu = mtu > ETHER_MAX_LEN ? ETHER_MAX_LEN : mtu;
758 
759  rte_eth_dev_set_mtu (xd->port_id, mtu);
760  }
761  /* *INDENT-ON* */
762 
763  if (nb_desc > dm->conf->num_mbufs)
764  dpdk_log_err ("%d mbufs allocated but total rx/tx ring size is %d\n",
765  dm->conf->num_mbufs, nb_desc);
766 
767  return 0;
768 }
769 
770 static void
772 {
773  clib_error_t *error;
774  u8 *pci_addr = 0;
775  int num_whitelisted = vec_len (conf->dev_confs);
776  vlib_pci_device_info_t *d = 0;
777  vlib_pci_addr_t *addr = 0, *addrs;
778 
779  addrs = vlib_pci_get_all_dev_addrs ();
780  /* *INDENT-OFF* */
781  vec_foreach (addr, addrs)
782  {
783  dpdk_device_config_t * devconf = 0;
784  vec_reset_length (pci_addr);
785  pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, addr, 0);
786  if (d)
787  {
789  d = 0;
790  }
791  d = vlib_pci_get_device_info (addr, &error);
792  if (error)
793  {
794  clib_error_report (error);
795  continue;
796  }
797 
799  continue;
800 
801  if (num_whitelisted)
802  {
803  uword * p = hash_get (conf->device_config_index_by_pci_addr, addr->as_u32);
804 
805  if (!p)
806  continue;
807 
808  devconf = pool_elt_at_index (conf->dev_confs, p[0]);
809  }
810 
811  /* virtio */
812  if (d->vendor_id == 0x1af4 &&
815  ;
816  /* vmxnet3 */
817  else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0)
818  ;
819  /* all Intel network devices */
820  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_NETWORK_ETHERNET)
821  ;
822  /* all Intel QAT devices VFs */
823  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_PROCESSOR_CO &&
824  (d->device_id == 0x0443 || d->device_id == 0x37c9 || d->device_id == 0x19e3))
825  ;
826  /* Cisco VIC */
827  else if (d->vendor_id == 0x1137 && d->device_id == 0x0043)
828  ;
829  /* Chelsio T4/T5 */
830  else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000)
831  ;
832  /* Amazen Elastic Network Adapter */
833  else if (d->vendor_id == 0x1d0f && d->device_id >= 0xec20 && d->device_id <= 0xec21)
834  ;
835  /* Cavium Network Adapter */
836  else if (d->vendor_id == 0x177d && d->device_id == 0x9712)
837  ;
838  /* Cavium FastlinQ QL41000 Series */
839  else if (d->vendor_id == 0x1077 && d->device_id >= 0x8070 && d->device_id <= 0x8090)
840  ;
841  /* Mellanox */
842  else if (d->vendor_id == 0x15b3 && d->device_id >= 0x1013 && d->device_id <= 0x101a)
843  {
844  continue;
845  }
846  else
847  {
848  dpdk_log_warn ("Unsupported PCI device 0x%04x:0x%04x found "
849  "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id,
850  pci_addr);
851  continue;
852  }
853 
854  error = vlib_pci_bind_to_uio (addr, (char *) conf->uio_driver_name);
855 
856  if (error)
857  {
858  if (devconf == 0)
859  {
860  pool_get (conf->dev_confs, devconf);
861  hash_set (conf->device_config_index_by_pci_addr, addr->as_u32,
862  devconf - conf->dev_confs);
863  devconf->pci_addr.as_u32 = addr->as_u32;
864  }
865  devconf->is_blacklisted = 1;
866  clib_error_report (error);
867  }
868  }
869  /* *INDENT-ON* */
870  vec_free (pci_addr);
872 }
873 
874 static clib_error_t *
875 dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
876  unformat_input_t * input, u8 is_default)
877 {
878  clib_error_t *error = 0;
879  uword *p;
880  dpdk_device_config_t *devconf;
881  unformat_input_t sub_input;
882 
883  if (is_default)
884  {
885  devconf = &conf->default_devconf;
886  }
887  else
888  {
889  p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32);
890 
891  if (!p)
892  {
893  pool_get (conf->dev_confs, devconf);
894  hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32,
895  devconf - conf->dev_confs);
896  }
897  else
898  return clib_error_return (0,
899  "duplicate configuration for PCI address %U",
900  format_vlib_pci_addr, &pci_addr);
901  }
902 
903  devconf->pci_addr.as_u32 = pci_addr.as_u32;
904  devconf->hqos_enabled = 0;
906 
907  if (!input)
908  return 0;
909 
912  {
913  if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues))
914  ;
915  else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues))
916  ;
917  else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc))
918  ;
919  else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc))
920  ;
921  else if (unformat (input, "workers %U", unformat_bitmap_list,
922  &devconf->workers))
923  ;
924  else
925  if (unformat
926  (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input))
927  {
928  error = unformat_rss_fn (&sub_input, &devconf->rss_fn);
929  if (error)
930  break;
931  }
932  else if (unformat (input, "vlan-strip-offload off"))
934  else if (unformat (input, "vlan-strip-offload on"))
936  else
937  if (unformat
938  (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input))
939  {
940  devconf->hqos_enabled = 1;
941  error = unformat_hqos (&sub_input, &devconf->hqos);
942  if (error)
943  break;
944  }
945  else if (unformat (input, "hqos"))
946  {
947  devconf->hqos_enabled = 1;
948  }
949  else
950  {
951  error = clib_error_return (0, "unknown input `%U'",
952  format_unformat_error, input);
953  break;
954  }
955  }
956 
957  if (error)
958  return error;
959 
960  if (devconf->workers && devconf->num_rx_queues == 0)
961  devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers);
962  else if (devconf->workers &&
963  clib_bitmap_count_set_bits (devconf->workers) !=
964  devconf->num_rx_queues)
965  error =
967  "%U: number of worker threadds must be "
968  "equal to number of rx queues", format_vlib_pci_addr,
969  &pci_addr);
970 
971  return error;
972 }
973 
974 static clib_error_t *
976 {
977  unformat_input_t input;
978  u8 *line, *s = 0;
979  int n, n_try;
980 
981  n = n_try = 4096;
982  while (n == n_try)
983  {
984  uword len = vec_len (s);
985  vec_resize (s, len + n_try);
986 
987  n = read (uf->file_descriptor, s + len, n_try);
988  if (n < 0 && errno != EAGAIN)
989  return clib_error_return_unix (0, "read");
990  _vec_len (s) = len + (n < 0 ? 0 : n);
991  }
992 
993  unformat_init_vector (&input, s);
994 
995  while (unformat_user (&input, unformat_line, &line))
996  {
997  dpdk_log_notice ("%v", line);
998  vec_free (line);
999  }
1000 
1001  unformat_free (&input);
1002  return 0;
1003 }
1004 
1005 static clib_error_t *
1007 {
1008  clib_error_t *error = 0;
1011  dpdk_device_config_t *devconf;
1012  vlib_pci_addr_t pci_addr;
1013  unformat_input_t sub_input;
1014  uword x;
1015  u8 *s, *tmp = 0;
1016  u32 log_level;
1017  int ret, i;
1018  int num_whitelisted = 0;
1019  u8 no_pci = 0;
1020  u8 no_huge = 0;
1021  u8 huge_dir = 0;
1022  u8 file_prefix = 0;
1023  u8 *socket_mem = 0;
1024  u8 *huge_dir_path = 0;
1025 
1026  huge_dir_path =
1027  format (0, "%s/hugepages%c", vlib_unix_get_runtime_dir (), 0);
1028 
1029  conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
1030  log_level = RTE_LOG_NOTICE;
1031 
1032  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1033  {
1034  /* Prime the pump */
1035  if (unformat (input, "no-hugetlb"))
1036  {
1037  vec_add1 (conf->eal_init_args, (u8 *) "--no-huge");
1038  no_huge = 1;
1039  }
1040 
1041  else if (unformat (input, "enable-tcp-udp-checksum"))
1042  conf->enable_tcp_udp_checksum = 1;
1043 
1044  else if (unformat (input, "no-tx-checksum-offload"))
1045  conf->no_tx_checksum_offload = 1;
1046 
1047  else if (unformat (input, "decimal-interface-names"))
1049 
1050  else if (unformat (input, "log-level %U", unformat_dpdk_log_level, &x))
1051  log_level = x;
1052 
1053  else if (unformat (input, "no-multi-seg"))
1054  conf->no_multi_seg = 1;
1055 
1056  else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
1057  &sub_input))
1058  {
1059  error =
1060  dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input,
1061  1);
1062 
1063  if (error)
1064  return error;
1065  }
1066  else
1067  if (unformat
1068  (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr,
1069  unformat_vlib_cli_sub_input, &sub_input))
1070  {
1071  error = dpdk_device_config (conf, pci_addr, &sub_input, 0);
1072 
1073  if (error)
1074  return error;
1075 
1076  num_whitelisted++;
1077  }
1078  else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr))
1079  {
1080  error = dpdk_device_config (conf, pci_addr, 0, 0);
1081 
1082  if (error)
1083  return error;
1084 
1085  num_whitelisted++;
1086  }
1087  else if (unformat (input, "num-mbufs %d", &conf->num_mbufs))
1088  ;
1089  else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
1090  ;
1091  else if (unformat (input, "socket-mem %s", &socket_mem))
1092  ;
1093  else if (unformat (input, "no-pci"))
1094  {
1095  no_pci = 1;
1096  tmp = format (0, "--no-pci%c", 0);
1097  vec_add1 (conf->eal_init_args, tmp);
1098  }
1099 
1100 #define _(a) \
1101  else if (unformat(input, #a)) \
1102  { \
1103  tmp = format (0, "--%s%c", #a, 0); \
1104  vec_add1 (conf->eal_init_args, tmp); \
1105  }
1107 #undef _
1108 #define _(a) \
1109  else if (unformat(input, #a " %s", &s)) \
1110  { \
1111  if (!strncmp(#a, "huge-dir", 8)) \
1112  huge_dir = 1; \
1113  else if (!strncmp(#a, "file-prefix", 11)) \
1114  file_prefix = 1; \
1115  tmp = format (0, "--%s%c", #a, 0); \
1116  vec_add1 (conf->eal_init_args, tmp); \
1117  vec_add1 (s, 0); \
1118  if (!strncmp(#a, "vdev", 4)) \
1119  if (strstr((char*)s, "af_packet")) \
1120  clib_warning ("af_packet obsoleted. Use CLI 'create host-interface'."); \
1121  vec_add1 (conf->eal_init_args, s); \
1122  }
1124 #undef _
1125 #define _(a,b) \
1126  else if (unformat(input, #a " %s", &s)) \
1127  { \
1128  tmp = format (0, "-%s%c", #b, 0); \
1129  vec_add1 (conf->eal_init_args, tmp); \
1130  vec_add1 (s, 0); \
1131  vec_add1 (conf->eal_init_args, s); \
1132  }
1134 #undef _
1135 #define _(a,b) \
1136  else if (unformat(input, #a " %s", &s)) \
1137  { \
1138  tmp = format (0, "-%s%c", #b, 0); \
1139  vec_add1 (conf->eal_init_args, tmp); \
1140  vec_add1 (s, 0); \
1141  vec_add1 (conf->eal_init_args, s); \
1142  conf->a##_set_manually = 1; \
1143  }
1145 #undef _
1146  else if (unformat (input, "default"))
1147  ;
1148 
1149  else if (unformat_skip_white_space (input))
1150  ;
1151  else
1152  {
1153  error = clib_error_return (0, "unknown input `%U'",
1154  format_unformat_error, input);
1155  goto done;
1156  }
1157  }
1158 
1159  if (!conf->uio_driver_name)
1160  conf->uio_driver_name = format (0, "auto%c", 0);
1161 
1162  /*
1163  * Use 1G huge pages if available.
1164  */
1165  if (!no_huge && !huge_dir)
1166  {
1167  u32 x, *mem_by_socket = 0;
1168  uword c = 0;
1169  int rv;
1170 
1171  umount ((char *) huge_dir_path);
1172 
1173  /* Process "socket-mem" parameter value */
1174  if (vec_len (socket_mem))
1175  {
1176  unformat_input_t in;
1177  unformat_init_vector (&in, socket_mem);
1179  {
1180  if (unformat (&in, "%u,", &x))
1181  ;
1182  else if (unformat (&in, "%u", &x))
1183  ;
1184  else if (unformat (&in, ","))
1185  x = 0;
1186  else
1187  break;
1188 
1189  vec_add1 (mem_by_socket, x);
1190  }
1191  /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */
1192  unformat_free (&in);
1193  socket_mem = 0;
1194  }
1195  else
1196  {
1197  /* *INDENT-OFF* */
1199  {
1200  vec_validate(mem_by_socket, c);
1201  mem_by_socket[c] = 64; /* default per-socket mem */
1202  }
1203  ));
1204  /* *INDENT-ON* */
1205  }
1206 
1207  /* *INDENT-OFF* */
1209  {
1210  clib_error_t *e;
1211 
1212  vec_validate(mem_by_socket, c);
1213 
1214  e = clib_sysfs_prealloc_hugepages(c, 2 << 10, mem_by_socket[c] / 2);
1215  if (e)
1216  clib_error_report (e);
1217  }));
1218  /* *INDENT-ON* */
1219 
1220  if (mem_by_socket == 0)
1221  {
1222  error = clib_error_return (0, "mem_by_socket NULL");
1223  goto done;
1224  }
1225  _vec_len (mem_by_socket) = c + 1;
1226 
1227  /* regenerate socket_mem string */
1228  vec_foreach_index (x, mem_by_socket)
1229  socket_mem = format (socket_mem, "%s%u",
1230  socket_mem ? "," : "", mem_by_socket[x]);
1231  socket_mem = format (socket_mem, "%c", 0);
1232 
1233  vec_free (mem_by_socket);
1234 
1235  error = vlib_unix_recursive_mkdir ((char *) huge_dir_path);
1236  if (error)
1237  {
1238  goto done;
1239  }
1240 
1241  rv = mount ("none", (char *) huge_dir_path, "hugetlbfs", 0, NULL);
1242 
1243  if (rv)
1244  {
1245  error = clib_error_return (0, "mount failed %d", errno);
1246  goto done;
1247  }
1248 
1249  tmp = format (0, "--huge-dir%c", 0);
1250  vec_add1 (conf->eal_init_args, tmp);
1251  tmp = format (0, "%s%c", huge_dir_path, 0);
1252  vec_add1 (conf->eal_init_args, tmp);
1253  if (!file_prefix)
1254  {
1255  tmp = format (0, "--file-prefix%c", 0);
1256  vec_add1 (conf->eal_init_args, tmp);
1257  tmp = format (0, "vpp%c", 0);
1258  vec_add1 (conf->eal_init_args, tmp);
1259  }
1260  }
1261 
1262  if (error)
1263  return error;
1264 
1265  /* I'll bet that -c and -n must be the first and second args... */
1266  if (!conf->coremask_set_manually)
1267  {
1269  uword *coremask = 0;
1270  int i;
1271 
1272  /* main thread core */
1273  coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
1274 
1275  for (i = 0; i < vec_len (tm->registrations); i++)
1276  {
1277  tr = tm->registrations[i];
1278  coremask = clib_bitmap_or (coremask, tr->coremask);
1279  }
1280 
1281  vec_insert (conf->eal_init_args, 2, 1);
1282  conf->eal_init_args[1] = (u8 *) "-c";
1283  tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
1284  conf->eal_init_args[2] = tmp;
1285  clib_bitmap_free (coremask);
1286  }
1287 
1288  if (!conf->nchannels_set_manually)
1289  {
1290  vec_insert (conf->eal_init_args, 2, 3);
1291  conf->eal_init_args[3] = (u8 *) "-n";
1292  tmp = format (0, "%d", conf->nchannels);
1293  conf->eal_init_args[4] = tmp;
1294  }
1295 
1296  if (no_pci == 0 && geteuid () == 0)
1297  dpdk_bind_devices_to_uio (conf);
1298 
1299 #define _(x) \
1300  if (devconf->x == 0 && conf->default_devconf.x > 0) \
1301  devconf->x = conf->default_devconf.x ;
1302 
1303  /* *INDENT-OFF* */
1304  pool_foreach (devconf, conf->dev_confs, ({
1305 
1306  /* default per-device config items */
1307  foreach_dpdk_device_config_item
1308 
1309  /* add DPDK EAL whitelist/blacklist entry */
1310  if (num_whitelisted > 0 && devconf->is_blacklisted == 0)
1311  {
1312  tmp = format (0, "-w%c", 0);
1313  vec_add1 (conf->eal_init_args, tmp);
1314  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1315  vec_add1 (conf->eal_init_args, tmp);
1316  }
1317  else if (num_whitelisted == 0 && devconf->is_blacklisted != 0)
1318  {
1319  tmp = format (0, "-b%c", 0);
1320  vec_add1 (conf->eal_init_args, tmp);
1321  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1322  vec_add1 (conf->eal_init_args, tmp);
1323  }
1324  }));
1325  /* *INDENT-ON* */
1326 
1327 #undef _
1328 
1329  /* set master-lcore */
1330  tmp = format (0, "--master-lcore%c", 0);
1331  vec_add1 (conf->eal_init_args, tmp);
1332  tmp = format (0, "%u%c", tm->main_lcore, 0);
1333  vec_add1 (conf->eal_init_args, tmp);
1334 
1335  /* set socket-mem */
1336  if (!no_huge)
1337  {
1338  tmp = format (0, "--socket-mem%c", 0);
1339  vec_add1 (conf->eal_init_args, tmp);
1340  tmp = format (0, "%s%c", socket_mem, 0);
1341  vec_add1 (conf->eal_init_args, tmp);
1342  }
1343 
1344  /* NULL terminate the "argv" vector, in case of stupidity */
1345  vec_add1 (conf->eal_init_args, 0);
1346  _vec_len (conf->eal_init_args) -= 1;
1347 
1348  /* Set up DPDK eal and packet mbuf pool early. */
1349 
1350  rte_log_set_global_level (log_level);
1351  int log_fds[2] = { 0 };
1352  if (pipe (log_fds) == 0)
1353  {
1354  FILE *f = fdopen (log_fds[1], "a");
1355  if (f && rte_openlog_stream (f) == 0)
1356  {
1357  clib_file_t t = { 0 };
1359  t.file_descriptor = log_fds[0];
1360  t.description = format (0, "DPDK logging pipe");
1361  clib_file_add (&file_main, &t);
1362  }
1363  }
1364 
1365  vm = vlib_get_main ();
1366 
1367  /* make copy of args as rte_eal_init tends to mess up with arg array */
1368  for (i = 1; i < vec_len (conf->eal_init_args); i++)
1369  conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ",
1370  conf->eal_init_args[i]);
1371 
1372  dpdk_log_warn ("EAL init args: %s", conf->eal_init_args_str);
1373  ret = rte_eal_init (vec_len (conf->eal_init_args),
1374  (char **) conf->eal_init_args);
1375 
1376  /* lazy umount hugepages */
1377  umount2 ((char *) huge_dir_path, MNT_DETACH);
1378  rmdir ((char *) huge_dir_path);
1379  vec_free (huge_dir_path);
1380 
1381  if (ret < 0)
1382  return clib_error_return (0, "rte_eal_init returned %d", ret);
1383 
1384  /* set custom ring memory allocator */
1385  {
1386  struct rte_mempool_ops *ops = NULL;
1387 
1388  ops = get_ops_by_name ("ring_sp_sc");
1389  ops->alloc = dpdk_ring_alloc;
1390 
1391  ops = get_ops_by_name ("ring_mp_sc");
1392  ops->alloc = dpdk_ring_alloc;
1393 
1394  ops = get_ops_by_name ("ring_sp_mc");
1395  ops->alloc = dpdk_ring_alloc;
1396 
1397  ops = get_ops_by_name ("ring_mp_mc");
1398  ops->alloc = dpdk_ring_alloc;
1399  }
1400 
1401  /* main thread 1st */
1402  error = dpdk_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ());
1403  if (error)
1404  return error;
1405 
1406  for (i = 0; i < RTE_MAX_LCORE; i++)
1407  {
1408  error = dpdk_buffer_pool_create (vm, conf->num_mbufs,
1409  rte_lcore_to_socket_id (i));
1410  if (error)
1411  return error;
1412  }
1413 
1414 done:
1415  return error;
1416 }
1417 
1419 
1420 void
1422 {
1423  vnet_main_t *vnm = vnet_get_main ();
1424  struct rte_eth_link prev_link = xd->link;
1425  u32 hw_flags = 0;
1426  u8 hw_flags_chg = 0;
1427 
1428  /* only update link state for PMD interfaces */
1429  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
1430  return;
1431 
1432  xd->time_last_link_update = now ? now : xd->time_last_link_update;
1433  memset (&xd->link, 0, sizeof (xd->link));
1434  rte_eth_link_get_nowait (xd->port_id, &xd->link);
1435 
1436  if (LINK_STATE_ELOGS)
1437  {
1439  ELOG_TYPE_DECLARE (e) =
1440  {
1441  .format =
1442  "update-link-state: sw_if_index %d, admin_up %d,"
1443  "old link_state %d new link_state %d",.format_args = "i4i1i1i1",};
1444 
1445  struct
1446  {
1447  u32 sw_if_index;
1448  u8 admin_up;
1449  u8 old_link_state;
1450  u8 new_link_state;
1451  } *ed;
1452  ed = ELOG_DATA (&vm->elog_main, e);
1453  ed->sw_if_index = xd->sw_if_index;
1454  ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
1455  ed->old_link_state = (u8)
1457  ed->new_link_state = (u8) xd->link.link_status;
1458  }
1459 
1460  if ((xd->flags & (DPDK_DEVICE_FLAG_ADMIN_UP | DPDK_DEVICE_FLAG_BOND_SLAVE))
1461  && ((xd->link.link_status != 0) ^
1463  {
1464  hw_flags_chg = 1;
1465  hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
1466  }
1467 
1468  if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex))
1469  {
1470  hw_flags_chg = 1;
1471  switch (xd->link.link_duplex)
1472  {
1473  case ETH_LINK_HALF_DUPLEX:
1475  break;
1476  case ETH_LINK_FULL_DUPLEX:
1478  break;
1479  default:
1480  break;
1481  }
1482  }
1483  if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed))
1484  {
1485  hw_flags_chg = 1;
1486  switch (xd->link.link_speed)
1487  {
1488  case ETH_SPEED_NUM_10M:
1490  break;
1491  case ETH_SPEED_NUM_100M:
1493  break;
1494  case ETH_SPEED_NUM_1G:
1495  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G;
1496  break;
1497  case ETH_SPEED_NUM_2_5G:
1499  break;
1500  case ETH_SPEED_NUM_5G:
1501  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_5G;
1502  break;
1503  case ETH_SPEED_NUM_10G:
1505  break;
1506  case ETH_SPEED_NUM_20G:
1508  break;
1509  case ETH_SPEED_NUM_25G:
1511  break;
1512  case ETH_SPEED_NUM_40G:
1514  break;
1515  case ETH_SPEED_NUM_50G:
1517  break;
1518  case ETH_SPEED_NUM_56G:
1520  break;
1521  case ETH_SPEED_NUM_100G:
1523  break;
1524  case 0:
1525  break;
1526  default:
1527  dpdk_log_warn ("unknown link speed %d", xd->link.link_speed);
1528  break;
1529  }
1530  }
1531  if (hw_flags_chg)
1532  {
1533  if (LINK_STATE_ELOGS)
1534  {
1536 
1537  ELOG_TYPE_DECLARE (e) =
1538  {
1539  .format =
1540  "update-link-state: sw_if_index %d, new flags %d",.format_args
1541  = "i4i4",};
1542 
1543  struct
1544  {
1545  u32 sw_if_index;
1546  u32 flags;
1547  } *ed;
1548  ed = ELOG_DATA (&vm->elog_main, e);
1549  ed->sw_if_index = xd->sw_if_index;
1550  ed->flags = hw_flags;
1551  }
1552  vnet_hw_interface_set_flags (vnm, xd->hw_if_index, hw_flags);
1553  }
1554 }
1555 
1556 static uword
1558 {
1559  clib_error_t *error;
1560  vnet_main_t *vnm = vnet_get_main ();
1561  dpdk_main_t *dm = &dpdk_main;
1563  dpdk_device_t *xd;
1565  int i;
1566  int j;
1567 
1568  error = dpdk_lib_init (dm);
1569 
1570  if (error)
1571  clib_error_report (error);
1572 
1573  tm->worker_thread_release = 1;
1574 
1575  f64 now = vlib_time_now (vm);
1576  vec_foreach (xd, dm->devices)
1577  {
1578  dpdk_update_link_state (xd, now);
1579  }
1580 
1581  {
1582  /*
1583  * Extra set up for bond interfaces:
1584  * 1. Setup MACs for bond interfaces and their slave links which was set
1585  * in dpdk_device_setup() but needs to be done again here to take
1586  * effect.
1587  * 2. Set up info and register slave link state change callback handling.
1588  * 3. Set up info for bond interface related CLI support.
1589  */
1590 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
1591  int nports = rte_eth_dev_count ();
1592 #else
1593  int nports = rte_eth_dev_count_avail ();
1594 #endif
1595  if (nports > 0)
1596  {
1597  /* *INDENT-OFF* */
1598  RTE_ETH_FOREACH_DEV(i)
1599  {
1600  xd = NULL;
1601  for (j = 0; j < nports; j++)
1602  {
1603  if (dm->devices[j].port_id == i)
1604  {
1605  xd = &dm->devices[j];
1606  }
1607  }
1608  ASSERT (xd != NULL);
1609 
1610  if (xd->pmd == VNET_DPDK_PMD_BOND)
1611  {
1612  u8 addr[6];
1613  dpdk_portid_t slink[16];
1614  int nlink = rte_eth_bond_slaves_get (i, slink, 16);
1615  if (nlink > 0)
1616  {
1617  vnet_hw_interface_t *bhi;
1618  ethernet_interface_t *bei;
1619  int rv;
1620 
1621  /* Get MAC of 1st slave link */
1622  rte_eth_macaddr_get
1623  (slink[0], (struct ether_addr *) addr);
1624 
1625  /* Set MAC of bounded interface to that of 1st slave link */
1626  dpdk_log_info ("Set MAC for bond port %d BondEthernet%d",
1627  i, xd->bond_instance_num);
1628  rv = rte_eth_bond_mac_address_set
1629  (i, (struct ether_addr *) addr);
1630  if (rv)
1631  dpdk_log_warn ("Set MAC addr failure rv=%d", rv);
1632 
1633  /* Populate MAC of bonded interface in VPP hw tables */
1634  bhi = vnet_get_hw_interface
1635  (vnm, dm->devices[i].hw_if_index);
1636  bei = pool_elt_at_index
1637  (em->interfaces, bhi->hw_instance);
1638  clib_memcpy (bhi->hw_address, addr, 6);
1639  clib_memcpy (bei->address, addr, 6);
1640 
1641  /* Init l3 packet size allowed on bonded interface */
1643  while (nlink >= 1)
1644  { /* for all slave links */
1645  int slave = slink[--nlink];
1646  dpdk_device_t *sdev = &dm->devices[slave];
1647  vnet_hw_interface_t *shi;
1648  vnet_sw_interface_t *ssi;
1649  ethernet_interface_t *sei;
1650  /* Add MAC to all slave links except the first one */
1651  if (nlink)
1652  {
1653  dpdk_log_info ("Add MAC for slave port %d",
1654  slave);
1655  rv = rte_eth_dev_mac_addr_add
1656  (slave, (struct ether_addr *) addr, 0);
1657  if (rv)
1658  dpdk_log_warn ("Add MAC addr failure rv=%d",
1659  rv);
1660  }
1661  /* Setup slave link state change callback handling */
1662  rte_eth_dev_callback_register
1663  (slave, RTE_ETH_EVENT_INTR_LSC,
1665  dpdk_device_t *sxd = &dm->devices[slave];
1666  sxd->flags |= DPDK_DEVICE_FLAG_BOND_SLAVE;
1667  sxd->bond_port = i;
1668  /* Set slaves bitmap for bonded interface */
1669  bhi->bond_info = clib_bitmap_set
1670  (bhi->bond_info, sdev->hw_if_index, 1);
1671  /* Set MACs and slave link flags on slave interface */
1672  shi = vnet_get_hw_interface (vnm, sdev->hw_if_index);
1673  ssi = vnet_get_sw_interface (vnm, sdev->sw_if_index);
1674  sei = pool_elt_at_index
1675  (em->interfaces, shi->hw_instance);
1678  clib_memcpy (shi->hw_address, addr, 6);
1679  clib_memcpy (sei->address, addr, 6);
1680  /* Set l3 packet size allowed as the lowest of slave */
1681  if (bhi->max_packet_bytes > shi->max_packet_bytes)
1682  bhi->max_packet_bytes = shi->max_packet_bytes;
1683  }
1684  }
1685  }
1686  }
1687  /* *INDENT-ON* */
1688  }
1689  }
1690 
1691  while (1)
1692  {
1693  /*
1694  * check each time through the loop in case intervals are changed
1695  */
1696  f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ?
1698 
1699  vlib_process_wait_for_event_or_clock (vm, min_wait);
1700 
1701  if (dm->admin_up_down_in_progress)
1702  /* skip the poll if an admin up down is in progress (on any interface) */
1703  continue;
1704 
1705  vec_foreach (xd, dm->devices)
1706  {
1707  f64 now = vlib_time_now (vm);
1708  if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval)
1709  dpdk_update_counters (xd, now);
1710  if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval)
1711  dpdk_update_link_state (xd, now);
1712 
1713  }
1714  }
1715 
1716  return 0;
1717 }
1718 
1719 /* *INDENT-OFF* */
1721  .function = dpdk_process,
1722  .type = VLIB_NODE_TYPE_PROCESS,
1723  .name = "dpdk-process",
1724  .process_log2_n_stack_bytes = 17,
1725 };
1726 /* *INDENT-ON* */
1727 
1728 static clib_error_t *
1730 {
1731  dpdk_main_t *dm = &dpdk_main;
1732  clib_error_t *error = 0;
1734 
1735  /* verify that structs are cacheline aligned */
1736  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
1737  "Cache line marker must be 1st element in dpdk_device_t");
1738  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) ==
1740  "Data in cache line 0 is bigger than cache line size");
1741  STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0,
1742  "Cache line marker must be 1st element in frame_queue_trace_t");
1743  STATIC_ASSERT (RTE_CACHE_LINE_SIZE == 1 << CLIB_LOG2_CACHE_LINE_BYTES,
1744  "DPDK RTE CACHE LINE SIZE does not match with 1<<CLIB_LOG2_CACHE_LINE_BYTES");
1745 
1746  dm->vlib_main = vm;
1747  dm->vnet_main = vnet_get_main ();
1748  dm->conf = &dpdk_config_main;
1749 
1750  dm->conf->nchannels = 4;
1751  dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
1752  vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
1753 
1754  vec_validate (dm->recycle, tm->n_thread_stacks - 1);
1755 
1756  /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
1757  dm->buffer_flags_template =
1758  (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID
1759  | VNET_BUFFER_F_L4_CHECKSUM_COMPUTED |
1760  VNET_BUFFER_F_L4_CHECKSUM_CORRECT | VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
1761 
1764 
1765  /* init CLI */
1766  if ((error = vlib_call_init_function (vm, dpdk_cli_init)))
1767  return error;
1768 
1769  dm->log_default = vlib_log_register_class ("dpdk", 0);
1770 
1771  return error;
1772 }
1773 
1775 
1776 
1777 /*
1778  * fd.io coding-style-patch-verification: ON
1779  *
1780  * Local Variables:
1781  * eval: (c-set-style "gnu")
1782  * End:
1783  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:221
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
static void dpdk_bind_devices_to_uio(dpdk_config_main_t *conf)
Definition: init.c:771
f64 time_last_link_update
Definition: dpdk.h:252
vmrglw vmrglh hi
format_function_t format_vlib_pci_addr
Definition: pci.h:288
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:255
#define VIRTIO_PCI_MODERN_DEVICEID_NET
Definition: pci_config.h:169
#define clib_min(x, y)
Definition: clib.h:289
#define VNET_HW_INTERFACE_FLAG_SPEED_1G
Definition: interface.h:471
ethernet_main_t ethernet_main
Definition: init.c:45
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:541
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
#define VNET_HW_INTERFACE_FLAG_SPEED_50G
Definition: interface.h:478
u8 interface_name_format_decimal
Definition: dpdk.h:367
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
#define NB_MBUF
Definition: dpdk.h:59
vnet_device_class_t dpdk_device_class
u32 sw_if_index
Definition: dpdk.h:204
#define DPDK_DEVICE_VLAN_STRIP_OFF
Definition: dpdk.h:334
#define NULL
Definition: clib.h:55
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:228
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:574
static uword * clib_bitmap_or(uword *ai, uword *bi)
Logical operator across two bitmaps.
static u32 dpdk_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 flags)
Definition: init.c:92
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1421
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_pci_addr_t *addr, clib_error_t **error)
Definition: pci.c:162
u16 flags
Definition: dpdk.h:212
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define LINK_STATE_ELOGS
Definition: init.c:45
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:53
static struct rte_mempool_ops * get_ops_by_name(char *ops_name)
Definition: init.c:137
u32 file_descriptor
Definition: file.h:54
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
dpdk_device_and_queue_t ** devices_by_hqos_cpu
Definition: dpdk.h:405
#define DPDK_NB_RX_DESC_VIRTIO
Definition: dpdk_priv.h:21
clib_error_t * errors
Definition: dpdk.h:266
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
u32 per_interface_next_index
Definition: dpdk.h:207
u8 enable_tcp_udp_checksum
Definition: dpdk.h:353
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:983
#define DPDK_DEVICE_VLAN_STRIP_ON
Definition: dpdk.h:335
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
#define VNET_HW_INTERFACE_FLAG_SPEED_100G
Definition: interface.h:480
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define DPDK_NB_TX_DESC_DEFAULT
Definition: dpdk_priv.h:20
u32 supported_flow_actions
Definition: dpdk.h:232
#define foreach_eal_double_hyphen_predicate_arg
Definition: dpdk_priv.h:32
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:287
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:458
int dpdk_port_state_callback(dpdk_portid_t port_id, enum rte_eth_event_type type, void *param, void *ret_param)
Definition: common.c:319
u16 bond_instance_num
Definition: dpdk.h:245
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
dpdk_device_config_hqos_t hqos
Definition: dpdk.h:342
vlib_pci_addr_t * vlib_pci_get_all_dev_addrs()
Definition: pci.c:1242
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
vhost_vring_addr_t addr
Definition: vhost_user.h:116
unsigned char u8
Definition: types.h:56
dpdk_config_main_t dpdk_config_main
Definition: init.c:43
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
#define ETHER_MAX_LEN
Maximum frame len, including CRC.
Definition: init.c:40
double f64
Definition: types.h:142
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define VNET_HW_INTERFACE_FLAG_SPEED_2_5G
Definition: interface.h:472
foreach_dpdk_device_config_item clib_bitmap_t * workers
Definition: dpdk.h:340
#define dpdk_log_warn(...)
Definition: dpdk.h:502
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
static clib_error_t * dpdk_log_read_ready(clib_file_t *uf)
Definition: init.c:975
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:156
dpdk_portid_t port_id
Definition: dpdk.h:201
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:649
dpdk_device_config_t default_devconf
Definition: dpdk.h:370
f64 stat_poll_interval
Definition: dpdk.h:435
static char * vlib_unix_get_runtime_dir(void)
Definition: unix.h:141
static dpdk_port_type_t port_type_from_speed_capa(struct rte_eth_dev_info *dev_info)
Definition: init.c:63
clib_error_t * vlib_pci_bind_to_uio(vlib_pci_addr_t *addr, char *uio_drv_name)
Definition: pci.c:345
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
u16 rx_q_used
Definition: dpdk.h:224
#define VNET_HW_INTERFACE_FLAG_SPEED_25G
Definition: interface.h:476
clib_file_main_t file_main
Definition: main.c:63
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:240
unsigned int u32
Definition: types.h:88
void dpdk_device_setup(dpdk_device_t *xd)
Definition: common.c:40
#define vlib_call_init_function(vm, x)
Definition: init.h:227
clib_error_t * unformat_rss_fn(unformat_input_t *input, uword *rss_fn)
Definition: format.c:835
#define DPDK_NB_TX_DESC_VIRTIO
Definition: dpdk_priv.h:22
struct rte_eth_conf port_conf
Definition: dpdk.h:228
static clib_error_t * dpdk_init(vlib_main_t *vm)
Definition: init.c:1729
#define fl(x, y)
u32 max_supported_packet_bytes
Definition: interface.h:534
f64 time_last_stats_update
Definition: dpdk.h:259
struct rte_eth_txconf tx_conf
Definition: dpdk.h:229
u8 * description
Definition: file.h:70
#define hash_get(h, key)
Definition: hash.h:249
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
#define vec_insert(V, N, M)
Insert N vector elements starting at element M, initialize new elements to zero (no header...
Definition: vec.h:687
vlib_pci_addr_t pci_addr
Definition: dpdk.h:330
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
Definition: buffer.c:542
static clib_error_t * dpdk_config(vlib_main_t *vm, unformat_input_t *input)
Definition: init.c:1006
#define foreach_eal_double_hyphen_arg
Definition: dpdk_priv.h:48
dpdk_portid_t bond_port
Definition: dpdk.h:249
#define ETHERNET_INTERFACE_FLAG_CONFIG_MTU(flags)
Definition: ethernet.h:153
u8 ** eal_init_args
Definition: dpdk.h:349
#define VNET_HW_INTERFACE_FLAG_SPEED_10M
Definition: interface.h:469
dpdk_per_thread_data_t * per_thread_data
Definition: dpdk.h:406
#define VNET_SW_INTERFACE_FLAG_BOND_SLAVE
Definition: interface.h:670
#define VNET_HW_INTERFACE_FLAG_SPEED_56G
Definition: interface.h:479
#define foreach_eal_single_hyphen_mandatory_arg
Definition: dpdk_priv.h:38
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
#define VNET_HW_INTERFACE_FLAG_HALF_DUPLEX
Definition: interface.h:461
#define clib_error_return_unix(e, args...)
Definition: error.h:102
#define foreach_dpdk_pmd
Definition: dpdk.h:64
vlib_buffer_t buffer_template
Definition: dpdk.h:388
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:439
#define ELOG_DATA(em, f)
Definition: elog.h:481
#define VIRTIO_PCI_LEGACY_DEVICEID_NET
Definition: pci_config.h:168
dpdk_port_type_t port_type
Definition: dpdk.h:260
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:164
signed char i8
Definition: types.h:45
u16 tx_q_used
Definition: dpdk.h:223
u16 nb_rx_desc
Definition: dpdk.h:225
uint16_t dpdk_portid_t
Definition: dpdk.h:121
#define dpdk_log_info(...)
Definition: dpdk.h:506
u32 hw_if_index
Definition: dpdk.h:203
u32 flags
Definition: vhost_user.h:110
void unformat_init_vector(unformat_input_t *input, u8 *vector_string)
Definition: unformat.c:1031
u32 ** recycle
Definition: dpdk.h:409
#define VNET_HW_INTERFACE_BOND_INFO_SLAVE
Definition: interface.h:554
u16 af_packet_instance_num
Definition: dpdk.h:244
#define foreach_eal_single_hyphen_arg
Definition: dpdk_priv.h:42
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
#define DPDK_NB_RX_DESC_DEFAULT
Definition: dpdk_priv.h:19
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
static clib_error_t * dpdk_device_config(dpdk_config_main_t *conf, vlib_pci_addr_t pci_addr, unformat_input_t *input, u8 is_default)
Definition: init.c:875
dpdk_device_t * devices
Definition: dpdk.h:404
vlib_main_t * vm
Definition: buffer.c:294
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:87
u8 nchannels_set_manually
Definition: dpdk.h:358
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
#define dpdk_log_err(...)
Definition: dpdk.h:500
volatile u32 ** lockp
Definition: dpdk.h:195
dpdk_device_config_t * dev_confs
Definition: dpdk.h:371
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
dpdk_pmd_t pmd
Definition: dpdk.h:209
format_function_t format_dpdk_device_errors
Definition: dpdk.h:513
elog_main_t elog_main
Definition: main.h:161
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:147
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD
Definition: interface.h:499
u8 coremask_set_manually
Definition: dpdk.h:357
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
#define VNET_HW_INTERFACE_FLAG_SPEED_10G
Definition: interface.h:474
static void dpdk_device_lock_init(dpdk_device_t *xd)
Definition: init.c:124
u8 * interface_name_suffix
Definition: dpdk.h:217
signed int i32
Definition: types.h:81
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define VNET_HW_INTERFACE_FLAG_FULL_DUPLEX
Definition: interface.h:462
#define ASSERT(truth)
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:205
format_function_t format_dpdk_device_name
Definition: dpdk.h:511
int hqos_cpu_count
Definition: dpdk.h:431
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:138
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
vlib_log_class_t log_default
Definition: dpdk.h:449
static uword dpdk_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: init.c:1557
Bitmaps built as vectors of machine words.
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:275
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
#define DPDK_LINK_POLL_INTERVAL
Definition: dpdk.h:272
size_t count
Definition: vapi.c:46
dpdk_main_t dpdk_main
Definition: init.c:42
uword * thread_registrations_by_name
Definition: threads.h:297
clib_error_t * dpdk_cli_init(vlib_main_t *vm)
Definition: cli.c:2090
dpdk_portid_t device_index
Definition: dpdk.h:198
struct rte_eth_link link
Definition: dpdk.h:251
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:247
dpdk_port_type_t
Definition: dpdk.h:99
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
Definition: defs.h:47
clib_error_t * vlib_unix_recursive_mkdir(char *path)
Definition: util.c:103
#define DPDK_STATS_POLL_INTERVAL
Definition: dpdk.h:269
#define VNET_HW_INTERFACE_FLAG_SPEED_100M
Definition: interface.h:470
unformat_function_t unformat_line
Definition: format.h:279
static vlib_node_registration_t dpdk_process_node
(constructor) VLIB_REGISTER_NODE (dpdk_process_node)
Definition: init.c:1720
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
uword unformat_vlib_cli_sub_input(unformat_input_t *i, va_list *args)
Definition: cli.c:153
u8 admin_up_down_in_progress
Definition: dpdk.h:427
#define STATIC_ASSERT(truth,...)
#define VNET_HW_INTERFACE_FLAG_SPEED_5G
Definition: interface.h:473
static struct rte_eth_conf port_conf_template
Definition: init.c:49
u8 no_tx_checksum_offload
Definition: dpdk.h:354
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
static clib_error_t * dpdk_lib_init(dpdk_main_t *dm)
Definition: init.c:191
#define hash_get_mem(h, key)
Definition: hash.h:269
u32 buffer_flags_template
Definition: dpdk.h:412
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:120
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
#define vnet_buffer(b)
Definition: buffer.h:360
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
#define VNET_HW_INTERFACE_FLAG_SPEED_40G
Definition: interface.h:477
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
int hqos_cpu_first_index
Definition: dpdk.h:430
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
#define ETHERNET_MAX_PACKET_BYTES
Definition: ethernet.h:140
#define vec_foreach(var, vec)
Vector iterator.
i8 cpu_socket
Definition: dpdk.h:210
#define ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC(flags)
Definition: ethernet.h:148
uword * cpu_socket_bitmap
Definition: threads.h:332
Definition: file.h:51
static int dpdk_ring_alloc(struct rte_mempool *mp)
Definition: init.c:151
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:662
u8 * uio_driver_name
Definition: dpdk.h:351
vlib_thread_registration_t ** registrations
Definition: threads.h:295
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
Definition: pci.h:107
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
unformat_function_t unformat_dpdk_log_level
Definition: dpdk.h:519
ethernet_interface_t * interfaces
Definition: ethernet.h:278
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
vnet_main_t * vnet_main
Definition: dpdk.h:439
u16 nb_tx_desc
Definition: dpdk.h:214
clib_error_t * unformat_hqos(unformat_input_t *input, dpdk_device_config_hqos_t *hqos)
Definition: format.c:872
u16 device_class
Definition: pci.h:72
#define VNET_HW_INTERFACE_FLAG_SPEED_20G
Definition: interface.h:475
uword * device_config_index_by_pci_addr
Definition: dpdk.h:372
uword unformat_skip_white_space(unformat_input_t *input)
Definition: unformat.c:815
static uword vnet_hw_interface_is_link_up(vnet_main_t *vnm, u32 hw_if_index)
volatile u32 worker_thread_release
Definition: threads.h:338
#define dpdk_log_notice(...)
Definition: dpdk.h:504
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
vnet_device_main_t vnet_device_main
Definition: devices.c:22
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
f64 link_state_poll_interval
Definition: dpdk.h:434
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
dpdk_config_main_t * conf
Definition: dpdk.h:440
vlib_main_t * vlib_main
Definition: dpdk.h:438