FD.io VPP  v18.07-rc0-415-g6c78436
Vector Packet Processing
init.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/error.h>
18 #include <vppinfra/format.h>
19 #include <vppinfra/bitmap.h>
20 #include <vppinfra/linux/sysfs.h>
21 #include <vlib/unix/unix.h>
22 #include <vlib/log.h>
23 
24 #include <vnet/ethernet/ethernet.h>
25 #include <dpdk/device/dpdk.h>
26 #include <vlib/pci/pci.h>
27 
28 #include <rte_ring.h>
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <unistd.h>
33 #include <sys/stat.h>
34 #include <sys/mount.h>
35 #include <string.h>
36 #include <fcntl.h>
37 
38 #include <dpdk/device/dpdk_priv.h>
39 
40 #define ETHER_MAX_LEN 1518 /**< Maximum frame len, including CRC. */
41 
44 
45 #define LINK_STATE_ELOGS 0
46 
47 /* Port configuration, mildly modified Intel app values */
48 
49 static struct rte_eth_conf port_conf_template = {
50  .rxmode = {
51  .split_hdr_size = 0,
52  .header_split = 0, /**< Header Split disabled */
53  .hw_ip_checksum = 0, /**< IP checksum offload disabled */
54  .hw_vlan_filter = 0, /**< VLAN filtering disabled */
55  .hw_strip_crc = 0, /**< CRC stripped by hardware */
56  },
57  .txmode = {
58  .mq_mode = ETH_MQ_TX_NONE,
59  },
60 };
61 
62 static dpdk_port_type_t
63 port_type_from_speed_capa (struct rte_eth_dev_info *dev_info)
64 {
65 
66  if (dev_info->speed_capa & ETH_LINK_SPEED_100G)
68  else if (dev_info->speed_capa & ETH_LINK_SPEED_56G)
70  else if (dev_info->speed_capa & ETH_LINK_SPEED_50G)
72  else if (dev_info->speed_capa & ETH_LINK_SPEED_40G)
74  else if (dev_info->speed_capa & ETH_LINK_SPEED_25G)
76  else if (dev_info->speed_capa & ETH_LINK_SPEED_20G)
78  else if (dev_info->speed_capa & ETH_LINK_SPEED_10G)
80  else if (dev_info->speed_capa & ETH_LINK_SPEED_5G)
82  else if (dev_info->speed_capa & ETH_LINK_SPEED_2_5G)
84  else if (dev_info->speed_capa & ETH_LINK_SPEED_1G)
86 
88 }
89 
90 
91 static u32
93 {
94  dpdk_main_t *dm = &dpdk_main;
96  u32 old = 0;
97 
99  {
100  old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0;
101 
103  xd->flags |= DPDK_DEVICE_FLAG_PROMISC;
104  else
105  xd->flags &= ~DPDK_DEVICE_FLAG_PROMISC;
106 
107  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
108  {
109  if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
110  rte_eth_promiscuous_enable (xd->port_id);
111  else
112  rte_eth_promiscuous_disable (xd->port_id);
113  }
114  }
115  else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
116  {
117  xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes;
118  dpdk_device_setup (xd);
119  }
120  return old;
121 }
122 
123 static void
125 {
126  int q;
127  vec_validate (xd->lockp, xd->tx_q_used - 1);
128  for (q = 0; q < xd->tx_q_used; q++)
129  {
132  memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES);
133  }
134 }
135 
136 static struct rte_mempool_ops *
137 get_ops_by_name (char *ops_name)
138 {
139  u32 i;
140 
141  for (i = 0; i < rte_mempool_ops_table.num_ops; i++)
142  {
143  if (!strcmp (ops_name, rte_mempool_ops_table.ops[i].name))
144  return &rte_mempool_ops_table.ops[i];
145  }
146 
147  return 0;
148 }
149 
150 static int
151 dpdk_ring_alloc (struct rte_mempool *mp)
152 {
153  u32 rg_flags = 0, count;
154  i32 ret;
155  char rg_name[RTE_RING_NAMESIZE];
156  struct rte_ring *r;
157 
158  ret = snprintf (rg_name, sizeof (rg_name), RTE_MEMPOOL_MZ_FORMAT, mp->name);
159  if (ret < 0 || ret >= (i32) sizeof (rg_name))
160  return -ENAMETOOLONG;
161 
162  /* ring flags */
163  if (mp->flags & MEMPOOL_F_SP_PUT)
164  rg_flags |= RING_F_SP_ENQ;
165  if (mp->flags & MEMPOOL_F_SC_GET)
166  rg_flags |= RING_F_SC_DEQ;
167 
168  count = rte_align32pow2 (mp->size + 1);
169  /*
170  * Allocate the ring that will be used to store objects.
171  * Ring functions will return appropriate errors if we are
172  * running as a secondary process etc., so no checks made
173  * in this function for that condition.
174  */
175  /* XXX can we get memory from the right socket? */
176  r = clib_mem_alloc_aligned (rte_ring_get_memsize (count),
178 
179  /* XXX rte_ring_lookup will not work */
180 
181  ret = rte_ring_init (r, rg_name, count, rg_flags);
182  if (ret)
183  return ret;
184 
185  mp->pool_data = r;
186 
187  return 0;
188 }
189 
190 static clib_error_t *
192 {
193  u32 nports;
194  u32 mtu, max_rx_frame;
195  u32 nb_desc = 0;
196  int i;
197  clib_error_t *error;
203  dpdk_device_t *xd;
204  vlib_pci_addr_t last_pci_addr;
205  u32 last_pci_addr_port = 0;
207  uword *p_hqos;
208 
209  u32 next_hqos_cpu = 0;
210  u8 af_packet_instance_num = 0;
211  u8 bond_ether_instance_num = 0;
212  last_pci_addr.as_u32 = ~0;
213 
214  dm->hqos_cpu_first_index = 0;
215  dm->hqos_cpu_count = 0;
216 
217  /* find out which cpus will be used for I/O TX */
218  p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads");
219  tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0;
220 
221  if (tr_hqos && tr_hqos->count > 0)
222  {
223  dm->hqos_cpu_first_index = tr_hqos->first_index;
224  dm->hqos_cpu_count = tr_hqos->count;
225  }
226 
229 
230 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
231  nports = rte_eth_dev_count ();
232 #else
233  nports = rte_eth_dev_count_avail ();
234 #endif
235 
236  if (nports < 1)
237  {
238  dpdk_log_notice ("DPDK drivers found no ports...");
239  }
240 
241  if (CLIB_DEBUG > 0)
242  dpdk_log_notice ("DPDK drivers found %d ports...", nports);
243 
244  if (dm->conf->enable_tcp_udp_checksum)
245  dm->buffer_flags_template &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT
246  | VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
247 
248  /* vlib_buffer_t template */
251  for (i = 0; i < tm->n_vlib_mains; i++)
252  {
255  fl = vlib_buffer_get_free_list (vm,
259  vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
260  }
261 
262  /* *INDENT-OFF* */
263  RTE_ETH_FOREACH_DEV(i)
264  {
265  u8 addr[6];
266  u8 vlan_strip = 0;
267  struct rte_eth_dev_info dev_info;
268  struct rte_pci_device *pci_dev;
269  struct rte_eth_link l;
270  dpdk_device_config_t *devconf = 0;
271  vlib_pci_addr_t pci_addr;
272  uword *p = 0;
273 
274  if (!rte_eth_dev_is_valid_port(i))
275  continue;
276 
277  rte_eth_link_get_nowait (i, &l);
278  rte_eth_dev_info_get (i, &dev_info);
279 
280 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
281  pci_dev = dev_info.pci_dev;
282 #else
283  if (dev_info.device == 0)
284  {
285  clib_warning ("DPDK bug: missing device info. Skipping %s device",
286  dev_info.driver_name);
287  continue;
288  }
289  pci_dev = RTE_DEV_TO_PCI (dev_info.device);
290 #endif
291 
292  if (pci_dev) /* bonded interface has no pci info */
293  {
294  pci_addr.domain = pci_dev->addr.domain;
295  pci_addr.bus = pci_dev->addr.bus;
296  pci_addr.slot = pci_dev->addr.devid;
297  pci_addr.function = pci_dev->addr.function;
298  p =
300  pci_addr.as_u32);
301  }
302 
303  if (p)
304  devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
305  else
306  devconf = &dm->conf->default_devconf;
307 
308  /* Create vnet interface */
312  xd->cpu_socket = (i8) rte_eth_dev_socket_id (i);
313 
314  /* Handle interface naming for devices with multiple ports sharing same PCI ID */
315  if (pci_dev)
316  {
317  struct rte_eth_dev_info di = { 0 };
318  struct rte_pci_device *next_pci_dev;
319  rte_eth_dev_info_get (i + 1, &di);
320 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
321  next_pci_dev = di.pci_dev;
322 #else
323  next_pci_dev = di.device ? RTE_DEV_TO_PCI (di.device) : 0;
324 #endif
325  if (pci_dev && next_pci_dev &&
326  pci_addr.as_u32 != last_pci_addr.as_u32 &&
327  memcmp (&pci_dev->addr, &next_pci_dev->addr,
328  sizeof (struct rte_pci_addr)) == 0)
329  {
330  xd->interface_name_suffix = format (0, "0");
331  last_pci_addr.as_u32 = pci_addr.as_u32;
332  last_pci_addr_port = i;
333  }
334  else if (pci_addr.as_u32 == last_pci_addr.as_u32)
335  {
337  format (0, "%u", i - last_pci_addr_port);
338  }
339  else
340  {
341  last_pci_addr.as_u32 = ~0;
342  }
343  }
344  else
345  last_pci_addr.as_u32 = ~0;
346 
347  clib_memcpy (&xd->tx_conf, &dev_info.default_txconf,
348  sizeof (struct rte_eth_txconf));
349 
350  if (dm->conf->no_multi_seg)
351  {
352  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
353  port_conf_template.rxmode.jumbo_frame = 0;
354  port_conf_template.rxmode.enable_scatter = 0;
355  }
356  else
357  {
358  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS;
359  port_conf_template.rxmode.jumbo_frame = 1;
360  port_conf_template.rxmode.enable_scatter = 1;
361  xd->flags |= DPDK_DEVICE_FLAG_MAYBE_MULTISEG;
362  }
363 
365  sizeof (struct rte_eth_conf));
366 
367  xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains);
368 
369  if (devconf->num_tx_queues > 0
370  && devconf->num_tx_queues < xd->tx_q_used)
371  xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues);
372 
373  if (devconf->num_rx_queues > 1
374  && dev_info.max_rx_queues >= devconf->num_rx_queues)
375  {
376  xd->rx_q_used = devconf->num_rx_queues;
377  xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
378  if (devconf->rss_fn == 0)
379  xd->port_conf.rx_adv_conf.rss_conf.rss_hf =
380  ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
381  else
382  xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn;
383  }
384  else
385  xd->rx_q_used = 1;
386 
387  xd->flags |= DPDK_DEVICE_FLAG_PMD;
388 
389  /* workaround for drivers not setting driver_name */
390  if ((!dev_info.driver_name) && (pci_dev))
391  dev_info.driver_name = pci_dev->driver->driver.name;
392 
393  ASSERT (dev_info.driver_name);
394 
395  if (!xd->pmd)
396  {
397 
398 
399 #define _(s,f) else if (dev_info.driver_name && \
400  !strcmp(dev_info.driver_name, s)) \
401  xd->pmd = VNET_DPDK_PMD_##f;
402  if (0)
403  ;
405 #undef _
406  else
408 
412 
413  switch (xd->pmd)
414  {
415  /* Drivers with valid speed_capa set */
416  case VNET_DPDK_PMD_E1000EM:
417  case VNET_DPDK_PMD_IGB:
418  case VNET_DPDK_PMD_IXGBE:
419  case VNET_DPDK_PMD_I40E:
420  xd->port_type = port_type_from_speed_capa (&dev_info);
421  xd->supported_flow_actions = VNET_FLOW_ACTION_MARK |
422  VNET_FLOW_ACTION_REDIRECT_TO_NODE |
423  VNET_FLOW_ACTION_BUFFER_ADVANCE |
424  VNET_FLOW_ACTION_COUNT | VNET_FLOW_ACTION_DROP;
425 
426  if (dm->conf->no_tx_checksum_offload == 0)
427  {
428  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOXSUMS;
429  xd->flags |=
430  DPDK_DEVICE_FLAG_TX_OFFLOAD |
431  DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM;
432  }
433 
434 
435  break;
436  case VNET_DPDK_PMD_CXGBE:
437  case VNET_DPDK_PMD_MLX4:
438  case VNET_DPDK_PMD_MLX5:
439  xd->port_type = port_type_from_speed_capa (&dev_info);
440  break;
441 
442  /* SR-IOV VFs */
443  case VNET_DPDK_PMD_IGBVF:
444  case VNET_DPDK_PMD_IXGBEVF:
445  case VNET_DPDK_PMD_I40EVF:
447  xd->port_conf.rxmode.hw_strip_crc = 1;
448  break;
449 
450  case VNET_DPDK_PMD_THUNDERX:
452  xd->port_conf.rxmode.hw_strip_crc = 1;
453  break;
454 
455  case VNET_DPDK_PMD_ENA:
457  xd->port_conf.rxmode.enable_scatter = 0;
458  break;
459 
460  case VNET_DPDK_PMD_DPAA2:
462  break;
463 
464  /* Cisco VIC */
465  case VNET_DPDK_PMD_ENIC:
466  if (l.link_speed == 40000)
468  else
470  break;
471 
472  /* Intel Red Rock Canyon */
473  case VNET_DPDK_PMD_FM10K:
475  xd->port_conf.rxmode.hw_strip_crc = 1;
476  break;
477 
478  /* virtio */
479  case VNET_DPDK_PMD_VIRTIO:
483  break;
484 
485  /* vmxnet3 */
486  case VNET_DPDK_PMD_VMXNET3:
488  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
489  break;
490 
491  case VNET_DPDK_PMD_AF_PACKET:
493  xd->af_packet_instance_num = af_packet_instance_num++;
494  break;
495 
496  case VNET_DPDK_PMD_BOND:
498  xd->bond_instance_num = bond_ether_instance_num++;
499  break;
500 
501  case VNET_DPDK_PMD_VIRTIO_USER:
503  break;
504 
505  case VNET_DPDK_PMD_VHOST_ETHER:
507  break;
508 
509  case VNET_DPDK_PMD_LIOVF_ETHER:
511  break;
512 
513  case VNET_DPDK_PMD_FAILSAFE:
515  xd->port_conf.intr_conf.lsc = 1;
516  break;
517 
518  default:
520  }
521 
522  if (devconf->num_rx_desc)
523  xd->nb_rx_desc = devconf->num_rx_desc;
524 
525  if (devconf->num_tx_desc)
526  xd->nb_tx_desc = devconf->num_tx_desc;
527  }
528 
529  if (xd->pmd == VNET_DPDK_PMD_AF_PACKET)
530  {
531  f64 now = vlib_time_now (vm);
532  u32 rnd;
533  rnd = (u32) (now * 1e6);
534  rnd = random_u32 (&rnd);
535  clib_memcpy (addr + 2, &rnd, sizeof (rnd));
536  addr[0] = 2;
537  addr[1] = 0xfe;
538  }
539  else
540  rte_eth_macaddr_get (i, (struct ether_addr *) addr);
541 
542  if (xd->tx_q_used < tm->n_vlib_mains)
544 
545  xd->port_id = i;
546  xd->device_index = xd - dm->devices;
547  xd->per_interface_next_index = ~0;
548 
549  /* assign interface to input thread */
550  int q;
551 
552  if (devconf->hqos_enabled)
553  {
554  xd->flags |= DPDK_DEVICE_FLAG_HQOS;
555 
556  int cpu;
557  if (devconf->hqos.hqos_thread_valid)
558  {
559  if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count)
560  return clib_error_return (0, "invalid HQoS thread index");
561 
562  cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread;
563  }
564  else
565  {
566  if (dm->hqos_cpu_count == 0)
567  return clib_error_return (0, "no HQoS threads available");
568 
569  cpu = dm->hqos_cpu_first_index + next_hqos_cpu;
570 
571  next_hqos_cpu++;
572  if (next_hqos_cpu == dm->hqos_cpu_count)
573  next_hqos_cpu = 0;
574 
575  devconf->hqos.hqos_thread_valid = 1;
576  devconf->hqos.hqos_thread = cpu;
577  }
578 
580  vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
581  dq->device = xd->device_index;
582  dq->queue_id = 0;
583  }
584 
585  /* count the number of descriptors used for this device */
586  nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
587 
589  (dm->vnet_main, dpdk_device_class.index, xd->device_index,
590  /* ethernet address */ addr,
592  if (error)
593  return error;
594 
595  /*
596  * Ensure default mtu is not > the mtu read from the hardware.
597  * Otherwise rte_eth_dev_configure() will fail and the port will
598  * not be available.
599  * Calculate max_frame_size and mtu supported by NIC
600  */
601  if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen)
602  {
603  /*
604  * This device does not support the platforms's max frame
605  * size. Use it's advertised mru instead.
606  */
607  max_rx_frame = dev_info.max_rx_pktlen;
608  mtu = dev_info.max_rx_pktlen - sizeof (ethernet_header_t);
609  }
610  else
611  {
612  /* VPP treats MTU and max_rx_pktlen both equal to
613  * ETHERNET_MAX_PACKET_BYTES, if dev_info.max_rx_pktlen >=
614  * ETHERNET_MAX_PACKET_BYTES + sizeof(ethernet_header_t)
615  */
616  if (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES +
617  sizeof (ethernet_header_t)))
618  {
620  max_rx_frame = ETHERNET_MAX_PACKET_BYTES;
621 
622  /*
623  * Some platforms do not account for Ethernet FCS (4 bytes) in
624  * MTU calculations. To interop with them increase mru but only
625  * if the device's settings can support it.
626  */
627  if (xd->port_conf.rxmode.hw_strip_crc &&
628  (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES +
629  sizeof (ethernet_header_t) +
630  4)))
631  {
632  max_rx_frame += 4;
633  }
634  }
635  else
636  {
637  max_rx_frame = ETHERNET_MAX_PACKET_BYTES;
639 
640  if (xd->port_conf.rxmode.hw_strip_crc &&
641  (dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)))
642  {
643  max_rx_frame += 4;
644  }
645  }
646  }
647 
648  if (xd->pmd == VNET_DPDK_PMD_FAILSAFE)
649  {
650  /* failsafe device numerables are reported with active device only,
651  * need to query the mtu for current device setup to overwrite
652  * reported value.
653  */
654  uint16_t dev_mtu;
655  if (!rte_eth_dev_get_mtu (i, &dev_mtu))
656  {
657  mtu = dev_mtu;
658  max_rx_frame = mtu + sizeof (ethernet_header_t);
659 
660  if (xd->port_conf.rxmode.hw_strip_crc)
661  {
662  max_rx_frame += 4;
663  }
664  }
665  }
666 
667  /*Set port rxmode config */
668  xd->port_conf.rxmode.max_rx_pkt_len = max_rx_frame;
669 
671  xd->sw_if_index = sw->sw_if_index;
673  dpdk_input_node.index);
674 
675  if (devconf->workers)
676  {
677  int i;
678  q = 0;
679  clib_bitmap_foreach (i, devconf->workers, ({
680  vnet_hw_interface_assign_rx_thread (dm->vnet_main, xd->hw_if_index, q++,
681  vdm->first_worker_thread_index + i);
682  }));
683  }
684  else
685  for (q = 0; q < xd->rx_q_used; q++)
686  {
688  ~1);
689  }
690 
691  /*Get vnet hardware interface */
693 
694  /*Override default max_packet_bytes and max_supported_bytes set in
695  * ethernet_register_interface() above*/
696  if (hi)
697  {
698  hi->max_packet_bytes = mtu;
699  hi->max_supported_packet_bytes = max_rx_frame;
700  }
701 
702  if (dm->conf->no_tx_checksum_offload == 0)
703  if (xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD)
705 
706  dpdk_device_setup (xd);
707 
708  if (vec_len (xd->errors))
709  dpdk_log_err ("setup failed for device %U. Errors:\n %U",
712 
713  if (devconf->hqos_enabled)
714  {
715  clib_error_t *rv;
716  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
717  if (rv)
718  return rv;
719  }
720 
721  /*
722  * For cisco VIC vNIC, set default to VLAN strip enabled, unless
723  * specified otherwise in the startup config.
724  * For other NICs default to VLAN strip disabled, unless specified
725  * otherwis in the startup config.
726  */
727  if (xd->pmd == VNET_DPDK_PMD_ENIC)
728  {
729  if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF)
730  vlan_strip = 1; /* remove vlan tag from VIC port by default */
731  else
732  dpdk_log_warn ("VLAN strip disabled for interface\n");
733  }
734  else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON)
735  vlan_strip = 1;
736 
737  if (vlan_strip)
738  {
739  int vlan_off;
740  vlan_off = rte_eth_dev_get_vlan_offload (xd->port_id);
741  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
742  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
743  if (rte_eth_dev_set_vlan_offload (xd->port_id, vlan_off) == 0)
744  dpdk_log_info ("VLAN strip enabled for interface\n");
745  else
746  dpdk_log_warn ("VLAN strip cannot be supported by interface\n");
747  }
748 
749  if (hi)
750  hi->max_packet_bytes = xd->port_conf.rxmode.max_rx_pkt_len
751  - sizeof (ethernet_header_t);
752  else
753  clib_warning ("hi NULL");
754 
755  if (dm->conf->no_multi_seg)
756  mtu = mtu > ETHER_MAX_LEN ? ETHER_MAX_LEN : mtu;
757 
758  rte_eth_dev_set_mtu (xd->port_id, mtu);
759  }
760  /* *INDENT-ON* */
761 
762  if (nb_desc > dm->conf->num_mbufs)
763  dpdk_log_err ("%d mbufs allocated but total rx/tx ring size is %d\n",
764  dm->conf->num_mbufs, nb_desc);
765 
766  return 0;
767 }
768 
769 static void
771 {
772  clib_error_t *error;
773  u8 *pci_addr = 0;
774  int num_whitelisted = vec_len (conf->dev_confs);
775  vlib_pci_device_info_t *d = 0;
776  vlib_pci_addr_t *addr = 0, *addrs;
777 
778  addrs = vlib_pci_get_all_dev_addrs ();
779  /* *INDENT-OFF* */
780  vec_foreach (addr, addrs)
781  {
782  dpdk_device_config_t * devconf = 0;
783  vec_reset_length (pci_addr);
784  pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, addr, 0);
785  if (d)
786  {
788  d = 0;
789  }
790  d = vlib_pci_get_device_info (addr, &error);
791  if (error)
792  {
793  clib_error_report (error);
794  continue;
795  }
796 
798  continue;
799 
800  if (num_whitelisted)
801  {
802  uword * p = hash_get (conf->device_config_index_by_pci_addr, addr->as_u32);
803 
804  if (!p)
805  continue;
806 
807  devconf = pool_elt_at_index (conf->dev_confs, p[0]);
808  }
809 
810  /* virtio */
811  if (d->vendor_id == 0x1af4 &&
814  ;
815  /* vmxnet3 */
816  else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0)
817  ;
818  /* all Intel network devices */
819  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_NETWORK_ETHERNET)
820  ;
821  /* all Intel QAT devices VFs */
822  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_PROCESSOR_CO &&
823  (d->device_id == 0x0443 || d->device_id == 0x37c9 || d->device_id == 0x19e3))
824  ;
825  /* Cisco VIC */
826  else if (d->vendor_id == 0x1137 && d->device_id == 0x0043)
827  ;
828  /* Chelsio T4/T5 */
829  else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000)
830  ;
831  /* Amazen Elastic Network Adapter */
832  else if (d->vendor_id == 0x1d0f && d->device_id >= 0xec20 && d->device_id <= 0xec21)
833  ;
834  /* Cavium Network Adapter */
835  else if (d->vendor_id == 0x177d && d->device_id == 0x9712)
836  ;
837  /* Mellanox */
838  else if (d->vendor_id == 0x15b3 && d->device_id >= 0x1013 && d->device_id <= 0x101a)
839  {
840  continue;
841  }
842  /* Cavium FastlinQ QL41000 Series */
843  else if (d->vendor_id == 0x1077 && d->device_id >= 0x8070 && d->device_id <= 0x8090)
844  ;
845  else
846  {
847  dpdk_log_warn ("Unsupported PCI device 0x%04x:0x%04x found "
848  "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id,
849  pci_addr);
850  continue;
851  }
852 
853  error = vlib_pci_bind_to_uio (addr, (char *) conf->uio_driver_name);
854 
855  if (error)
856  {
857  if (devconf == 0)
858  {
859  pool_get (conf->dev_confs, devconf);
860  hash_set (conf->device_config_index_by_pci_addr, addr->as_u32,
861  devconf - conf->dev_confs);
862  devconf->pci_addr.as_u32 = addr->as_u32;
863  }
864  devconf->is_blacklisted = 1;
865  clib_error_report (error);
866  }
867  }
868  /* *INDENT-ON* */
869  vec_free (pci_addr);
871 }
872 
873 static clib_error_t *
874 dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
875  unformat_input_t * input, u8 is_default)
876 {
877  clib_error_t *error = 0;
878  uword *p;
879  dpdk_device_config_t *devconf;
880  unformat_input_t sub_input;
881 
882  if (is_default)
883  {
884  devconf = &conf->default_devconf;
885  }
886  else
887  {
888  p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32);
889 
890  if (!p)
891  {
892  pool_get (conf->dev_confs, devconf);
893  hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32,
894  devconf - conf->dev_confs);
895  }
896  else
897  return clib_error_return (0,
898  "duplicate configuration for PCI address %U",
899  format_vlib_pci_addr, &pci_addr);
900  }
901 
902  devconf->pci_addr.as_u32 = pci_addr.as_u32;
903  devconf->hqos_enabled = 0;
905 
906  if (!input)
907  return 0;
908 
911  {
912  if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues))
913  ;
914  else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues))
915  ;
916  else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc))
917  ;
918  else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc))
919  ;
920  else if (unformat (input, "workers %U", unformat_bitmap_list,
921  &devconf->workers))
922  ;
923  else
924  if (unformat
925  (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input))
926  {
927  error = unformat_rss_fn (&sub_input, &devconf->rss_fn);
928  if (error)
929  break;
930  }
931  else if (unformat (input, "vlan-strip-offload off"))
933  else if (unformat (input, "vlan-strip-offload on"))
935  else
936  if (unformat
937  (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input))
938  {
939  devconf->hqos_enabled = 1;
940  error = unformat_hqos (&sub_input, &devconf->hqos);
941  if (error)
942  break;
943  }
944  else if (unformat (input, "hqos"))
945  {
946  devconf->hqos_enabled = 1;
947  }
948  else
949  {
950  error = clib_error_return (0, "unknown input `%U'",
951  format_unformat_error, input);
952  break;
953  }
954  }
955 
956  if (error)
957  return error;
958 
959  if (devconf->workers && devconf->num_rx_queues == 0)
960  devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers);
961  else if (devconf->workers &&
962  clib_bitmap_count_set_bits (devconf->workers) !=
963  devconf->num_rx_queues)
964  error =
966  "%U: number of worker threadds must be "
967  "equal to number of rx queues", format_vlib_pci_addr,
968  &pci_addr);
969 
970  return error;
971 }
972 
973 static clib_error_t *
975 {
976  unformat_input_t input;
977  u8 *line, *s = 0;
978  int n, n_try;
979 
980  n = n_try = 4096;
981  while (n == n_try)
982  {
983  uword len = vec_len (s);
984  vec_resize (s, len + n_try);
985 
986  n = read (uf->file_descriptor, s + len, n_try);
987  if (n < 0 && errno != EAGAIN)
988  return clib_error_return_unix (0, "read");
989  _vec_len (s) = len + (n < 0 ? 0 : n);
990  }
991 
992  unformat_init_vector (&input, s);
993 
994  while (unformat_user (&input, unformat_line, &line))
995  {
996  dpdk_log_notice ("%v", line);
997  vec_free (line);
998  }
999 
1000  unformat_free (&input);
1001  return 0;
1002 }
1003 
1004 static clib_error_t *
1006 {
1007  clib_error_t *error = 0;
1010  dpdk_device_config_t *devconf;
1011  vlib_pci_addr_t pci_addr;
1012  unformat_input_t sub_input;
1013  uword x;
1014  u8 *s, *tmp = 0;
1015  u8 *rte_cmd = 0, *ethname = 0;
1016  u32 log_level;
1017  int ret, i;
1018  int num_whitelisted = 0;
1019  u8 no_pci = 0;
1020  u8 no_huge = 0;
1021  u8 huge_dir = 0;
1022  u8 file_prefix = 0;
1023  u8 *socket_mem = 0;
1024  u8 *huge_dir_path = 0;
1025 
1026  huge_dir_path =
1027  format (0, "%s/hugepages%c", vlib_unix_get_runtime_dir (), 0);
1028 
1029  conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
1030  log_level = RTE_LOG_NOTICE;
1031 
1032  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1033  {
1034  /* Prime the pump */
1035  if (unformat (input, "no-hugetlb"))
1036  {
1037  vec_add1 (conf->eal_init_args, (u8 *) "--no-huge");
1038  no_huge = 1;
1039  }
1040 
1041  else if (unformat (input, "enable-tcp-udp-checksum"))
1042  conf->enable_tcp_udp_checksum = 1;
1043 
1044  else if (unformat (input, "no-tx-checksum-offload"))
1045  conf->no_tx_checksum_offload = 1;
1046 
1047  else if (unformat (input, "decimal-interface-names"))
1049 
1050  else if (unformat (input, "log-level %U", unformat_dpdk_log_level, &x))
1051  log_level = x;
1052 
1053  else if (unformat (input, "no-multi-seg"))
1054  conf->no_multi_seg = 1;
1055 
1056  else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
1057  &sub_input))
1058  {
1059  error =
1060  dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input,
1061  1);
1062 
1063  if (error)
1064  return error;
1065  }
1066  else
1067  if (unformat
1068  (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr,
1069  unformat_vlib_cli_sub_input, &sub_input))
1070  {
1071  error = dpdk_device_config (conf, pci_addr, &sub_input, 0);
1072 
1073  if (error)
1074  return error;
1075 
1076  num_whitelisted++;
1077  }
1078  else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr))
1079  {
1080  error = dpdk_device_config (conf, pci_addr, 0, 0);
1081 
1082  if (error)
1083  return error;
1084 
1085  num_whitelisted++;
1086  }
1087  else if (unformat (input, "num-mbufs %d", &conf->num_mbufs))
1088  ;
1089  else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
1090  ;
1091  else if (unformat (input, "socket-mem %s", &socket_mem))
1092  ;
1093  else if (unformat (input, "no-pci"))
1094  {
1095  no_pci = 1;
1096  tmp = format (0, "--no-pci%c", 0);
1097  vec_add1 (conf->eal_init_args, tmp);
1098  }
1099 
1100 #define _(a) \
1101  else if (unformat(input, #a)) \
1102  { \
1103  tmp = format (0, "--%s%c", #a, 0); \
1104  vec_add1 (conf->eal_init_args, tmp); \
1105  }
1107 #undef _
1108 #define _(a) \
1109  else if (unformat(input, #a " %s", &s)) \
1110  { \
1111  if (!strncmp(#a, "huge-dir", 8)) \
1112  huge_dir = 1; \
1113  else if (!strncmp(#a, "file-prefix", 11)) \
1114  file_prefix = 1; \
1115  tmp = format (0, "--%s%c", #a, 0); \
1116  vec_add1 (conf->eal_init_args, tmp); \
1117  vec_add1 (s, 0); \
1118  if (!strncmp(#a, "vdev", 4)) \
1119  if (strstr((char*)s, "af_packet")) \
1120  clib_warning ("af_packet obsoleted. Use CLI 'create host-interface'."); \
1121  vec_add1 (conf->eal_init_args, s); \
1122  }
1124 #undef _
1125 #define _(a,b) \
1126  else if (unformat(input, #a " %s", &s)) \
1127  { \
1128  tmp = format (0, "-%s%c", #b, 0); \
1129  vec_add1 (conf->eal_init_args, tmp); \
1130  vec_add1 (s, 0); \
1131  vec_add1 (conf->eal_init_args, s); \
1132  }
1134 #undef _
1135 #define _(a,b) \
1136  else if (unformat(input, #a " %s", &s)) \
1137  { \
1138  tmp = format (0, "-%s%c", #b, 0); \
1139  vec_add1 (conf->eal_init_args, tmp); \
1140  vec_add1 (s, 0); \
1141  vec_add1 (conf->eal_init_args, s); \
1142  conf->a##_set_manually = 1; \
1143  }
1145 #undef _
1146  else if (unformat (input, "default"))
1147  ;
1148 
1149  else if (unformat_skip_white_space (input))
1150  ;
1151  else
1152  {
1153  error = clib_error_return (0, "unknown input `%U'",
1154  format_unformat_error, input);
1155  goto done;
1156  }
1157  }
1158 
1159  if (!conf->uio_driver_name)
1160  conf->uio_driver_name = format (0, "auto%c", 0);
1161 
1162  /*
1163  * Use 1G huge pages if available.
1164  */
1165  if (!no_huge && !huge_dir)
1166  {
1167  u32 x, *mem_by_socket = 0;
1168  uword c = 0;
1169  int rv;
1170 
1171  umount ((char *) huge_dir_path);
1172 
1173  /* Process "socket-mem" parameter value */
1174  if (vec_len (socket_mem))
1175  {
1176  unformat_input_t in;
1177  unformat_init_vector (&in, socket_mem);
1179  {
1180  if (unformat (&in, "%u,", &x))
1181  ;
1182  else if (unformat (&in, "%u", &x))
1183  ;
1184  else if (unformat (&in, ","))
1185  x = 0;
1186  else
1187  break;
1188 
1189  vec_add1 (mem_by_socket, x);
1190  }
1191  /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */
1192  unformat_free (&in);
1193  socket_mem = 0;
1194  }
1195  else
1196  {
1197  /* *INDENT-OFF* */
1199  {
1200  vec_validate(mem_by_socket, c);
1201  mem_by_socket[c] = 64; /* default per-socket mem */
1202  }
1203  ));
1204  /* *INDENT-ON* */
1205  }
1206 
1207  /* *INDENT-OFF* */
1209  {
1210  clib_error_t *e;
1211 
1212  vec_validate(mem_by_socket, c);
1213 
1214  e = clib_sysfs_prealloc_hugepages(c, 2 << 10, mem_by_socket[c] / 2);
1215  if (e)
1216  clib_error_report (e);
1217  }));
1218  /* *INDENT-ON* */
1219 
1220  if (mem_by_socket == 0)
1221  {
1222  error = clib_error_return (0, "mem_by_socket NULL");
1223  goto done;
1224  }
1225  _vec_len (mem_by_socket) = c + 1;
1226 
1227  /* regenerate socket_mem string */
1228  vec_foreach_index (x, mem_by_socket)
1229  socket_mem = format (socket_mem, "%s%u",
1230  socket_mem ? "," : "", mem_by_socket[x]);
1231  socket_mem = format (socket_mem, "%c", 0);
1232 
1233  vec_free (mem_by_socket);
1234 
1235  error = vlib_unix_recursive_mkdir ((char *) huge_dir_path);
1236  if (error)
1237  {
1238  goto done;
1239  }
1240 
1241  rv = mount ("none", (char *) huge_dir_path, "hugetlbfs", 0, NULL);
1242 
1243  if (rv)
1244  {
1245  error = clib_error_return (0, "mount failed %d", errno);
1246  goto done;
1247  }
1248 
1249  tmp = format (0, "--huge-dir%c", 0);
1250  vec_add1 (conf->eal_init_args, tmp);
1251  tmp = format (0, "%s%c", huge_dir_path, 0);
1252  vec_add1 (conf->eal_init_args, tmp);
1253  if (!file_prefix)
1254  {
1255  tmp = format (0, "--file-prefix%c", 0);
1256  vec_add1 (conf->eal_init_args, tmp);
1257  tmp = format (0, "vpp%c", 0);
1258  vec_add1 (conf->eal_init_args, tmp);
1259  }
1260  }
1261 
1262  vec_free (rte_cmd);
1263  vec_free (ethname);
1264 
1265  if (error)
1266  return error;
1267 
1268  /* I'll bet that -c and -n must be the first and second args... */
1269  if (!conf->coremask_set_manually)
1270  {
1272  uword *coremask = 0;
1273  int i;
1274 
1275  /* main thread core */
1276  coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
1277 
1278  for (i = 0; i < vec_len (tm->registrations); i++)
1279  {
1280  tr = tm->registrations[i];
1281  coremask = clib_bitmap_or (coremask, tr->coremask);
1282  }
1283 
1284  vec_insert (conf->eal_init_args, 2, 1);
1285  conf->eal_init_args[1] = (u8 *) "-c";
1286  tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
1287  conf->eal_init_args[2] = tmp;
1288  clib_bitmap_free (coremask);
1289  }
1290 
1291  if (!conf->nchannels_set_manually)
1292  {
1293  vec_insert (conf->eal_init_args, 2, 3);
1294  conf->eal_init_args[3] = (u8 *) "-n";
1295  tmp = format (0, "%d", conf->nchannels);
1296  conf->eal_init_args[4] = tmp;
1297  }
1298 
1299  if (no_pci == 0 && geteuid () == 0)
1300  dpdk_bind_devices_to_uio (conf);
1301 
1302 #define _(x) \
1303  if (devconf->x == 0 && conf->default_devconf.x > 0) \
1304  devconf->x = conf->default_devconf.x ;
1305 
1306  /* *INDENT-OFF* */
1307  pool_foreach (devconf, conf->dev_confs, ({
1308 
1309  /* default per-device config items */
1310  foreach_dpdk_device_config_item
1311 
1312  /* add DPDK EAL whitelist/blacklist entry */
1313  if (num_whitelisted > 0 && devconf->is_blacklisted == 0)
1314  {
1315  tmp = format (0, "-w%c", 0);
1316  vec_add1 (conf->eal_init_args, tmp);
1317  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1318  vec_add1 (conf->eal_init_args, tmp);
1319  }
1320  else if (num_whitelisted == 0 && devconf->is_blacklisted != 0)
1321  {
1322  tmp = format (0, "-b%c", 0);
1323  vec_add1 (conf->eal_init_args, tmp);
1324  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1325  vec_add1 (conf->eal_init_args, tmp);
1326  }
1327  }));
1328  /* *INDENT-ON* */
1329 
1330 #undef _
1331 
1332  /* set master-lcore */
1333  tmp = format (0, "--master-lcore%c", 0);
1334  vec_add1 (conf->eal_init_args, tmp);
1335  tmp = format (0, "%u%c", tm->main_lcore, 0);
1336  vec_add1 (conf->eal_init_args, tmp);
1337 
1338  /* set socket-mem */
1339  if (!no_huge)
1340  {
1341  tmp = format (0, "--socket-mem%c", 0);
1342  vec_add1 (conf->eal_init_args, tmp);
1343  tmp = format (0, "%s%c", socket_mem, 0);
1344  vec_add1 (conf->eal_init_args, tmp);
1345  }
1346 
1347  /* NULL terminate the "argv" vector, in case of stupidity */
1348  vec_add1 (conf->eal_init_args, 0);
1349  _vec_len (conf->eal_init_args) -= 1;
1350 
1351  /* Set up DPDK eal and packet mbuf pool early. */
1352 
1353  rte_log_set_global_level (log_level);
1354  int log_fds[2] = { 0 };
1355  if (pipe (log_fds) == 0)
1356  {
1357  FILE *f = fdopen (log_fds[1], "a");
1358  if (f && rte_openlog_stream (f) == 0)
1359  {
1360  clib_file_t t = { 0 };
1362  t.file_descriptor = log_fds[0];
1363  t.description = format (0, "DPDK logging pipe");
1364  clib_file_add (&file_main, &t);
1365  }
1366  }
1367 
1368  vm = vlib_get_main ();
1369 
1370  /* make copy of args as rte_eal_init tends to mess up with arg array */
1371  for (i = 1; i < vec_len (conf->eal_init_args); i++)
1372  conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ",
1373  conf->eal_init_args[i]);
1374 
1375  dpdk_log_warn ("EAL init args: %s", conf->eal_init_args_str);
1376  ret = rte_eal_init (vec_len (conf->eal_init_args),
1377  (char **) conf->eal_init_args);
1378 
1379  /* lazy umount hugepages */
1380  umount2 ((char *) huge_dir_path, MNT_DETACH);
1381  rmdir ((char *) huge_dir_path);
1382  vec_free (huge_dir_path);
1383 
1384  if (ret < 0)
1385  return clib_error_return (0, "rte_eal_init returned %d", ret);
1386 
1387  /* set custom ring memory allocator */
1388  {
1389  struct rte_mempool_ops *ops = NULL;
1390 
1391  ops = get_ops_by_name ("ring_sp_sc");
1392  ops->alloc = dpdk_ring_alloc;
1393 
1394  ops = get_ops_by_name ("ring_mp_sc");
1395  ops->alloc = dpdk_ring_alloc;
1396 
1397  ops = get_ops_by_name ("ring_sp_mc");
1398  ops->alloc = dpdk_ring_alloc;
1399 
1400  ops = get_ops_by_name ("ring_mp_mc");
1401  ops->alloc = dpdk_ring_alloc;
1402  }
1403 
1404  /* main thread 1st */
1405  error = dpdk_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ());
1406  if (error)
1407  return error;
1408 
1409  for (i = 0; i < RTE_MAX_LCORE; i++)
1410  {
1411  error = dpdk_buffer_pool_create (vm, conf->num_mbufs,
1412  rte_lcore_to_socket_id (i));
1413  if (error)
1414  return error;
1415  }
1416 
1417 done:
1418  return error;
1419 }
1420 
1422 
1423 void
1425 {
1426  vnet_main_t *vnm = vnet_get_main ();
1427  struct rte_eth_link prev_link = xd->link;
1428  u32 hw_flags = 0;
1429  u8 hw_flags_chg = 0;
1430 
1431  /* only update link state for PMD interfaces */
1432  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
1433  return;
1434 
1435  xd->time_last_link_update = now ? now : xd->time_last_link_update;
1436  memset (&xd->link, 0, sizeof (xd->link));
1437  rte_eth_link_get_nowait (xd->port_id, &xd->link);
1438 
1439  if (LINK_STATE_ELOGS)
1440  {
1442  ELOG_TYPE_DECLARE (e) =
1443  {
1444  .format =
1445  "update-link-state: sw_if_index %d, admin_up %d,"
1446  "old link_state %d new link_state %d",.format_args = "i4i1i1i1",};
1447 
1448  struct
1449  {
1450  u32 sw_if_index;
1451  u8 admin_up;
1452  u8 old_link_state;
1453  u8 new_link_state;
1454  } *ed;
1455  ed = ELOG_DATA (&vm->elog_main, e);
1456  ed->sw_if_index = xd->sw_if_index;
1457  ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
1458  ed->old_link_state = (u8)
1460  ed->new_link_state = (u8) xd->link.link_status;
1461  }
1462 
1463  if ((xd->flags & (DPDK_DEVICE_FLAG_ADMIN_UP | DPDK_DEVICE_FLAG_BOND_SLAVE))
1464  && ((xd->link.link_status != 0) ^
1466  {
1467  hw_flags_chg = 1;
1468  hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
1469  }
1470 
1471  if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex))
1472  {
1473  hw_flags_chg = 1;
1474  switch (xd->link.link_duplex)
1475  {
1476  case ETH_LINK_HALF_DUPLEX:
1478  break;
1479  case ETH_LINK_FULL_DUPLEX:
1481  break;
1482  default:
1483  break;
1484  }
1485  }
1486  if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed))
1487  {
1488  hw_flags_chg = 1;
1489  switch (xd->link.link_speed)
1490  {
1491  case ETH_SPEED_NUM_10M:
1493  break;
1494  case ETH_SPEED_NUM_100M:
1496  break;
1497  case ETH_SPEED_NUM_1G:
1498  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G;
1499  break;
1500  case ETH_SPEED_NUM_2_5G:
1502  break;
1503  case ETH_SPEED_NUM_5G:
1504  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_5G;
1505  break;
1506  case ETH_SPEED_NUM_10G:
1508  break;
1509  case ETH_SPEED_NUM_20G:
1511  break;
1512  case ETH_SPEED_NUM_25G:
1514  break;
1515  case ETH_SPEED_NUM_40G:
1517  break;
1518  case ETH_SPEED_NUM_50G:
1520  break;
1521  case ETH_SPEED_NUM_56G:
1523  break;
1524  case ETH_SPEED_NUM_100G:
1526  break;
1527  case 0:
1528  break;
1529  default:
1530  dpdk_log_warn ("unknown link speed %d", xd->link.link_speed);
1531  break;
1532  }
1533  }
1534  if (hw_flags_chg)
1535  {
1536  if (LINK_STATE_ELOGS)
1537  {
1539 
1540  ELOG_TYPE_DECLARE (e) =
1541  {
1542  .format =
1543  "update-link-state: sw_if_index %d, new flags %d",.format_args
1544  = "i4i4",};
1545 
1546  struct
1547  {
1548  u32 sw_if_index;
1549  u32 flags;
1550  } *ed;
1551  ed = ELOG_DATA (&vm->elog_main, e);
1552  ed->sw_if_index = xd->sw_if_index;
1553  ed->flags = hw_flags;
1554  }
1555  vnet_hw_interface_set_flags (vnm, xd->hw_if_index, hw_flags);
1556  }
1557 }
1558 
1559 static uword
1561 {
1562  clib_error_t *error;
1563  vnet_main_t *vnm = vnet_get_main ();
1564  dpdk_main_t *dm = &dpdk_main;
1566  dpdk_device_t *xd;
1568  int i;
1569  int j;
1570 
1571  error = dpdk_lib_init (dm);
1572 
1573  if (error)
1574  clib_error_report (error);
1575 
1576  tm->worker_thread_release = 1;
1577 
1578  f64 now = vlib_time_now (vm);
1579  vec_foreach (xd, dm->devices)
1580  {
1581  dpdk_update_link_state (xd, now);
1582  }
1583 
1584  {
1585  /*
1586  * Extra set up for bond interfaces:
1587  * 1. Setup MACs for bond interfaces and their slave links which was set
1588  * in dpdk_device_setup() but needs to be done again here to take
1589  * effect.
1590  * 2. Set up info and register slave link state change callback handling.
1591  * 3. Set up info for bond interface related CLI support.
1592  */
1593 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
1594  int nports = rte_eth_dev_count ();
1595 #else
1596  int nports = rte_eth_dev_count_avail ();
1597 #endif
1598  if (nports > 0)
1599  {
1600  /* *INDENT-OFF* */
1601  RTE_ETH_FOREACH_DEV(i)
1602  {
1603  xd = NULL;
1604  for (j = 0; j < nports; j++)
1605  {
1606  if (dm->devices[j].port_id == i)
1607  {
1608  xd = &dm->devices[j];
1609  }
1610  }
1611  ASSERT (xd != NULL);
1612 
1613  if (xd->pmd == VNET_DPDK_PMD_BOND)
1614  {
1615  u8 addr[6];
1616  dpdk_portid_t slink[16];
1617  int nlink = rte_eth_bond_slaves_get (i, slink, 16);
1618  if (nlink > 0)
1619  {
1620  vnet_hw_interface_t *bhi;
1621  ethernet_interface_t *bei;
1622  int rv;
1623 
1624  /* Get MAC of 1st slave link */
1625  rte_eth_macaddr_get
1626  (slink[0], (struct ether_addr *) addr);
1627 
1628  /* Set MAC of bounded interface to that of 1st slave link */
1629  dpdk_log_info ("Set MAC for bond port %d BondEthernet%d",
1630  i, xd->bond_instance_num);
1631  rv = rte_eth_bond_mac_address_set
1632  (i, (struct ether_addr *) addr);
1633  if (rv)
1634  dpdk_log_warn ("Set MAC addr failure rv=%d", rv);
1635 
1636  /* Populate MAC of bonded interface in VPP hw tables */
1637  bhi = vnet_get_hw_interface
1638  (vnm, dm->devices[i].hw_if_index);
1639  bei = pool_elt_at_index
1640  (em->interfaces, bhi->hw_instance);
1641  clib_memcpy (bhi->hw_address, addr, 6);
1642  clib_memcpy (bei->address, addr, 6);
1643 
1644  /* Init l3 packet size allowed on bonded interface */
1646  while (nlink >= 1)
1647  { /* for all slave links */
1648  int slave = slink[--nlink];
1649  dpdk_device_t *sdev = &dm->devices[slave];
1650  vnet_hw_interface_t *shi;
1651  vnet_sw_interface_t *ssi;
1652  ethernet_interface_t *sei;
1653  /* Add MAC to all slave links except the first one */
1654  if (nlink)
1655  {
1656  dpdk_log_info ("Add MAC for slave port %d",
1657  slave);
1658  rv = rte_eth_dev_mac_addr_add
1659  (slave, (struct ether_addr *) addr, 0);
1660  if (rv)
1661  dpdk_log_warn ("Add MAC addr failure rv=%d",
1662  rv);
1663  }
1664  /* Setup slave link state change callback handling */
1665  rte_eth_dev_callback_register
1666  (slave, RTE_ETH_EVENT_INTR_LSC,
1668  dpdk_device_t *sxd = &dm->devices[slave];
1669  sxd->flags |= DPDK_DEVICE_FLAG_BOND_SLAVE;
1670  sxd->bond_port = i;
1671  /* Set slaves bitmap for bonded interface */
1672  bhi->bond_info = clib_bitmap_set
1673  (bhi->bond_info, sdev->hw_if_index, 1);
1674  /* Set MACs and slave link flags on slave interface */
1675  shi = vnet_get_hw_interface (vnm, sdev->hw_if_index);
1676  ssi = vnet_get_sw_interface (vnm, sdev->sw_if_index);
1677  sei = pool_elt_at_index
1678  (em->interfaces, shi->hw_instance);
1681  clib_memcpy (shi->hw_address, addr, 6);
1682  clib_memcpy (sei->address, addr, 6);
1683  /* Set l3 packet size allowed as the lowest of slave */
1684  if (bhi->max_packet_bytes > shi->max_packet_bytes)
1685  bhi->max_packet_bytes = shi->max_packet_bytes;
1686  }
1687  }
1688  }
1689  }
1690  /* *INDENT-ON* */
1691  }
1692  }
1693 
1694  while (1)
1695  {
1696  /*
1697  * check each time through the loop in case intervals are changed
1698  */
1699  f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ?
1701 
1702  vlib_process_wait_for_event_or_clock (vm, min_wait);
1703 
1704  if (dm->admin_up_down_in_progress)
1705  /* skip the poll if an admin up down is in progress (on any interface) */
1706  continue;
1707 
1708  vec_foreach (xd, dm->devices)
1709  {
1710  f64 now = vlib_time_now (vm);
1711  if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval)
1712  dpdk_update_counters (xd, now);
1713  if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval)
1714  dpdk_update_link_state (xd, now);
1715 
1716  }
1717  }
1718 
1719  return 0;
1720 }
1721 
1722 /* *INDENT-OFF* */
1724  .function = dpdk_process,
1725  .type = VLIB_NODE_TYPE_PROCESS,
1726  .name = "dpdk-process",
1727  .process_log2_n_stack_bytes = 17,
1728 };
1729 /* *INDENT-ON* */
1730 
1731 static clib_error_t *
1733 {
1734  dpdk_main_t *dm = &dpdk_main;
1735  clib_error_t *error = 0;
1737 
1738  /* verify that structs are cacheline aligned */
1739  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
1740  "Cache line marker must be 1st element in dpdk_device_t");
1741  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) ==
1743  "Data in cache line 0 is bigger than cache line size");
1744  STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0,
1745  "Cache line marker must be 1st element in frame_queue_trace_t");
1746  STATIC_ASSERT (RTE_CACHE_LINE_SIZE == 1 << CLIB_LOG2_CACHE_LINE_BYTES,
1747  "DPDK RTE CACHE LINE SIZE does not match with 1<<CLIB_LOG2_CACHE_LINE_BYTES");
1748 
1749  dm->vlib_main = vm;
1750  dm->vnet_main = vnet_get_main ();
1751  dm->conf = &dpdk_config_main;
1752 
1753  dm->conf->nchannels = 4;
1754  dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
1755  vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
1756 
1757  vec_validate (dm->recycle, tm->n_thread_stacks - 1);
1758 
1759  /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
1760  dm->buffer_flags_template =
1761  (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID
1762  | VNET_BUFFER_F_L4_CHECKSUM_COMPUTED |
1763  VNET_BUFFER_F_L4_CHECKSUM_CORRECT | VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
1764 
1767 
1768  /* init CLI */
1769  if ((error = vlib_call_init_function (vm, dpdk_cli_init)))
1770  return error;
1771 
1772  dm->log_default = vlib_log_register_class ("dpdk", 0);
1773 
1774  return error;
1775 }
1776 
1778 
1779 
1780 /*
1781  * fd.io coding-style-patch-verification: ON
1782  *
1783  * Local Variables:
1784  * eval: (c-set-style "gnu")
1785  * End:
1786  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:221
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
static void dpdk_bind_devices_to_uio(dpdk_config_main_t *conf)
Definition: init.c:770
f64 time_last_link_update
Definition: dpdk.h:251
vmrglw vmrglh hi
static u8 * format_bitmap_hex(u8 *s, va_list *args)
Format a bitmap as a string of hex bytes.
Definition: bitmap.h:765
format_function_t format_vlib_pci_addr
Definition: pci.h:288
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:255
#define VIRTIO_PCI_MODERN_DEVICEID_NET
Definition: pci_config.h:169
#define clib_min(x, y)
Definition: clib.h:289
#define VNET_HW_INTERFACE_FLAG_SPEED_1G
Definition: interface.h:471
ethernet_main_t ethernet_main
Definition: init.c:45
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:541
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
#define VNET_HW_INTERFACE_FLAG_SPEED_50G
Definition: interface.h:478
u8 interface_name_format_decimal
Definition: dpdk.h:366
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
#define NB_MBUF
Definition: dpdk.h:59
vnet_device_class_t dpdk_device_class
u32 sw_if_index
Definition: dpdk.h:203
#define DPDK_DEVICE_VLAN_STRIP_OFF
Definition: dpdk.h:333
#define NULL
Definition: clib.h:55
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:225
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:574
static uword * clib_bitmap_or(uword *ai, uword *bi)
Logical operator across two bitmaps.
static u32 dpdk_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 flags)
Definition: init.c:92
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1424
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_pci_addr_t *addr, clib_error_t **error)
Definition: pci.c:162
u16 flags
Definition: dpdk.h:211
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define LINK_STATE_ELOGS
Definition: init.c:45
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:53
static struct rte_mempool_ops * get_ops_by_name(char *ops_name)
Definition: init.c:137
u32 file_descriptor
Definition: file.h:54
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
dpdk_device_and_queue_t ** devices_by_hqos_cpu
Definition: dpdk.h:404
#define DPDK_NB_RX_DESC_VIRTIO
Definition: dpdk_priv.h:21
clib_error_t * errors
Definition: dpdk.h:265
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
u32 per_interface_next_index
Definition: dpdk.h:206
u8 enable_tcp_udp_checksum
Definition: dpdk.h:352
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:983
#define DPDK_DEVICE_VLAN_STRIP_ON
Definition: dpdk.h:334
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
#define VNET_HW_INTERFACE_FLAG_SPEED_100G
Definition: interface.h:480
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define DPDK_NB_TX_DESC_DEFAULT
Definition: dpdk_priv.h:20
u32 supported_flow_actions
Definition: dpdk.h:231
#define foreach_eal_double_hyphen_predicate_arg
Definition: dpdk_priv.h:32
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:287
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:458
int dpdk_port_state_callback(dpdk_portid_t port_id, enum rte_eth_event_type type, void *param, void *ret_param)
Definition: common.c:319
u16 bond_instance_num
Definition: dpdk.h:244
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
dpdk_device_config_hqos_t hqos
Definition: dpdk.h:341
vlib_pci_addr_t * vlib_pci_get_all_dev_addrs()
Definition: pci.c:1240
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:227
unsigned char u8
Definition: types.h:56
dpdk_config_main_t dpdk_config_main
Definition: init.c:43
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
#define ETHER_MAX_LEN
Maximum frame len, including CRC.
Definition: init.c:40
double f64
Definition: types.h:142
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define VNET_HW_INTERFACE_FLAG_SPEED_2_5G
Definition: interface.h:472
foreach_dpdk_device_config_item clib_bitmap_t * workers
Definition: dpdk.h:339
#define dpdk_log_warn(...)
Definition: dpdk.h:501
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:440
static clib_error_t * dpdk_log_read_ready(clib_file_t *uf)
Definition: init.c:974
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:156
dpdk_portid_t port_id
Definition: dpdk.h:200
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:649
dpdk_device_config_t default_devconf
Definition: dpdk.h:369
f64 stat_poll_interval
Definition: dpdk.h:434
static char * vlib_unix_get_runtime_dir(void)
Definition: unix.h:141
static dpdk_port_type_t port_type_from_speed_capa(struct rte_eth_dev_info *dev_info)
Definition: init.c:63
clib_error_t * vlib_pci_bind_to_uio(vlib_pci_addr_t *addr, char *uio_drv_name)
Definition: pci.c:345
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
u16 rx_q_used
Definition: dpdk.h:223
#define VNET_HW_INTERFACE_FLAG_SPEED_25G
Definition: interface.h:476
clib_file_main_t file_main
Definition: main.c:63
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:240
unsigned int u32
Definition: types.h:88
void dpdk_device_setup(dpdk_device_t *xd)
Definition: common.c:40
#define vlib_call_init_function(vm, x)
Definition: init.h:227
clib_error_t * unformat_rss_fn(unformat_input_t *input, uword *rss_fn)
Definition: format.c:811
#define DPDK_NB_TX_DESC_VIRTIO
Definition: dpdk_priv.h:22
struct rte_eth_conf port_conf
Definition: dpdk.h:227
static clib_error_t * dpdk_init(vlib_main_t *vm)
Definition: init.c:1732
#define fl(x, y)
u32 max_supported_packet_bytes
Definition: interface.h:534
f64 time_last_stats_update
Definition: dpdk.h:258
struct rte_eth_txconf tx_conf
Definition: dpdk.h:228
u8 * description
Definition: file.h:70
#define hash_get(h, key)
Definition: hash.h:249
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:461
#define vec_insert(V, N, M)
Insert N vector elements starting at element M, initialize new elements to zero (no header...
Definition: vec.h:687
vlib_pci_addr_t pci_addr
Definition: dpdk.h:329
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
Definition: buffer.c:542
static clib_error_t * dpdk_config(vlib_main_t *vm, unformat_input_t *input)
Definition: init.c:1005
#define foreach_eal_double_hyphen_arg
Definition: dpdk_priv.h:48
dpdk_portid_t bond_port
Definition: dpdk.h:248
#define ETHERNET_INTERFACE_FLAG_CONFIG_MTU(flags)
Definition: ethernet.h:153
u8 ** eal_init_args
Definition: dpdk.h:348
#define VNET_HW_INTERFACE_FLAG_SPEED_10M
Definition: interface.h:469
dpdk_per_thread_data_t * per_thread_data
Definition: dpdk.h:405
#define VNET_SW_INTERFACE_FLAG_BOND_SLAVE
Definition: interface.h:669
#define VNET_HW_INTERFACE_FLAG_SPEED_56G
Definition: interface.h:479
#define foreach_eal_single_hyphen_mandatory_arg
Definition: dpdk_priv.h:38
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
#define VNET_HW_INTERFACE_FLAG_HALF_DUPLEX
Definition: interface.h:461
#define clib_error_return_unix(e, args...)
Definition: error.h:102
#define foreach_dpdk_pmd
Definition: dpdk.h:64
vlib_buffer_t buffer_template
Definition: dpdk.h:387
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:439
#define ELOG_DATA(em, f)
Definition: elog.h:481
#define VIRTIO_PCI_LEGACY_DEVICEID_NET
Definition: pci_config.h:168
dpdk_port_type_t port_type
Definition: dpdk.h:259
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:164
signed char i8
Definition: types.h:45
u16 tx_q_used
Definition: dpdk.h:222
u16 nb_rx_desc
Definition: dpdk.h:224
uint16_t dpdk_portid_t
Definition: dpdk.h:120
#define dpdk_log_info(...)
Definition: dpdk.h:505
u32 hw_if_index
Definition: dpdk.h:202
void unformat_init_vector(unformat_input_t *input, u8 *vector_string)
Definition: unformat.c:1031
u32 ** recycle
Definition: dpdk.h:408
#define VNET_HW_INTERFACE_BOND_INFO_SLAVE
Definition: interface.h:554
u16 af_packet_instance_num
Definition: dpdk.h:243
#define foreach_eal_single_hyphen_arg
Definition: dpdk_priv.h:42
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
#define DPDK_NB_RX_DESC_DEFAULT
Definition: dpdk_priv.h:19
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
svmdb_client_t * c
static clib_error_t * dpdk_device_config(dpdk_config_main_t *conf, vlib_pci_addr_t pci_addr, unformat_input_t *input, u8 is_default)
Definition: init.c:874
dpdk_device_t * devices
Definition: dpdk.h:403
vlib_main_t * vm
Definition: buffer.c:294
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:87
u8 nchannels_set_manually
Definition: dpdk.h:357
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
#define dpdk_log_err(...)
Definition: dpdk.h:499
volatile u32 ** lockp
Definition: dpdk.h:194
dpdk_device_config_t * dev_confs
Definition: dpdk.h:370
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
dpdk_pmd_t pmd
Definition: dpdk.h:208
format_function_t format_dpdk_device_errors
Definition: dpdk.h:512
elog_main_t elog_main
Definition: main.h:158
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:147
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD
Definition: interface.h:499
u8 coremask_set_manually
Definition: dpdk.h:356
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
#define VNET_HW_INTERFACE_FLAG_SPEED_10G
Definition: interface.h:474
static void dpdk_device_lock_init(dpdk_device_t *xd)
Definition: init.c:124
u8 * interface_name_suffix
Definition: dpdk.h:216
signed int i32
Definition: types.h:81
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define VNET_HW_INTERFACE_FLAG_FULL_DUPLEX
Definition: interface.h:462
#define ASSERT(truth)
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:205
format_function_t format_dpdk_device_name
Definition: dpdk.h:510
int hqos_cpu_count
Definition: dpdk.h:430
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:138
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
vlib_log_class_t log_default
Definition: dpdk.h:448
static uword dpdk_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: init.c:1560
Bitmaps built as vectors of machine words.
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:273
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
#define DPDK_LINK_POLL_INTERVAL
Definition: dpdk.h:271
size_t count
Definition: vapi.c:42
dpdk_main_t dpdk_main
Definition: init.c:42
uword * thread_registrations_by_name
Definition: threads.h:297
clib_error_t * dpdk_cli_init(vlib_main_t *vm)
Definition: cli.c:2090
dpdk_portid_t device_index
Definition: dpdk.h:197
struct rte_eth_link link
Definition: dpdk.h:250
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:247
dpdk_port_type_t
Definition: dpdk.h:98
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
Definition: defs.h:47
clib_error_t * vlib_unix_recursive_mkdir(char *path)
Definition: util.c:103
#define DPDK_STATS_POLL_INTERVAL
Definition: dpdk.h:268
#define VNET_HW_INTERFACE_FLAG_SPEED_100M
Definition: interface.h:470
unformat_function_t unformat_line
Definition: format.h:278
static vlib_node_registration_t dpdk_process_node
(constructor) VLIB_REGISTER_NODE (dpdk_process_node)
Definition: init.c:1723
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
uword unformat_vlib_cli_sub_input(unformat_input_t *i, va_list *args)
Definition: cli.c:153
u8 admin_up_down_in_progress
Definition: dpdk.h:426
#define STATIC_ASSERT(truth,...)
#define VNET_HW_INTERFACE_FLAG_SPEED_5G
Definition: interface.h:473
static struct rte_eth_conf port_conf_template
Definition: init.c:49
u8 no_tx_checksum_offload
Definition: dpdk.h:353
static uword unformat_bitmap_list(unformat_input_t *input, va_list *va)
unformat a list of bit ranges into a bitmap (eg "0-3,5-7,11" )
Definition: bitmap.h:714
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
static clib_error_t * dpdk_lib_init(dpdk_main_t *dm)
Definition: init.c:191
#define hash_get_mem(h, key)
Definition: hash.h:269
u32 buffer_flags_template
Definition: dpdk.h:411
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:120
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
#define vnet_buffer(b)
Definition: buffer.h:360
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
#define VNET_HW_INTERFACE_FLAG_SPEED_40G
Definition: interface.h:477
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
int hqos_cpu_first_index
Definition: dpdk.h:429
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
#define ETHERNET_MAX_PACKET_BYTES
Definition: ethernet.h:140
#define vec_foreach(var, vec)
Vector iterator.
i8 cpu_socket
Definition: dpdk.h:209
#define ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC(flags)
Definition: ethernet.h:148
uword * cpu_socket_bitmap
Definition: threads.h:332
Definition: file.h:51
static int dpdk_ring_alloc(struct rte_mempool *mp)
Definition: init.c:151
vhost_vring_addr_t addr
Definition: vhost-user.h:83
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:662
u8 * uio_driver_name
Definition: dpdk.h:350
vlib_thread_registration_t ** registrations
Definition: threads.h:295
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
Definition: pci.h:107
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
unformat_function_t unformat_dpdk_log_level
Definition: dpdk.h:518
ethernet_interface_t * interfaces
Definition: ethernet.h:278
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
vnet_main_t * vnet_main
Definition: dpdk.h:438
u16 nb_tx_desc
Definition: dpdk.h:213
clib_error_t * unformat_hqos(unformat_input_t *input, dpdk_device_config_hqos_t *hqos)
Definition: format.c:848
u16 device_class
Definition: pci.h:72
#define VNET_HW_INTERFACE_FLAG_SPEED_20G
Definition: interface.h:475
uword * device_config_index_by_pci_addr
Definition: dpdk.h:371
uword unformat_skip_white_space(unformat_input_t *input)
Definition: unformat.c:815
static uword vnet_hw_interface_is_link_up(vnet_main_t *vnm, u32 hw_if_index)
volatile u32 worker_thread_release
Definition: threads.h:338
#define dpdk_log_notice(...)
Definition: dpdk.h:503
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
vnet_device_main_t vnet_device_main
Definition: devices.c:22
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
f64 link_state_poll_interval
Definition: dpdk.h:433
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
dpdk_config_main_t * conf
Definition: dpdk.h:439
vlib_main_t * vlib_main
Definition: dpdk.h:437