FD.io VPP  v17.04.2-2-ga8f93f8
Vector Packet Processing
init.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/error.h>
18 #include <vppinfra/format.h>
19 #include <vppinfra/bitmap.h>
20 
21 #include <vnet/ethernet/ethernet.h>
22 #include <dpdk/device/dpdk.h>
23 #include <vlib/unix/physmem.h>
24 #include <vlib/pci/pci.h>
25 
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <unistd.h>
29 #include <sys/stat.h>
30 #include <sys/mount.h>
31 #include <string.h>
32 #include <fcntl.h>
33 
34 #include <dpdk/device/dpdk_priv.h>
35 
37 
38 #include <vlibapi/api.h>
39 #include <vlibmemory/api.h>
40 
41 /* define message IDs */
42 #include <dpdk/api/dpdk_msg_enum.h>
43 
44 #define vl_typedefs /* define message structures */
46 #undef vl_typedefs
47 
48 #define vl_endianfun /* define message structures */
50 #undef vl_endianfun
51 
52 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
53 
54 /* Get the API version number. */
55 #define vl_api_version(n,v) static u32 api_version=(v);
57 #undef vl_api_version
58 
59 /* Macro to finish up custom dump fns */
60 #define FINISH \
61  vec_add1 (s, 0); \
62  vl_print (handle, (char *)s); \
63  vec_free (s); \
64  return handle;
65 
67 
68 static void
71 {
73  int rv = 0;
74 
75  dpdk_main_t *dm = &dpdk_main;
76  dpdk_device_t *xd;
77 
78  u32 sw_if_index = ntohl (mp->sw_if_index);
79  u32 subport = ntohl (mp->subport);
80  u32 pipe = ntohl (mp->pipe);
81  u32 profile = ntohl (mp->profile);
83 
85 
86  /* hw_if & dpdk device */
87  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
88 
89  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
90 
91  rv = rte_sched_pipe_config (xd->hqos_ht->hqos, subport, pipe, profile);
92 
94 
95  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_PIPE_REPLY);
96 }
97 
100 {
101  u8 *s;
102 
103  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_pipe ");
104 
105  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
106 
107  s = format (s, "subport %u pipe %u profile %u ",
108  ntohl (mp->subport), ntohl (mp->pipe), ntohl (mp->profile));
109 
110  FINISH;
111 }
112 
113 static void
116 {
118  int rv = 0;
119 
120  dpdk_main_t *dm = &dpdk_main;
121  dpdk_device_t *xd;
122  struct rte_sched_subport_params p;
123 
124  u32 sw_if_index = ntohl (mp->sw_if_index);
125  u32 subport = ntohl (mp->subport);
126  p.tb_rate = ntohl (mp->tb_rate);
127  p.tb_size = ntohl (mp->tb_size);
128  p.tc_rate[0] = ntohl (mp->tc_rate[0]);
129  p.tc_rate[1] = ntohl (mp->tc_rate[1]);
130  p.tc_rate[2] = ntohl (mp->tc_rate[2]);
131  p.tc_rate[3] = ntohl (mp->tc_rate[3]);
132  p.tc_period = ntohl (mp->tc_period);
133 
135 
137 
138  /* hw_if & dpdk device */
139  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
140 
141  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
142 
143  rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport, &p);
144 
146 
147  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_SUBPORT_REPLY);
148 }
149 
152 {
153  u8 *s;
154 
155  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_subport ");
156 
157  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
158 
159  s =
160  format (s,
161  "subport %u rate %u bkt_size %u tc0 %u tc1 %u tc2 %u tc3 %u period %u",
162  ntohl (mp->subport), ntohl (mp->tb_rate), ntohl (mp->tb_size),
163  ntohl (mp->tc_rate[0]), ntohl (mp->tc_rate[1]),
164  ntohl (mp->tc_rate[2]), ntohl (mp->tc_rate[3]),
165  ntohl (mp->tc_period));
166 
167  FINISH;
168 }
169 
170 static void
173 {
175  int rv = 0;
176 
177  dpdk_main_t *dm = &dpdk_main;
179  dpdk_device_t *xd;
180 
181  u32 sw_if_index = ntohl (mp->sw_if_index);
182  u32 entry = ntohl (mp->entry);
183  u32 tc = ntohl (mp->tc);
184  u32 queue = ntohl (mp->queue);
185  u32 val, i;
186 
188 
190 
191  /* hw_if & dpdk device */
192  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
193 
194  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
195 
196  if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
197  {
198  clib_warning ("invalid traffic class !!");
199  rv = VNET_API_ERROR_INVALID_VALUE;
200  goto done;
201  }
202  if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS)
203  {
204  clib_warning ("invalid queue !!");
205  rv = VNET_API_ERROR_INVALID_VALUE;
206  goto done;
207  }
208 
209  /* Detect the set of worker threads */
210  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
211 
212  if (p == 0)
213  {
214  clib_warning ("worker thread registration AWOL !!");
215  rv = VNET_API_ERROR_INVALID_VALUE_2;
216  goto done;
217  }
218 
220  int worker_thread_first = tr->first_index;
221  int worker_thread_count = tr->count;
222 
223  val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue;
224  for (i = 0; i < worker_thread_count; i++)
225  xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val;
226 
228 done:
229 
230  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_TCTBL_REPLY);
231 }
232 
235 {
236  u8 *s;
237 
238  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_tctbl ");
239 
240  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
241 
242  s = format (s, "entry %u tc %u queue %u",
243  ntohl (mp->entry), ntohl (mp->tc), ntohl (mp->queue));
244 
245  FINISH;
246 }
247 
248 #define foreach_dpdk_plugin_api_msg \
249 _(SW_INTERFACE_SET_DPDK_HQOS_PIPE, sw_interface_set_dpdk_hqos_pipe) \
250 _(SW_INTERFACE_SET_DPDK_HQOS_SUBPORT, sw_interface_set_dpdk_hqos_subport) \
251 _(SW_INTERFACE_SET_DPDK_HQOS_TCTBL, sw_interface_set_dpdk_hqos_tctbl)
252 
253 /* Set up the API message handling tables */
254 static clib_error_t *
256 {
257  dpdk_main_t *dm __attribute__ ((unused)) = &dpdk_main;
258 #define _(N,n) \
259  vl_msg_api_set_handlers((VL_API_##N + dm->msg_id_base), \
260  #n, \
261  vl_api_##n##_t_handler, \
262  vl_noop_handler, \
263  vl_api_##n##_t_endian, \
264  vl_api_##n##_t_print, \
265  sizeof(vl_api_##n##_t), 1);
267 #undef _
268  return 0;
269 }
270 
271 #define vl_msg_name_crc_list
272 #include <dpdk/api/dpdk_all_api_h.h>
273 #undef vl_msg_name_crc_list
274 
275 static void
277 {
278 #define _(id,n,crc) \
279  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + dm->msg_id_base);
280  foreach_vl_msg_name_crc_dpdk;
281 #undef _
282 }
283 
284 // TODO
285 /*
286 static void plugin_custom_dump_configure (dpdk_main_t * dm)
287 {
288 #define _(n,f) dm->api_main->msg_print_handlers \
289  [VL_API_##n + dm->msg_id_base] \
290  = (void *) vl_api_##f##_t_print;
291  foreach_dpdk_plugin_api_msg;
292 #undef _
293 }
294 */
295 /* force linker to link functions used by vlib and declared weak */
297  &rte_pktmbuf_init,
298  &rte_pktmbuf_pool_init,
299 };
300 
301 #define LINK_STATE_ELOGS 0
302 
303 #define DEFAULT_HUGE_DIR "/run/vpp/hugepages"
304 #define VPP_RUN_DIR "/run/vpp"
305 
306 /* Port configuration, mildly modified Intel app values */
307 
308 static struct rte_eth_conf port_conf_template = {
309  .rxmode = {
310  .split_hdr_size = 0,
311  .header_split = 0, /**< Header Split disabled */
312  .hw_ip_checksum = 0, /**< IP checksum offload disabled */
313  .hw_vlan_filter = 0, /**< VLAN filtering disabled */
314  .hw_strip_crc = 0, /**< CRC stripped by hardware */
315  },
316  .txmode = {
317  .mq_mode = ETH_MQ_TX_NONE,
318  },
319 };
320 
321 clib_error_t *
323 {
324  int rv;
325  int j;
326 
327  ASSERT (os_get_cpu_number () == 0);
328 
330  {
332  rte_eth_dev_stop (xd->device_index);
333  }
334 
335  rv = rte_eth_dev_configure (xd->device_index, xd->rx_q_used,
336  xd->tx_q_used, &xd->port_conf);
337 
338  if (rv < 0)
339  return clib_error_return (0, "rte_eth_dev_configure[%d]: err %d",
340  xd->device_index, rv);
341 
342  /* Set up one TX-queue per worker thread */
343  for (j = 0; j < xd->tx_q_used; j++)
344  {
345  rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
346  xd->cpu_socket, &xd->tx_conf);
347 
348  /* retry with any other CPU socket */
349  if (rv < 0)
350  rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
351  SOCKET_ID_ANY, &xd->tx_conf);
352  if (rv < 0)
353  break;
354  }
355 
356  if (rv < 0)
357  return clib_error_return (0, "rte_eth_tx_queue_setup[%d]: err %d",
358  xd->device_index, rv);
359 
360  for (j = 0; j < xd->rx_q_used; j++)
361  {
362 
363  rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
364  xd->cpu_socket, 0,
365  dm->
366  pktmbuf_pools[xd->cpu_socket_id_by_queue
367  [j]]);
368 
369  /* retry with any other CPU socket */
370  if (rv < 0)
371  rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
372  SOCKET_ID_ANY, 0,
373  dm->
374  pktmbuf_pools[xd->cpu_socket_id_by_queue
375  [j]]);
376  if (rv < 0)
377  return clib_error_return (0, "rte_eth_rx_queue_setup[%d]: err %d",
378  xd->device_index, rv);
379  }
380 
382  {
383  int rv;
384  rv = rte_eth_dev_start (xd->device_index);
385  if (!rv && xd->default_mac_address)
386  rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
387  (struct ether_addr *)
388  xd->default_mac_address);
389  if (rv < 0)
390  clib_warning ("rte_eth_dev_start %d returned %d",
391  xd->device_index, rv);
392  }
393  return 0;
394 }
395 
396 static u32
398 {
399  dpdk_main_t *dm = &dpdk_main;
401  u32 old = 0;
402 
404  {
405  old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0;
406 
409  else
411 
413  {
415  rte_eth_promiscuous_enable (xd->device_index);
416  else
417  rte_eth_promiscuous_disable (xd->device_index);
418  }
419  }
420  else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
421  {
422  int rv;
423 
424  xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes;
425 
427  rte_eth_dev_stop (xd->device_index);
428 
429  rv = rte_eth_dev_configure
430  (xd->device_index, xd->rx_q_used, xd->tx_q_used, &xd->port_conf);
431 
432  if (rv < 0)
434  "rte_eth_dev_configure[%d]: err %d",
435  xd->device_index, rv);
436 
437  rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
438 
440  {
441  int rv = rte_eth_dev_start (xd->device_index);
442  if (!rv && xd->default_mac_address)
443  rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
444  (struct ether_addr *)
445  xd->default_mac_address);
446  if (rv < 0)
447  clib_warning ("rte_eth_dev_start %d returned %d",
448  xd->device_index, rv);
449  }
450 
451  }
452  return old;
453 }
454 
455 void
457 {
458  int q;
459  vec_validate (xd->lockp, xd->tx_q_used - 1);
460  for (q = 0; q < xd->tx_q_used; q++)
461  {
464  memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES);
465  }
466 }
467 
468 void
470 {
471  int q;
472 
473  for (q = 0; q < vec_len (xd->lockp); q++)
474  clib_mem_free ((void *) xd->lockp[q]);
475  vec_free (xd->lockp);
476  xd->lockp = 0;
477 }
478 
479 static clib_error_t *
481 {
482  u32 nports;
483  u32 nb_desc = 0;
484  int i;
485  clib_error_t *error;
490  dpdk_device_t *xd;
491  vlib_pci_addr_t last_pci_addr;
492  u32 last_pci_addr_port = 0;
493  vlib_thread_registration_t *tr, *tr_hqos;
494  uword *p, *p_hqos;
495 
496  u32 next_cpu = 0, next_hqos_cpu = 0;
497  u8 af_packet_port_id = 0;
498  u8 bond_ether_port_id = 0;
499  last_pci_addr.as_u32 = ~0;
500 
501  dm->input_cpu_first_index = 0;
502  dm->input_cpu_count = 1;
503 
504  /* find out which cpus will be used for input */
505  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
506  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
507 
508  if (tr && tr->count > 0)
509  {
511  dm->input_cpu_count = tr->count;
512  }
513 
516 
517  dm->hqos_cpu_first_index = 0;
518  dm->hqos_cpu_count = 0;
519 
520  /* find out which cpus will be used for I/O TX */
521  p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads");
522  tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0;
523 
524  if (tr_hqos && tr_hqos->count > 0)
525  {
526  dm->hqos_cpu_first_index = tr_hqos->first_index;
527  dm->hqos_cpu_count = tr_hqos->count;
528  }
529 
532 
533  nports = rte_eth_dev_count ();
534  if (nports < 1)
535  {
536  clib_warning ("DPDK drivers found no ports...");
537  }
538 
539  if (CLIB_DEBUG > 0)
540  clib_warning ("DPDK drivers found %d ports...", nports);
541 
542  /*
543  * All buffers are all allocated from the same rte_mempool.
544  * Thus they all have the same number of data bytes.
545  */
549  "dpdk rx");
550 
551  if (dm->conf->enable_tcp_udp_checksum)
554 
555  /* vlib_buffer_t template */
558  for (i = 0; i < tm->n_vlib_mains; i++)
559  {
562  fl = vlib_buffer_get_free_list (vm,
565  bt->flags = dm->buffer_flags_template;
566  bt->current_data = -RTE_PKTMBUF_HEADROOM;
567  vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0;
568  }
569 
570  for (i = 0; i < nports; i++)
571  {
572  u8 addr[6];
573  u8 vlan_strip = 0;
574  int j;
575  struct rte_eth_dev_info dev_info;
576  clib_error_t *rv;
577  struct rte_eth_link l;
578  dpdk_device_config_t *devconf = 0;
579  vlib_pci_addr_t pci_addr;
580  uword *p = 0;
581 
582  rte_eth_dev_info_get (i, &dev_info);
583  if (dev_info.pci_dev) /* bonded interface has no pci info */
584  {
585  pci_addr.domain = dev_info.pci_dev->addr.domain;
586  pci_addr.bus = dev_info.pci_dev->addr.bus;
587  pci_addr.slot = dev_info.pci_dev->addr.devid;
588  pci_addr.function = dev_info.pci_dev->addr.function;
589  p =
591  pci_addr.as_u32);
592  }
593 
594  if (p)
595  devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
596  else
597  devconf = &dm->conf->default_devconf;
598 
599  /* Create vnet interface */
603  xd->cpu_socket = (i8) rte_eth_dev_socket_id (i);
604 
605  /* Handle interface naming for devices with multiple ports sharing same PCI ID */
606  if (dev_info.pci_dev)
607  {
608  struct rte_eth_dev_info di = { 0 };
609  rte_eth_dev_info_get (i + 1, &di);
610  if (di.pci_dev && pci_addr.as_u32 != last_pci_addr.as_u32 &&
611  memcmp (&dev_info.pci_dev->addr, &di.pci_dev->addr,
612  sizeof (struct rte_pci_addr)) == 0)
613  {
614  xd->interface_name_suffix = format (0, "0");
615  last_pci_addr.as_u32 = pci_addr.as_u32;
616  last_pci_addr_port = i;
617  }
618  else if (pci_addr.as_u32 == last_pci_addr.as_u32)
619  {
621  format (0, "%u", i - last_pci_addr_port);
622  }
623  else
624  {
625  last_pci_addr.as_u32 = ~0;
626  }
627  }
628  else
629  last_pci_addr.as_u32 = ~0;
630 
631  clib_memcpy (&xd->tx_conf, &dev_info.default_txconf,
632  sizeof (struct rte_eth_txconf));
633  if (dm->conf->no_multi_seg)
634  {
635  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
636  port_conf_template.rxmode.jumbo_frame = 0;
637  port_conf_template.rxmode.enable_scatter = 0;
638  }
639  else
640  {
641  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS;
642  port_conf_template.rxmode.jumbo_frame = 1;
643  port_conf_template.rxmode.enable_scatter = 1;
645  }
646 
647  clib_memcpy (&xd->port_conf, &port_conf_template,
648  sizeof (struct rte_eth_conf));
649 
650  xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains);
651 
652  if (devconf->num_tx_queues > 0
653  && devconf->num_tx_queues < xd->tx_q_used)
654  xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues);
655 
656  if (devconf->num_rx_queues > 1 && dm->use_rss == 0)
657  {
658  dm->use_rss = 1;
659  }
660 
661  if (devconf->num_rx_queues > 1
662  && dev_info.max_rx_queues >= devconf->num_rx_queues)
663  {
664  xd->rx_q_used = devconf->num_rx_queues;
665  xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
666  if (devconf->rss_fn == 0)
667  xd->port_conf.rx_adv_conf.rss_conf.rss_hf =
668  ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
669  else
670  xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn;
671  }
672  else
673  xd->rx_q_used = 1;
674 
676 
677  /* workaround for drivers not setting driver_name */
678  if ((!dev_info.driver_name) && (dev_info.pci_dev))
679  dev_info.driver_name = dev_info.pci_dev->driver->driver.name;
680 
681  ASSERT (dev_info.driver_name);
682 
683  if (!xd->pmd)
684  {
685 
686 
687 #define _(s,f) else if (dev_info.driver_name && \
688  !strcmp(dev_info.driver_name, s)) \
689  xd->pmd = VNET_DPDK_PMD_##f;
690  if (0)
691  ;
693 #undef _
694  else
696 
700 
701  switch (xd->pmd)
702  {
703  /* 1G adapters */
704  case VNET_DPDK_PMD_E1000EM:
705  case VNET_DPDK_PMD_IGB:
706  case VNET_DPDK_PMD_IGBVF:
708  break;
709 
710  /* 10G adapters */
711  case VNET_DPDK_PMD_IXGBE:
712  case VNET_DPDK_PMD_IXGBEVF:
713  case VNET_DPDK_PMD_THUNDERX:
715  break;
716  case VNET_DPDK_PMD_DPAA2:
718  break;
719 
720  /* Cisco VIC */
721  case VNET_DPDK_PMD_ENIC:
722  rte_eth_link_get_nowait (i, &l);
724  if (l.link_speed == 40000)
726  else
728  break;
729 
730  /* Intel Fortville */
731  case VNET_DPDK_PMD_I40E:
732  case VNET_DPDK_PMD_I40EVF:
735 
736  switch (dev_info.pci_dev->id.device_id)
737  {
741  break;
742  case I40E_DEV_ID_QSFP_A:
743  case I40E_DEV_ID_QSFP_B:
744  case I40E_DEV_ID_QSFP_C:
746  break;
747  case I40E_DEV_ID_VF:
748  rte_eth_link_get_nowait (i, &l);
749  xd->port_type = l.link_speed == 10000 ?
751  break;
752  default:
754  }
755  break;
756 
757  case VNET_DPDK_PMD_CXGBE:
758  switch (dev_info.pci_dev->id.device_id)
759  {
760  case 0x540d: /* T580-CR */
761  case 0x5410: /* T580-LP-cr */
763  break;
764  case 0x5403: /* T540-CR */
766  break;
767  default:
769  }
770  break;
771 
772  case VNET_DPDK_PMD_MLX5:
773  {
774  char *pn_100g[] = { "MCX415A-CCAT", "MCX416A-CCAT", 0 };
775  char *pn_40g[] = { "MCX413A-BCAT", "MCX414A-BCAT",
776  "MCX415A-BCAT", "MCX416A-BCAT", "MCX4131A-BCAT", 0
777  };
778  char *pn_10g[] = { "MCX4111A-XCAT", "MCX4121A-XCAT", 0 };
779 
780  vlib_pci_device_t *pd = vlib_get_pci_device (&pci_addr);
781  u8 *pn = 0;
782  char **c;
783  int found = 0;
784  pn = format (0, "%U%c",
785  format_vlib_pci_vpd, pd->vpd_r, "PN", 0);
786 
787  if (!pn)
788  break;
789 
790  c = pn_100g;
791  while (!found && c[0])
792  {
793  if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
794  {
796  break;
797  }
798  c++;
799  }
800 
801  c = pn_40g;
802  while (!found && c[0])
803  {
804  if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
805  {
807  break;
808  }
809  c++;
810  }
811 
812  c = pn_10g;
813  while (!found && c[0])
814  {
815  if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
816  {
818  break;
819  }
820  c++;
821  }
822 
823  vec_free (pn);
824  }
825 
826  break;
827  /* Intel Red Rock Canyon */
828  case VNET_DPDK_PMD_FM10K:
830  break;
831 
832  /* virtio */
833  case VNET_DPDK_PMD_VIRTIO:
837  break;
838 
839  /* vmxnet3 */
840  case VNET_DPDK_PMD_VMXNET3:
842  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
843  break;
844 
845  case VNET_DPDK_PMD_AF_PACKET:
847  xd->port_id = af_packet_port_id++;
848  break;
849 
850  case VNET_DPDK_PMD_BOND:
853  xd->port_id = bond_ether_port_id++;
854  break;
855 
856  default:
858  }
859 
860  if (devconf->num_rx_desc)
861  xd->nb_rx_desc = devconf->num_rx_desc;
862 
863  if (devconf->num_tx_desc)
864  xd->nb_tx_desc = devconf->num_tx_desc;
865  }
866 
867  /*
868  * Ensure default mtu is not > the mtu read from the hardware.
869  * Otherwise rte_eth_dev_configure() will fail and the port will
870  * not be available.
871  */
872  if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen)
873  {
874  /*
875  * This device does not support the platforms's max frame
876  * size. Use it's advertised mru instead.
877  */
878  xd->port_conf.rxmode.max_rx_pkt_len = dev_info.max_rx_pktlen;
879  }
880  else
881  {
882  xd->port_conf.rxmode.max_rx_pkt_len = ETHERNET_MAX_PACKET_BYTES;
883 
884  /*
885  * Some platforms do not account for Ethernet FCS (4 bytes) in
886  * MTU calculations. To interop with them increase mru but only
887  * if the device's settings can support it.
888  */
889  if ((dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)) &&
890  xd->port_conf.rxmode.hw_strip_crc)
891  {
892  /*
893  * Allow additional 4 bytes (for Ethernet FCS). These bytes are
894  * stripped by h/w and so will not consume any buffer memory.
895  */
896  xd->port_conf.rxmode.max_rx_pkt_len += 4;
897  }
898  }
899 
900  if (xd->pmd == VNET_DPDK_PMD_AF_PACKET)
901  {
902  f64 now = vlib_time_now (vm);
903  u32 rnd;
904  rnd = (u32) (now * 1e6);
905  rnd = random_u32 (&rnd);
906  clib_memcpy (addr + 2, &rnd, sizeof (rnd));
907  addr[0] = 2;
908  addr[1] = 0xfe;
909  }
910  else
911  rte_eth_macaddr_get (i, (struct ether_addr *) addr);
912 
913  if (xd->tx_q_used < tm->n_vlib_mains)
915 
916  xd->device_index = xd - dm->devices;
917  ASSERT (i == xd->device_index);
918  xd->per_interface_next_index = ~0;
919 
920  /* assign interface to input thread */
922  int q;
923 
924  if (devconf->workers)
925  {
926  int i;
927  q = 0;
928  /* *INDENT-OFF* */
929  clib_bitmap_foreach (i, devconf->workers, ({
930  int cpu = dm->input_cpu_first_index + i;
931  unsigned lcore = vlib_worker_threads[cpu].lcore_id;
932  vec_validate(xd->cpu_socket_id_by_queue, q);
933  xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id(lcore);
934  vec_add2(dm->devices_by_cpu[cpu], dq, 1);
935  dq->device = xd->device_index;
936  dq->queue_id = q++;
937  }));
938  /* *INDENT-ON* */
939  }
940  else
941  for (q = 0; q < xd->rx_q_used; q++)
942  {
943  int cpu = dm->input_cpu_first_index + next_cpu;
944  unsigned lcore = vlib_worker_threads[cpu].lcore_id;
945 
946  /*
947  * numa node for worker thread handling this queue
948  * needed for taking buffers from the right mempool
949  */
951  xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id (lcore);
952 
953  /*
954  * construct vector of (device,queue) pairs for each worker thread
955  */
956  vec_add2 (dm->devices_by_cpu[cpu], dq, 1);
957  dq->device = xd->device_index;
958  dq->queue_id = q;
959 
960  next_cpu++;
961  if (next_cpu == dm->input_cpu_count)
962  next_cpu = 0;
963  }
964 
965 
966  if (devconf->hqos_enabled)
967  {
969 
970  if (devconf->hqos.hqos_thread_valid)
971  {
972  int cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread;
973 
974  if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count)
975  return clib_error_return (0, "invalid HQoS thread index");
976 
977  vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
978  dq->device = xd->device_index;
979  dq->queue_id = 0;
980  }
981  else
982  {
983  int cpu = dm->hqos_cpu_first_index + next_hqos_cpu;
984 
985  if (dm->hqos_cpu_count == 0)
986  return clib_error_return (0, "no HQoS threads available");
987 
988  vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
989  dq->device = xd->device_index;
990  dq->queue_id = 0;
991 
992  next_hqos_cpu++;
993  if (next_hqos_cpu == dm->hqos_cpu_count)
994  next_hqos_cpu = 0;
995 
996  devconf->hqos.hqos_thread_valid = 1;
997  devconf->hqos.hqos_thread = cpu;
998  }
999  }
1000 
1003  for (j = 0; j < tm->n_vlib_mains; j++)
1004  {
1005  vec_validate_ha (xd->tx_vectors[j], xd->nb_tx_desc,
1007  vec_reset_length (xd->tx_vectors[j]);
1008  }
1009 
1012  for (j = 0; j < xd->rx_q_used; j++)
1013  {
1016  vec_reset_length (xd->rx_vectors[j]);
1017  }
1018 
1021 
1022  rv = dpdk_port_setup (dm, xd);
1023 
1024  if (rv)
1025  return rv;
1026 
1027  if (devconf->hqos_enabled)
1028  {
1029  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
1030  if (rv)
1031  return rv;
1032  }
1033 
1034  /* count the number of descriptors used for this device */
1035  nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
1036 
1038  (dm->vnet_main, dpdk_device_class.index, xd->device_index,
1039  /* ethernet address */ addr,
1041  if (error)
1042  return error;
1043 
1045  xd->vlib_sw_if_index = sw->sw_if_index;
1047 
1048  /*
1049  * For cisco VIC vNIC, set default to VLAN strip enabled, unless
1050  * specified otherwise in the startup config.
1051  * For other NICs default to VLAN strip disabled, unless specified
1052  * otherwis in the startup config.
1053  */
1054  if (xd->pmd == VNET_DPDK_PMD_ENIC)
1055  {
1056  if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF)
1057  vlan_strip = 1; /* remove vlan tag from VIC port by default */
1058  else
1059  clib_warning ("VLAN strip disabled for interface\n");
1060  }
1061  else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON)
1062  vlan_strip = 1;
1063 
1064  if (vlan_strip)
1065  {
1066  int vlan_off;
1067  vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index);
1068  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
1069  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
1070  if (rte_eth_dev_set_vlan_offload (xd->device_index, vlan_off) == 0)
1071  clib_warning ("VLAN strip enabled for interface\n");
1072  else
1073  clib_warning ("VLAN strip cannot be supported by interface\n");
1074  }
1075 
1077  xd->port_conf.rxmode.max_rx_pkt_len - sizeof (ethernet_header_t);
1078 
1079  rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
1080  }
1081 
1082  if (nb_desc > dm->conf->num_mbufs)
1083  clib_warning ("%d mbufs allocated but total rx/tx ring size is %d\n",
1084  dm->conf->num_mbufs, nb_desc);
1085 
1086  return 0;
1087 }
1088 
1089 static void
1091 {
1092  vlib_pci_main_t *pm = &pci_main;
1093  clib_error_t *error;
1094  vlib_pci_device_t *d;
1095  u8 *pci_addr = 0;
1096  int num_whitelisted = vec_len (conf->dev_confs);
1097 
1098  /* *INDENT-OFF* */
1099  pool_foreach (d, pm->pci_devs, ({
1100  dpdk_device_config_t * devconf = 0;
1101  vec_reset_length (pci_addr);
1102  pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, &d->bus_address, 0);
1103 
1104  if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && d->device_class != PCI_CLASS_PROCESSOR_CO)
1105  continue;
1106 
1107  if (num_whitelisted)
1108  {
1109  uword * p = hash_get (conf->device_config_index_by_pci_addr, d->bus_address.as_u32);
1110 
1111  if (!p)
1112  continue;
1113 
1114  devconf = pool_elt_at_index (conf->dev_confs, p[0]);
1115  }
1116 
1117  /* virtio */
1118  if (d->vendor_id == 0x1af4 && d->device_id == 0x1000)
1119  ;
1120  /* vmxnet3 */
1121  else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0)
1122  ;
1123  /* all Intel network devices */
1124  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_NETWORK_ETHERNET)
1125  ;
1126  /* all Intel QAT devices VFs */
1127  else if (d->vendor_id == 0x8086 && d->device_class == PCI_CLASS_PROCESSOR_CO &&
1128  (d->device_id == 0x0443 || d->device_id == 0x37c9 || d->device_id == 0x19e3))
1129  ;
1130  /* Cisco VIC */
1131  else if (d->vendor_id == 0x1137 && d->device_id == 0x0043)
1132  ;
1133  /* Chelsio T4/T5 */
1134  else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000)
1135  ;
1136  else
1137  {
1138  clib_warning ("Unsupported PCI device 0x%04x:0x%04x found "
1139  "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id,
1140  pci_addr);
1141  continue;
1142  }
1143 
1144  error = vlib_pci_bind_to_uio (d, (char *) conf->uio_driver_name);
1145 
1146  if (error)
1147  {
1148  if (devconf == 0)
1149  {
1150  pool_get (conf->dev_confs, devconf);
1152  devconf - conf->dev_confs);
1153  devconf->pci_addr.as_u32 = d->bus_address.as_u32;
1154  }
1155  devconf->is_blacklisted = 1;
1156  clib_error_report (error);
1157  }
1158  }));
1159  /* *INDENT-ON* */
1160  vec_free (pci_addr);
1161 }
1162 
1163 static clib_error_t *
1164 dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
1165  unformat_input_t * input, u8 is_default)
1166 {
1167  clib_error_t *error = 0;
1168  uword *p;
1169  dpdk_device_config_t *devconf;
1170  unformat_input_t sub_input;
1171 
1172  if (is_default)
1173  {
1174  devconf = &conf->default_devconf;
1175  }
1176  else
1177  {
1178  p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32);
1179 
1180  if (!p)
1181  {
1182  pool_get (conf->dev_confs, devconf);
1183  hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32,
1184  devconf - conf->dev_confs);
1185  }
1186  else
1187  return clib_error_return (0,
1188  "duplicate configuration for PCI address %U",
1189  format_vlib_pci_addr, &pci_addr);
1190  }
1191 
1192  devconf->pci_addr.as_u32 = pci_addr.as_u32;
1193  devconf->hqos_enabled = 0;
1195 
1196  if (!input)
1197  return 0;
1198 
1199  unformat_skip_white_space (input);
1200  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1201  {
1202  if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues))
1203  ;
1204  else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues))
1205  ;
1206  else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc))
1207  ;
1208  else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc))
1209  ;
1210  else if (unformat (input, "workers %U", unformat_bitmap_list,
1211  &devconf->workers))
1212  ;
1213  else
1214  if (unformat
1215  (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input))
1216  {
1217  error = unformat_rss_fn (&sub_input, &devconf->rss_fn);
1218  if (error)
1219  break;
1220  }
1221  else if (unformat (input, "vlan-strip-offload off"))
1223  else if (unformat (input, "vlan-strip-offload on"))
1225  else
1226  if (unformat
1227  (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input))
1228  {
1229  devconf->hqos_enabled = 1;
1230  error = unformat_hqos (&sub_input, &devconf->hqos);
1231  if (error)
1232  break;
1233  }
1234  else if (unformat (input, "hqos"))
1235  {
1236  devconf->hqos_enabled = 1;
1237  }
1238  else
1239  {
1240  error = clib_error_return (0, "unknown input `%U'",
1241  format_unformat_error, input);
1242  break;
1243  }
1244  }
1245 
1246  if (error)
1247  return error;
1248 
1249  if (devconf->workers && devconf->num_rx_queues == 0)
1250  devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers);
1251  else if (devconf->workers &&
1252  clib_bitmap_count_set_bits (devconf->workers) !=
1253  devconf->num_rx_queues)
1254  error =
1255  clib_error_return (0,
1256  "%U: number of worker threadds must be "
1257  "equal to number of rx queues", format_vlib_pci_addr,
1258  &pci_addr);
1259 
1260  return error;
1261 }
1262 
1263 static clib_error_t *
1265 {
1266  clib_error_t *error = 0;
1267  dpdk_main_t *dm = &dpdk_main;
1270  dpdk_device_config_t *devconf;
1271  vlib_pci_addr_t pci_addr;
1272  unformat_input_t sub_input;
1273  u8 *s, *tmp = 0;
1274  u8 *rte_cmd = 0, *ethname = 0;
1275  u32 log_level;
1276  int ret, i;
1277  int num_whitelisted = 0;
1278  u8 no_pci = 0;
1279  u8 no_huge = 0;
1280  u8 huge_dir = 0;
1281  u8 file_prefix = 0;
1282  u8 *socket_mem = 0;
1283 
1284  conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
1285 
1286  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1287  {
1288  /* Prime the pump */
1289  if (unformat (input, "no-hugetlb"))
1290  {
1291  vec_add1 (conf->eal_init_args, (u8 *) "no-huge");
1292  no_huge = 1;
1293  }
1294 
1295  else if (unformat (input, "enable-tcp-udp-checksum"))
1296  conf->enable_tcp_udp_checksum = 1;
1297 
1298  else if (unformat (input, "decimal-interface-names"))
1300 
1301  else if (unformat (input, "no-multi-seg"))
1302  conf->no_multi_seg = 1;
1303 
1304  else if (unformat (input, "enable-cryptodev"))
1305  conf->cryptodev = 1;
1306 
1307  else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
1308  &sub_input))
1309  {
1310  error =
1311  dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input,
1312  1);
1313 
1314  if (error)
1315  return error;
1316  }
1317  else
1318  if (unformat
1319  (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr,
1320  unformat_vlib_cli_sub_input, &sub_input))
1321  {
1322  error = dpdk_device_config (conf, pci_addr, &sub_input, 0);
1323 
1324  if (error)
1325  return error;
1326 
1327  num_whitelisted++;
1328  }
1329  else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr))
1330  {
1331  error = dpdk_device_config (conf, pci_addr, 0, 0);
1332 
1333  if (error)
1334  return error;
1335 
1336  num_whitelisted++;
1337  }
1338  else if (unformat (input, "num-mbufs %d", &conf->num_mbufs))
1339  ;
1340  else if (unformat (input, "kni %d", &conf->num_kni))
1341  ;
1342  else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
1343  ;
1344  else if (unformat (input, "socket-mem %s", &socket_mem))
1345  ;
1346  else if (unformat (input, "no-pci"))
1347  {
1348  no_pci = 1;
1349  tmp = format (0, "--no-pci%c", 0);
1350  vec_add1 (conf->eal_init_args, tmp);
1351  }
1352  else if (unformat (input, "poll-sleep %d", &dm->poll_sleep_usec))
1353  ;
1354 
1355 #define _(a) \
1356  else if (unformat(input, #a)) \
1357  { \
1358  tmp = format (0, "--%s%c", #a, 0); \
1359  vec_add1 (conf->eal_init_args, tmp); \
1360  }
1362 #undef _
1363 #define _(a) \
1364  else if (unformat(input, #a " %s", &s)) \
1365  { \
1366  if (!strncmp(#a, "huge-dir", 8)) \
1367  huge_dir = 1; \
1368  else if (!strncmp(#a, "file-prefix", 11)) \
1369  file_prefix = 1; \
1370  tmp = format (0, "--%s%c", #a, 0); \
1371  vec_add1 (conf->eal_init_args, tmp); \
1372  vec_add1 (s, 0); \
1373  if (!strncmp(#a, "vdev", 4)) \
1374  if (strstr((char*)s, "af_packet")) \
1375  clib_warning ("af_packet obsoleted. Use CLI 'create host-interface'."); \
1376  vec_add1 (conf->eal_init_args, s); \
1377  }
1379 #undef _
1380 #define _(a,b) \
1381  else if (unformat(input, #a " %s", &s)) \
1382  { \
1383  tmp = format (0, "-%s%c", #b, 0); \
1384  vec_add1 (conf->eal_init_args, tmp); \
1385  vec_add1 (s, 0); \
1386  vec_add1 (conf->eal_init_args, s); \
1387  }
1389 #undef _
1390 #define _(a,b) \
1391  else if (unformat(input, #a " %s", &s)) \
1392  { \
1393  tmp = format (0, "-%s%c", #b, 0); \
1394  vec_add1 (conf->eal_init_args, tmp); \
1395  vec_add1 (s, 0); \
1396  vec_add1 (conf->eal_init_args, s); \
1397  conf->a##_set_manually = 1; \
1398  }
1400 #undef _
1401  else if (unformat (input, "default"))
1402  ;
1403 
1404  else if (unformat_skip_white_space (input))
1405  ;
1406  else
1407  {
1408  error = clib_error_return (0, "unknown input `%U'",
1409  format_unformat_error, input);
1410  goto done;
1411  }
1412  }
1413 
1414  if (!conf->uio_driver_name)
1415  conf->uio_driver_name = format (0, "uio_pci_generic%c", 0);
1416 
1417  /*
1418  * Use 1G huge pages if available.
1419  */
1420  if (!no_huge && !huge_dir)
1421  {
1422  u32 x, *mem_by_socket = 0;
1423  uword c = 0;
1424  u8 use_1g = 1;
1425  u8 use_2m = 1;
1426  u8 less_than_1g = 1;
1427  int rv;
1428 
1429  umount (DEFAULT_HUGE_DIR);
1430 
1431  /* Process "socket-mem" parameter value */
1432  if (vec_len (socket_mem))
1433  {
1434  unformat_input_t in;
1435  unformat_init_vector (&in, socket_mem);
1437  {
1438  if (unformat (&in, "%u,", &x))
1439  ;
1440  else if (unformat (&in, "%u", &x))
1441  ;
1442  else if (unformat (&in, ","))
1443  x = 0;
1444  else
1445  break;
1446 
1447  vec_add1 (mem_by_socket, x);
1448 
1449  if (x > 1023)
1450  less_than_1g = 0;
1451  }
1452  /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */
1453  unformat_free (&in);
1454  socket_mem = 0;
1455  }
1456  else
1457  {
1458  /* *INDENT-OFF* */
1460  {
1461  vec_validate(mem_by_socket, c);
1462  mem_by_socket[c] = 256; /* default per-socket mem */
1463  }
1464  ));
1465  /* *INDENT-ON* */
1466  }
1467 
1468  /* check if available enough 1GB pages for each socket */
1469  /* *INDENT-OFF* */
1471  {
1472  int pages_avail, page_size, mem;
1473 
1474  vec_validate(mem_by_socket, c);
1475  mem = mem_by_socket[c];
1476 
1477  page_size = 1024;
1478  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
1479 
1480  if (pages_avail < 0 || page_size * pages_avail < mem)
1481  use_1g = 0;
1482 
1483  page_size = 2;
1484  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
1485 
1486  if (pages_avail < 0 || page_size * pages_avail < mem)
1487  use_2m = 0;
1488  }));
1489  /* *INDENT-ON* */
1490 
1491  if (mem_by_socket == 0)
1492  {
1493  error = clib_error_return (0, "mem_by_socket NULL");
1494  goto done;
1495  }
1496  _vec_len (mem_by_socket) = c + 1;
1497 
1498  /* regenerate socket_mem string */
1499  vec_foreach_index (x, mem_by_socket)
1500  socket_mem = format (socket_mem, "%s%u",
1501  socket_mem ? "," : "", mem_by_socket[x]);
1502  socket_mem = format (socket_mem, "%c", 0);
1503 
1504  vec_free (mem_by_socket);
1505 
1506  rv = mkdir (VPP_RUN_DIR, 0755);
1507  if (rv && errno != EEXIST)
1508  {
1509  error = clib_error_return (0, "mkdir '%s' failed errno %d",
1510  VPP_RUN_DIR, errno);
1511  goto done;
1512  }
1513 
1514  rv = mkdir (DEFAULT_HUGE_DIR, 0755);
1515  if (rv && errno != EEXIST)
1516  {
1517  error = clib_error_return (0, "mkdir '%s' failed errno %d",
1518  DEFAULT_HUGE_DIR, errno);
1519  goto done;
1520  }
1521 
1522  if (use_1g && !(less_than_1g && use_2m))
1523  {
1524  rv =
1525  mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, "pagesize=1G");
1526  }
1527  else if (use_2m)
1528  {
1529  rv = mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, NULL);
1530  }
1531  else
1532  {
1533  return clib_error_return (0, "not enough free huge pages");
1534  }
1535 
1536  if (rv)
1537  {
1538  error = clib_error_return (0, "mount failed %d", errno);
1539  goto done;
1540  }
1541 
1542  tmp = format (0, "--huge-dir%c", 0);
1543  vec_add1 (conf->eal_init_args, tmp);
1544  tmp = format (0, "%s%c", DEFAULT_HUGE_DIR, 0);
1545  vec_add1 (conf->eal_init_args, tmp);
1546  if (!file_prefix)
1547  {
1548  tmp = format (0, "--file-prefix%c", 0);
1549  vec_add1 (conf->eal_init_args, tmp);
1550  tmp = format (0, "vpp%c", 0);
1551  vec_add1 (conf->eal_init_args, tmp);
1552  }
1553  }
1554 
1555  vec_free (rte_cmd);
1556  vec_free (ethname);
1557 
1558  if (error)
1559  return error;
1560 
1561  /* I'll bet that -c and -n must be the first and second args... */
1562  if (!conf->coremask_set_manually)
1563  {
1565  uword *coremask = 0;
1566  int i;
1567 
1568  /* main thread core */
1569  coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
1570 
1571  for (i = 0; i < vec_len (tm->registrations); i++)
1572  {
1573  tr = tm->registrations[i];
1574  coremask = clib_bitmap_or (coremask, tr->coremask);
1575  }
1576 
1577  vec_insert (conf->eal_init_args, 2, 1);
1578  conf->eal_init_args[1] = (u8 *) "-c";
1579  tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
1580  conf->eal_init_args[2] = tmp;
1581  clib_bitmap_free (coremask);
1582  }
1583 
1584  if (!conf->nchannels_set_manually)
1585  {
1586  vec_insert (conf->eal_init_args, 2, 3);
1587  conf->eal_init_args[3] = (u8 *) "-n";
1588  tmp = format (0, "%d", conf->nchannels);
1589  conf->eal_init_args[4] = tmp;
1590  }
1591 
1592  if (no_pci == 0 && geteuid () == 0)
1593  dpdk_bind_devices_to_uio (conf);
1594 
1595 #define _(x) \
1596  if (devconf->x == 0 && conf->default_devconf.x > 0) \
1597  devconf->x = conf->default_devconf.x ;
1598 
1599  /* *INDENT-OFF* */
1600  pool_foreach (devconf, conf->dev_confs, ({
1601 
1602  /* default per-device config items */
1603  foreach_dpdk_device_config_item
1604 
1605  /* add DPDK EAL whitelist/blacklist entry */
1606  if (num_whitelisted > 0 && devconf->is_blacklisted == 0)
1607  {
1608  tmp = format (0, "-w%c", 0);
1609  vec_add1 (conf->eal_init_args, tmp);
1610  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1611  vec_add1 (conf->eal_init_args, tmp);
1612  }
1613  else if (num_whitelisted == 0 && devconf->is_blacklisted != 0)
1614  {
1615  tmp = format (0, "-b%c", 0);
1616  vec_add1 (conf->eal_init_args, tmp);
1617  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1618  vec_add1 (conf->eal_init_args, tmp);
1619  }
1620  }));
1621  /* *INDENT-ON* */
1622 
1623 #undef _
1624 
1625  /* set master-lcore */
1626  tmp = format (0, "--master-lcore%c", 0);
1627  vec_add1 (conf->eal_init_args, tmp);
1628  tmp = format (0, "%u%c", tm->main_lcore, 0);
1629  vec_add1 (conf->eal_init_args, tmp);
1630 
1631  /* set socket-mem */
1632  tmp = format (0, "--socket-mem%c", 0);
1633  vec_add1 (conf->eal_init_args, tmp);
1634  tmp = format (0, "%s%c", socket_mem, 0);
1635  vec_add1 (conf->eal_init_args, tmp);
1636 
1637  /* NULL terminate the "argv" vector, in case of stupidity */
1638  vec_add1 (conf->eal_init_args, 0);
1639  _vec_len (conf->eal_init_args) -= 1;
1640 
1641  /* Set up DPDK eal and packet mbuf pool early. */
1642 
1643  log_level = (CLIB_DEBUG > 0) ? RTE_LOG_DEBUG : RTE_LOG_NOTICE;
1644 
1645  rte_set_log_level (log_level);
1646 
1647  vm = vlib_get_main ();
1648 
1649  /* make copy of args as rte_eal_init tends to mess up with arg array */
1650  for (i = 1; i < vec_len (conf->eal_init_args); i++)
1651  conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ",
1652  conf->eal_init_args[i]);
1653 
1654  ret =
1655  rte_eal_init (vec_len (conf->eal_init_args),
1656  (char **) conf->eal_init_args);
1657 
1658  /* lazy umount hugepages */
1659  umount2 (DEFAULT_HUGE_DIR, MNT_DETACH);
1660 
1661  if (ret < 0)
1662  return clib_error_return (0, "rte_eal_init returned %d", ret);
1663 
1664  /* Dump the physical memory layout prior to creating the mbuf_pool */
1665  fprintf (stdout, "DPDK physical memory layout:\n");
1666  rte_dump_physmem_layout (stdout);
1667 
1668  /* main thread 1st */
1669  error = vlib_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ());
1670  if (error)
1671  return error;
1672 
1673  for (i = 0; i < RTE_MAX_LCORE; i++)
1674  {
1675  error = vlib_buffer_pool_create (vm, conf->num_mbufs,
1676  rte_lcore_to_socket_id (i));
1677  if (error)
1678  return error;
1679  }
1680 
1681 done:
1682  return error;
1683 }
1684 
1686 
1687 void
1689 {
1690  vnet_main_t *vnm = vnet_get_main ();
1691  struct rte_eth_link prev_link = xd->link;
1692  u32 hw_flags = 0;
1693  u8 hw_flags_chg = 0;
1694 
1695  /* only update link state for PMD interfaces */
1696  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
1697  return;
1698 
1699  xd->time_last_link_update = now ? now : xd->time_last_link_update;
1700  memset (&xd->link, 0, sizeof (xd->link));
1701  rte_eth_link_get_nowait (xd->device_index, &xd->link);
1702 
1703  if (LINK_STATE_ELOGS)
1704  {
1706  ELOG_TYPE_DECLARE (e) =
1707  {
1708  .format =
1709  "update-link-state: sw_if_index %d, admin_up %d,"
1710  "old link_state %d new link_state %d",.format_args = "i4i1i1i1",};
1711 
1712  struct
1713  {
1714  u32 sw_if_index;
1715  u8 admin_up;
1716  u8 old_link_state;
1717  u8 new_link_state;
1718  } *ed;
1719  ed = ELOG_DATA (&vm->elog_main, e);
1720  ed->sw_if_index = xd->vlib_sw_if_index;
1721  ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
1722  ed->old_link_state = (u8)
1724  ed->new_link_state = (u8) xd->link.link_status;
1725  }
1726 
1727  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) &&
1728  ((xd->link.link_status != 0) ^
1730  {
1731  hw_flags_chg = 1;
1732  hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
1733  }
1734 
1735  if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex))
1736  {
1737  hw_flags_chg = 1;
1738  switch (xd->link.link_duplex)
1739  {
1740  case ETH_LINK_HALF_DUPLEX:
1742  break;
1743  case ETH_LINK_FULL_DUPLEX:
1745  break;
1746  default:
1747  break;
1748  }
1749  }
1750  if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed))
1751  {
1752  hw_flags_chg = 1;
1753  switch (xd->link.link_speed)
1754  {
1755  case ETH_SPEED_NUM_10M:
1757  break;
1758  case ETH_SPEED_NUM_100M:
1760  break;
1761  case ETH_SPEED_NUM_1G:
1762  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G;
1763  break;
1764  case ETH_SPEED_NUM_10G:
1766  break;
1767  case ETH_SPEED_NUM_40G:
1769  break;
1770  case 0:
1771  break;
1772  default:
1773  clib_warning ("unknown link speed %d", xd->link.link_speed);
1774  break;
1775  }
1776  }
1777  if (hw_flags_chg)
1778  {
1779  if (LINK_STATE_ELOGS)
1780  {
1782 
1783  ELOG_TYPE_DECLARE (e) =
1784  {
1785  .format =
1786  "update-link-state: sw_if_index %d, new flags %d",.format_args
1787  = "i4i4",};
1788 
1789  struct
1790  {
1791  u32 sw_if_index;
1792  u32 flags;
1793  } *ed;
1794  ed = ELOG_DATA (&vm->elog_main, e);
1795  ed->sw_if_index = xd->vlib_sw_if_index;
1796  ed->flags = hw_flags;
1797  }
1798  vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, hw_flags);
1799  }
1800 }
1801 
1802 static uword
1804 {
1805  clib_error_t *error;
1806  vnet_main_t *vnm = vnet_get_main ();
1807  dpdk_main_t *dm = &dpdk_main;
1809  dpdk_device_t *xd;
1811  int i;
1812 
1813  error = dpdk_lib_init (dm);
1814 
1815  /*
1816  * Turn on the input node if we found some devices to drive
1817  * and we're not running worker threads or i/o threads
1818  */
1819 
1820  if (error == 0 && vec_len (dm->devices) > 0)
1821  {
1822  if (tm->n_vlib_mains == 1)
1824  VLIB_NODE_STATE_POLLING);
1825  else
1826  for (i = 0; i < tm->n_vlib_mains; i++)
1827  if (vec_len (dm->devices_by_cpu[i]) > 0)
1829  VLIB_NODE_STATE_POLLING);
1830  }
1831 
1832  if (error)
1833  clib_error_report (error);
1834 
1835  tm->worker_thread_release = 1;
1836 
1837  f64 now = vlib_time_now (vm);
1838  vec_foreach (xd, dm->devices)
1839  {
1840  dpdk_update_link_state (xd, now);
1841  }
1842 
1843  {
1844  /*
1845  * Extra set up for bond interfaces:
1846  * 1. Setup MACs for bond interfaces and their slave links which was set
1847  * in dpdk_port_setup() but needs to be done again here to take effect.
1848  * 2. Set up info for bond interface related CLI support.
1849  */
1850  int nports = rte_eth_dev_count ();
1851  if (nports > 0)
1852  {
1853  for (i = 0; i < nports; i++)
1854  {
1855  xd = &dm->devices[i];
1856  ASSERT (i == xd->device_index);
1857  if (xd->pmd == VNET_DPDK_PMD_BOND)
1858  {
1859  u8 addr[6];
1860  u8 slink[16];
1861  int nlink = rte_eth_bond_slaves_get (i, slink, 16);
1862  if (nlink > 0)
1863  {
1864  vnet_hw_interface_t *bhi;
1865  ethernet_interface_t *bei;
1866  int rv;
1867 
1868  /* Get MAC of 1st slave link */
1869  rte_eth_macaddr_get
1870  (slink[0], (struct ether_addr *) addr);
1871 
1872  /* Set MAC of bounded interface to that of 1st slave link */
1873  clib_warning ("Set MAC for bond dev# %d", i);
1874  rv = rte_eth_bond_mac_address_set
1875  (i, (struct ether_addr *) addr);
1876  if (rv)
1877  clib_warning ("Set MAC addr failure rv=%d", rv);
1878 
1879  /* Populate MAC of bonded interface in VPP hw tables */
1880  bhi = vnet_get_hw_interface
1881  (vnm, dm->devices[i].vlib_hw_if_index);
1882  bei = pool_elt_at_index
1883  (em->interfaces, bhi->hw_instance);
1884  clib_memcpy (bhi->hw_address, addr, 6);
1885  clib_memcpy (bei->address, addr, 6);
1886 
1887  /* Init l3 packet size allowed on bonded interface */
1892  while (nlink >= 1)
1893  { /* for all slave links */
1894  int slave = slink[--nlink];
1895  dpdk_device_t *sdev = &dm->devices[slave];
1896  vnet_hw_interface_t *shi;
1897  vnet_sw_interface_t *ssi;
1898  ethernet_interface_t *sei;
1899  /* Add MAC to all slave links except the first one */
1900  if (nlink)
1901  {
1902  clib_warning ("Add MAC for slave dev# %d", slave);
1903  rv = rte_eth_dev_mac_addr_add
1904  (slave, (struct ether_addr *) addr, 0);
1905  if (rv)
1906  clib_warning ("Add MAC addr failure rv=%d", rv);
1907  }
1908  /* Set slaves bitmap for bonded interface */
1909  bhi->bond_info = clib_bitmap_set
1910  (bhi->bond_info, sdev->vlib_hw_if_index, 1);
1911  /* Set slave link flags on slave interface */
1912  shi = vnet_get_hw_interface
1913  (vnm, sdev->vlib_hw_if_index);
1914  ssi = vnet_get_sw_interface
1915  (vnm, sdev->vlib_sw_if_index);
1916  sei = pool_elt_at_index
1917  (em->interfaces, shi->hw_instance);
1918 
1921  clib_memcpy (shi->hw_address, addr, 6);
1922  clib_memcpy (sei->address, addr, 6);
1923 
1924  /* Set l3 packet size allowed as the lowest of slave */
1925  if (bhi->max_l3_packet_bytes[VLIB_RX] >
1930 
1931  /* Set max packet size allowed as the lowest of slave */
1932  if (bhi->max_packet_bytes > shi->max_packet_bytes)
1933  bhi->max_packet_bytes = shi->max_packet_bytes;
1934  }
1935  }
1936  }
1937  }
1938  }
1939  }
1940 
1941  while (1)
1942  {
1943  /*
1944  * check each time through the loop in case intervals are changed
1945  */
1946  f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ?
1948 
1949  vlib_process_wait_for_event_or_clock (vm, min_wait);
1950 
1951  if (dm->admin_up_down_in_progress)
1952  /* skip the poll if an admin up down is in progress (on any interface) */
1953  continue;
1954 
1955  vec_foreach (xd, dm->devices)
1956  {
1957  f64 now = vlib_time_now (vm);
1958  if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval)
1959  dpdk_update_counters (xd, now);
1960  if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval)
1961  dpdk_update_link_state (xd, now);
1962 
1963  }
1964  }
1965 
1966  return 0;
1967 }
1968 
1969 /* *INDENT-OFF* */
1970 VLIB_REGISTER_NODE (dpdk_process_node,static) = {
1971  .function = dpdk_process,
1972  .type = VLIB_NODE_TYPE_PROCESS,
1973  .name = "dpdk-process",
1974  .process_log2_n_stack_bytes = 17,
1975 };
1976 /* *INDENT-ON* */
1977 
1978 int
1980 {
1981  if (interval < DPDK_MIN_STATS_POLL_INTERVAL)
1982  return (VNET_API_ERROR_INVALID_VALUE);
1983 
1984  dpdk_main.stat_poll_interval = interval;
1985 
1986  return 0;
1987 }
1988 
1989 int
1991 {
1992  if (interval < DPDK_MIN_LINK_POLL_INTERVAL)
1993  return (VNET_API_ERROR_INVALID_VALUE);
1994 
1995  dpdk_main.link_state_poll_interval = interval;
1996 
1997  return 0;
1998 }
1999 
2000 clib_error_t *
2002 {
2003  dpdk_main_t *dm = &dpdk_main;
2004  vlib_node_t *ei;
2005  clib_error_t *error = 0;
2007 
2008  /* verify that structs are cacheline aligned */
2009  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
2010  "Cache line marker must be 1st element in dpdk_device_t");
2011  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) ==
2013  "Data in cache line 0 is bigger than cache line size");
2014  STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0,
2015  "Cache line marker must be 1st element in frame_queue_trace_t");
2016 
2017  u8 *name;
2018  name = format (0, "dpdk_%08x%c", api_version, 0);
2019 
2020  /* Ask for a correctly-sized block of API message decode slots */
2022  ((char *) name, VL_MSG_FIRST_AVAILABLE);
2023  vec_free (name);
2024 
2025  dm->vlib_main = vm;
2026  dm->vnet_main = vnet_get_main ();
2027  dm->conf = &dpdk_config_main;
2028 
2029  error = dpdk_plugin_api_hookup (vm);
2030 
2031  /* Add our API messages to the global name_crc hash table */
2033 
2034 // TODO
2035 // plugin_custom_dump_configure (dm);
2036 
2037  ei = vlib_get_node_by_name (vm, (u8 *) "ethernet-input");
2038  if (ei == 0)
2039  return clib_error_return (0, "ethernet-input node AWOL");
2040 
2041  dm->ethernet_input_node_index = ei->index;
2042 
2043  dm->conf->nchannels = 4;
2044  dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
2045  vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
2046 
2047  dm->dpdk_device_by_kni_port_id = hash_create (0, sizeof (uword));
2048  dm->vu_sw_if_index_by_listener_fd = hash_create (0, sizeof (uword));
2049  dm->vu_sw_if_index_by_sock_fd = hash_create (0, sizeof (uword));
2050 
2051  /* $$$ use n_thread_stacks since it's known-good at this point */
2052  vec_validate (dm->recycle, tm->n_thread_stacks - 1);
2053 
2054  /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
2055  dm->buffer_flags_template =
2058 
2061 
2062  /* init CLI */
2063  if ((error = vlib_call_init_function (vm, dpdk_cli_init)))
2064  return error;
2065 
2066  return error;
2067 }
2068 
2070 
2071 
2072 /*
2073  * fd.io coding-style-patch-verification: ON
2074  *
2075  * Local Variables:
2076  * eval: (c-set-style "gnu")
2077  * End:
2078  */
u32 ** d_trace_buffers
Definition: dpdk.h:189
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
#define DPDK_DEVICE_FLAG_PROMISC
Definition: dpdk.h:196
DPDK interface HQoS subport parameters set request.
Definition: dpdk.api:52
u8 * default_mac_address
Definition: dpdk.h:238
static void dpdk_bind_devices_to_uio(dpdk_config_main_t *conf)
Definition: init.c:1090
f64 time_last_link_update
Definition: dpdk.h:227
vmrglw vmrglh hi
static u8 * format_bitmap_hex(u8 *s, va_list *args)
Format a bitmap as a string of hex bytes.
Definition: bitmap.h:744
format_function_t format_vlib_pci_addr
Definition: pci.h:238
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
#define clib_min(x, y)
Definition: clib.h:332
#define VNET_HW_INTERFACE_FLAG_SPEED_1G
Definition: interface.h:392
ethernet_main_t ethernet_main
Definition: init.c:45
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:530
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:683
u8 interface_name_format_decimal
Definition: dpdk.h:340
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
#define NB_MBUF
Definition: dpdk.h:63
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
u8 use_rss
Definition: dpdk.h:393
vnet_device_class_t dpdk_device_class
#define DPDK_DEVICE_VLAN_STRIP_OFF
Definition: dpdk.h:306
#define I40E_DEV_ID_QSFP_B
Definition: dpdk_priv.h:26
#define NULL
Definition: clib.h:55
u32 index
Definition: node.h:237
clib_error_t * vlib_pci_bind_to_uio(vlib_pci_device_t *d, char *uio_driver_name)
Definition: linux_pci.c:95
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:185
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:573
static uword * clib_bitmap_or(uword *ai, uword *bi)
Logical operator across two bitmaps.
static u32 dpdk_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 flags)
Definition: init.c:397
static void * vl_api_sw_interface_set_dpdk_hqos_tctbl_t_print(vl_api_sw_interface_set_dpdk_hqos_tctbl_t *mp, void *handle)
Definition: init.c:234
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1688
u16 flags
Definition: dpdk.h:194
static void vl_api_sw_interface_set_dpdk_hqos_subport_t_handler(vl_api_sw_interface_set_dpdk_hqos_subport_t *mp)
Definition: init.c:115
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define LINK_STATE_ELOGS
Definition: init.c:301
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
dpdk_device_and_queue_t ** devices_by_hqos_cpu
Definition: dpdk.h:357
#define DPDK_NB_RX_DESC_VIRTIO
Definition: dpdk_priv.h:21
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:561
#define DPDK_DEVICE_FLAG_HQOS
Definition: dpdk.h:201
u32 per_interface_next_index
Definition: dpdk.h:182
u8 enable_tcp_udp_checksum
Definition: dpdk.h:325
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
#define DPDK_DEVICE_VLAN_STRIP_ON
Definition: dpdk.h:307
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
vlib_buffer_t * buffer_templates
Definition: dpdk.h:363
struct rte_sched_port * hqos
Definition: dpdk.h:162
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
#define DPDK_NB_TX_DESC_DEFAULT
Definition: dpdk_priv.h:20
#define foreach_eal_double_hyphen_predicate_arg
Definition: dpdk_priv.h:32
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:237
int dpdk_set_stat_poll_interval(f64 interval)
Definition: init.c:1979
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:379
void * vlib_weakly_linked_functions[]
Definition: init.c:296
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:447
u16 device_id
Definition: pci.h:80
dpdk_device_config_hqos_t hqos
Definition: dpdk.h:314
#define VPP_RUN_DIR
Definition: init.c:304
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:200
vlib_pci_addr_t bus_address
Definition: pci.h:58
vlib_main_t ** vlib_mains
Definition: buffer.c:285
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define DPDK_DEVICE_FLAG_PMD
Definition: dpdk.h:197
u8 port_id
Definition: dpdk.h:224
clib_error_t * vlib_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
Definition: buffer.c:426
#define I40E_DEV_ID_QSFP_A
Definition: dpdk_priv.h:25
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
api_main_t api_main
Definition: api_shared.c:35
#define I40E_DEV_ID_SFP_XL710
Definition: dpdk_priv.h:24
struct rte_mbuf *** tx_vectors
Definition: dpdk.h:185
foreach_dpdk_device_config_item clib_bitmap_t * workers
Definition: dpdk.h:312
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:376
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:111
dpdk_config_main_t dpdk_config_main
Definition: dpdk.h:349
#define I40E_DEV_ID_QSFP_C
Definition: dpdk_priv.h:27
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:655
#define IP_BUFFER_L4_CHECKSUM_CORRECT
Definition: buffer.h:50
dpdk_device_config_t default_devconf
Definition: dpdk.h:343
f64 stat_poll_interval
Definition: dpdk.h:405
clib_error_t * dpdk_port_setup(dpdk_main_t *dm, dpdk_device_t *xd)
Definition: init.c:322
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
Definition: buffer.h:89
int input_cpu_first_index
Definition: dpdk.h:396
char i8
Definition: types.h:45
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:111
u16 rx_q_used
Definition: dpdk.h:213
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:391
DPDK interface HQoS subport parameters set reply.
Definition: dpdk.api:67
DPDK interface HQoS pipe profile set reply.
Definition: dpdk.api:37
#define DPDK_DEVICE_FLAG_MAYBE_MULTISEG
Definition: dpdk.h:199
u16 vendor_id
Definition: pci.h:79
#define vlib_call_init_function(vm, x)
Definition: init.h:162
clib_error_t * unformat_rss_fn(unformat_input_t *input, uword *rss_fn)
Definition: format.c:706
#define DPDK_NB_TX_DESC_VIRTIO
Definition: dpdk_priv.h:22
u32 device_index
Definition: dpdk.h:176
struct rte_eth_conf port_conf
Definition: dpdk.h:216
DPDK interface HQoS tctbl entry set reply.
Definition: dpdk.api:93
dpdk_device_hqos_per_worker_thread_t * hqos_wt
Definition: dpdk.h:220
u16 device_class
Definition: pci.h:78
#define I40E_DEV_ID_10G_BASE_T
Definition: dpdk_priv.h:28
f64 time_last_stats_update
Definition: dpdk.h:234
u32 vlib_sw_if_index
Definition: dpdk.h:179
struct rte_eth_txconf tx_conf
Definition: dpdk.h:217
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:35
#define hash_get(h, key)
Definition: hash.h:248
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:397
#define vec_insert(V, N, M)
Insert N vector elements starting at element M, initialize new elements to zero (no header...
Definition: vec.h:686
vlib_pci_addr_t pci_addr
Definition: dpdk.h:302
static clib_error_t * dpdk_config(vlib_main_t *vm, unformat_input_t *input)
Definition: init.c:1264
#define foreach_eal_double_hyphen_arg
Definition: dpdk_priv.h:48
dpdk_device_and_queue_t ** devices_by_cpu
Definition: dpdk.h:356
#define ETHERNET_INTERFACE_FLAG_CONFIG_MTU(flags)
Definition: ethernet.h:119
u8 ** eal_init_args
Definition: dpdk.h:321
#define VNET_HW_INTERFACE_FLAG_SPEED_10M
Definition: interface.h:390
u32 vlib_hw_if_index
Definition: dpdk.h:178
#define VNET_SW_INTERFACE_FLAG_BOND_SLAVE
Definition: interface.h:545
#define foreach_eal_single_hyphen_mandatory_arg
Definition: dpdk_priv.h:38
vlib_pci_device_t * pci_devs
Definition: pci.h:116
struct _unformat_input_t unformat_input_t
#define VNET_HW_INTERFACE_FLAG_HALF_DUPLEX
Definition: interface.h:382
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
#define foreach_dpdk_pmd
Definition: dpdk.h:70
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:390
#define ELOG_DATA(em, f)
Definition: elog.h:392
dpdk_port_type_t port_type
Definition: dpdk.h:235
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:119
#define REPLY_MACRO(t)
static void vl_api_sw_interface_set_dpdk_hqos_pipe_t_handler(vl_api_sw_interface_set_dpdk_hqos_pipe_t *mp)
Definition: init.c:70
#define DPDK_MIN_STATS_POLL_INTERVAL
Definition: dpdk.h:242
int input_cpu_count
Definition: dpdk.h:397
#define VLIB_FRAME_SIZE
Definition: node.h:328
u16 tx_q_used
Definition: dpdk.h:212
u16 nb_rx_desc
Definition: dpdk.h:214
clib_error_t * dpdk_init(vlib_main_t *vm)
Definition: init.c:2001
void unformat_init_vector(unformat_input_t *input, u8 *vector_string)
Definition: unformat.c:1030
#define DPDK_DEVICE_FLAG_ADMIN_UP
Definition: dpdk.h:195
static void * vl_api_sw_interface_set_dpdk_hqos_subport_t_print(vl_api_sw_interface_set_dpdk_hqos_subport_t *mp, void *handle)
Definition: init.c:151
u32 ** recycle
Definition: dpdk.h:360
#define VNET_HW_INTERFACE_BOND_INFO_SLAVE
Definition: interface.h:465
#define BAD_SW_IF_INDEX_LABEL
#define foreach_eal_single_hyphen_arg
Definition: dpdk_priv.h:42
#define DPDK_NB_RX_DESC_DEFAULT
Definition: dpdk_priv.h:19
void dpdk_device_lock_free(dpdk_device_t *xd)
Definition: init.c:469
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
svmdb_client_t * c
static clib_error_t * dpdk_device_config(dpdk_config_main_t *conf, vlib_pci_addr_t pci_addr, unformat_input_t *input, u8 is_default)
Definition: init.c:1164
dpdk_device_t * devices
Definition: dpdk.h:355
vlib_main_t * vm
Definition: buffer.c:276
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:433
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:78
u8 nchannels_set_manually
Definition: dpdk.h:330
u16 * cpu_socket_id_by_queue
Definition: dpdk.h:215
static void setup_message_id_table(dpdk_main_t *dm, api_main_t *am)
Definition: init.c:276
vlib_pci_main_t pci_main
Definition: pci.c:53
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:340
volatile u32 ** lockp
Definition: dpdk.h:173
#define FINISH
Definition: init.c:60
dpdk_device_config_t * dev_confs
Definition: dpdk.h:344
struct rte_mbuf *** rx_vectors
Definition: dpdk.h:186
DPDK interface HQoS tctbl entry set request.
Definition: dpdk.api:80
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:69
dpdk_pmd_t pmd
Definition: dpdk.h:191
elog_main_t elog_main
Definition: main.h:141
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:113
#define DEFAULT_HUGE_DIR
Definition: init.c:303
u8 coremask_set_manually
Definition: dpdk.h:329
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:350
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
Definition: node.c:45
#define VNET_HW_INTERFACE_FLAG_SPEED_10G
Definition: interface.h:393
#define vec_validate_ha(V, I, H, A)
Make sure vector is long enough for given index (general version).
Definition: vec.h:416
u8 * interface_name_suffix
Definition: dpdk.h:206
#define hash_create(elts, value_bytes)
Definition: hash.h:658
#define VNET_HW_INTERFACE_FLAG_FULL_DUPLEX
Definition: interface.h:383
u32 max_l3_packet_bytes[VLIB_N_RX_TX]
Definition: interface.h:451
#define ASSERT(truth)
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:208
#define DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE
Definition: dpdk.h:198
unsigned int u32
Definition: types.h:88
int hqos_cpu_count
Definition: dpdk.h:401
#define IP_BUFFER_L4_CHECKSUM_COMPUTED
Definition: buffer.h:49
u32 poll_sleep_usec
Definition: dpdk.h:408
static uword dpdk_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: init.c:1803
#define I40E_DEV_ID_VF
Definition: dpdk_priv.h:29
static void clib_mem_free(void *p)
Definition: mem.h:176
void dpdk_device_lock_init(dpdk_device_t *xd)
Definition: init.c:456
Bitmaps built as vectors of machine words.
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:246
dpdk_device_hqos_per_hqos_thread_t * hqos_ht
Definition: dpdk.h:221
#define clib_error_report(e)
Definition: error.h:125
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:146
#define DPDK_LINK_POLL_INTERVAL
Definition: dpdk.h:244
dpdk_main_t dpdk_main
Definition: init.c:36
uword * thread_registrations_by_name
Definition: threads.h:276
clib_error_t * dpdk_cli_init(vlib_main_t *vm)
Definition: cli.c:2066
struct rte_eth_link link
Definition: dpdk.h:226
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:250
u64 uword
Definition: types.h:112
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:441
Definition: defs.h:47
#define DPDK_STATS_POLL_INTERVAL
Definition: dpdk.h:241
#define VNET_HW_INTERFACE_FLAG_SPEED_100M
Definition: interface.h:391
u32 ethernet_input_node_index
Definition: dpdk.h:372
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
#define foreach_dpdk_plugin_api_msg
Definition: init.c:248
uword unformat_vlib_cli_sub_input(unformat_input_t *i, va_list *args)
Definition: cli.c:151
u8 admin_up_down_in_progress
Definition: dpdk.h:391
#define STATIC_ASSERT(truth,...)
static struct rte_eth_conf port_conf_template
Definition: init.c:308
static uword unformat_bitmap_list(unformat_input_t *input, va_list *va)
unformat a list of bit ranges into a bitmap (eg "0-3,5-7,11" )
Definition: bitmap.h:693
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
static clib_error_t * dpdk_lib_init(dpdk_main_t *dm)
Definition: init.c:480
static void vl_api_sw_interface_set_dpdk_hqos_tctbl_t_handler(vl_api_sw_interface_set_dpdk_hqos_tctbl_t *mp)
Definition: init.c:172
int dpdk_set_link_state_poll_interval(f64 interval)
Definition: init.c:1990
#define DPDK_MIN_LINK_POLL_INTERVAL
Definition: dpdk.h:245
#define hash_get_mem(h, key)
Definition: hash.h:268
u32 buffer_flags_template
Definition: dpdk.h:366
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:117
#define VLIB_BUFFER_EXT_HDR_VALID
Definition: buffer.h:93
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:777
#define vnet_buffer(b)
Definition: buffer.h:294
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
#define VNET_HW_INTERFACE_FLAG_SPEED_40G
Definition: interface.h:394
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 vlib_buffer_free_list_index
Definition: dpdk.h:369
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
int hqos_cpu_first_index
Definition: dpdk.h:400
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
DPDK interface HQoS pipe profile set request.
Definition: dpdk.api:24
#define ETHERNET_MAX_PACKET_BYTES
Definition: ethernet.h:106
#define vec_foreach(var, vec)
Vector iterator.
i8 cpu_socket
Definition: dpdk.h:192
u16 msg_id_base
Definition: dpdk.h:419
#define ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC(flags)
Definition: ethernet.h:114
uword * cpu_socket_bitmap
Definition: threads.h:311
u16 vl_msg_api_get_msg_ids(char *name, int n)
Definition: api_shared.c:831
vhost_vring_addr_t addr
Definition: vhost-user.h:84
format_function_t format_vlib_pci_vpd
Definition: pci.h:241
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:385
static void * vl_api_sw_interface_set_dpdk_hqos_pipe_t_print(vl_api_sw_interface_set_dpdk_hqos_pipe_t *mp, void *handle)
Definition: init.c:99
u8 * uio_driver_name
Definition: dpdk.h:323
vlib_thread_registration_t ** registrations
Definition: threads.h:274
u32 flags
Definition: vhost-user.h:78
static clib_error_t * dpdk_plugin_api_hookup(vlib_main_t *vm)
Definition: init.c:255
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
ethernet_interface_t * interfaces
Definition: ethernet.h:243
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
vnet_main_t * vnet_main
Definition: dpdk.h:412
u16 nb_tx_desc
Definition: dpdk.h:203
clib_error_t * unformat_hqos(unformat_input_t *input, dpdk_device_config_hqos_t *hqos)
Definition: format.c:729
u8 * vpd_r
Definition: pci.h:84
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:577
uword * device_config_index_by_pci_addr
Definition: dpdk.h:345
uword unformat_skip_white_space(unformat_input_t *input)
Definition: unformat.c:814
static uword vnet_hw_interface_is_link_up(vnet_main_t *vnm, u32 hw_if_index)
volatile u32 worker_thread_release
Definition: threads.h:317
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:971
uword * dpdk_device_by_kni_port_id
Definition: dpdk.h:382
Definition: defs.h:46
uword * vu_sw_if_index_by_listener_fd
Definition: dpdk.h:383
f64 link_state_poll_interval
Definition: dpdk.h:404
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
#define VALIDATE_SW_IF_INDEX(mp)
dpdk_config_main_t * conf
Definition: dpdk.h:413
vlib_pci_device_t * vlib_get_pci_device(vlib_pci_addr_t *addr)
Definition: pci.c:56
uword * vu_sw_if_index_by_sock_fd
Definition: dpdk.h:384
vlib_main_t * vlib_main
Definition: dpdk.h:411