FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
vmxnet3.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vppinfra/types.h>
17 #include <vlib/vlib.h>
18 #include <vlib/pci/pci.h>
19 #include <vnet/ethernet/ethernet.h>
20 #include <vnet/plugin/plugin.h>
21 #include <vpp/app/version.h>
22 
23 #include <vmxnet3/vmxnet3.h>
24 
25 #define PCI_VENDOR_ID_VMWARE 0x15ad
26 #define PCI_DEVICE_ID_VMWARE_VMXNET3 0x07b0
27 
29 
30 static pci_device_id_t vmxnet3_pci_device_ids[] = {
31  {
33  .device_id = PCI_DEVICE_ID_VMWARE_VMXNET3},
34  {0},
35 };
36 
37 static clib_error_t *
39  u32 flags)
40 {
41  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
44  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
45 
46  if (vd->flags & VMXNET3_DEVICE_F_ERROR)
47  return clib_error_return (0, "device is in error state");
48 
49  if (is_up)
50  {
53  vd->flags |= VMXNET3_DEVICE_F_ADMIN_UP;
54  }
55  else
56  {
58  vd->flags &= ~VMXNET3_DEVICE_F_ADMIN_UP;
59  }
60  return 0;
61 }
62 
63 static clib_error_t *
66 {
68  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
70  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
71 
73  rxq->int_mode = 0;
74  else
75  rxq->int_mode = 1;
76 
77  return 0;
78 }
79 
80 static void
82  u32 node_index)
83 {
85  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
87 
88  /* Shut off redirection */
89  if (node_index == ~0)
90  {
91  vd->per_interface_next_index = node_index;
92  return;
93  }
94 
97  node_index);
98 }
99 
101 #define _(n,s) s,
103 #undef _
104 };
105 
106 /* *INDENT-OFF* */
108 {
109  .name = "VMXNET3 interface",
110  .format_device = format_vmxnet3_device,
111  .format_device_name = format_vmxnet3_device_name,
112  .admin_up_down_function = vmxnet3_interface_admin_up_down,
113  .rx_mode_change_function = vmxnet3_interface_rx_mode_change,
114  .rx_redirect_to_node = vmxnet3_set_interface_next_node,
115  .tx_function_n_errors = VMXNET3_TX_N_ERROR,
116  .tx_function_error_strings = vmxnet3_tx_func_error_strings,
117 };
118 /* *INDENT-ON* */
119 
120 static u32
122 {
123  return 0;
124 }
125 
126 static void
128 {
129  u32 val;
130 
131  memcpy (&val, vd->mac_addr, 4);
132  vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACL, val);
133 
134  val = 0;
135  memcpy (&val, vd->mac_addr + 4, 2);
136  vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACH, val);
137 }
138 
139 static clib_error_t *
141 {
142  vmxnet3_main_t *vmxm = &vmxnet3_main;
143  vmxnet3_shared *shared;
144  vmxnet3_queues *q;
145  u64 shared_dma;
146  clib_error_t *error;
147  u16 qid = 0, rid;
148  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
149  vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, qid);
150 
151  vd->dma = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
152  sizeof (*vd->dma), 512);
153  if (error)
154  return error;
155 
156  memset (vd->dma, 0, sizeof (*vd->dma));
157 
158  q = &vd->dma->queues;
159  q->tx.cfg.desc_address = vmxnet3_dma_addr (vm, vd, txq->tx_desc);
160  q->tx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, txq->tx_comp);
161  q->tx.cfg.num_desc = txq->size;
162  q->tx.cfg.num_comp = txq->size;
163  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
164  {
165  q->rx.cfg.desc_address[rid] = vmxnet3_dma_addr (vm, vd,
166  rxq->rx_desc[rid]);
167  q->rx.cfg.num_desc[rid] = rxq->size;
168  }
169  q->rx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, rxq->rx_comp);
170  q->rx.cfg.num_comp = rxq->size;
171 
172  shared = &vd->dma->shared;
173  shared->magic = VMXNET3_SHARED_MAGIC;
174  shared->misc.version = VMXNET3_VERSION_MAGIC;
175  if (sizeof (void *) == 4)
176  shared->misc.guest_info = VMXNET3_GOS_BITS_32;
177  else
178  shared->misc.guest_info = VMXNET3_GOS_BITS_64;
179  shared->misc.guest_info |= VMXNET3_GOS_TYPE_LINUX;
180  shared->misc.version_support = VMXNET3_VERSION_SELECT;
181  shared->misc.upt_version_support = VMXNET3_UPT_VERSION_SELECT;
182  shared->misc.queue_desc_address = vmxnet3_dma_addr (vm, vd, q);
183  shared->misc.queue_desc_len = sizeof (*q);
184  shared->misc.mtu = VMXNET3_MTU;
185  shared->misc.num_tx_queues = vd->num_tx_queues;
186  shared->misc.num_rx_queues = vd->num_rx_queues;
187  shared->interrupt.num_intrs = vd->num_intrs;
188  shared->interrupt.event_intr_index = 1;
189  shared->interrupt.control = VMXNET3_IC_DISABLE_ALL;
190  shared->rx_filter.mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST |
191  VMXNET3_RXMODE_ALL_MULTI;
192  shared_dma = vmxnet3_dma_addr (vm, vd, shared);
193 
194  vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAL, shared_dma);
195  vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAH, shared_dma >> 32);
196 
197  return 0;
198 }
199 
200 static inline void
202 {
203  int i;
204  vmxnet3_shared *shared = &vd->dma->shared;
205 
206  shared->interrupt.control &= ~VMXNET3_IC_DISABLE_ALL;
207  for (i = 0; i < vd->num_intrs; i++)
208  vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 0);
209 }
210 
211 static inline void
213 {
214  int i;
215  vmxnet3_shared *shared = &vd->dma->shared;
216 
217  shared->interrupt.control |= VMXNET3_IC_DISABLE_ALL;
218  for (i = 0; i < vd->num_intrs; i++)
219  vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 1);
220 }
221 
222 static clib_error_t *
224 {
225  vmxnet3_main_t *vmxm = &vmxnet3_main;
226  vmxnet3_rxq_t *rxq;
227  clib_error_t *error;
228  u16 rid;
229 
231  rxq = vec_elt_at_index (vd->rxqs, qid);
232  memset (rxq, 0, sizeof (*rxq));
233  rxq->size = qsz;
234  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
235  {
236  rxq->rx_desc[rid] =
238  &error, qsz * sizeof (*rxq->rx_desc[rid]),
239  512);
240  if (error)
241  return error;
242  memset (rxq->rx_desc[rid], 0, qsz * sizeof (*rxq->rx_desc[rid]));
243  }
244  rxq->rx_comp = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
245  qsz * sizeof (*rxq->rx_comp),
246  512);
247  if (error)
248  return error;
249  memset (rxq->rx_comp, 0, qsz * sizeof (*rxq->rx_comp));
250  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
251  {
252  vmxnet3_rx_ring *ring;
253 
254  ring = &rxq->rx_ring[rid];
255  ring->gen = VMXNET3_RXF_GEN;
256  ring->rid = rid;
258  }
260 
261  return 0;
262 }
263 
264 static clib_error_t *
266 {
267  vmxnet3_main_t *vmxm = &vmxnet3_main;
268  vmxnet3_txq_t *txq;
269  clib_error_t *error;
270 
271  if (qid >= vd->num_tx_queues)
272  {
273  qid = qid % vd->num_tx_queues;
274  txq = vec_elt_at_index (vd->txqs, qid);
275  if (txq->lock == 0)
276  clib_spinlock_init (&txq->lock);
277  vd->flags |= VMXNET3_DEVICE_F_SHARED_TXQ_LOCK;
278  return 0;
279  }
280 
282  txq = vec_elt_at_index (vd->txqs, qid);
283  memset (txq, 0, sizeof (*txq));
284  txq->size = qsz;
285  txq->tx_desc = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
286  qsz * sizeof (*txq->tx_desc),
287  512);
288  if (error)
289  return error;
290  memset (txq->tx_desc, 0, qsz * sizeof (*txq->tx_desc));
291  txq->tx_comp = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
292  qsz * sizeof (*txq->tx_comp),
293  512);
294  if (error)
295  return error;
296  memset (txq->tx_comp, 0, qsz * sizeof (*txq->tx_comp));
298  txq->tx_ring.gen = VMXNET3_TXF_GEN;
300 
301  return 0;
302 }
303 
304 static clib_error_t *
307 {
308  clib_error_t *error = 0;
309  u32 ret, i;
310  vmxnet3_main_t *vmxm = &vmxnet3_main;
312 
313  vd->num_tx_queues = 1;
314  vd->num_rx_queues = 1;
315  vd->num_intrs = 2;
316 
317  /* Quiesce the device */
318  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
319  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
320  if (ret != 0)
321  {
322  error = clib_error_return (0, "error on quiescing device rc (%u)", ret);
323  return error;
324  }
325 
326  /* Reset the device */
327  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
328  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
329  if (ret != 0)
330  {
331  error = clib_error_return (0, "error on resetting device rc (%u)", ret);
332  return error;
333  }
334 
335  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_VRRS);
336  vd->version = count_leading_zeros (ret);
337  vd->version = uword_bits - vd->version;
338 
339  if (vd->version == 0 || vd->version > 3)
340  {
341  error = clib_error_return (0, "unsupported hardware version %u",
342  vd->version);
343  return error;
344  }
345 
346  vmxnet3_reg_write (vd, 1, VMXNET3_REG_VRRS, 1 << (vd->version - 1));
347 
348  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_UVRS);
349  if (ret & 1)
351  else
352  {
353  error = clib_error_return (0, "unsupported upt version %u", ret);
354  return error;
355  }
356 
357  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
358  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
359  if (ret & 1)
360  {
361  vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
362  vd->link_speed = ret >> 16;
363  }
364  else
365  {
366  vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
367  }
368 
369  /* Get the mac address */
370  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACL);
371  clib_memcpy (vd->mac_addr, &ret, 4);
372  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACH);
373  clib_memcpy (vd->mac_addr + 4, &ret, 2);
374 
375  if (vmxm->physmem_region_alloc == 0)
376  {
378  error =
379  vlib_physmem_region_alloc (vm, "vmxnet3 descriptors", 4 << 20, 0,
380  flags, &vmxm->physmem_region);
381  if (error)
382  return error;
383  vmxm->physmem_region_alloc = 1;
384  }
385 
386  error = vmxnet3_rxq_init (vm, vd, 0, args->rxq_size);
387  if (error)
388  return error;
389 
390  for (i = 0; i < tm->n_vlib_mains; i++)
391  {
392  error = vmxnet3_txq_init (vm, vd, i, args->txq_size);
393  if (error)
394  return error;
395  }
396 
397  error = vmxnet3_provision_driver_shared (vm, vd);
398  if (error)
399  return error;
400 
401  vmxnet3_write_mac (vd);
402 
403  /* Activate device */
404  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
405  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
406  if (ret != 0)
407  {
408  error =
409  clib_error_return (0, "error on activating device rc (%u)", ret);
410  return error;
411  }
412 
413  /* Disable interrupts */
415 
416  vec_foreach_index (i, vd->rxqs)
417  {
418  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
419 
420  vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
421  vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
422  }
423  vd->flags |= VMXNET3_DEVICE_F_INITIALIZED;
424 
426 
427  return error;
428 }
429 
430 static void
432 {
433  vnet_main_t *vnm = vnet_get_main ();
434  vmxnet3_main_t *vmxm = &vmxnet3_main;
436  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
437  u16 qid = line;
438 
439  if (vec_len (vd->rxqs) > qid && vd->rxqs[qid].int_mode != 0)
441 }
442 
443 static void
445 {
446  vnet_main_t *vnm = vnet_get_main ();
447  vmxnet3_main_t *vmxm = &vmxnet3_main;
449  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
450  u32 ret;
451 
452  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
453  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
454  if (ret & 1)
455  {
456  vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
457  vd->link_speed = ret >> 16;
460  }
461  else
462  {
463  vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
465  }
466 }
467 
468 static u8
470 {
471  if (qsz < 64 || qsz > 4096)
472  return 0;
473  if ((qsz % 64) != 0)
474  return 0;
475  return 1;
476 }
477 
478 void
480 {
481  vnet_main_t *vnm = vnet_get_main ();
482  vmxnet3_main_t *vmxm = &vmxnet3_main;
483  vmxnet3_device_t *vd;
485  clib_error_t *error = 0;
486 
487  if (args->rxq_size == 0)
489  if (args->txq_size == 0)
491 
492  if (!vmxnet3_queue_size_valid (args->rxq_size) ||
494  {
495  args->rv = VNET_API_ERROR_INVALID_VALUE;
496  args->error =
497  clib_error_return (error,
498  "queue size must be <= 4096, >= 64, "
499  "and multiples of 64");
500  vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
501  format_vlib_pci_addr, &args->addr,
502  "queue size must be <= 4096, >= 64, and multiples of 64");
503  return;
504  }
505 
506  /* *INDENT-OFF* */
507  pool_foreach (vd, vmxm->devices, ({
508  if (vd->pci_addr.as_u32 == args->addr.as_u32)
509  {
510  args->rv = VNET_API_ERROR_INVALID_VALUE;
511  args->error =
512  clib_error_return (error, "PCI address in use");
513  vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
514  format_vlib_pci_addr, &args->addr, "pci address in use");
515  return;
516  }
517  }));
518  /* *INDENT-ON* */
519 
520  pool_get (vmxm->devices, vd);
521  vd->dev_instance = vd - vmxm->devices;
522  vd->per_interface_next_index = ~0;
523  vd->pci_addr = args->addr;
524 
525  if (args->enable_elog)
526  vd->flags |= VMXNET3_DEVICE_F_ELOG;
527 
528  if ((error =
529  vlib_pci_device_open (&args->addr, vmxnet3_pci_device_ids, &h)))
530  {
531  pool_put (vmxm->devices, vd);
532  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
533  args->error =
534  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
535  &args->addr);
536  vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
537  format_vlib_pci_addr, &args->addr,
538  "error encountered on pci device open");
539  return;
540  }
541 
542  /*
543  * Do not use vmxnet3_log_error prior to this line since the macro
544  * references vd->pci_dev_handle
545  */
546  vd->pci_dev_handle = h;
547  vlib_pci_set_private_data (h, vd->dev_instance);
548 
549  if ((error = vlib_pci_bus_master_enable (h)))
550  {
551  vmxnet3_log_error (vd, "error encountered on pci bus master enable");
552  goto error;
553  }
554 
555  if ((error = vlib_pci_map_region (h, 0, (void **) &vd->bar[0])))
556  {
557  vmxnet3_log_error (vd, "error encountered on pci map region for bar 0");
558  goto error;
559  }
560 
561  if ((error = vlib_pci_map_region (h, 1, (void **) &vd->bar[1])))
562  {
563  vmxnet3_log_error (vd, "error encountered on pci map region for bar 1");
564  goto error;
565  }
566 
567  if ((error = vlib_pci_register_msix_handler (h, 0, 1,
569  {
570  vmxnet3_log_error (vd,
571  "error encountered on pci register msix handler 0");
572  goto error;
573  }
574 
575  if ((error = vlib_pci_register_msix_handler (h, 1, 1,
577  {
578  vmxnet3_log_error (vd,
579  "error encountered on pci register msix handler 1");
580  goto error;
581  }
582 
583  if ((error = vlib_pci_enable_msix_irq (h, 0, 2)))
584  {
585  vmxnet3_log_error (vd, "error encountered on pci enable msix irq");
586  goto error;
587  }
588 
589  if ((error = vlib_pci_intr_enable (h)))
590  {
591  vmxnet3_log_error (vd, "error encountered on pci interrupt enable");
592  goto error;
593  }
594 
595  if ((error = vmxnet3_device_init (vm, vd, args)))
596  {
597  vmxnet3_log_error (vd, "error encountered on device init");
598  goto error;
599  }
600 
601  /* create interface */
603  vd->dev_instance, vd->mac_addr,
604  &vd->hw_if_index, vmxnet3_flag_change);
605 
606  if (error)
607  {
608  vmxnet3_log_error (vd,
609  "error encountered on ethernet register interface");
610  goto error;
611  }
612 
613  vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vd->hw_if_index);
614  vd->sw_if_index = sw->sw_if_index;
615  args->sw_if_index = sw->sw_if_index;
616 
617  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vd->hw_if_index);
619  vnet_hw_interface_set_input_node (vnm, vd->hw_if_index,
620  vmxnet3_input_node.index);
621  vnet_hw_interface_assign_rx_thread (vnm, vd->hw_if_index, 0, ~0);
622  if (vd->flags & VMXNET3_DEVICE_F_LINK_UP)
623  vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
625  else
626  vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
627  return;
628 
629 error:
630  vmxnet3_delete_if (vm, vd);
631  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
632  args->error = error;
633 }
634 
635 void
637 {
638  vnet_main_t *vnm = vnet_get_main ();
639  vmxnet3_main_t *vmxm = &vmxnet3_main;
640  u32 i, bi;
641  u16 desc_idx;
642 
643  /* Quiesce the device */
644  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
645 
646  /* Reset the device */
647  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
648 
649  if (vd->hw_if_index)
650  {
654  }
655 
657 
658  /* *INDENT-OFF* */
659  vec_foreach_index (i, vd->rxqs)
660  {
661  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
662  u16 mask = rxq->size - 1;
663  u16 rid;
664 
665  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
666  {
667  vmxnet3_rx_ring *ring;
668 
669  ring = &rxq->rx_ring[rid];
670  desc_idx = (ring->consume + 1) & mask;
671  vlib_buffer_free_from_ring (vm, ring->bufs, desc_idx, rxq->size,
672  ring->fill);
673  vec_free (ring->bufs);
674  vlib_physmem_free (vm, vmxm->physmem_region, rxq->rx_desc[rid]);
675  }
676  vlib_physmem_free (vm, vmxm->physmem_region, rxq->rx_comp);
677  }
678  /* *INDENT-ON* */
679  vec_free (vd->rxqs);
680 
681  /* *INDENT-OFF* */
682  vec_foreach_index (i, vd->txqs)
683  {
684  vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, i);
685  u16 mask = txq->size - 1;
686  u16 end_idx;
687 
688  desc_idx = txq->tx_ring.consume;
689  end_idx = txq->tx_ring.produce;
690  while (desc_idx != end_idx)
691  {
692  bi = txq->tx_ring.bufs[desc_idx];
693  vlib_buffer_free_no_next (vm, &bi, 1);
694  desc_idx++;
695  desc_idx &= mask;
696  }
697  clib_spinlock_free (&txq->lock);
698  vec_free (txq->tx_ring.bufs);
699  vlib_physmem_free (vm, vmxm->physmem_region, txq->tx_desc);
700  vlib_physmem_free (vm, vmxm->physmem_region, txq->tx_comp);
701  }
702  /* *INDENT-ON* */
703  vec_free (vd->txqs);
704 
705  vlib_physmem_free (vm, vmxm->physmem_region, vd->dma);
706 
707  clib_error_free (vd->error);
708  memset (vd, 0, sizeof (*vd));
709  pool_put (vmxm->devices, vd);
710 }
711 
712 /*
713  * fd.io coding-style-patch-verification: ON
714  *
715  * Local Variables:
716  * eval: (c-set-style "gnu")
717  * End:
718  */
static char * vmxnet3_tx_func_error_strings[]
Definition: vmxnet3.c:100
vmrglw vmrglh hi
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring0(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:591
void vmxnet3_delete_if(vlib_main_t *vm, vmxnet3_device_t *vd)
Definition: vmxnet3.c:636
format_function_t format_vlib_pci_addr
Definition: pci.h:288
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define VLIB_PHYSMEM_F_INIT_MHEAP
Definition: physmem.h:57
vlib_physmem_region_index_t physmem_region
Definition: vmxnet3.h:496
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:469
format_function_t format_vmxnet3_device_name
Definition: vmxnet3.h:532
#define VMXNET3_GOS_BITS_32
Definition: vmxnet3.h:109
#define VLIB_PHYSMEM_F_HUGETLB
Definition: physmem.h:58
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:323
#define VMXNET3_REG_VRRS
Definition: vmxnet3.h:68
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
#define count_leading_zeros(x)
Definition: clib.h:134
vmxnet3_rx_desc * rx_desc[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:423
static_always_inline uword vmxnet3_dma_addr(vlib_main_t *vm, vmxnet3_device_t *vd, void *p)
Definition: vmxnet3.h:571
unsigned long u64
Definition: types.h:89
vlib_pci_dev_handle_t pci_dev_handle
Definition: vmxnet3.h:470
#define VMXNET3_REG_DSAL
Definition: vmxnet3.h:70
clib_spinlock_t lock
Definition: vmxnet3.h:448
#define VMXNET3_REG_MACL
Definition: vmxnet3.h:73
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define foreach_vmxnet3_tx_func_error
Definition: vmxnet3.h:19
#define VMXNET3_MTU
Definition: vmxnet3.h:80
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
int i
#define VMXNET3_NUM_TX_DESC
Definition: vmxnet3.h:97
void vmxnet3_create_if(vlib_main_t *vm, vmxnet3_create_if_args_t *args)
Definition: vmxnet3.c:479
static void vmxnet3_irq_1_handler(vlib_pci_dev_handle_t h, u16 line)
Definition: vmxnet3.c:444
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:494
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
static void * vlib_physmem_alloc_aligned(vlib_main_t *vm, vlib_physmem_region_index_t idx, clib_error_t **error, uword n_bytes, uword alignment)
Definition: physmem_funcs.h:97
vmxnet3_dma * dma
Definition: vmxnet3.h:488
static clib_error_t * vmxnet3_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: vmxnet3.c:38
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
static void vmxnet3_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: vmxnet3.c:81
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1118
unsigned char u8
Definition: types.h:56
clib_error_t * vlib_pci_register_msix_handler(vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:766
static_always_inline void vmxnet3_reg_write(vmxnet3_device_t *vd, u8 bar, u32 addr, u32 val)
Definition: vmxnet3.h:553
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
#define VMXNET3_RXCF_GEN
Definition: vmxnet3.h:86
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
static_always_inline u32 vmxnet3_reg_read(vmxnet3_device_t *vd, u8 bar, u32 addr)
Definition: vmxnet3.h:560
#define VMXNET3_VERSION_SELECT
Definition: vmxnet3.h:104
vmxnet3_rxq_t * rxqs
Definition: vmxnet3.h:475
vlib_log_class_t log_default
Definition: vmxnet3.h:499
vnet_hw_interface_rx_mode
Definition: interface.h:51
memset(h->entries, 0, sizeof(h->entries[0])*entries)
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
static void vmxnet3_irq_0_handler(vlib_pci_dev_handle_t h, u16 line)
Definition: vmxnet3.c:431
clib_error_t * vlib_pci_device_open(vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1046
unsigned int u32
Definition: types.h:88
vlib_pci_addr_t addr
Definition: vmxnet3.h:506
#define VMXNET3_REG_UVRS
Definition: vmxnet3.h:69
#define VMXNET3_REG_MACH
Definition: vmxnet3.h:74
static clib_error_t * vmxnet3_rxq_init(vlib_main_t *vm, vmxnet3_device_t *vd, u16 qid, u16 qsz)
Definition: vmxnet3.c:223
#define VMXNET3_REG_IMR
Definition: vmxnet3.h:61
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:602
static clib_error_t * vmxnet3_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
Definition: vmxnet3.c:64
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
#define VMXNET3_REG_CMD
Definition: vmxnet3.h:72
static void vmxnet3_write_mac(vmxnet3_device_t *vd)
Definition: vmxnet3.c:127
static u8 vmxnet3_queue_size_valid(u16 qsz)
Definition: vmxnet3.c:469
unsigned short u16
Definition: types.h:57
#define VMXNET3_GOS_TYPE_LINUX
Definition: vmxnet3.h:111
#define VMXNET3_GOS_BITS_64
Definition: vmxnet3.h:110
vmxnet3_rx_comp_ring rx_comp_ring
Definition: vmxnet3.h:425
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
vmxnet3_tx_comp_ring tx_comp_ring
Definition: vmxnet3.h:453
#define VMXNET3_UPT_VERSION_SELECT
Definition: vmxnet3.h:105
#define VMXNET3_RX_RING_SIZE
Definition: vmxnet3.h:95
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
vmxnet3_tx_comp * tx_comp
Definition: vmxnet3.h:451
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE
Definition: interface.h:532
static clib_error_t * vlib_pci_bus_master_enable(vlib_pci_dev_handle_t h)
Definition: pci.h:244
#define VMXNET3_VERSION_MAGIC
Definition: vmxnet3.h:102
u32 flags
Definition: vhost_user.h:115
static void vmxnet3_enable_interrupt(vmxnet3_device_t *vd)
Definition: vmxnet3.c:201
#define VMXNET3_TXF_GEN
Definition: vmxnet3.h:89
#define VMXNET3_IC_DISABLE_ALL
Definition: vmxnet3.h:107
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:568
static void vmxnet3_disable_interrupt(vmxnet3_device_t *vd)
Definition: vmxnet3.c:212
vlib_main_t * vm
Definition: buffer.c:294
#define uword_bits
Definition: types.h:102
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
#define clib_memcpy(a, b, c)
Definition: string.h:75
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:138
u32 physmem_region_alloc
Definition: vmxnet3.h:497
u8 mac_addr[6]
Definition: vmxnet3.h:483
#define PCI_VENDOR_ID_VMWARE
Definition: vmxnet3.c:25
u32 per_interface_next_index
Definition: vmxnet3.h:465
VNET_DEVICE_CLASS(vmxnet3_device_class,)
vmxnet3_txq_t * txqs
Definition: vmxnet3.h:476
clib_error_t * error
Definition: vmxnet3.h:486
static clib_error_t * vmxnet3_device_init(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_create_if_args_t *args)
Definition: vmxnet3.c:305
void vlib_pci_device_close(vlib_pci_dev_handle_t h)
Definition: pci.c:1091
#define VMXNET3_RXF_GEN
Definition: vmxnet3.h:83
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:138
format_function_t format_vmxnet3_device
Definition: vmxnet3.h:531
#define VMXNET3_REG_DSAH
Definition: vmxnet3.h:71
#define VMXNET3_TXCF_GEN
Definition: vmxnet3.h:92
static void vlib_physmem_free(vlib_main_t *vm, vlib_physmem_region_index_t idx, void *mem)
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:277
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static clib_error_t * vmxnet3_provision_driver_shared(vlib_main_t *vm, vmxnet3_device_t *vd)
Definition: vmxnet3.c:140
vlib_node_registration_t vmxnet3_input_node
(constructor) VLIB_REGISTER_NODE (vmxnet3_input_node)
Definition: input.c:369
#define VMXNET3_SHARED_MAGIC
Definition: vmxnet3.h:103
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define PCI_DEVICE_ID_VMWARE_VMXNET3
Definition: vmxnet3.c:26
uword vlib_pci_get_private_data(vlib_pci_dev_handle_t h)
Definition: pci.c:134
vmxnet3_tx_desc * tx_desc
Definition: vmxnet3.h:450
u64 uword
Definition: types.h:112
vnet_device_class_t vmxnet3_device_class
static u32 vmxnet3_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: vmxnet3.c:121
void vlib_pci_set_private_data(vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:141
vmxnet3_rx_comp * rx_comp
Definition: vmxnet3.h:424
#define clib_error_free(e)
Definition: error.h:86
vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:422
clib_error_t * vlib_pci_map_region(vlib_pci_dev_handle_t h, u32 resource, void **result)
Definition: pci.c:1033
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:187
clib_error_t * vlib_pci_enable_msix_irq(vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:822
static clib_error_t * vlib_pci_intr_enable(vlib_pci_dev_handle_t h)
Definition: pci.h:212
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
vmxnet3_device_t * devices
Definition: vmxnet3.h:495
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring1(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:633
#define vmxnet3_log_error(dev, f,...)
Definition: vmxnet3.h:540
#define VMXNET3_NUM_RX_DESC
Definition: vmxnet3.h:99
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
vmxnet3_tx_ring tx_ring
Definition: vmxnet3.h:452
clib_error_t * error
Definition: vmxnet3.h:513
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
static clib_error_t * vmxnet3_txq_init(vlib_main_t *vm, vmxnet3_device_t *vd, u16 qid, u16 qsz)
Definition: vmxnet3.c:265
u16 vendor_id
Definition: pci.h:120