FD.io VPP  v18.07-rc0-415-g6c78436
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 
23 #include <avf/avf.h>
24 
25 #define AVF_MBOX_LEN 64
26 #define AVF_MBOX_BUF_SZ 512
27 #define AVF_RXQ_SZ 512
28 #define AVF_TXQ_SZ 512
29 #define AVF_ITR_INT 8160
30 
31 #define PCI_VENDOR_ID_INTEL 0x8086
32 #define PCI_DEVICE_ID_INTEL_AVF 0x1889
33 #define PCI_DEVICE_ID_INTEL_X710_VF 0x154c
34 #define PCI_DEVICE_ID_INTEL_X722_VF 0x37cd
35 
37 
38 static pci_device_id_t avf_pci_device_ids[] = {
40  {.vendor_id = PCI_VENDOR_ID_INTEL,.device_id = PCI_DEVICE_ID_INTEL_X710_VF},
41  {.vendor_id = PCI_VENDOR_ID_INTEL,.device_id = PCI_DEVICE_ID_INTEL_X722_VF},
42  {0},
43 };
44 
45 static inline void
47 {
48  u32 dyn_ctl0 = 0, icr0_ena = 0;
49 
50  dyn_ctl0 |= (3 << 3); /* 11b = No ITR update */
51 
52  avf_reg_write (ad, AVFINT_ICR0_ENA1, icr0_ena);
53  avf_reg_write (ad, AVFINT_DYN_CTL0, dyn_ctl0);
54  avf_reg_flush (ad);
55 }
56 
57 static inline void
59 {
60  u32 dyn_ctl0 = 0, icr0_ena = 0;
61 
62  icr0_ena |= (1 << 30); /* [30] Admin Queue Enable */
63 
64  dyn_ctl0 |= (1 << 0); /* [0] Interrupt Enable */
65  dyn_ctl0 |= (1 << 1); /* [1] Clear PBA */
66  //dyn_ctl0 |= (3 << 3); /* [4:3] ITR Index, 11b = No ITR update */
67  dyn_ctl0 |= ((AVF_ITR_INT / 2) << 5); /* [16:5] ITR Interval in 2us steps */
68 
69  avf_irq_0_disable (ad);
70  avf_reg_write (ad, AVFINT_ICR0_ENA1, icr0_ena);
71  avf_reg_write (ad, AVFINT_DYN_CTL0, dyn_ctl0);
72  avf_reg_flush (ad);
73 }
74 
75 static inline void
77 {
78  u32 dyn_ctln = 0;
79 
80  avf_reg_write (ad, AVFINT_DYN_CTLN (line), dyn_ctln);
81  avf_reg_flush (ad);
82 }
83 
84 static inline void
86 {
87  u32 dyn_ctln = 0;
88 
89  dyn_ctln |= (1 << 0); /* [0] Interrupt Enable */
90  dyn_ctln |= (1 << 1); /* [1] Clear PBA */
91  dyn_ctln |= ((AVF_ITR_INT / 2) << 5); /* [16:5] ITR Interval in 2us steps */
92 
93  avf_irq_n_disable (ad, line);
94  avf_reg_write (ad, AVFINT_DYN_CTLN (line), dyn_ctln);
95  avf_reg_flush (ad);
96 }
97 
98 
101  void *data, int len)
102 {
103  avf_main_t *am = &avf_main;
104  clib_error_t *err = 0;
105  avf_aq_desc_t *d, dc;
106  int n_retry = 5;
107 
108  d = &ad->atq[ad->atq_next_slot];
109  clib_memcpy (d, dt, sizeof (avf_aq_desc_t));
110  d->flags |= AVF_AQ_F_RD | AVF_AQ_F_SI;
111  if (len)
112  d->datalen = len;
113  if (len)
114  {
115  u64 pa;
116  pa = ad->atq_bufs_pa + ad->atq_next_slot * AVF_MBOX_BUF_SZ;
117  d->addr_hi = (u32) (pa >> 32);
118  d->addr_lo = (u32) pa;
120  len);
121  d->flags |= AVF_AQ_F_BUF;
122  }
123 
124  if (ad->flags & AVF_DEVICE_F_ELOG)
125  clib_memcpy (&dc, d, sizeof (avf_aq_desc_t));
126 
128  vlib_log_debug (am->log_class, "%U", format_hexdump, data, len);
129  ad->atq_next_slot = (ad->atq_next_slot + 1) % AVF_MBOX_LEN;
131  avf_reg_flush (ad);
132 
133 retry:
134  vlib_process_suspend (vm, 10e-6);
135 
136  if (((d->flags & AVF_AQ_F_DD) == 0) || ((d->flags & AVF_AQ_F_CMP) == 0))
137  {
138  if (--n_retry == 0)
139  {
140  err = clib_error_return (0, "adminq enqueue timeout [opcode 0x%x]",
141  d->opcode);
142  goto done;
143  }
144  goto retry;
145  }
146 
147  clib_memcpy (dt, d, sizeof (avf_aq_desc_t));
148  if (d->flags & AVF_AQ_F_ERR)
149  return clib_error_return (0, "adminq enqueue error [opcode 0x%x, retval "
150  "%d]", d->opcode, d->retval);
151 
152 done:
153  if (ad->flags & AVF_DEVICE_F_ELOG)
154  {
155  /* *INDENT-OFF* */
156  ELOG_TYPE_DECLARE (el) =
157  {
158  .format = "avf[%d] aq enq: s_flags 0x%x r_flags 0x%x opcode 0x%x "
159  "datalen %d retval %d",
160  .format_args = "i4i2i2i2i2i2",
161  };
162  struct
163  {
164  u32 dev_instance;
165  u16 s_flags;
166  u16 r_flags;
167  u16 opcode;
168  u16 datalen;
169  u16 retval;
170  } *ed;
171  ed = ELOG_DATA (&vm->elog_main, el);
172  ed->dev_instance = ad->dev_instance;
173  ed->s_flags = dc.flags;
174  ed->r_flags = d->flags;
175  ed->opcode = dc.opcode;
176  ed->datalen = dc.datalen;
177  ed->retval = d->retval;
178  /* *INDENT-ON* */
179  }
180 
181  return err;
182 }
183 
184 clib_error_t *
186  u32 val)
187 {
188  clib_error_t *err;
189  avf_aq_desc_t d = {.opcode = 0x207,.param1 = reg,.param3 = val };
190  err = avf_aq_desc_enq (vm, ad, &d, 0, 0);
191 
192  if (ad->flags & AVF_DEVICE_F_ELOG)
193  {
194  /* *INDENT-OFF* */
195  ELOG_TYPE_DECLARE (el) =
196  {
197  .format = "avf[%d] rx ctl reg write: reg 0x%x val 0x%x ",
198  .format_args = "i4i4i4",
199  };
200  struct
201  {
202  u32 dev_instance;
203  u32 reg;
204  u32 val;
205  } *ed;
206  ed = ELOG_DATA (&vm->elog_main, el);
207  ed->dev_instance = ad->dev_instance;
208  ed->reg = reg;
209  ed->val = val;
210  /* *INDENT-ON* */
211  }
212  return err;
213 }
214 
215 clib_error_t *
217 {
218  avf_main_t *am = &avf_main;
219  avf_rxq_t *rxq;
220  clib_error_t *error = 0;
221  u32 n_alloc, i;
222 
224  rxq = vec_elt_at_index (ad->rxqs, qid);
225  rxq->size = AVF_RXQ_SZ;
226  rxq->next = 0;
227  rxq->descs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error,
228  rxq->size * sizeof (avf_rx_desc_t),
230  memset (rxq->descs, 0, rxq->size * sizeof (avf_rx_desc_t));
232  rxq->qrx_tail = ad->bar0 + AVF_QRX_TAIL (qid);
233 
234  n_alloc = vlib_buffer_alloc (vm, rxq->bufs, rxq->size - 8);
235 
236  if (n_alloc == 0)
237  return clib_error_return (0, "buffer allocation error");
238 
239  rxq->n_bufs = n_alloc;
240  avf_rx_desc_t *d = rxq->descs;
241  for (i = 0; i < n_alloc; i++)
242  {
243  if (ad->flags & AVF_DEVICE_F_IOVA)
244  {
245  vlib_buffer_t *b = vlib_get_buffer (vm, rxq->bufs[i]);
246  d->qword[0] = pointer_to_uword (b->data);
247  }
248  else
249  d->qword[0] =
251  d++;
252  }
253  return 0;
254 }
255 
256 clib_error_t *
258 {
259  avf_main_t *am = &avf_main;
260  avf_txq_t *txq;
261  clib_error_t *error = 0;
262 
263  if (qid >= ad->num_queue_pairs)
264  {
265  qid = qid % ad->num_queue_pairs;
266  txq = vec_elt_at_index (ad->txqs, qid);
267  if (txq->lock == 0)
268  clib_spinlock_init (&txq->lock);
269  ad->flags |= AVF_DEVICE_F_SHARED_TXQ_LOCK;
270  return 0;
271  }
272 
274  txq = vec_elt_at_index (ad->txqs, qid);
275  txq->size = AVF_TXQ_SZ;
276  txq->next = 0;
277  txq->descs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error,
278  txq->size * sizeof (avf_tx_desc_t),
281  txq->qtx_tail = ad->bar0 + AVF_QTX_TAIL (qid);
282  return 0;
283 }
284 
285 typedef struct
286 {
290 
291 void
293 {
294  avf_aq_desc_t *d;
295  u64 pa = ad->arq_bufs_pa + slot * AVF_MBOX_BUF_SZ;
296  d = &ad->arq[slot];
297  memset (d, 0, sizeof (avf_aq_desc_t));
298  d->flags = AVF_AQ_F_BUF;
300  d->addr_hi = (u32) (pa >> 32);
301  d->addr_lo = (u32) pa;
302 }
303 
304 static inline uword
306 {
307  avf_main_t *am = &avf_main;
308  return (ad->flags & AVF_DEVICE_F_IOVA) ?
309  pointer_to_uword (p) :
311 }
312 
313 static void
315 {
316  u64 pa;
317  int i;
318 
319  /* VF MailBox Transmit */
320  memset (ad->atq, 0, sizeof (avf_aq_desc_t) * AVF_MBOX_LEN);
321  ad->atq_bufs_pa = avf_dma_addr (vm, ad, ad->atq_bufs);
322 
323  pa = avf_dma_addr (vm, ad, ad->atq);
324  avf_reg_write (ad, AVF_ATQT, 0); /* Tail */
325  avf_reg_write (ad, AVF_ATQH, 0); /* Head */
326  avf_reg_write (ad, AVF_ATQLEN, AVF_MBOX_LEN | (1 << 31)); /* len & ena */
327  avf_reg_write (ad, AVF_ATQBAL, (u32) pa); /* Base Address Low */
328  avf_reg_write (ad, AVF_ATQBAH, (u32) (pa >> 32)); /* Base Address High */
329 
330  /* VF MailBox Receive */
331  memset (ad->arq, 0, sizeof (avf_aq_desc_t) * AVF_MBOX_LEN);
332  ad->arq_bufs_pa = avf_dma_addr (vm, ad, ad->arq_bufs);
333 
334  for (i = 0; i < AVF_MBOX_LEN; i++)
335  avf_arq_slot_init (ad, i);
336 
337  pa = avf_dma_addr (vm, ad, ad->arq);
338 
339  avf_reg_write (ad, AVF_ARQH, 0); /* Head */
340  avf_reg_write (ad, AVF_ARQT, 0); /* Head */
341  avf_reg_write (ad, AVF_ARQLEN, AVF_MBOX_LEN | (1 << 31)); /* len & ena */
342  avf_reg_write (ad, AVF_ARQBAL, (u32) pa); /* Base Address Low */
343  avf_reg_write (ad, AVF_ARQBAH, (u32) (pa >> 32)); /* Base Address High */
344  avf_reg_write (ad, AVF_ARQT, AVF_MBOX_LEN - 1); /* Tail */
345 
346  ad->atq_next_slot = 0;
347  ad->arq_next_slot = 0;
348 }
349 
350 clib_error_t *
352  void *in, int in_len, void *out, int out_len)
353 {
354  clib_error_t *err;
355  avf_aq_desc_t *d, dt = {.opcode = 0x801,.v_opcode = op };
356  u32 head;
357  int n_retry = 5;
358 
359 
360  /* supppres interrupt in the next adminq receive slot
361  as we are going to wait for response
362  we only need interrupts when event is received */
363  d = &ad->arq[ad->arq_next_slot];
364  d->flags |= AVF_AQ_F_SI;
365 
366  if ((err = avf_aq_desc_enq (vm, ad, &dt, in, in_len)))
367  return err;
368 
369 retry:
370  head = avf_get_u32 (ad->bar0, AVF_ARQH);
371 
372  if (ad->arq_next_slot == head)
373  {
374  if (--n_retry == 0)
375  return clib_error_return (0, "timeout");
376  vlib_process_suspend (vm, 10e-3);
377  goto retry;
378  }
379 
380  d = &ad->arq[ad->arq_next_slot];
381 
382  if (d->v_opcode == VIRTCHNL_OP_EVENT)
383  {
384  void *buf = ad->arq_bufs + ad->arq_next_slot * AVF_MBOX_BUF_SZ;
386 
387  if ((d->datalen != sizeof (virtchnl_pf_event_t)) ||
388  ((d->flags & AVF_AQ_F_BUF) == 0))
389  return clib_error_return (0, "event message error");
390 
391  vec_add2 (ad->events, e, 1);
392  clib_memcpy (e, buf, sizeof (virtchnl_pf_event_t));
394  ad->arq_next_slot++;
395  n_retry = 5;
396  goto retry;
397  }
398 
399  if (d->v_opcode != op)
400  {
401  err = clib_error_return (0, "unexpected message receiver [v_opcode = %u"
402  "expected %u]", d->v_opcode, op);
403  goto done;
404  }
405 
406  if (d->v_retval)
407  {
408  err = clib_error_return (0, "error [v_opcode = %u, v_retval %d]",
409  d->v_opcode, d->v_retval);
410  goto done;
411  }
412 
413  if (d->flags & AVF_AQ_F_BUF)
414  {
415  void *buf = ad->arq_bufs + ad->arq_next_slot * AVF_MBOX_BUF_SZ;
416  clib_memcpy (out, buf, out_len);
417  }
418 
421  avf_reg_flush (ad);
422  ad->arq_next_slot = (ad->arq_next_slot + 1) % AVF_MBOX_LEN;
423 
424 done:
425 
426  if (ad->flags & AVF_DEVICE_F_ELOG)
427  {
428  /* *INDENT-OFF* */
429  ELOG_TYPE_DECLARE (el) =
430  {
431  .format = "avf[%d] send to pf: v_opcode %s (%d) v_retval 0x%x",
432  .format_args = "i4t4i4i4",
433  .n_enum_strings = VIRTCHNL_N_OPS,
434  .enum_strings = {
435 #define _(v, n) [v] = #n,
437 #undef _
438  },
439  };
440  struct
441  {
442  u32 dev_instance;
443  u32 v_opcode;
444  u32 v_opcode_val;
445  u32 v_retval;
446  } *ed;
447  ed = ELOG_DATA (&vm->elog_main, el);
448  ed->dev_instance = ad->dev_instance;
449  ed->v_opcode = op;
450  ed->v_opcode_val = op;
451  ed->v_retval = d->v_retval;
452  /* *INDENT-ON* */
453  }
454  return err;
455 }
456 
457 clib_error_t *
460 {
461  clib_error_t *err = 0;
462  virtchnl_version_info_t myver = {
464  .minor = VIRTCHNL_VERSION_MINOR,
465  };
466 
467  err = avf_send_to_pf (vm, ad, VIRTCHNL_OP_VERSION, &myver,
468  sizeof (virtchnl_version_info_t), ver,
469  sizeof (virtchnl_version_info_t));
470 
471  if (err)
472  return err;
473 
474  return err;
475 }
476 
477 clib_error_t *
480 {
481  u32 bitmap = (VIRTCHNL_VF_OFFLOAD_L2 | VIRTCHNL_VF_OFFLOAD_RSS_PF |
482  VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_VLAN |
483  VIRTCHNL_VF_OFFLOAD_RX_POLLING);
484 
485  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_GET_VF_RESOURCES, &bitmap,
486  sizeof (u32), res, sizeof (virtchnl_vf_resource_t));
487 }
488 
489 clib_error_t *
491 {
492  int msg_len = sizeof (virtchnl_rss_lut_t) + ad->rss_lut_size - 1;
493  u8 msg[msg_len];
494  virtchnl_rss_lut_t *rl;
495 
496  memset (msg, 0, msg_len);
497  rl = (virtchnl_rss_lut_t *) msg;
498  rl->vsi_id = ad->vsi_id;
499  rl->lut_entries = ad->rss_lut_size;
500 
501  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_RSS_LUT, msg, msg_len, 0,
502  0);
503 }
504 
505 clib_error_t *
507 {
508  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, 0, 0, 0,
509  0);
510 }
511 
512 clib_error_t *
514 {
515  virtchnl_promisc_info_t pi = { 0 };
516 
517  pi.vsi_id = ad->vsi_id;
518  pi.flags = 1;
519  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, &pi,
520  sizeof (virtchnl_promisc_info_t), 0, 0);
521 }
522 
523 
524 clib_error_t *
526 {
527  int i;
528  int n_qp = clib_max (vec_len (ad->rxqs), vec_len (ad->txqs));
529  int msg_len = sizeof (virtchnl_vsi_queue_config_info_t) + n_qp *
531  u8 msg[msg_len];
533 
534  memset (msg, 0, msg_len);
536  ci->vsi_id = ad->vsi_id;
537  ci->num_queue_pairs = n_qp;
538 
539  for (i = 0; i < n_qp; i++)
540  {
541  virtchnl_txq_info_t *txq = &ci->qpair[i].txq;
542  virtchnl_rxq_info_t *rxq = &ci->qpair[i].rxq;
543 
544  rxq->vsi_id = ad->vsi_id;
545  rxq->queue_id = i;
546  rxq->max_pkt_size = 1518;
547  if (i < vec_len (ad->rxqs))
548  {
549  avf_rxq_t *q = vec_elt_at_index (ad->rxqs, i);
550  rxq->ring_len = q->size;
552  rxq->dma_ring_addr = avf_dma_addr (vm, ad, q->descs);
553  avf_reg_write (ad, AVF_QRX_TAIL (i), q->size - 1);
554  }
555 
556  avf_txq_t *q = vec_elt_at_index (ad->txqs, i);
557  txq->vsi_id = ad->vsi_id;
558  if (i < vec_len (ad->txqs))
559  {
560  txq->queue_id = i;
561  txq->ring_len = q->size;
562  txq->dma_ring_addr = avf_dma_addr (vm, ad, q->descs);
563  }
564  }
565 
566  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_VSI_QUEUES, msg, msg_len,
567  0, 0);
568 }
569 
570 clib_error_t *
572 {
573  int count = 1;
574  int msg_len = sizeof (virtchnl_irq_map_info_t) +
575  count * sizeof (virtchnl_vector_map_t);
576  u8 msg[msg_len];
578 
579  memset (msg, 0, msg_len);
580  imi = (virtchnl_irq_map_info_t *) msg;
581  imi->num_vectors = count;
582 
583  imi->vecmap[0].vector_id = 1;
584  imi->vecmap[0].vsi_id = ad->vsi_id;
585  imi->vecmap[0].rxq_map = 1;
586  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_IRQ_MAP, msg, msg_len, 0,
587  0);
588 }
589 
590 clib_error_t *
592 {
593  int msg_len =
594  sizeof (virtchnl_ether_addr_list_t) +
595  count * sizeof (virtchnl_ether_addr_t);
596  u8 msg[msg_len];
598  int i;
599 
600  memset (msg, 0, msg_len);
601  al = (virtchnl_ether_addr_list_t *) msg;
602  al->vsi_id = ad->vsi_id;
603  al->num_elements = count;
604  for (i = 0; i < count; i++)
605  clib_memcpy (&al->list[i].addr, macs + i * 6, 6);
606  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ADD_ETH_ADDR, msg, msg_len, 0,
607  0);
608 }
609 
610 clib_error_t *
612 {
613  virtchnl_queue_select_t qs = { 0 };
614  qs.vsi_id = ad->vsi_id;
615  qs.rx_queues = rx;
616  qs.tx_queues = tx;
617  avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, 0);
618  avf_reg_write (ad, AVF_QRX_TAIL (0), rxq->n_bufs);
619  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ENABLE_QUEUES, &qs,
620  sizeof (virtchnl_queue_select_t), 0, 0);
621 }
622 
623 clib_error_t *
626 {
627  virtchnl_queue_select_t qs = { 0 };
628  qs.vsi_id = ad->vsi_id;
629  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_GET_STATS,
630  &qs, sizeof (virtchnl_queue_select_t),
631  es, sizeof (virtchnl_eth_stats_t));
632 }
633 
634 clib_error_t *
636 {
637  avf_aq_desc_t d = { 0 };
638  clib_error_t *error;
639  u32 rstat;
640  int n_retry = 20;
641 
642  d.opcode = 0x801;
643  d.v_opcode = VIRTCHNL_OP_RESET_VF;
644  if ((error = avf_aq_desc_enq (vm, ad, &d, 0, 0)))
645  return error;
646 
647 retry:
648  vlib_process_suspend (vm, 10e-3);
649  rstat = avf_get_u32 (ad->bar0, AVFGEN_RSTAT);
650 
651  if (rstat == 2 || rstat == 3)
652  return 0;
653 
654  if (--n_retry == 0)
655  return clib_error_return (0, "reset failed (timeout)");
656 
657  goto retry;
658 }
659 
660 clib_error_t *
662 {
663  virtchnl_version_info_t ver = { 0 };
664  virtchnl_vf_resource_t res = { 0 };
665  clib_error_t *error;
667  int i;
668 
669  avf_adminq_init (vm, ad);
670 
671  if ((error = avf_device_reset (vm, ad)))
672  return error;
673 
674  avf_adminq_init (vm, ad);
675 
676  /*
677  * OP_VERSION
678  */
679  if ((error = avf_op_version (vm, ad, &ver)))
680  return error;
681 
682  if (ver.major != VIRTCHNL_VERSION_MAJOR ||
684  return clib_error_return (0, "incompatible protocol version "
685  "(remote %d.%d)", ver.major, ver.minor);
686 
687  /*
688  * OP_GET_VF_RESOUCES
689  */
690  if ((error = avf_op_get_vf_resources (vm, ad, &res)))
691  return error;
692 
693  if (res.num_vsis != 1 || res.vsi_res[0].vsi_type != VIRTCHNL_VSI_SRIOV)
694  return clib_error_return (0, "unexpected GET_VF_RESOURCE reply received");
695 
696  ad->vsi_id = res.vsi_res[0].vsi_id;
699  ad->max_vectors = res.max_vectors;
700  ad->max_mtu = res.max_mtu;
701  ad->rss_key_size = res.rss_key_size;
702  ad->rss_lut_size = res.rss_lut_size;
703 
704  clib_memcpy (ad->hwaddr, res.vsi_res[0].default_mac_addr, 6);
705 
706  /*
707  * Disable VLAN stripping
708  */
709  if ((error = avf_op_disable_vlan_stripping (vm, ad)))
710  return error;
711 
712  if ((error = avf_config_promisc_mode (vm, ad)))
713  return error;
714 
715  if ((ad->feature_bitmap & VIRTCHNL_VF_OFFLOAD_RSS_PF) &&
716  (error = avf_op_config_rss_lut (vm, ad)))
717  return error;
718 
719  /*
720  * Init Queues
721  */
722  if ((error = avf_rxq_init (vm, ad, 0)))
723  return error;
724 
725  for (i = 0; i < tm->n_vlib_mains; i++)
726  if ((error = avf_txq_init (vm, ad, i)))
727  return error;
728 
729  if ((error = avf_op_config_vsi_queues (vm, ad)))
730  return error;
731 
732  if ((error = avf_op_config_irq_map (vm, ad)))
733  return error;
734 
735  avf_irq_0_enable (ad);
736  avf_irq_n_enable (ad, 0);
737 
738  if ((error = avf_op_add_eth_addr (vm, ad, 1, ad->hwaddr)))
739  return error;
740 
741  if ((error = avf_op_enable_queues (vm, ad, 1, 0)))
742  return error;
743 
744  if ((error = avf_op_enable_queues (vm, ad, 0, 1)))
745  return error;
746 
747  ad->flags |= AVF_DEVICE_F_INITIALIZED;
748  return error;
749 }
750 
751 void
753 {
754  avf_main_t *am = &avf_main;
755  vnet_main_t *vnm = vnet_get_main ();
757  u32 r;
758 
759  if (ad->flags & AVF_DEVICE_F_ERROR)
760  return;
761 
762  if ((ad->flags & AVF_DEVICE_F_INITIALIZED) == 0)
763  return;
764 
765  ASSERT (ad->error == 0);
766 
767  r = avf_get_u32 (ad->bar0, AVF_ARQLEN);
768  if ((r & 0xf0000000) != (1 << 31))
769  {
770  ad->error = clib_error_return (0, "arq not enabled, arqlen = 0x%x", r);
771  goto error;
772  }
773 
774  r = avf_get_u32 (ad->bar0, AVF_ATQLEN);
775  if ((r & 0xf0000000) != (1 << 31))
776  {
777  ad->error = clib_error_return (0, "atq not enabled, atqlen = 0x%x", r);
778  goto error;
779  }
780 
781  if (is_irq == 0)
782  avf_op_get_stats (vm, ad, &ad->eth_stats);
783 
784  /* *INDENT-OFF* */
785  vec_foreach (e, ad->events)
786  {
788  {
789  int link_up = e->event_data.link_event.link_status;
790  virtchnl_link_speed_t speed = e->event_data.link_event.link_speed;
791  u32 flags = 0;
792 
793  if (link_up && (ad->flags & AVF_DEVICE_F_LINK_UP) == 0)
794  {
795  ad->flags |= AVF_DEVICE_F_LINK_UP;
798  if (speed == VIRTCHNL_LINK_SPEED_40GB)
800  else if (speed == VIRTCHNL_LINK_SPEED_25GB)
802  else if (speed == VIRTCHNL_LINK_SPEED_10GB)
804  else if (speed == VIRTCHNL_LINK_SPEED_1GB)
806  else if (speed == VIRTCHNL_LINK_SPEED_100MB)
808  vnet_hw_interface_set_flags (vnm, ad->hw_if_index, flags);
809  ad->link_speed = speed;
810  }
811  else if (!link_up && (ad->flags & AVF_DEVICE_F_LINK_UP) != 0)
812  {
813  ad->flags &= ~AVF_DEVICE_F_LINK_UP;
814  ad->link_speed = 0;
815  }
816 
817  if (ad->flags & AVF_DEVICE_F_ELOG)
818  {
819  ELOG_TYPE_DECLARE (el) =
820  {
821  .format = "avf[%d] link change: link_status %d "
822  "link_speed %d",
823  .format_args = "i4i1i1",
824  };
825  struct
826  {
827  u32 dev_instance;
828  u8 link_status;
829  u8 link_speed;
830  } *ed;
831  ed = ELOG_DATA (&vm->elog_main, el);
832  ed->dev_instance = ad->dev_instance;
833  ed->link_status = link_up;
834  ed->link_speed = speed;
835  }
836  }
837  else
838  {
839  if (ad->flags & AVF_DEVICE_F_ELOG)
840  {
841  ELOG_TYPE_DECLARE (el) =
842  {
843  .format = "avf[%d] unknown event: event %d severity %d",
844  .format_args = "i4i4i1i1",
845  };
846  struct
847  {
848  u32 dev_instance;
849  u32 event;
850  u32 severity;
851  } *ed;
852  ed = ELOG_DATA (&vm->elog_main, el);
853  ed->dev_instance = ad->dev_instance;
854  ed->event = e->event;
855  ed->severity = e->severity;
856  }
857  }
858  }
859  /* *INDENT-ON* */
860  vec_reset_length (ad->events);
861 
862  return;
863 
864 error:
865  ad->flags |= AVF_DEVICE_F_ERROR;
866  ASSERT (ad->error != 0);
867  vlib_log_err (am->log_class, "%U", format_clib_error, ad->error);
868 }
869 
870 static u32
872 {
873  avf_main_t *am = &avf_main;
874  vlib_log_warn (am->log_class, "TODO");
875  return 0;
876 }
877 
878 static uword
880 {
881  avf_main_t *am = &avf_main;
882  avf_device_t *ad;
883  uword *event_data = 0, event_type;
884  int enabled = 0, irq;
885  f64 last_run_duration = 0;
886  f64 last_periodic_time = 0;
887 
888  while (1)
889  {
890  if (enabled)
891  vlib_process_wait_for_event_or_clock (vm, 5.0 - last_run_duration);
892  else
894 
895  event_type = vlib_process_get_events (vm, &event_data);
896  vec_reset_length (event_data);
897  irq = 0;
898 
899  switch (event_type)
900  {
901  case ~0:
902  last_periodic_time = vlib_time_now (vm);
903  break;
905  enabled = 1;
906  break;
908  enabled = 0;
909  continue;
911  irq = 1;
912  break;
913  default:
914  ASSERT (0);
915  }
916 
917  /* *INDENT-OFF* */
918  pool_foreach (ad, am->devices,
919  {
920  avf_process_one_device (vm, ad, irq);
921  });
922  /* *INDENT-ON* */
923  last_run_duration = vlib_time_now (vm) - last_periodic_time;
924  }
925  return 0;
926 }
927 
928 /* *INDENT-OFF* */
930  .function = avf_process,
931  .type = VLIB_NODE_TYPE_PROCESS,
932  .name = "avf-process",
933 };
934 /* *INDENT-ON* */
935 
936 static void
938 {
940  avf_main_t *am = &avf_main;
942  avf_device_t *ad = pool_elt_at_index (am->devices, pd);
943  u32 icr0;
944 
945  icr0 = avf_reg_read (ad, AVFINT_ICR0);
946 
947  if (ad->flags & AVF_DEVICE_F_ELOG)
948  {
949  /* *INDENT-OFF* */
950  ELOG_TYPE_DECLARE (el) =
951  {
952  .format = "avf[%d] irq 0: icr0 0x%x",
953  .format_args = "i4i4",
954  };
955  /* *INDENT-ON* */
956  struct
957  {
958  u32 dev_instance;
959  u32 icr0;
960  } *ed;
961 
962  ed = ELOG_DATA (&vm->elog_main, el);
963  ed->dev_instance = ad->dev_instance;
964  ed->icr0 = icr0;
965  }
966 
967  avf_irq_0_enable (ad);
968 
969  /* bit 30 - Send/Receive Admin queue interrupt indication */
970  if (icr0 & (1 << 30))
973 }
974 
975 static void
977 {
979  avf_main_t *am = &avf_main;
981  avf_device_t *ad = pool_elt_at_index (am->devices, pd);
982 
983  if (ad->flags & AVF_DEVICE_F_ELOG)
984  {
985  /* *INDENT-OFF* */
986  ELOG_TYPE_DECLARE (el) =
987  {
988  .format = "avf[%d] irq %d: received",
989  .format_args = "i4i2",
990  };
991  /* *INDENT-ON* */
992  struct
993  {
994  u32 dev_instance;
995  u16 line;
996  } *ed;
997 
998  ed = ELOG_DATA (&vm->elog_main, el);
999  ed->dev_instance = ad->dev_instance;
1000  ed->line = line;
1001  }
1002 
1003  avf_irq_n_enable (ad, 0);
1004 }
1005 
1006 void
1008 {
1009  vnet_main_t *vnm = vnet_get_main ();
1010  avf_main_t *am = &avf_main;
1011  int i;
1012 
1013  if (ad->hw_if_index)
1014  {
1018  }
1019 
1021 
1022  vlib_physmem_free (vm, am->physmem_region, ad->atq);
1023  vlib_physmem_free (vm, am->physmem_region, ad->arq);
1024  vlib_physmem_free (vm, am->physmem_region, ad->atq_bufs);
1025  vlib_physmem_free (vm, am->physmem_region, ad->arq_bufs);
1026 
1027  /* *INDENT-OFF* */
1028  vec_foreach_index (i, ad->rxqs)
1029  {
1030  avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
1031  vlib_physmem_free (vm, am->physmem_region, rxq->descs);
1032  if (rxq->n_bufs)
1033  vlib_buffer_free_from_ring (vm, rxq->bufs, rxq->next, rxq->size,
1034  rxq->n_bufs);
1035  vec_free (rxq->bufs);
1036  }
1037  /* *INDENT-ON* */
1038  vec_free (ad->rxqs);
1039 
1040  /* *INDENT-OFF* */
1041  vec_foreach_index (i, ad->txqs)
1042  {
1043  avf_txq_t *txq = vec_elt_at_index (ad->txqs, i);
1044  vlib_physmem_free (vm, am->physmem_region, txq->descs);
1045  if (txq->n_bufs)
1046  {
1047  u16 first = (txq->next - txq->n_bufs) & (txq->size -1);
1048  vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
1049  txq->n_bufs);
1050  }
1051  vec_free (txq->bufs);
1052  }
1053  /* *INDENT-ON* */
1054  vec_free (ad->txqs);
1055 
1056  clib_error_free (ad->error);
1057  memset (ad, 0, sizeof (*ad));
1058  pool_put (am->devices, ad);
1059 }
1060 
1061 void
1063 {
1064  vnet_main_t *vnm = vnet_get_main ();
1065  avf_main_t *am = &avf_main;
1066  avf_device_t *ad;
1068  clib_error_t *error = 0;
1069 
1070  pool_get (am->devices, ad);
1071  ad->dev_instance = ad - am->devices;
1072  ad->per_interface_next_index = ~0;
1073 
1074  if (args->enable_elog)
1075  ad->flags |= AVF_DEVICE_F_ELOG;
1076 
1077  if ((error = vlib_pci_device_open (&args->addr, avf_pci_device_ids, &h)))
1078  {
1079  pool_put (am->devices, ad);
1080  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1081  args->error =
1082  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1083  &args->addr);
1084  return;
1085  }
1086  ad->pci_dev_handle = h;
1087 
1089 
1090  if ((error = vlib_pci_bus_master_enable (h)))
1091  goto error;
1092 
1093  if ((error = vlib_pci_map_region (h, 0, &ad->bar0)))
1094  goto error;
1095 
1096  if ((error = vlib_pci_register_msix_handler (h, 0, 1, &avf_irq_0_handler)))
1097  goto error;
1098 
1099  if ((error = vlib_pci_register_msix_handler (h, 1, 1, &avf_irq_n_handler)))
1100  goto error;
1101 
1102  if ((error = vlib_pci_enable_msix_irq (h, 0, 2)))
1103  goto error;
1104 
1105  if (am->physmem_region_alloc == 0)
1106  {
1108  error = vlib_physmem_region_alloc (vm, "avf descriptors", 4 << 20, 0,
1109  flags, &am->physmem_region);
1110  if (error)
1111  goto error;
1112  am->physmem_region_alloc = 1;
1113  }
1114  ad->atq = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error,
1115  sizeof (avf_aq_desc_t) * AVF_MBOX_LEN,
1116  64);
1117  if (error)
1118  goto error;
1119 
1120  ad->arq = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error,
1121  sizeof (avf_aq_desc_t) * AVF_MBOX_LEN,
1122  64);
1123  if (error)
1124  goto error;
1125 
1126  ad->atq_bufs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error,
1127  AVF_MBOX_BUF_SZ * AVF_MBOX_LEN,
1128  64);
1129  if (error)
1130  goto error;
1131 
1132  ad->arq_bufs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error,
1133  AVF_MBOX_BUF_SZ * AVF_MBOX_LEN,
1134  64);
1135  if (error)
1136  goto error;
1137 
1138  if ((error = vlib_pci_intr_enable (h)))
1139  goto error;
1140 
1141  /* FIXME detect */
1142  ad->flags |= AVF_DEVICE_F_IOVA;
1143 
1144  if ((error = avf_device_init (vm, ad)))
1145  goto error;
1146 
1147  /* create interface */
1148  error = ethernet_register_interface (vnm, avf_device_class.index,
1149  ad->dev_instance, ad->hwaddr,
1151 
1152  if (error)
1153  goto error;
1154 
1156  ad->sw_if_index = sw->sw_if_index;
1157 
1159  avf_input_node.index);
1160 
1161  if (pool_elts (am->devices) == 1)
1164 
1165  return;
1166 
1167 error:
1168  avf_delete_if (vm, ad);
1169  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1170  args->error = clib_error_return (error, "pci-addr %U",
1171  format_vlib_pci_addr, &args->addr);
1172  vlib_log_err (am->log_class, "%U", format_clib_error, args->error);
1173 }
1174 
1175 static clib_error_t *
1177 {
1178  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
1179  avf_main_t *am = &avf_main;
1181  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1182 
1183  if (ad->flags & AVF_DEVICE_F_ERROR)
1184  return clib_error_return (0, "device is in error state");
1185 
1186  if (is_up)
1187  {
1190  ad->flags |= AVF_DEVICE_F_ADMIN_UP;
1192  }
1193  else
1194  {
1196  ad->flags &= ~AVF_DEVICE_F_ADMIN_UP;
1197  }
1198  return 0;
1199 }
1200 
1201 /* *INDENT-OFF* */
1203 {
1204  .name = "Adaptive Virtual Function (AVF) interface",
1205  .tx_function = avf_interface_tx,
1206  .format_device = format_avf_device,
1207  .format_device_name = format_avf_device_name,
1208  .admin_up_down_function = avf_interface_admin_up_down,
1209 };
1210 /* *INDENT-ON* */
1211 
1212 clib_error_t *
1214 {
1215  avf_main_t *am = &avf_main;
1216  clib_error_t *error;
1218  int i;
1219 
1220  if ((error = vlib_call_init_function (vm, pci_bus_init)))
1221  return error;
1222 
1225 
1226  /* initialize ptype based loopup table */
1228 
1229  /* *INDENT-OFF* */
1230  vec_foreach_index (i, am->ptypes)
1231  {
1232  avf_ptype_t *p = vec_elt_at_index (am->ptypes, i);
1233  if ((i >= 22) && (i <= 87))
1234  {
1236  p->flags = VNET_BUFFER_F_IS_IP4;
1237  }
1238  else if ((i >= 88) && (i <= 153))
1239  {
1241  p->flags = VNET_BUFFER_F_IS_IP6;
1242  }
1243  else
1246  p->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1247  }
1248  /* *INDENT-ON* */
1249 
1250  am->log_class = vlib_log_register_class ("avf_plugin", 0);
1251  vlib_log_debug (am->log_class, "initialized");
1252 
1253  return 0;
1254 }
1255 
1257 
1258 /*
1259  * fd.io coding-style-patch-verification: ON
1260  *
1261  * Local Variables:
1262  * eval: (c-set-style "gnu")
1263  * End:
1264  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:221
u8 next_node
Definition: avf.h:152
vmrglw vmrglh hi
format_function_t format_vlib_pci_addr
Definition: pci.h:288
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define AVF_ARQLEN
Definition: virtchnl.h:34
virtchnl_queue_pair_info_t qpair[1]
Definition: virtchnl.h:270
u32 hw_if_index
Definition: avf.h:87
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
#define AVF_ATQH
Definition: virtchnl.h:27
#define vlib_log_warn(...)
Definition: log.h:51
#define VLIB_PHYSMEM_F_INIT_MHEAP
Definition: physmem.h:57
#define VNET_HW_INTERFACE_FLAG_SPEED_1G
Definition: interface.h:471
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:541
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
clib_error_t * avf_init(vlib_main_t *vm)
Definition: device.c:1213
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:619
clib_error_t * avf_send_to_pf(vlib_main_t *vm, avf_device_t *ad, virtchnl_ops_t op, void *in, int in_len, void *out, int out_len)
Definition: device.c:351
#define VLIB_PHYSMEM_F_HUGETLB
Definition: physmem.h:58
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:319
avf_ptype_t * ptypes
Definition: avf.h:169
#define AVF_ATQBAH
Definition: virtchnl.h:32
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
void avf_arq_slot_init(avf_device_t *ad, u16 slot)
Definition: device.c:292
clib_error_t * error
Definition: avf.h:120
u64 atq_bufs_pa
Definition: avf.h:100
virtchnl_vsi_type_t vsi_type
Definition: virtchnl.h:136
clib_error_t * avf_rxq_init(vlib_main_t *vm, avf_device_t *ad, u16 qid)
Definition: device.c:216
unsigned long u64
Definition: types.h:89
virtchnl_vector_map_t vecmap[1]
Definition: virtchnl.h:298
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:225
virtchnl_link_speed_t link_speed
Definition: avf.h:114
#define AVF_ARQBAH
Definition: virtchnl.h:26
static void avf_adminq_init(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:314
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define AVF_QRX_TAIL(q)
Definition: virtchnl.h:38
#define AVF_ARQT
Definition: virtchnl.h:30
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
format_function_t format_avf_device
Definition: avf.h:192
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
vlib_pci_addr_t addr
Definition: avf.h:176
#define AVF_AQ_F_SI
Definition: virtchnl.h:48
u32 dev_instance
Definition: avf.h:85
#define AVF_QTX_TAIL(q)
Definition: virtchnl.h:37
virtchnl_link_speed_t
Definition: virtchnl.h:169
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:458
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
static void * vlib_physmem_alloc_aligned(vlib_main_t *vm, vlib_physmem_region_index_t idx, clib_error_t **error, uword n_bytes, uword alignment)
Definition: physmem_funcs.h:97
avf_device_t * devices
Definition: avf.h:161
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:227
static u64 vlib_get_buffer_data_physical_address(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:306
volatile u32 * qtx_tail
Definition: avf.h:70
clib_error_t * avf_op_config_irq_map(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:571
static vlib_node_registration_t avf_process_node
(constructor) VLIB_REGISTER_NODE (avf_process_node)
Definition: device.c:929
#define AVF_ATQLEN
Definition: virtchnl.h:28
unsigned char u8
Definition: types.h:56
vnet_device_class_t avf_device_class
#define AVF_ARQH
Definition: virtchnl.h:31
clib_error_t * avf_device_reset(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:635
clib_error_t * vlib_pci_register_msix_handler(vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:764
clib_error_t * avf_op_disable_vlan_stripping(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:506
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
virtchnl_ops_t
Definition: virtchnl.h:85
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
static uword vlib_process_suspend(vlib_main_t *vm, f64 dt)
Suspend a vlib cooperative multi-tasking thread for a period of time.
Definition: node_funcs.h:448
static uword avf_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:879
#define AVF_AQ_F_DD
Definition: virtchnl.h:40
#define AVFINT_ICR0_ENA1
Definition: virtchnl.h:24
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:440
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:156
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:542
clib_spinlock_t lock
Definition: avf.h:73
#define PCI_DEVICE_ID_INTEL_AVF
Definition: device.c:32
static u32 avf_reg_read(avf_device_t *ad, u32 addr)
Definition: avf.h:243
#define AVF_MBOX_BUF_SZ
Definition: device.c:26
volatile u32 * qrx_tail
Definition: avf.h:59
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
#define VNET_HW_INTERFACE_FLAG_SPEED_25G
Definition: interface.h:476
clib_error_t * vlib_pci_device_open(vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1044
clib_error_t * avf_config_promisc_mode(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:513
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:440
int physmem_region_alloc
Definition: avf.h:164
unsigned int u32
Definition: types.h:88
vlib_pci_dev_handle_t pci_dev_handle
Definition: avf.h:88
#define vlib_call_init_function(vm, x)
Definition: init.h:227
virtchnl_ether_addr_t list[1]
Definition: virtchnl.h:313
#define vlib_log_debug(...)
Definition: log.h:54
void * arq_bufs
Definition: avf.h:99
avf_main_t avf_main
Definition: device.c:36
avf_aq_desc_t * arq
Definition: avf.h:97
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:589
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
void avf_process_one_device(vlib_main_t *vm, avf_device_t *ad, int is_irq)
Definition: device.c:752
static void avf_irq_n_handler(vlib_pci_dev_handle_t h, u16 line)
Definition: device.c:976
const u32 device_input_next_node_advance[((VNET_DEVICE_INPUT_N_NEXT_NODES/CLIB_CACHE_LINE_BYTES)+1)*CLIB_CACHE_LINE_BYTES]
Definition: devices.c:47
i8 buffer_advance
Definition: avf.h:153
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:461
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:952
static u32 avf_get_u32(void *start, int offset)
Definition: avf.h:197
virtchnl_txq_info_t txq
Definition: virtchnl.h:261
#define AVF_ATQT
Definition: virtchnl.h:35
unsigned short u16
Definition: types.h:57
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:273
u64 qword[4]
Definition: avf.h:40
#define ELOG_DATA(em, f)
Definition: elog.h:481
#define AVF_AQ_F_RD
Definition: virtchnl.h:45
#define VIRTCHNL_VERSION_MAJOR
Definition: virtchnl.h:18
clib_error_t * avf_op_enable_queues(vlib_main_t *vm, avf_device_t *ad, u32 rx, u32 tx)
Definition: device.c:611
#define AVF_ITR_INT
Definition: device.c:29
static void avf_reg_flush(avf_device_t *ad)
Definition: avf.h:249
#define AVF_RXQ_SZ
Definition: device.c:27
#define AVF_MBOX_LEN
Definition: device.c:25
#define AVFINT_ICR0
Definition: virtchnl.h:23
static clib_error_t * vlib_pci_bus_master_enable(vlib_pci_dev_handle_t h)
Definition: pci.h:244
void avf_delete_if(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:1007
u8 hwaddr[6]
Definition: avf.h:108
u16 atq_next_slot
Definition: avf.h:102
static u32 avf_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: device.c:871
static void avf_irq_0_enable(avf_device_t *ad)
Definition: device.c:58
#define AVF_AQ_F_BUF
Definition: virtchnl.h:47
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
vlib_main_t * vm
Definition: buffer.c:294
vlib_node_registration_t avf_input_node
(constructor) VLIB_REGISTER_NODE (avf_input_node)
Definition: input.c:535
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
clib_error_t * pci_bus_init(vlib_main_t *vm)
Definition: pci.c:251
static void avf_irq_0_disable(avf_device_t *ad)
Definition: device.c:46
Definition: avf.h:56
clib_error_t * avf_op_get_stats(vlib_main_t *vm, avf_device_t *ad, virtchnl_eth_stats_t *es)
Definition: device.c:624
u32 flags
Definition: avf.h:154
#define clib_memcpy(a, b, c)
Definition: string.h:75
vlib_log_class_t log_class
Definition: avf.h:166
elog_main_t elog_main
Definition: main.h:158
avf_tx_desc_t * descs
Definition: avf.h:74
u8 * format_hexdump(u8 *s, va_list *va)
Definition: std-formats.c:281
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
clib_error_t * avf_aq_desc_enq(vlib_main_t *vm, avf_device_t *ad, avf_aq_desc_t *dt, void *data, int len)
Definition: device.c:100
virtchnl_ops_t v_opcode
Definition: virtchnl.h:208
#define AVFINT_DYN_CTL0
Definition: virtchnl.h:25
#define VNET_HW_INTERFACE_FLAG_SPEED_10G
Definition: interface.h:474
u16 vsi_id
Definition: avf.h:106
u32 per_interface_next_index
Definition: avf.h:83
clib_error_t * avf_op_add_eth_addr(vlib_main_t *vm, avf_device_t *ad, u8 count, u8 *macs)
Definition: device.c:591
static clib_error_t * avf_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:1176
u32 feature_bitmap
Definition: avf.h:107
virtchnl_status_code_t v_retval
Definition: virtchnl.h:213
u32 * bufs
Definition: avf.h:75
#define VNET_HW_INTERFACE_FLAG_FULL_DUPLEX
Definition: interface.h:462
void vlib_pci_device_close(vlib_pci_dev_handle_t h)
Definition: pci.c:1089
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
Definition: interface.h:660
#define ASSERT(truth)
struct virtchnl_pf_event_t::@371::@372 link_event
avf_aq_desc_t * atq
Definition: avf.h:96
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:138
u32 flags
Definition: avf.h:82
#define PCI_DEVICE_ID_INTEL_X722_VF
Definition: device.c:34
#define AVFINT_DYN_CTLN(x)
Definition: virtchnl.h:22
Definition: avf.h:67
u32 * bufs
Definition: avf.h:63
clib_error_t * avf_op_config_rss_lut(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:490
static void avf_irq_n_enable(avf_device_t *ad, u8 line)
Definition: device.c:85
void * bar0
Definition: avf.h:89
#define PCI_DEVICE_ID_INTEL_X710_VF
Definition: device.c:33
static void vlib_physmem_free(vlib_main_t *vm, vlib_physmem_region_index_t idx, void *mem)
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:273
VNET_DEVICE_CLASS(bond_dev_class)
virtchnl_pf_event_t * events
Definition: avf.h:104
size_t count
Definition: vapi.c:42
union virtchnl_pf_event_t::@371 event_data
virtchnl_event_codes_t event
Definition: virtchnl.h:179
static void avf_reg_write(avf_device_t *ad, u32 addr, u32 val)
Definition: avf.h:237
clib_error_t * avf_op_version(vlib_main_t *vm, avf_device_t *ad, virtchnl_version_info_t *ver)
Definition: device.c:458
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static uword avf_dma_addr(vlib_main_t *vm, avf_device_t *ad, void *p)
Definition: device.c:305
#define clib_max(x, y)
Definition: clib.h:282
virtchnl_eth_stats_t eth_stats
Definition: avf.h:117
void * atq_bufs
Definition: avf.h:98
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define AVFGEN_RSTAT
Definition: virtchnl.h:36
u16 num_queue_pairs
Definition: avf.h:109
u16 next
Definition: avf.h:71
static void avf_irq_0_handler(vlib_pci_dev_handle_t h, u16 line)
Definition: device.c:937
virtchnl_rxq_info_t rxq
Definition: virtchnl.h:262
#define AVF_ARQBAL
Definition: virtchnl.h:29
u32 rss_lut_size
Definition: avf.h:113
clib_error_t * avf_device_init(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:661
#define VNET_HW_INTERFACE_FLAG_SPEED_100M
Definition: interface.h:470
#define AVF_AQ_F_CMP
Definition: virtchnl.h:41
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
format_function_t format_avf_device_name
Definition: avf.h:193
uword vlib_pci_get_private_data(vlib_pci_dev_handle_t h)
Definition: pci.c:134
#define foreach_virtchnl_op
Definition: virtchnl.h:53
static u64 vlib_physmem_virtual_to_physical(vlib_main_t *vm, vlib_physmem_region_index_t idx, void *mem)
u64 uword
Definition: types.h:112
u16 size
Definition: avf.h:61
void vlib_pci_set_private_data(vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:141
u16 arq_next_slot
Definition: avf.h:103
#define clib_error_free(e)
Definition: error.h:86
clib_error_t * vlib_pci_map_region(vlib_pci_dev_handle_t h, u32 resource, void **result)
Definition: pci.c:1031
avf_rxq_t * rxqs
Definition: avf.h:92
virtchnl_vsi_resource_t vsi_res[1]
Definition: virtchnl.h:150
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:187
#define AVF_TXQ_SZ
Definition: device.c:28
clib_error_t * vlib_pci_enable_msix_irq(vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:820
clib_error_t * error
Definition: avf.h:180
static clib_error_t * vlib_pci_intr_enable(vlib_pci_dev_handle_t h)
Definition: pci.h:212
#define VNET_HW_INTERFACE_FLAG_SPEED_40G
Definition: interface.h:477
avf_per_thread_data_t * per_thread_data
Definition: avf.h:162
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
u16 size
Definition: avf.h:72
u8 data[0]
Packet data.
Definition: buffer.h:172
u32 sw_if_index
Definition: avf.h:86
#define vec_foreach(var, vec)
Vector iterator.
#define vlib_log_err(...)
Definition: log.h:50
u64 arq_bufs_pa
Definition: avf.h:101
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:109
void avf_create_if(vlib_main_t *vm, avf_create_if_args_t *args)
Definition: device.c:1062
#define AVF_ATQBAL
Definition: virtchnl.h:33
u16 n_bufs
Definition: avf.h:76
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
#define VIRTCHNL_VERSION_MINOR
Definition: virtchnl.h:19
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:490
static void avf_irq_n_disable(avf_device_t *ad, u8 line)
Definition: device.c:76
#define PCI_VENDOR_ID_INTEL
Definition: device.c:31
clib_error_t * avf_cmd_rx_ctl_reg_write(vlib_main_t *vm, avf_device_t *ad, u32 reg, u32 val)
Definition: device.c:185
vlib_physmem_region_index_t physmem_region
Definition: avf.h:163
u16 next
Definition: avf.h:60
uword avf_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:38
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
avf_txq_t * txqs
Definition: avf.h:93
avf_rx_desc_t * descs
Definition: avf.h:62
clib_error_t * avf_op_config_vsi_queues(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:525
u16 vendor_id
Definition: pci.h:120
clib_error_t * avf_txq_init(vlib_main_t *vm, avf_device_t *ad, u16 qid)
Definition: device.c:257
u16 n_bufs
Definition: avf.h:64
#define AVF_AQ_F_ERR
Definition: virtchnl.h:42
u16 max_vectors
Definition: avf.h:110
clib_error_t * avf_op_get_vf_resources(vlib_main_t *vm, avf_device_t *ad, virtchnl_vf_resource_t *res)
Definition: device.c:478
u32 rss_key_size
Definition: avf.h:112
u16 max_mtu
Definition: avf.h:111
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128