FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
vmxnet3.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef __included_vmnet_vmnet_h__
17 #define __included_vmnet_vmnet_h__
18 
19 #define foreach_vmxnet3_tx_func_error \
20  _(ERROR_PACKETS, "error packets") \
21  _(LINK_DOWN, "link down") \
22  _(NO_FREE_SLOTS, "no free tx slots")
23 
24 typedef enum
25 {
26 #define _(f,s) VMXNET3_TX_ERROR_##f,
28 #undef _
31 
32 #define foreach_vmxnet3_rxmode_flags \
33  _(0, UCAST, "unicast") \
34  _(1, MCAST, "multicast") \
35  _(2, BCAST, "broadcast") \
36  _(3, ALL_MULTI, "all multicast") \
37  _(4, PROMISC, "promiscuous")
38 
39 enum
40 {
41 #define _(a, b, c) VMXNET3_RXMODE_##b = (1 << a),
43 #undef _
44 };
45 
46 #define foreach_vmxnet3_show_entry \
47  _(RX_COMP, "rx comp") \
48  _(RX_DESC0, "rx desc 0") \
49  _(RX_DESC1, "rx desc 1") \
50  _(TX_COMP, "tx comp") \
51  _(TX_DESC, "tx desc")
52 
53 enum
54 {
55 #define _(a, b) VMXNET3_SHOW_##a,
57 #undef _
58 };
59 
60 #define foreach_vmxnet3_feature_flags \
61  _(0, RXCSUM, "rx checksum") \
62  _(1, RSS, "RSS") \
63  _(2, RXVLAN, "rx VLAN") \
64  _(3, LRO, "LRO")
65 
66 enum
67 {
68 #define _(a, b, c) VMXNET3_F_##b = (1 << a),
70 #undef _
71 };
72 
73 #define foreach_vmxnet3_rss_hash_type \
74  _(0, IPV4, "ipv4") \
75  _(1, TCP_IPV4, "tcp ipv4") \
76  _(2, IPV6, "ipv6") \
77  _(3, TCP_IPV6, "tcp ipv6")
78 
79 enum
80 {
81 #define _(a, b, c) VMXNET3_RSS_HASH_TYPE_##b = (1 << a),
83 #undef _
84 };
85 
86 #define VMXNET3_RSS_HASH_FUNC_TOEPLITZ 1
87 #define VMXNET3_RSS_MAX_KEY_SZ 40
88 #define VMXNET3_RSS_MAX_IND_TABLE_SZ 128
89 
90 #define VMXNET3_TXQ_MAX 8
91 #define VMXNET3_RXQ_MAX 16
92 #define VMXNET3_TX_START(vd) ((vd)->queues)
93 #define VMXNET3_RX_START(vd) \
94  ((vd)->queues + (vd)->num_tx_queues * sizeof (vmxnet3_tx_queue))
95 
96 /* BAR 0 */
97 #define VMXNET3_REG_IMR 0x0000 /* Interrupt Mask Register */
98 #define VMXNET3_REG_TXPROD 0x0600 /* Tx Producer Index */
99 #define VMXNET3_REG_RXPROD 0x0800 /* Rx Producer Index for ring 1 */
100 #define VMXNET3_REG_RXPROD2 0x0A00 /* Rx Producer Index for ring 2 */
101 
102 
103 /* BAR 1 */
104 #define VMXNET3_REG_VRRS 0x0000 /* VMXNET3 Revision Report Selection */
105 #define VMXNET3_REG_UVRS 0x0008 /* UPT Version Report Selection */
106 #define VMXNET3_REG_DSAL 0x0010 /* Driver Shared Address Low */
107 #define VMXNET3_REG_DSAH 0x0018 /* Driver Shared Address High */
108 #define VMXNET3_REG_CMD 0x0020 /* Command */
109 #define VMXNET3_REG_MACL 0x0028 /* MAC Address Low */
110 #define VMXNET3_REG_MACH 0x0030 /* MAC Address High */
111 #define VMXNET3_REG_ICR 0x0038 /* Interrupt Cause Register */
112 #define VMXNET3_REG_ECR 0x0040 /* Event Cause Register */
113 
114 #define VMXNET3_VLAN_LEN 4
115 #define VMXNET3_FCS_LEN 4
116 #define VMXNET3_MTU (1514 + VMXNET3_VLAN_LEN + VMXNET3_FCS_LEN)
117 
118 #define VMXNET3_RXF_BTYPE (1 << 14) /* rx body buffer type */
119 #define VMXNET3_RXF_GEN (1 << 31) /* rx generation */
120 
121 #define VMXNET3_RXCF_CKSUM_MASK (0xFFFF) /* rx checksum mask */
122 #define VMXNET3_RXCF_TUC (1 << 16) /* rx udp/tcp checksum correct */
123 #define VMXNET3_RXCF_UDP (1 << 17) /* rx udp packet */
124 #define VMXNET3_RXCF_TCP (1 << 18) /* rx tcp packet */
125 #define VMXNET3_RXCF_IPC (1 << 19) /* rx ip checksum correct */
126 #define VMXNET3_RXCF_IP6 (1 << 20) /* rx ip6 packet */
127 #define VMXNET3_RXCF_IP4 (1 << 21) /* rx ip4 packet */
128 #define VMXNET3_RXCF_CT (0x7F << 24) /* rx completion type 24-30, 7 bits */
129 #define VMXNET3_RXCF_GEN (1 << 31) /* rx completion generation */
130 
131 #define VMXNET3_RXC_INDEX (0xFFF) /* rx completion index mask */
132 
133 #define foreach_vmxnet3_offload \
134  _(0, NONE, "none") \
135  _(2, CSUM, "checksum") \
136  _(3, TSO, "tso")
137 
138 enum
139 {
140 #define _(a, b, c) VMXNET3_OM_##b = (a),
142 #undef _
143 };
144 
145 /* tx desc flag 0 */
146 #define VMXNET3_TXF_GEN (1 << 14) /* tx generation */
147 
148 /* tx desc flag 1 */
149 #define VMXNET3_TXF_OM(x) ((x) << 10) /* tx offload mode */
150 #define VMXNET3_TXF_MSSCOF(x) ((x) << 18) /* tx MSS checksum offset, flags */
151 #define VMXNET3_TXF_EOP (1 << 12) /* tx end of packet */
152 #define VMXNET3_TXF_CQ (1 << 13) /* tx completion request */
153 
154 /* tx completion flag */
155 #define VMXNET3_TXCF_GEN (1 << 31) /* tx completion generation */
156 #define VMXNET3_TXC_INDEX (0xFFF) /* tx completion index mask */
157 
158 #define VMXNET3_RX_RING_SIZE 2
159 #define VMXNET3_INPUT_REFILL_THRESHOLD 32
160 #define VMXNET3_NUM_TX_DESC 1024
161 #define VMXNET3_NUM_TX_COMP VMXNET3_NUM_TX_DESC
162 #define VMXNET3_NUM_RX_DESC 1024
163 #define VMXNET3_NUM_RX_COMP VMXNET3_NUM_RX_DESC
164 
165 #define VMXNET3_VERSION_MAGIC 0x69505845
166 #define VMXNET3_SHARED_MAGIC 0xbabefee1
167 #define VMXNET3_VERSION_SELECT 1
168 #define VMXNET3_UPT_VERSION_SELECT 1
169 #define VMXNET3_MAX_INTRS 25
170 #define VMXNET3_IC_DISABLE_ALL 0x1
171 
172 #define VMXNET3_GOS_BITS_32 (1 << 0)
173 #define VMXNET3_GOS_BITS_64 (2 << 0)
174 #define VMXNET3_GOS_TYPE_LINUX (1 << 2)
175 #define VMXNET3_RXCL_LEN_MASK (0x3FFF) // 14 bits
176 #define VMXNET3_RXCL_ERROR (1 << 14)
177 
178 #define VMXNET3_RXCI_EOP (1 << 14) /* end of packet */
179 #define VMXNET3_RXCI_SOP (1 << 15) /* start of packet */
180 #define VMXNET3_RXCI_CNC (1 << 30) /* Checksum not calculated */
181 
182 #define VMXNET3_RXCOMP_TYPE (3 << 24) /* RX completion descriptor */
183 #define VMXNET3_RXCOMP_TYPE_LRO (4 << 24) /* RX completion descriptor for LRO */
184 
185 #define VMXNET3_RXECF_MSS_MASK (0xFFFF) // 16 bits
186 
187 #define foreach_vmxnet3_device_flags \
188  _(0, INITIALIZED, "initialized") \
189  _(1, ERROR, "error") \
190  _(2, ADMIN_UP, "admin-up") \
191  _(3, IOVA, "iova") \
192  _(4, LINK_UP, "link-up") \
193  _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \
194  _(6, ELOG, "elog")
195 
196 enum
197 {
198 #define _(a, b, c) VMXNET3_DEVICE_F_##b = (1 << a),
200 #undef _
201 };
202 
203 #define foreach_vmxnet3_set_cmds \
204  _(0, ACTIVATE_DEV, "activate device") \
205  _(1, QUIESCE_DEV, "quiesce device") \
206  _(2, RESET_DEV, "reset device") \
207  _(3, UPDATE_RX_MODE, "update rx mode") \
208  _(4, UPDATE_MAC_FILTERS, "update mac filters") \
209  _(5, UPDATE_VLAN_FILTERS, "update vlan filters") \
210  _(6, UPDATE_RSSIDT, "update rss idt") \
211  _(7, UPDATE_IML, "update iml") \
212  _(8, UPDATE_PMCFG, "update pm cfg") \
213  _(9, UPDATE_FEATURE, "update feature") \
214  _(10, STOP_EMULATION, "stop emulation") \
215  _(11, LOAD_PLUGIN, "load plugin") \
216  _(12, ACTIVATE_VF, "activate vf") \
217  _(13, RESERVED3, "reserved 3") \
218  _(14, RESERVED4, "reservced 4") \
219  _(15, REGISTER_MEMREGS, "register mem regs")
220 
221 enum
222 {
223 #define _(a, b, c) VMXNET3_CMD_##b = (a + 0xCAFE0000),
225 #undef _
226 };
227 
228 #define foreach_vmxnet3_get_cmds \
229  _(0, GET_QUEUE_STATUS, "get queue status") \
230  _(1, GET_STATS, "get stats") \
231  _(2, GET_LINK, "get link") \
232  _(3, GET_PERM_MAC_LO, "get perm mac lo") \
233  _(4, GET_PERM_MAC_HI, "get perm mac hi") \
234  _(5, GET_DID_LO, "get did lo") \
235  _(6, GET_DID_HI, "get did hi") \
236  _(7, GET_DEV_EXTRA_INFO, "get dev extra info") \
237  _(8, GET_CONF_INTR, "get conf intr") \
238  _(9, GET_ADAPTIVE_RING_INFO, "get adaptive ring info") \
239  _(10, GET_TXDATA_DESC_SIZE, "get txdata desc size") \
240  _(11, RESERVED5, "reserved5")
241 
242 enum
243 {
244 #define _(a, b, c) VMXNET3_CMD_##b = (a + 0xF00D0000),
246 #undef _
247 };
248 
249 typedef CLIB_PACKED (struct
250  {
251  u32 version; u32 guest_info; u32 version_support;
252  u32 upt_version_support; u64 upt_features;
253  u64 driver_data_address; u64 queue_desc_address;
254  u32 driver_data_len; u32 queue_desc_len;
255  u32 mtu;
256  u16 max_num_rx_sg; u8 num_tx_queues; u8 num_rx_queues;
257  u32 pad[4];
258  }) vmxnet3_misc_config;
259 
260 typedef CLIB_PACKED (struct
261  {
262  u8 mask_mode;
263  u8 num_intrs;
264  u8 event_intr_index;
265  u8 moderation_level[VMXNET3_MAX_INTRS]; u32 control;
266  u32 pad[2];
267  }) vmxnet3_interrupt_config;
268 
269 typedef CLIB_PACKED (struct
270  {
271  u32 mode; u16 multicast_len; u16 pad;
272  u64 multicast_address; u8 vlan_filter[512];
273  }) vmxnet3_rx_filter_config;
274 
275 typedef CLIB_PACKED (struct
276  {
278  u64 address;
279  }) vmxnet3_variable_config;
280 
281 typedef CLIB_PACKED (struct
282  {
283  u32 magic;
284  u32 pad;
285  vmxnet3_misc_config misc;
286  vmxnet3_interrupt_config interrupt;
287  vmxnet3_rx_filter_config rx_filter;
288  vmxnet3_variable_config rss;
289  vmxnet3_variable_config pattern;
290  vmxnet3_variable_config plugin; u32 ecr;
291  u32 pad1[5];
292  }) vmxnet3_shared;
293 
294 typedef CLIB_PACKED (struct
295  {
296  u8 stopped;
297  u8 pad[3];
298  u32 error;
299  }) vmxnet3_queue_status;
300 
301 typedef CLIB_PACKED (struct
302  {
303  u32 num_deferred; u32 threshold;
304  u64 pad;
305  }) vmxnet3_tx_queue_control;
306 
307 typedef CLIB_PACKED (struct
308  {
309  u64 desc_address;
310  u64 data_address;
311  u64 comp_address; u64 driver_data_address; u64 pad;
312  u32 num_desc;
313  u32 num_data; u32 num_comp; u32 driver_data_len;
314  u8 intr_index;
315  u8 pad1; u16 data_address_size; u8 pad2[4];
316  }) vmxnet3_tx_queue_config;
317 
318 typedef CLIB_PACKED (struct
319  {
320  u64 tso_pkts;
321  u64 tso_bytes;
322  u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
323  u64 mcast_bytes;
324  u64 bcast_pkts; u64 bcast_bytes; u64 error_pkts;
325  u64 discard_pkts;
326  }) vmxnet3_tx_stats;
327 
328 typedef CLIB_PACKED (struct
329  {
330  vmxnet3_tx_queue_control ctrl;
331  vmxnet3_tx_queue_config cfg;
332  vmxnet3_queue_status status; vmxnet3_tx_stats stats;
333  u8 pad[88];
334  }) vmxnet3_tx_queue;
335 
336 typedef CLIB_PACKED (struct
337  {
338  u8 update_prod; u8 pad[7];
339  u64 pad1;
340  }) vmxnet3_rx_queue_control;
341 
342 typedef CLIB_PACKED (struct
343  {
344  u64 desc_address[2];
345  u64 comp_address; u64 driver_data_address;
346  u64 data_address; u32 num_desc[2];
347  u32 num_comp;
348  u32 driver_data_len; u8 intr_index; u8 pad1;
349  u16 data_address_size; u8 pad2[4];
350  }) vmxnet3_rx_queue_config;
351 
352 typedef CLIB_PACKED (struct
353  {
354  u64 lro_pkts;
355  u64 lro_bytes;
356  u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
357  u64 mcast_bytes;
358  u64 bcast_pkts; u64 bcast_bytes; u64 nobuf_pkts;
359  u64 error_pkts;
360  }) vmxnet3_rx_stats;
361 
362 typedef CLIB_PACKED (struct
363  {
364  vmxnet3_rx_queue_control ctrl;
365  vmxnet3_rx_queue_config cfg;
366  vmxnet3_queue_status status; vmxnet3_rx_stats stats;
367  u8 pad[88];
368  }) vmxnet3_rx_queue;
369 
370 /*
371  * flags:
372  * buffer length -- bits 0-13
373  * buffer type -- bit 14
374  * descriptor type -- bit 15
375  * reserved -- bits 16-30
376  * generation -- bit 31
377  */
378 typedef CLIB_PACKED (struct
379  {
380  u64 address;
381  u32 flags;
382  u32 pad;
383  }) vmxnet3_rx_desc;
384 
385 /*
386  * index:
387  * RX desc index -- bits 0-11
388  * ext1 -- bits 12-13
389  * end of packet -- bit 14
390  * start of packet -- bit 15
391  * ring ID -- bits 16-25
392  * RSS hash type -- bits 26-29
393  * checksum not calculated -- bit 30
394  * ext2 -- bit 31
395  *
396  * rss: RSS hash value
397  *
398  * len:
399  * data length -- bits 0-13
400  * error -- bit 14
401  * tag is stripped -- bit 15
402  * tag stripped -- bits 16-31
403  *
404  * flags:
405  * checksum -- bits 0 - 15
406  * tcp/udp checksum correct-- bit 16
407  * udp packet -- bit 17
408  * tcp packet -- bit 18
409  * ip checksum correct -- bit 19
410  * ipv6 -- bit 20
411  * ipv4 -- bit 21
412  * ip fragment -- bit 22
413  * frame crc correct -- bit 23
414  * completion type -- bits 24-30
415  * generation -- bit 31
416  */
417 typedef CLIB_PACKED (struct
418  {
419  u32 index; u32 rss;
420  u32 len;
421  u32 flags;
422  }) vmxnet3_rx_comp;
423 
424 /*
425  * flags:
426  * mss -- bits 0 - 15
427  * tcp/udp checksum correct-- bit 16
428  * udp packet -- bit 17
429  * tcp packet -- bit 18
430  * ip checksum correct -- bit 19
431  * ipv6 -- bit 20
432  * ipv4 -- bit 21
433  * ip fragment -- bit 22
434  * frame crc correct -- bit 23
435  * completion type -- bits 24-30
436  * generation -- bit 31
437  */
438 typedef CLIB_PACKED (struct
439  {
440  u32 dword1;
441  u8 seg_cnt; u8 dup_ack_cnt; u16 ts_delta; u32 dword2;
442  u32 flags;
443  }) vmxnet3_rx_comp_ext;
444 
445 /*
446  * index:
447  * TX desc index -- bits 0-11
448  * ext1 -- bits 12-31
449  *
450  * flags:
451  * reserved -- bits 0-23
452  * completion type -- bits 24-30
453  * generation -- bit 31
454  */
455 typedef CLIB_PACKED (struct
456  {
457  u32 index;
458  u32 pad[2];
459  u32 flags;
460  }) vmxnet3_tx_comp;
461 
462 /*
463  * flags[0]:
464  * length -- bits 0-13
465  * generation -- bit 14
466  * reserved -- bit 15
467  * descriptor type -- bit 16
468  * ext1 -- bit 17
469  * MSS, checksum offset -- bits 18-31
470  * flags[1]:
471  * header length -- bits 0-9
472  * offload mode -- bits 10-11
473  * end of packet -- bit 12
474  * completion request -- bit 13
475  * ext2 -- bit 14
476  * vlan tag insertion -- bit 15
477  * tag to insert -- bits 16-31
478  */
479 typedef CLIB_PACKED (struct
480  {
481  u64 address;
482  u32 flags[2];
483  }) vmxnet3_tx_desc;
484 
485 typedef CLIB_PACKED (struct
486  {
487  u16 hash_type;
488  u16 hash_func;
489  u16 hash_key_sz;
490  u16 ind_table_sz;
491  u8 hash_key[VMXNET3_RSS_MAX_KEY_SZ];
492  u8 ind_table[VMXNET3_RSS_MAX_IND_TABLE_SZ];
493  }) vmxnet3_rss_shared;
494 
495 typedef struct
496 {
497  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
505 
506 typedef struct
507 {
508  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
512 
513 typedef struct
514 {
515  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
521  vmxnet3_rx_desc *rx_desc[VMXNET3_RX_RING_SIZE];
522  vmxnet3_rx_comp *rx_comp;
524 } vmxnet3_rxq_t;
525 
526 typedef struct
527 {
528  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
534 
535 typedef struct
536 {
537  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
541 
542 typedef struct
543 {
544  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
548 
549  vmxnet3_tx_desc *tx_desc;
550  vmxnet3_tx_comp *tx_comp;
553 } vmxnet3_txq_t;
554 
555 typedef struct
556 {
557  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
560 
566  vlib_pci_addr_t pci_addr;
567  void *bar[2];
568 
569  /* queues */
572 
576 
579 
581 
582  vmxnet3_shared *driver_shared;
583  void *queues;
584  vmxnet3_rss_shared *rss;
587  vmxnet3_tx_stats *tx_stats;
588  vmxnet3_rx_stats *rx_stats;
590 
591 typedef struct
592 {
597 
599 
600 typedef struct
601 {
602  vlib_pci_addr_t addr;
610  /* return */
615 
616 typedef struct
617 {
622 
625 
629 
630 /* format.c */
634 
635 #define vmxnet3_log_debug(dev, f, ...) \
636  vlib_log (VLIB_LOG_LEVEL_DEBUG, vmxnet3_main.log_default, "%U: " f, \
637  format_vlib_pci_addr, &dev->pci_addr, \
638  ## __VA_ARGS__)
639 
640 #define vmxnet3_log_error(dev, f, ...) \
641  vlib_log (VLIB_LOG_LEVEL_ERR, vmxnet3_main.log_default, "%U: " f, \
642  format_vlib_pci_addr, &dev->pci_addr, \
643  ## __VA_ARGS__)
644 
645 /* no log version, called by data plane */
648 {
649  *(volatile u32 *) ((u8 *) vd->bar[bar] + addr) = val;
650 }
651 
654 {
655  vmxnet3_log_debug (vd, "reg wr bar %u addr 0x%x val 0x%x", bar, addr, val);
656  vmxnet3_reg_write_inline (vd, bar, addr, val);
657 }
658 
661 {
662  u32 val;
663 
664  val = *(volatile u32 *) (vd->bar[bar] + addr);
665  vmxnet3_log_debug (vd, "reg rd bar %u addr 0x%x val 0x%x", bar, addr, val);
666 
667  return val;
668 }
669 
672 {
673  return (vd->flags & VMXNET3_DEVICE_F_IOVA) ? pointer_to_uword (p) :
674  vlib_physmem_get_pa (vm, p);
675 }
676 
679 {
680  ring->produce++;
681  if (PREDICT_FALSE (ring->produce == rxq->size))
682  {
683  ring->produce = 0;
684  ring->gen ^= VMXNET3_RXF_GEN;
685  }
686 }
687 
690  vmxnet3_rxq_t * rxq)
691 {
692  vmxnet3_rx_desc *rxd;
693  u16 n_refill, n_alloc;
694  vmxnet3_rx_ring *ring;
695  vmxnet3_rx_queue *rx;
696 
697  ring = &rxq->rx_ring[0];
698  n_refill = rxq->size - ring->fill;
699 
701  return 0;
702 
703  n_alloc =
705  rxq->size, n_refill,
706  rxq->buffer_pool_index);
707  if (PREDICT_FALSE (n_alloc != n_refill))
708  {
709  if (n_alloc)
710  vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
711  n_alloc);
712  return clib_error_return (0, "buffer alloc failed");
713  }
714 
715  while (n_alloc)
716  {
717  vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
718  rxd = &rxq->rx_desc[0][ring->produce];
719  rxd->address = vlib_buffer_get_pa (vm, b);
720  rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm);
721 
723  ring->fill++;
724  n_alloc--;
725  }
726 
727  rx = VMXNET3_RX_START (vd);
728  if (PREDICT_FALSE (rx->ctrl.update_prod))
730 
731  return 0;
732 }
733 
736  vmxnet3_rxq_t * rxq)
737 {
738  vmxnet3_rx_desc *rxd;
739  u16 n_refill, n_alloc;
740  vmxnet3_rx_ring *ring;
741  vmxnet3_rx_queue *rx;
742 
743  ring = &rxq->rx_ring[1];
744  n_refill = rxq->size - ring->fill;
745 
747  return 0;
748 
749  n_alloc =
751  rxq->size, n_refill,
752  rxq->buffer_pool_index);
753  if (PREDICT_FALSE (n_alloc != n_refill))
754  {
755  if (n_alloc)
756  vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
757  n_alloc);
758  return clib_error_return (0, "buffer alloc failed");
759  }
760 
761  while (n_alloc)
762  {
763  vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
764  rxd = &rxq->rx_desc[1][ring->produce];
765  rxd->address = vlib_buffer_get_pa (vm, b);
766  rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm) |
768 
770  ring->fill++;
771  n_alloc--;
772  }
773 
774  rx = VMXNET3_RX_START (vd);
775  if (PREDICT_FALSE (rx->ctrl.update_prod))
777 
778  return 0;
779 }
780 
781 #endif /* __included_vmnet_vmnet_h__ */
782 /*
783  * fd.io coding-style-patch-verification: ON
784  *
785  * Local Variables:
786  * eval: (c-set-style "gnu")
787  * End:
788  */
vmxnet3_device_t::gso_enable
u8 gso_enable
Definition: vmxnet3.h:586
vmxnet3_device_t::dev_instance
u32 dev_instance
Definition: vmxnet3.h:561
vmxnet3_device_t::flags
u32 flags
Definition: vmxnet3.h:558
foreach_vmxnet3_tx_func_error
#define foreach_vmxnet3_tx_func_error
Definition: vmxnet3.h:19
vmxnet3_rx_ring::gen
u32 gen
Definition: vmxnet3.h:499
vmxnet3_rxq_refill_ring1
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring1(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:735
vmxnet3_device_t::num_rx_queues
u16 num_rx_queues
Definition: vmxnet3.h:574
format_vmxnet3_device_name
format_function_t format_vmxnet3_device_name
Definition: vmxnet3.h:632
vmxnet3_dma_addr
static_always_inline uword vmxnet3_dma_addr(vlib_main_t *vm, vmxnet3_device_t *vd, void *p)
Definition: vmxnet3.h:671
foreach_vmxnet3_show_entry
#define foreach_vmxnet3_show_entry
Definition: vmxnet3.h:46
vmxnet3_create_if_args_t::bind
u8 bind
Definition: vmxnet3.h:608
VMXNET3_MAX_INTRS
#define VMXNET3_MAX_INTRS
Definition: vmxnet3.h:169
vnet_device_class_t
struct _vnet_device_class vnet_device_class_t
vmxnet3_rx_ring::rid
u16 rid
Definition: vmxnet3.h:501
vmxnet3_device_t::pci_dev_handle
vlib_pci_dev_handle_t pci_dev_handle
Definition: vmxnet3.h:565
vlib_buffer_alloc_to_ring_from_pool
static __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring_from_pool(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers into ring from specific buffer pool.
Definition: buffer_funcs.h:753
vmxnet3_input_node
vlib_node_registration_t vmxnet3_input_node
(constructor) VLIB_REGISTER_NODE (vmxnet3_input_node)
Definition: input.c:491
vmxnet3_rxq_refill_ring0
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring0(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:689
vmxnet3_txq_t::lock
clib_spinlock_t lock
Definition: vmxnet3.h:547
vmxnet3_rxq_t::queue_index
u32 queue_index
Definition: vmxnet3.h:519
vmxnet3_reg_read
static_always_inline u32 vmxnet3_reg_read(vmxnet3_device_t *vd, u8 bar, u32 addr)
Definition: vmxnet3.h:660
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
vmxnet3_log_debug
#define vmxnet3_log_debug(dev, f,...)
Definition: vmxnet3.h:635
pointer_to_uword
static uword pointer_to_uword(const void *p)
Definition: types.h:131
vmxnet3_rx_ring::fill
u16 fill
Definition: vmxnet3.h:500
vmxnet3_txq_t::tx_comp
vmxnet3_tx_comp * tx_comp
Definition: vmxnet3.h:550
vmxnet3_tx_comp_ring
Definition: vmxnet3.h:535
CLIB_CACHE_LINE_ALIGN_MARK
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
vlib_log_class_t
u32 vlib_log_class_t
Definition: vlib.h:52
vmxnet3_reg_write_inline
static_always_inline void vmxnet3_reg_write_inline(vmxnet3_device_t *vd, u8 bar, u32 addr, u32 val)
Definition: vmxnet3.h:647
foreach_vmxnet3_device_flags
#define foreach_vmxnet3_device_flags
Definition: vmxnet3.h:187
vmxnet3_device_t::tx_stats
vmxnet3_tx_stats * tx_stats
Definition: vmxnet3.h:587
foreach_vmxnet3_set_cmds
#define foreach_vmxnet3_set_cmds
Definition: vmxnet3.h:203
clib_error_return
#define clib_error_return(e, args...)
Definition: error.h:99
vlib_pci_dev_handle_t
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
vmxnet3_device_t::pci_addr
vlib_pci_addr_t pci_addr
Definition: vmxnet3.h:566
vmxnet3_reg_write
static_always_inline void vmxnet3_reg_write(vmxnet3_device_t *vd, u8 bar, u32 addr, u32 val)
Definition: vmxnet3.h:653
vmxnet3_create_if_args_t::addr
vlib_pci_addr_t addr
Definition: vmxnet3.h:602
VMXNET3_RSS_MAX_KEY_SZ
#define VMXNET3_RSS_MAX_KEY_SZ
Definition: vmxnet3.h:87
u16
unsigned short u16
Definition: types.h:57
vmxnet3_rxq_t::int_mode
u8 int_mode
Definition: vmxnet3.h:517
foreach_vmxnet3_rss_hash_type
#define foreach_vmxnet3_rss_hash_type
Definition: vmxnet3.h:73
mode
vl_api_tunnel_mode_t mode
Definition: gre.api:48
vmxnet3_plugin_api_hookup
clib_error_t * vmxnet3_plugin_api_hookup(vlib_main_t *vm)
Definition: vmxnet3_api.c:236
vmxnet3_rx_comp_ring
Definition: vmxnet3.h:506
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
vmxnet3_main_t
Definition: vmxnet3.h:591
VMXNET3_RX_START
#define VMXNET3_RX_START(vd)
Definition: vmxnet3.h:93
vmxnet3_device_t::error
clib_error_t * error
Definition: vmxnet3.h:580
stats
vl_api_ikev2_sa_stats_t stats
Definition: ikev2_types.api:162
vlib_buffer_get_pa
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:488
vmxnet3_tx_ring
Definition: vmxnet3.h:526
addr
vhost_vring_addr_t addr
Definition: vhost_user.h:130
vmxnet3_device_t::queues
void * queues
Definition: vmxnet3.h:583
vmxnet3_create_if
void vmxnet3_create_if(vlib_main_t *vm, vmxnet3_create_if_args_t *args)
Definition: vmxnet3.c:606
vmxnet3_tx_func_error_t
vmxnet3_tx_func_error_t
Definition: vmxnet3.h:24
error
Definition: cJSON.c:88
vmxnet3_create_if_args_t::rxq_size
u16 rxq_size
Definition: vmxnet3.h:604
vmxnet3_rx_ring::produce
u16 produce
Definition: vmxnet3.h:502
i32
signed int i32
Definition: types.h:77
vmxnet3_rxq_t::size
u16 size
Definition: vmxnet3.h:516
vmxnet3_main_t::msg_id_base
u16 msg_id_base
Definition: vmxnet3.h:594
vmxnet3_input_trace_t::buffer
vlib_buffer_t buffer
Definition: vmxnet3.h:620
vlib_buffer_free_from_ring
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:1029
vmxnet3_rx_ring
Definition: vmxnet3.h:495
vlib_physmem_get_pa
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
Definition: physmem_funcs.h:103
vmxnet3_create_if_args_t::rv
i32 rv
Definition: vmxnet3.h:611
vmxnet3_rxq_t::buffer_pool_index
u8 buffer_pool_index
Definition: vmxnet3.h:518
len
u8 len
Definition: ip_types.api:103
vmxnet3_delete_if
void vmxnet3_delete_if(vlib_main_t *vm, vmxnet3_device_t *ad)
Definition: vmxnet3.c:858
vmxnet3_create_if_args_t
Definition: vmxnet3.h:600
foreach_vmxnet3_get_cmds
#define foreach_vmxnet3_get_cmds
Definition: vmxnet3.h:228
vmxnet3_create_if_args_t::txq_num
u16 txq_num
Definition: vmxnet3.h:607
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
vmxnet3_main_t::log_default
vlib_log_class_t log_default
Definition: vmxnet3.h:595
vmxnet3_txq_t::tx_comp_ring
vmxnet3_tx_comp_ring tx_comp_ring
Definition: vmxnet3.h:552
static_always_inline
#define static_always_inline
Definition: clib.h:112
clib_spinlock_s
Definition: lock.h:51
vmxnet3_input_trace_t::next_index
u32 next_index
Definition: vmxnet3.h:618
uword
u64 uword
Definition: types.h:112
vmxnet3_rx_ring::bufs
u32 * bufs
Definition: vmxnet3.h:498
vmxnet3_rxq_t::rx_comp_ring
vmxnet3_rx_comp_ring rx_comp_ring
Definition: vmxnet3.h:523
vmxnet3_device_t::driver_shared
vmxnet3_shared * driver_shared
Definition: vmxnet3.h:582
vmxnet3_tx_ring::produce
u16 produce
Definition: vmxnet3.h:531
vmxnet3_rx_ring::consume
u16 consume
Definition: vmxnet3.h:503
VMXNET3_REG_RXPROD
#define VMXNET3_REG_RXPROD
Definition: vmxnet3.h:99
vmxnet3_rx_comp_ring::gen
u32 gen
Definition: vmxnet3.h:509
vmxnet3_input_trace_t
Definition: vmxnet3.h:616
address
manual_print typedef address
Definition: ip_types.api:96
vmxnet3_device_t::bar
void * bar[2]
Definition: vmxnet3.h:567
vmxnet3_create_if_args_t::txq_size
u16 txq_size
Definition: vmxnet3.h:606
vlib_node_registration_t
struct _vlib_node_registration vlib_node_registration_t
vmxnet3_device_t::num_intrs
u16 num_intrs
Definition: vmxnet3.h:575
vmxnet3_rxq_t::rx_comp
vmxnet3_rx_comp * rx_comp
Definition: vmxnet3.h:522
vmxnet3_device_t::rss
vmxnet3_rss_shared * rss
Definition: vmxnet3.h:584
mac_addr
vl_api_mac_address_t mac_addr
Definition: l2.api:210
format_vmxnet3_device
format_function_t format_vmxnet3_device
Definition: vmxnet3.h:631
vmxnet3_tx_comp_ring::gen
u32 gen
Definition: vmxnet3.h:538
format_function_t
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
VMXNET3_RX_RING_SIZE
#define VMXNET3_RX_RING_SIZE
Definition: vmxnet3.h:158
index
u32 index
Definition: flow_types.api:221
vmxnet3_device_t::per_interface_next_index
u32 per_interface_next_index
Definition: vmxnet3.h:559
foreach_vmxnet3_rxmode_flags
#define foreach_vmxnet3_rxmode_flags
Definition: vmxnet3.h:32
u64
unsigned long u64
Definition: types.h:89
vmxnet3_device_t::rxqs
vmxnet3_rxq_t * rxqs
Definition: vmxnet3.h:570
version
option version
Definition: sample.api:19
vmxnet3_txq_t::tx_ring
vmxnet3_tx_ring tx_ring
Definition: vmxnet3.h:551
vmxnet3_create_if_args_t::enable_elog
u32 enable_elog
Definition: vmxnet3.h:603
VMXNET3_RSS_MAX_IND_TABLE_SZ
#define VMXNET3_RSS_MAX_IND_TABLE_SZ
Definition: vmxnet3.h:88
vmxnet3_tx_comp_ring::next
u16 next
Definition: vmxnet3.h:539
vlib_buffer_get_default_data_size
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:122
u32
unsigned int u32
Definition: types.h:88
vmxnet3_txq_t
Definition: vmxnet3.h:542
vmxnet3_create_if_args_t::sw_if_index
u32 sw_if_index
Definition: vmxnet3.h:612
format_vmxnet3_input_trace
format_function_t format_vmxnet3_input_trace
Definition: vmxnet3.h:633
VMXNET3_TX_N_ERROR
@ VMXNET3_TX_N_ERROR
Definition: vmxnet3.h:29
vmxnet3_rxq_t::rx_ring
vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:520
CLIB_PACKED
typedef CLIB_PACKED(struct { u32 version;u32 guest_info;u32 version_support;u32 upt_version_support;u64 upt_features;u64 driver_data_address;u64 queue_desc_address;u32 driver_data_len;u32 queue_desc_len;u32 mtu;u16 max_num_rx_sg;u8 num_tx_queues;u8 num_rx_queues;u32 pad[4];}) vmxnet3_misc_config
vmxnet3_tx_ring::bufs
u32 * bufs
Definition: vmxnet3.h:529
vmxnet3_device_t::rx_stats
vmxnet3_rx_stats * rx_stats
Definition: vmxnet3.h:588
vmxnet3_create_if_args_t::enable_gso
u8 enable_gso
Definition: vmxnet3.h:609
length
char const int length
Definition: cJSON.h:163
vmxnet3_device_t::txqs
vmxnet3_txq_t * txqs
Definition: vmxnet3.h:571
vmxnet3_device_t::sw_if_index
u32 sw_if_index
Definition: vmxnet3.h:562
vlib_main_t
Definition: main.h:102
vmxnet3_rxq_t
Definition: vmxnet3.h:513
foreach_vmxnet3_feature_flags
#define foreach_vmxnet3_feature_flags
Definition: vmxnet3.h:60
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
vmxnet3_create_if_args_t::error
clib_error_t * error
Definition: vmxnet3.h:613
VMXNET3_RXF_GEN
#define VMXNET3_RXF_GEN
Definition: vmxnet3.h:119
vmxnet3_rx_ring_advance_produce
static_always_inline void vmxnet3_rx_ring_advance_produce(vmxnet3_rxq_t *rxq, vmxnet3_rx_ring *ring)
Definition: vmxnet3.h:678
vmxnet3_txq_t::tx_desc
vmxnet3_tx_desc * tx_desc
Definition: vmxnet3.h:549
vmxnet3_device_t::numa_node
u32 numa_node
Definition: vmxnet3.h:564
pad
u8 pad[3]
log2 (size of the packing page block)
Definition: bihash_doc.h:61
vmxnet3_device_t::link_speed
u32 link_speed
Definition: vmxnet3.h:585
vmxnet3_txq_t::reg_txprod
u32 reg_txprod
Definition: vmxnet3.h:546
vmxnet3_input_trace_t::hw_if_index
u32 hw_if_index
Definition: vmxnet3.h:619
PREDICT_TRUE
#define PREDICT_TRUE(x)
Definition: clib.h:125
vmxnet3_device_t::version
u8 version
Definition: vmxnet3.h:577
vmxnet3_txq_t::size
u16 size
Definition: vmxnet3.h:545
vmxnet3_device_t
Definition: vmxnet3.h:555
vmxnet3_device_t::num_tx_queues
u16 num_tx_queues
Definition: vmxnet3.h:573
vmxnet3_tx_ring::gen
u32 gen
Definition: vmxnet3.h:530
vmxnet3_main_t::devices
vmxnet3_device_t * devices
Definition: vmxnet3.h:593
VMXNET3_REG_RXPROD2
#define VMXNET3_REG_RXPROD2
Definition: vmxnet3.h:100
vmxnet3_main
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
VMXNET3_INPUT_REFILL_THRESHOLD
#define VMXNET3_INPUT_REFILL_THRESHOLD
Definition: vmxnet3.h:159
vmxnet3_device_t::hw_if_index
u32 hw_if_index
Definition: vmxnet3.h:563
vmxnet3_rx_comp_ring::next
u16 next
Definition: vmxnet3.h:510
vmxnet3_create_if_args_t::rxq_num
u16 rxq_num
Definition: vmxnet3.h:605
VMXNET3_RXF_BTYPE
#define VMXNET3_RXF_BTYPE
Definition: vmxnet3.h:118
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
vmxnet3_rxq_t::rx_desc
vmxnet3_rx_desc * rx_desc[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:521
vmxnet3_device_class
vnet_device_class_t vmxnet3_device_class
vmxnet3_tx_ring::consume
u16 consume
Definition: vmxnet3.h:532
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105
foreach_vmxnet3_offload
#define foreach_vmxnet3_offload
Definition: vmxnet3.h:133