FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
virtio_inline.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __VIRTIO_INLINE_H__
16 #define __VIRTIO_INLINE_H__
17 
18 #define foreach_virtio_input_error \
19  _ (BUFFER_ALLOC, "buffer alloc error") \
20  _ (UNKNOWN, "unknown")
21 
22 typedef enum
23 {
24 #define _(f, s) VIRTIO_INPUT_ERROR_##f,
26 #undef _
29 
33  const int hdr_sz, u32 node_index)
34 {
35  u16 used, next, avail, n_slots, n_refill;
36  u16 sz = vring->size;
37  u16 mask = sz - 1;
38 
39 more:
40  used = vring->desc_in_use;
41 
42  if (sz - used < sz / 8)
43  return;
44 
45  /* deliver free buffers in chunks of 64 */
46  n_refill = clib_min (sz - used, 64);
47 
48  next = vring->desc_next;
49  avail = vring->avail->idx;
51  vm, vring->buffers, next, vring->size, n_refill, vring->buffer_pool_index);
52 
53  if (PREDICT_FALSE (n_slots != n_refill))
54  {
55  vlib_error_count (vm, node_index, VIRTIO_INPUT_ERROR_BUFFER_ALLOC,
56  n_refill - n_slots);
57  if (n_slots == 0)
58  return;
59  }
60 
61  while (n_slots)
62  {
63  vring_desc_t *d = &vring->desc[next];
64  ;
66  /*
67  * current_data may not be initialized with 0 and may contain
68  * previous offset. Here we want to make sure, it should be 0
69  * initialized.
70  */
71  b->current_data = -hdr_sz;
72  clib_memset (vlib_buffer_get_current (b), 0, hdr_sz);
73  d->addr = ((type == VIRTIO_IF_TYPE_PCI) ?
78  vring->avail->ring[avail & mask] = next;
79  avail++;
80  next = (next + 1) & mask;
81  n_slots--;
82  used++;
83  }
84  clib_atomic_store_seq_cst (&vring->avail->idx, avail);
85  vring->desc_next = next;
86  vring->desc_in_use = used;
87  if ((clib_atomic_load_seq_cst (&vring->used->flags) &
89  {
90  virtio_kick (vm, vring, vif);
91  }
92  goto more;
93 }
94 
98  const int hdr_sz, u32 node_index)
99 {
100  u16 used, next, n_slots, n_refill, flags = 0, first_desc_flags;
101  u16 sz = vring->size;
102 
103 more:
104  used = vring->desc_in_use;
105 
106  if (sz == used)
107  return;
108 
109  /* deliver free buffers in chunks of 64 */
110  n_refill = clib_min (sz - used, 64);
111 
112  next = vring->desc_next;
113  first_desc_flags = vring->packed_desc[next].flags;
115  vm, vring->buffers, next, sz, n_refill, vring->buffer_pool_index);
116 
117  if (PREDICT_FALSE (n_slots != n_refill))
118  {
119  vlib_error_count (vm, node_index, VIRTIO_INPUT_ERROR_BUFFER_ALLOC,
120  n_refill - n_slots);
121  if (n_slots == 0)
122  return;
123  }
124 
125  while (n_slots)
126  {
127  vring_packed_desc_t *d = &vring->packed_desc[next];
129  /*
130  * current_data may not be initialized with 0 and may contain
131  * previous offset. Here we want to make sure, it should be 0
132  * initialized.
133  */
134  b->current_data = -hdr_sz;
135  clib_memset (vlib_buffer_get_current (b), 0, hdr_sz);
136  d->addr = ((type == VIRTIO_IF_TYPE_PCI) ?
139  d->len = vlib_buffer_get_default_data_size (vm) + hdr_sz;
140 
141  if (vring->avail_wrap_counter)
143  else
145 
146  d->id = next;
147  if (vring->desc_next == next)
148  first_desc_flags = flags;
149  else
150  d->flags = flags;
151 
152  next++;
153  if (next >= sz)
154  {
155  next = 0;
156  vring->avail_wrap_counter ^= 1;
157  }
158  n_slots--;
159  used++;
160  }
162  vring->packed_desc[vring->desc_next].flags = first_desc_flags;
163  vring->desc_next = next;
164  vring->desc_in_use = used;
166  if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
167  {
168  virtio_kick (vm, vring, vif);
169  }
170 
171  goto more;
172 }
173 
174 #endif
175 
176 /*
177  * fd.io coding-style-patch-verification: ON
178  *
179  * Local Variables:
180  * eval: (c-set-style "gnu")
181  * End:
182  */
vlib_buffer_get_current_pa
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:494
vlib_buffer_alloc_to_ring_from_pool
static __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring_from_pool(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers into ring from specific buffer pool.
Definition: buffer_funcs.h:753
virtio_vring_t
Definition: virtio.h:63
vring_used_t::flags
u16 flags
Definition: virtio_std.h:117
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
pointer_to_uword
static uword pointer_to_uword(const void *p)
Definition: types.h:131
vring_avail_t::ring
u16 ring[0]
Definition: virtio_std.h:105
virtio_kick
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
Definition: virtio.h:251
next
u16 * next
Definition: nat44_ei_out2in.c:718
u16
unsigned short u16
Definition: types.h:57
vring_desc_t::addr
u64 addr
Definition: virtio_std.h:95
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
VRING_DESC_F_USED
#define VRING_DESC_F_USED
Definition: virtio_std.h:76
node_index
node node_index
Definition: interface_output.c:420
VRING_DESC_F_WRITE
#define VRING_DESC_F_WRITE
Definition: virtio_std.h:72
vlib_error_count
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
virtio_vring_t::size
u16 size
Definition: virtio.h:83
virtio_vring_t::avail
vring_avail_t * avail
Definition: virtio.h:73
vlib_buffer_t::current_data
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:119
virtio_vring_t::buffer_pool_index
u8 buffer_pool_index
Definition: virtio.h:107
virtio_refill_vring_packed
static_always_inline void virtio_refill_vring_packed(vlib_main_t *vm, virtio_if_t *vif, virtio_if_type_t type, virtio_vring_t *vring, const int hdr_sz, u32 node_index)
Definition: virtio_inline.h:96
CLIB_MEMORY_STORE_BARRIER
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:140
virtio_vring_t::desc_next
u16 desc_next
Definition: virtio.h:87
virtio_vring_t::used
vring_used_t * used
Definition: virtio.h:72
vring_desc_t::flags
u16 flags
Definition: virtio_std.h:97
vring_desc_t
Definition: virtio_std.h:93
VRING_DESC_F_AVAIL
#define VRING_DESC_F_AVAIL
Definition: virtio_std.h:75
virtio_vring_t::desc
vring_desc_t * desc
Definition: virtio.h:71
virtio_if_type_t
virtio_if_type_t
Definition: virtio.h:50
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
static_always_inline
#define static_always_inline
Definition: clib.h:112
mask
vl_api_pnat_mask_t mask
Definition: pnat.api:45
vring_desc_t::len
u32 len
Definition: virtio_std.h:96
clib_atomic_load_seq_cst
#define clib_atomic_load_seq_cst(a)
Definition: atomics.h:55
CLIB_MEMORY_BARRIER
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:137
VRING_USED_F_NO_NOTIFY
#define VRING_USED_F_NO_NOTIFY
Definition: virtio_std.h:90
clib_min
#define clib_min(x, y)
Definition: clib.h:342
virtio_vring_t::avail_wrap_counter
u16 avail_wrap_counter
Definition: virtio.h:100
vlib_buffer_get_default_data_size
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:122
u32
unsigned int u32
Definition: types.h:88
virtio_vring_t::packed_desc
vring_packed_desc_t * packed_desc
Definition: virtio.h:77
clib_atomic_store_seq_cst
#define clib_atomic_store_seq_cst(a, b)
Definition: atomics.h:53
virtio_if_t
Definition: virtio.h:128
VIRTIO_INPUT_N_ERROR
@ VIRTIO_INPUT_N_ERROR
Definition: virtio_inline.h:27
clib_memset
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vlib_main_t
Definition: main.h:102
virtio_vring_t::buffers
u32 * buffers
Definition: virtio.h:82
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
vlib_buffer_get_current
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:257
virtio_vring_t::desc_in_use
u16 desc_in_use
Definition: virtio.h:86
virtio_refill_vring_split
static_always_inline void virtio_refill_vring_split(vlib_main_t *vm, virtio_if_t *vif, virtio_if_type_t type, virtio_vring_t *vring, const int hdr_sz, u32 node_index)
Definition: virtio_inline.h:31
foreach_virtio_input_error
#define foreach_virtio_input_error
Definition: virtio_inline.h:18
virtio_vring_t::device_event
vring_desc_event_t * device_event
Definition: virtio.h:79
vring_avail_t::idx
u16 idx
Definition: virtio_std.h:104
type
vl_api_fib_path_type_t type
Definition: fib_types.api:123
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
virtio_input_error_t
virtio_input_error_t
Definition: virtio_inline.h:22
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105