FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 
24 #include <avf/avf.h>
25 
26 #define AVF_TXQ_DESC_CMD(x) (1 << (x + 4))
27 #define AVF_TXQ_DESC_CMD_EOP AVF_TXQ_DESC_CMD(0)
28 #define AVF_TXQ_DESC_CMD_RS AVF_TXQ_DESC_CMD(1)
29 #define AVF_TXQ_DESC_CMD_RSV AVF_TXQ_DESC_CMD(2)
30 
33 {
34  return d->qword[1] & 0x0f;
35 }
36 
38  vlib_node_runtime_t * node,
39  vlib_frame_t * frame)
40 {
41  avf_main_t *am = &avf_main;
42  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
44  u32 thread_index = vm->thread_index;
45  u8 qid = thread_index;
46  avf_txq_t *txq = vec_elt_at_index (ad->txqs, qid % ad->num_queue_pairs);
47  avf_tx_desc_t *d0, *d1, *d2, *d3;
48  u32 *buffers = vlib_frame_args (frame);
49  u32 bi0, bi1, bi2, bi3;
50  u16 n_left, n_left_to_send, n_in_batch;
51  vlib_buffer_t *b0, *b1, *b2, *b3;
52  u16 next;
53  u16 n_retry = 5;
54  u16 mask = txq->size - 1;
57 
59 
60  n_left_to_send = frame->n_vectors;
61  next = txq->next;
62 
63 retry:
64  /* release consumed bufs */
65  if (txq->n_enqueued)
66  {
67  u16 first, slot, n_free = 0;
68  first = slot = (next - txq->n_enqueued) & mask;
69  d0 = txq->descs + slot;
70  while (n_free < txq->n_enqueued && avf_tx_desc_get_dtyp (d0) == 0x0F)
71  {
72  n_free++;
73  slot = (slot + 1) & mask;
74  d0 = txq->descs + slot;
75  }
76 
77  if (n_free)
78  {
79  txq->n_enqueued -= n_free;
80  vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
81  n_free);
82  }
83  }
84 
85  n_in_batch = clib_min (n_left_to_send, txq->size - txq->n_enqueued - 8);
86  n_left = n_in_batch;
87 
88  while (n_left >= 8)
89  {
90  u16 slot0, slot1, slot2, slot3;
91 
92  vlib_prefetch_buffer_with_index (vm, buffers[4], LOAD);
93  vlib_prefetch_buffer_with_index (vm, buffers[5], LOAD);
94  vlib_prefetch_buffer_with_index (vm, buffers[6], LOAD);
95  vlib_prefetch_buffer_with_index (vm, buffers[7], LOAD);
96 
97  slot0 = next;
98  slot1 = (next + 1) & mask;
99  slot2 = (next + 2) & mask;
100  slot3 = (next + 3) & mask;
101 
102  d0 = txq->descs + slot0;
103  d1 = txq->descs + slot1;
104  d2 = txq->descs + slot2;
105  d3 = txq->descs + slot3;
106 
107  bi0 = buffers[0];
108  bi1 = buffers[1];
109  bi2 = buffers[2];
110  bi3 = buffers[3];
111 
112  txq->bufs[slot0] = bi0;
113  txq->bufs[slot1] = bi1;
114  txq->bufs[slot2] = bi2;
115  txq->bufs[slot3] = bi3;
116  b0 = vlib_get_buffer (vm, bi0);
117  b1 = vlib_get_buffer (vm, bi1);
118  b2 = vlib_get_buffer (vm, bi2);
119  b3 = vlib_get_buffer (vm, bi3);
120 
121 #if 0
122  d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
123  b0->current_data;
124 #else
125  d0->qword[0] = pointer_to_uword (b0->data) + b0->current_data;
126  d1->qword[0] = pointer_to_uword (b1->data) + b1->current_data;
127  d2->qword[0] = pointer_to_uword (b2->data) + b2->current_data;
128  d3->qword[0] = pointer_to_uword (b3->data) + b3->current_data;
129 
130 #endif
131  d0->qword[1] = ((u64) b0->current_length) << 34 | bits;
132  d1->qword[1] = ((u64) b1->current_length) << 34 | bits;
133  d2->qword[1] = ((u64) b2->current_length) << 34 | bits;
134  d3->qword[1] = ((u64) b3->current_length) << 34 | bits;
135 
136  next = (next + 4) & mask;
137  txq->n_enqueued += 4;
138  buffers += 4;
139  n_left -= 4;
140  }
141 
142  while (n_left)
143  {
144  d0 = txq->descs + next;
145  bi0 = buffers[0];
146  txq->bufs[next] = bi0;
147  b0 = vlib_get_buffer (vm, bi0);
148 
149 #if 0
150  d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
151  b0->current_data;
152 #else
153  d0->qword[0] = pointer_to_uword (b0->data) + b0->current_data;
154 #endif
155  d0->qword[1] = (((u64) b0->current_length) << 34) | bits;
156 
157  next = (next + 1) & mask;
158  txq->n_enqueued++;
159  buffers++;
160  n_left--;
161  }
162 
164  *(txq->qtx_tail) = txq->next = next;
165 
166  n_left_to_send -= n_in_batch;
167 
168  if (n_left_to_send)
169  {
170  if (n_retry--)
171  goto retry;
172 
173  vlib_buffer_free (vm, buffers, n_left_to_send);
174  vlib_error_count (vm, node->node_index,
175  AVF_TX_ERROR_NO_FREE_SLOTS, n_left_to_send);
176  }
177 
179 
180  return frame->n_vectors - n_left;
181 }
182 
183 /*
184  * fd.io coding-style-patch-verification: ON
185  *
186  * Local Variables:
187  * eval: (c-set-style "gnu")
188  * End:
189  */
#define clib_min(x, y)
Definition: clib.h:291
#define AVF_TXQ_DESC_CMD_RS
Definition: output.c:28
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:547
unsigned long u64
Definition: types.h:89
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
u32 thread_index
Definition: main.h:179
avf_device_t * devices
Definition: avf.h:189
static u64 vlib_get_buffer_data_physical_address(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:319
volatile u32 * qtx_tail
Definition: avf.h:94
unsigned char u8
Definition: types.h:56
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:109
#define static_always_inline
Definition: clib.h:95
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:337
clib_spinlock_t lock
Definition: avf.h:97
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
VNET_DEVICE_CLASS_TX_FN() avf_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:37
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:286
unsigned int u32
Definition: types.h:88
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:602
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
unsigned short u16
Definition: types.h:57
#define AVF_TXQ_DESC_CMD_RSV
Definition: output.c:29
vlib_main_t * vm
Definition: buffer.c:294
avf_tx_desc_t * descs
Definition: avf.h:98
static_always_inline u8 avf_tx_desc_get_dtyp(avf_tx_desc_t *d)
Definition: output.c:32
u32 * bufs
Definition: avf.h:99
Definition: avf.h:91
#define AVF_TXQ_DESC_CMD_EOP
Definition: output.c:27
u16 n_enqueued
Definition: avf.h:100
static void * vlib_frame_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:284
static uword pointer_to_uword(const void *p)
Definition: types.h:131
avf_main_t avf_main
Definition: device.c:36
u16 next
Definition: avf.h:95
u64 qword[2]
Definition: avf.h:70
u16 size
Definition: avf.h:96
u8 data[0]
Packet data.
Definition: buffer.h:175
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:111
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:58