FD.io VPP  v18.04-17-g3a0d853
Vector Packet Processing
output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 
24 #include <avf/avf.h>
25 
26 #define AVF_TXQ_DESC_CMD(x) (1 << (x + 4))
27 #define AVF_TXQ_DESC_CMD_EOP AVF_TXQ_DESC_CMD(0)
28 #define AVF_TXQ_DESC_CMD_RS AVF_TXQ_DESC_CMD(1)
29 
32 {
33  return d->qword[1] & 0x0f;
34 }
35 
36 uword
38  vlib_node_runtime_t * node,
39  vlib_frame_t * frame)
40 {
41  avf_main_t *am = &avf_main;
42  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
44  u32 thread_index = vlib_get_thread_index ();
45  u8 qid = thread_index;
46  avf_txq_t *txq = vec_elt_at_index (ad->txqs, qid % ad->num_queue_pairs);
47  avf_tx_desc_t *d0, *d1, *d2, *d3;
48  u32 *buffers = vlib_frame_args (frame);
49  u32 bi0, bi1, bi2, bi3;
50  u16 n_left = frame->n_vectors;
51  vlib_buffer_t *b0, *b1, *b2, *b3;
52  u16 mask = txq->size - 1;
53 
55 
56  /* release cosumed bufs */
57  if (txq->n_bufs)
58  {
59  u16 first, slot, n_free = 0;
60  first = slot = (txq->next - txq->n_bufs) & mask;
61  d0 = txq->descs + slot;
62  while (n_free < txq->n_bufs && avf_tx_desc_get_dtyp (d0) == 0x0F)
63  {
64  n_free++;
65  slot = (slot + 1) & mask;
66  d0 = txq->descs + slot;
67  }
68 
69  if (n_free)
70  {
71  txq->n_bufs -= n_free;;
72  vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
73  n_free);
74  }
75  }
76 
77  while (n_left >= 7)
78  {
79  u16 slot0, slot1, slot2, slot3;
80 
81  vlib_prefetch_buffer_with_index (vm, buffers[4], LOAD);
82  vlib_prefetch_buffer_with_index (vm, buffers[5], LOAD);
83  vlib_prefetch_buffer_with_index (vm, buffers[6], LOAD);
84  vlib_prefetch_buffer_with_index (vm, buffers[7], LOAD);
85 
86  slot0 = txq->next;
87  slot1 = (txq->next + 1) & mask;
88  slot2 = (txq->next + 2) & mask;
89  slot3 = (txq->next + 3) & mask;
90 
91  d0 = txq->descs + slot0;
92  d1 = txq->descs + slot1;
93  d2 = txq->descs + slot2;
94  d3 = txq->descs + slot3;
95 
96  bi0 = buffers[0];
97  bi1 = buffers[1];
98  bi2 = buffers[2];
99  bi3 = buffers[3];
100 
101  txq->bufs[slot0] = bi0;
102  txq->bufs[slot1] = bi1;
103  txq->bufs[slot2] = bi2;
104  txq->bufs[slot3] = bi3;
105  b0 = vlib_get_buffer (vm, bi0);
106  b1 = vlib_get_buffer (vm, bi1);
107  b2 = vlib_get_buffer (vm, bi2);
108  b3 = vlib_get_buffer (vm, bi3);
109 
110 #if 0
111  d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
112  b0->current_data;
113 #else
114  d0->qword[0] = pointer_to_uword (b0->data);
115  d1->qword[0] = pointer_to_uword (b1->data);
116  d2->qword[0] = pointer_to_uword (b2->data);
117  d3->qword[0] = pointer_to_uword (b3->data);
118 
119 #endif
121  d0->qword[1] = ((u64) b0->current_length) << 34 | bits;
122  d1->qword[1] = ((u64) b1->current_length) << 34 | bits;
123  d2->qword[1] = ((u64) b2->current_length) << 34 | bits;
124  d3->qword[1] = ((u64) b3->current_length) << 34 | bits;
125 
126  txq->next = (txq->next + 4) & mask;
127  txq->n_bufs += 4;
128  buffers += 4;
129  n_left -= 4;
130  }
131 
132  while (n_left)
133  {
134  d0 = txq->descs + txq->next;
135  bi0 = buffers[0];
136  txq->bufs[txq->next] = bi0;
137  b0 = vlib_get_buffer (vm, bi0);
138 
139 #if 0
140  d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
141  b0->current_data;
142 #else
143  d0->qword[0] = pointer_to_uword (b0->data);
144 
145 #endif
146  d0->qword[1] = ((u64) b0->current_length) << 34;
148 
149  txq->next = (txq->next + 1) & mask;
150  txq->n_bufs++;
151  buffers++;
152  n_left--;
153  }
155  *(txq->qtx_tail) = txq->next;
156 
158 
159  return frame->n_vectors - n_left;
160 }
161 
162 #ifndef CLIB_MULTIARCH_VARIANT
163 #if __x86_64__
166 static void __clib_constructor
168 {
169  if (avf_interface_tx_avx512 && clib_cpu_supports_avx512f ())
171  else if (avf_interface_tx_avx2 && clib_cpu_supports_avx2 ())
173 }
174 #endif
175 #endif
176 
177 /*
178  * fd.io coding-style-patch-verification: ON
179  *
180  * Local Variables:
181  * eval: (c-set-style "gnu")
182  * End:
183  */
#define AVF_TXQ_DESC_CMD_RS
Definition: output.c:28
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
avf_device_t * devices
Definition: avf.h:160
static u64 vlib_get_buffer_data_physical_address(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:163
volatile u32 * qtx_tail
Definition: avf.h:68
vnet_device_class_t avf_device_class
vlib_node_function_t __clib_weak avf_interface_tx_avx512
Definition: output.c:164
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:104
#define static_always_inline
Definition: clib.h:93
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:181
clib_spinlock_t lock
Definition: avf.h:71
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
unsigned long u64
Definition: types.h:89
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:446
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:461
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
static void __clib_constructor avf_interface_tx_multiarch_select(void)
Definition: output.c:167
uword( vlib_node_function_t)(struct vlib_main_t *vm, struct vlib_node_runtime_t *node, struct vlib_frame_t *frame)
Definition: node.h:54
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
vlib_main_t * vm
Definition: buffer.c:294
avf_tx_desc_t * descs
Definition: avf.h:72
static_always_inline u8 avf_tx_desc_get_dtyp(avf_tx_desc_t *d)
Definition: output.c:31
u32 * bufs
Definition: avf.h:73
unsigned int u32
Definition: types.h:88
Definition: avf.h:65
#define AVF_TXQ_DESC_CMD_EOP
Definition: output.c:27
static void * vlib_frame_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:284
avf_main_t avf_main
Definition: device.c:36
u64 uword
Definition: types.h:112
vlib_node_function_t __clib_weak avf_interface_tx_avx2
Definition: output.c:165
u16 next
Definition: avf.h:69
unsigned short u16
Definition: types.h:57
unsigned char u8
Definition: types.h:56
u64 qword[2]
Definition: avf.h:47
uword CLIB_MULTIARCH_FN() avf_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:37
u16 size
Definition: avf.h:70
u8 data[0]
Packet data.
Definition: buffer.h:179
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:109
u16 n_bufs
Definition: avf.h:74
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:59