FD.io VPP  v17.04.2-2-ga8f93f8
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * af_packet.c - linux kernel packet interface
4  *
5  * Copyright (c) 2016 Cisco and/or its affiliates.
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at:
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *------------------------------------------------------------------
18  */
19 
20 #include <linux/if_packet.h>
21 
22 #include <vlib/vlib.h>
23 #include <vlib/unix/unix.h>
24 #include <vnet/ip/ip.h>
25 #include <vnet/ethernet/ethernet.h>
26 #include <vnet/devices/devices.h>
27 #include <vnet/feature/feature.h>
28 
30 
31 #define foreach_af_packet_input_error
32 
33 typedef enum
34 {
35 #define _(f,s) AF_PACKET_INPUT_ERROR_##f,
37 #undef _
40 
42 #define _(n,s) s,
44 #undef _
45 };
46 
47 typedef struct
48 {
51  int block;
52  struct tpacket2_hdr tph;
54 
55 static u8 *
56 format_af_packet_input_trace (u8 * s, va_list * args)
57 {
58  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
59  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
60  af_packet_input_trace_t *t = va_arg (*args, af_packet_input_trace_t *);
61  uword indent = format_get_indent (s);
62 
63  s = format (s, "af_packet: hw_if_index %d next-index %d",
64  t->hw_if_index, t->next_index);
65 
66  s =
67  format (s,
68  "\n%Utpacket2_hdr:\n%Ustatus 0x%x len %u snaplen %u mac %u net %u"
69  "\n%Usec 0x%x nsec 0x%x vlan %U"
70 #ifdef TP_STATUS_VLAN_TPID_VALID
71  " vlan_tpid %u"
72 #endif
73  ,
74  format_white_space, indent + 2,
75  format_white_space, indent + 4,
76  t->tph.tp_status,
77  t->tph.tp_len,
78  t->tph.tp_snaplen,
79  t->tph.tp_mac,
80  t->tph.tp_net,
81  format_white_space, indent + 4,
82  t->tph.tp_sec,
83  t->tph.tp_nsec, format_ethernet_vlan_tci, t->tph.tp_vlan_tci
84 #ifdef TP_STATUS_VLAN_TPID_VALID
85  , t->tph.tp_vlan_tpid
86 #endif
87  );
88  return s;
89 }
90 
91 always_inline void
92 buffer_add_to_chain (vlib_main_t * vm, u32 bi, u32 first_bi, u32 prev_bi)
93 {
94  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
95  vlib_buffer_t *first_b = vlib_get_buffer (vm, first_bi);
96  vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_bi);
97 
98  /* update first buffer */
100 
101  /* update previous buffer */
102  prev_b->next_buffer = bi;
103  prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
104 
105  /* update current buffer */
106  b->next_buffer = 0;
107 }
108 
111  vlib_frame_t * frame, af_packet_if_t * apif)
112 {
114  struct tpacket2_hdr *tph;
116  u32 block = 0;
117  u32 rx_frame;
118  u32 n_free_bufs;
119  u32 n_rx_packets = 0;
120  u32 n_rx_bytes = 0;
121  u32 *to_next = 0;
122  u32 block_size = apif->rx_req->tp_block_size;
123  u32 frame_size = apif->rx_req->tp_frame_size;
124  u32 frame_num = apif->rx_req->tp_frame_nr;
125  u8 *block_start = apif->rx_ring + block * block_size;
126  uword n_trace = vlib_get_trace_count (vm, node);
127  u32 cpu_index = os_get_cpu_number ();
128  u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm,
130  u32 min_bufs = apif->rx_req->tp_frame_size / n_buffer_bytes;
131 
132  if (apif->per_interface_next_index != ~0)
133  next_index = apif->per_interface_next_index;
134 
135  n_free_bufs = vec_len (apm->rx_buffers[cpu_index]);
136  if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE))
137  {
138  vec_validate (apm->rx_buffers[cpu_index],
139  VLIB_FRAME_SIZE + n_free_bufs - 1);
140  n_free_bufs +=
141  vlib_buffer_alloc (vm, &apm->rx_buffers[cpu_index][n_free_bufs],
143  _vec_len (apm->rx_buffers[cpu_index]) = n_free_bufs;
144  }
145 
146  rx_frame = apif->next_rx_frame;
147  tph = (struct tpacket2_hdr *) (block_start + rx_frame * frame_size);
148  while ((tph->tp_status & TP_STATUS_USER) && (n_free_bufs > min_bufs))
149  {
150  vlib_buffer_t *b0 = 0, *first_b0 = 0;
151  u32 next0 = next_index;
152 
153  u32 n_left_to_next;
154  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
155  while ((tph->tp_status & TP_STATUS_USER) && (n_free_bufs > min_bufs) &&
156  n_left_to_next)
157  {
158  u32 data_len = tph->tp_snaplen;
159  u32 offset = 0;
160  u32 bi0 = 0, first_bi0 = 0, prev_bi0;
161 
162  while (data_len)
163  {
164  /* grab free buffer */
165  u32 last_empty_buffer =
166  vec_len (apm->rx_buffers[cpu_index]) - 1;
167  prev_bi0 = bi0;
168  bi0 = apm->rx_buffers[cpu_index][last_empty_buffer];
169  b0 = vlib_get_buffer (vm, bi0);
170  _vec_len (apm->rx_buffers[cpu_index]) = last_empty_buffer;
171  n_free_bufs--;
172 
173  /* copy data */
174  u32 bytes_to_copy =
175  data_len > n_buffer_bytes ? n_buffer_bytes : data_len;
176  b0->current_data = 0;
178  (u8 *) tph + tph->tp_mac + offset, bytes_to_copy);
179 
180  /* fill buffer header */
181  b0->current_length = bytes_to_copy;
182 
183  if (offset == 0)
184  {
187  vnet_buffer (b0)->sw_if_index[VLIB_RX] = apif->sw_if_index;
188  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
189  first_bi0 = bi0;
190  first_b0 = vlib_get_buffer (vm, first_bi0);
191  }
192  else
193  buffer_add_to_chain (vm, bi0, first_bi0, prev_bi0);
194 
195  offset += bytes_to_copy;
196  data_len -= bytes_to_copy;
197  }
198  n_rx_packets++;
199  n_rx_bytes += tph->tp_snaplen;
200  to_next[0] = first_bi0;
201  to_next += 1;
202  n_left_to_next--;
203 
204  /* trace */
206  if (PREDICT_FALSE (n_trace > 0))
207  {
209  vlib_trace_buffer (vm, node, next0, first_b0, /* follow_chain */
210  0);
211  vlib_set_trace_count (vm, node, --n_trace);
212  tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr));
213  tr->next_index = next0;
214  tr->hw_if_index = apif->hw_if_index;
215  clib_memcpy (&tr->tph, tph, sizeof (struct tpacket2_hdr));
216  }
217 
218  /* redirect if feature path enabled */
220 
221  /* enque and take next packet */
222  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
223  n_left_to_next, first_bi0, next0);
224 
225  /* next packet */
226  tph->tp_status = TP_STATUS_KERNEL;
227  rx_frame = (rx_frame + 1) % frame_num;
228  tph = (struct tpacket2_hdr *) (block_start + rx_frame * frame_size);
229  }
230 
231  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
232  }
233 
234  apif->next_rx_frame = rx_frame;
235 
237  (vnet_get_main ()->interface_main.combined_sw_if_counters
239  os_get_cpu_number (), apif->hw_if_index, n_rx_packets, n_rx_bytes);
240 
241  vnet_device_increment_rx_packets (cpu_index, n_rx_packets);
242  return n_rx_packets;
243 }
244 
245 static uword
247  vlib_frame_t * frame)
248 {
249  u32 n_rx_packets = 0;
251  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
253 
255  {
256  af_packet_if_t *apif;
257  apif = vec_elt_at_index (apm->interfaces, dq->dev_instance);
258  if (apif->is_admin_up)
259  n_rx_packets += af_packet_device_input_fn (vm, node, frame, apif);
260  }
261 
262  return n_rx_packets;
263 }
264 
265 /* *INDENT-OFF* */
267  .function = af_packet_input_fn,
268  .name = "af-packet-input",
269  .sibling_of = "device-input",
270  .format_trace = format_af_packet_input_trace,
271  .type = VLIB_NODE_TYPE_INPUT,
272  .state = VLIB_NODE_STATE_INTERRUPT,
273  .n_errors = AF_PACKET_INPUT_N_ERROR,
274  .error_strings = af_packet_input_error_strings,
275 };
276 
278 /* *INDENT-ON* */
279 
280 
281 /*
282  * fd.io coding-style-patch-verification: ON
283  *
284  * Local Variables:
285  * eval: (c-set-style "gnu")
286  * End:
287  */
u32 ** rx_buffers
Definition: af_packet.h:50
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:67
#define CLIB_UNUSED(x)
Definition: clib.h:79
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:143
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:463
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:459
af_packet_input_error_t
Definition: node.c:33
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
struct tpacket_req * rx_req
Definition: af_packet.h:26
vlib_node_registration_t af_packet_input_node
(constructor) VLIB_REGISTER_NODE (af_packet_input_node)
Definition: node.c:266
static uword af_packet_input_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: node.c:246
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:104
af_packet_if_t * interfaces
Definition: af_packet.h:44
static void vnet_device_increment_rx_packets(u32 cpu_index, u64 count)
Definition: devices.h:98
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:87
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
#define always_inline
Definition: clib.h:84
static uword format_get_indent(u8 *s)
Definition: format.h:72
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
u32 next_rx_frame
Definition: af_packet.h:34
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
Definition: buffer.h:89
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:390
#define PREDICT_FALSE(x)
Definition: clib.h:97
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:399
#define VLIB_FRAME_SIZE
Definition: node.h:328
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:216
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:350
static char * af_packet_input_error_strings[]
Definition: node.c:41
#define foreach_af_packet_input_error
Definition: node.c:31
static u8 * format_af_packet_input_trace(u8 *s, va_list *args)
Definition: node.c:56
vlib_main_t * vm
Definition: buffer.c:276
#define clib_memcpy(a, b, c)
Definition: string.h:69
u32 per_interface_next_index
Definition: af_packet.h:37
unsigned int u32
Definition: types.h:88
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 cpu_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:211
struct tpacket2_hdr tph
Definition: node.c:52
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
af_packet_main_t af_packet_main
Definition: af_packet.h:56
static void buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi)
Definition: node.c:92
static uword af_packet_device_input_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, af_packet_if_t *apif)
Definition: node.c:110
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:103
template key/value backing page structure
Definition: bihash_doc.h:44
Definition: defs.h:47
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
u8 * format_ethernet_vlan_tci(u8 *s, va_list *va)
Definition: format.c:73
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:484
#define vnet_buffer(b)
Definition: buffer.h:294
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:227
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
#define vec_foreach(var, vec)
Vector iterator.
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:159
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:245
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
VLIB_NODE_FUNCTION_MULTIARCH(ethernet_input_not_l2_node, ethernet_input_not_l2)
Definition: node.c:1170
Definition: defs.h:46