FD.io VPP  v19.04.4-rc0-5-ge88582fac
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * ethernet_node.c: ethernet packet processing
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <vlib/vlib.h>
41 #include <vnet/pg/pg.h>
42 #include <vnet/ethernet/ethernet.h>
44 #include <vnet/devices/pipe/pipe.h>
45 #include <vppinfra/sparse_vec.h>
46 #include <vnet/l2/l2_bvi.h>
47 
48 #define foreach_ethernet_input_next \
49  _ (PUNT, "error-punt") \
50  _ (DROP, "error-drop") \
51  _ (LLC, "llc-input") \
52  _ (IP4_INPUT, "ip4-input") \
53  _ (IP4_INPUT_NCS, "ip4-input-no-checksum")
54 
55 typedef enum
56 {
57 #define _(s,n) ETHERNET_INPUT_NEXT_##s,
59 #undef _
62 
63 typedef struct
64 {
65  u8 packet_data[32];
69 
70 static u8 *
71 format_ethernet_input_trace (u8 * s, va_list * va)
72 {
73  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
74  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
76  u32 indent = format_get_indent (s);
77 
78  if (t->frame_flags)
79  {
80  s = format (s, "frame: flags 0x%x", t->frame_flags);
82  s = format (s, ", hw-if-index %u, sw-if-index %u",
84  s = format (s, "\n%U", format_white_space, indent);
85  }
86  s = format (s, "%U", format_ethernet_header, t->packet_data);
87 
88  return s;
89 }
90 
92 
93 typedef enum
94 {
99 
100 
101 // Parse the ethernet header to extract vlan tags and innermost ethertype
104  vlib_buffer_t * b0,
105  u16 * type,
106  u16 * orig_type,
107  u16 * outer_id, u16 * inner_id, u32 * match_flags)
108 {
109  u8 vlan_count;
110 
111  if (variant == ETHERNET_INPUT_VARIANT_ETHERNET
112  || variant == ETHERNET_INPUT_VARIANT_NOT_L2)
113  {
114  ethernet_header_t *e0;
115 
116  e0 = (void *) (b0->data + b0->current_data);
117 
118  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
119  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
120 
121  vlib_buffer_advance (b0, sizeof (e0[0]));
122 
123  *type = clib_net_to_host_u16 (e0->type);
124  }
125  else if (variant == ETHERNET_INPUT_VARIANT_ETHERNET_TYPE)
126  {
127  // here when prior node was LLC/SNAP processing
128  u16 *e0;
129 
130  e0 = (void *) (b0->data + b0->current_data);
131 
132  vlib_buffer_advance (b0, sizeof (e0[0]));
133 
134  *type = clib_net_to_host_u16 (e0[0]);
135  }
136 
137  // save for distinguishing between dot1q and dot1ad later
138  *orig_type = *type;
139 
140  // default the tags to 0 (used if there is no corresponding tag)
141  *outer_id = 0;
142  *inner_id = 0;
143 
145  vlan_count = 0;
146 
147  // check for vlan encaps
148  if (ethernet_frame_is_tagged (*type))
149  {
151  u16 tag;
152 
154 
155  h0 = (void *) (b0->data + b0->current_data);
156 
157  tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
158 
159  *outer_id = tag & 0xfff;
160  if (0 == *outer_id)
161  *match_flags &= ~SUBINT_CONFIG_MATCH_1_TAG;
162 
163  *type = clib_net_to_host_u16 (h0->type);
164 
165  vlib_buffer_advance (b0, sizeof (h0[0]));
166  vlan_count = 1;
167 
168  if (*type == ETHERNET_TYPE_VLAN)
169  {
170  // Double tagged packet
172 
173  h0 = (void *) (b0->data + b0->current_data);
174 
175  tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
176 
177  *inner_id = tag & 0xfff;
178 
179  *type = clib_net_to_host_u16 (h0->type);
180 
181  vlib_buffer_advance (b0, sizeof (h0[0]));
182  vlan_count = 2;
183  if (*type == ETHERNET_TYPE_VLAN)
184  {
185  // More than double tagged packet
187 
188  vlib_buffer_advance (b0, sizeof (h0[0]));
189  vlan_count = 3; // "unknown" number, aka, 3-or-more
190  }
191  }
192  }
193  ethernet_buffer_set_vlan_count (b0, vlan_count);
194 }
195 
196 // Determine the subinterface for this packet, given the result of the
197 // vlan table lookups and vlan header parsing. Check the most specific
198 // matches first.
201  vlib_buffer_t * b0,
202  u32 match_flags,
203  main_intf_t * main_intf,
204  vlan_intf_t * vlan_intf,
205  qinq_intf_t * qinq_intf,
206  u32 * new_sw_if_index, u8 * error0, u32 * is_l2)
207 {
208  u32 matched;
209 
210  matched = eth_identify_subint (hi, match_flags, main_intf, vlan_intf,
211  qinq_intf, new_sw_if_index, error0, is_l2);
212 
213  if (matched)
214  {
215 
216  // Perform L3 my-mac filter
217  // A unicast packet arriving on an L3 interface must have a dmac matching the interface mac.
218  // This is required for promiscuous mode, else we will forward packets we aren't supposed to.
219  if (!(*is_l2))
220  {
221  ethernet_header_t *e0;
222  e0 = (void *) (b0->data + vnet_buffer (b0)->l2_hdr_offset);
223 
224  if (!(ethernet_address_cast (e0->dst_address)))
225  {
226  if (!ethernet_mac_address_equal ((u8 *) e0, hi->hw_address))
227  {
228  *error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
229  }
230  }
231  }
232 
233  // Check for down subinterface
234  *error0 = (*new_sw_if_index) != ~0 ? (*error0) : ETHERNET_ERROR_DOWN;
235  }
236 }
237 
240  ethernet_input_variant_t variant,
241  u32 is_l20,
242  u32 type0, vlib_buffer_t * b0, u8 * error0, u8 * next0)
243 {
244  vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
245  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
246 
247  if (PREDICT_FALSE (*error0 != ETHERNET_ERROR_NONE))
248  {
249  // some error occurred
250  *next0 = ETHERNET_INPUT_NEXT_DROP;
251  }
252  else if (is_l20)
253  {
254  // record the L2 len and reset the buffer so the L2 header is preserved
255  u32 eth_start = vnet_buffer (b0)->l2_hdr_offset;
256  vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start;
257  *next0 = em->l2_next;
258  ASSERT (vnet_buffer (b0)->l2.l2_len ==
260  vlib_buffer_advance (b0, -(vnet_buffer (b0)->l2.l2_len));
261 
262  // check for common IP/MPLS ethertypes
263  }
264  else if (type0 == ETHERNET_TYPE_IP4)
265  {
266  *next0 = em->l3_next.input_next_ip4;
267  }
268  else if (type0 == ETHERNET_TYPE_IP6)
269  {
270  *next0 = em->l3_next.input_next_ip6;
271  }
272  else if (type0 == ETHERNET_TYPE_MPLS)
273  {
274  *next0 = em->l3_next.input_next_mpls;
275 
276  }
277  else if (em->redirect_l3)
278  {
279  // L3 Redirect is on, the cached common next nodes will be
280  // pointing to the redirect node, catch the uncommon types here
281  *next0 = em->redirect_l3_next;
282  }
283  else
284  {
285  // uncommon ethertype, check table
286  u32 i0;
287  i0 = sparse_vec_index (em->l3_next.input_next_by_type, type0);
288  *next0 = vec_elt (em->l3_next.input_next_by_type, i0);
289  *error0 =
290  i0 ==
291  SPARSE_VEC_INVALID_INDEX ? ETHERNET_ERROR_UNKNOWN_TYPE : *error0;
292 
293  // The table is not populated with LLC values, so check that now.
294  // If variant is variant_ethernet then we came from LLC processing. Don't
295  // go back there; drop instead using by keeping the drop/bad table result.
296  if ((type0 < 0x600) && (variant == ETHERNET_INPUT_VARIANT_ETHERNET))
297  {
298  *next0 = ETHERNET_INPUT_NEXT_LLC;
299  }
300  }
301 }
302 
303 
304 /* following vector code relies on following assumptions */
305 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_data, 0);
306 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_length, 2);
309  STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l3_hdr_offset) - 2,
310  "l3_hdr_offset must follow l2_hdr_offset");
311 
314 {
315  i16 adv = sizeof (ethernet_header_t);
316  u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
317  VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
318 
319 #ifdef CLIB_HAVE_VEC256
320  /* to reduce number of small loads/stores we are loading first 64 bits
321  of each buffer metadata into 256-bit register so we can advance
322  current_data, current_length and flags.
323  Observed saving of this code is ~2 clocks per packet */
324  u64x4 r, radv;
325 
326  /* vector if signed 16 bit integers used in signed vector add operation
327  to advnce current_data and current_length */
328  u32x8 flags4 = { 0, flags, 0, flags, 0, flags, 0, flags };
329  i16x16 adv4 = {
330  adv, -adv, 0, 0, adv, -adv, 0, 0,
331  adv, -adv, 0, 0, adv, -adv, 0, 0
332  };
333 
334  /* load 4 x 64 bits */
335  r = u64x4_gather (b[0], b[1], b[2], b[3]);
336 
337  /* set flags */
338  r |= (u64x4) flags4;
339 
340  /* advance buffer */
341  radv = (u64x4) ((i16x16) r + adv4);
342 
343  /* write 4 x 64 bits */
344  u64x4_scatter (is_l3 ? radv : r, b[0], b[1], b[2], b[3]);
345 
346  /* use old current_data as l2_hdr_offset and new current_data as
347  l3_hdr_offset */
348  r = (u64x4) u16x16_blend (r, radv << 16, 0xaa);
349 
350  /* store both l2_hdr_offset and l3_hdr_offset in single store operation */
351  u32x8_scatter_one ((u32x8) r, 0, &vnet_buffer (b[0])->l2_hdr_offset);
352  u32x8_scatter_one ((u32x8) r, 2, &vnet_buffer (b[1])->l2_hdr_offset);
353  u32x8_scatter_one ((u32x8) r, 4, &vnet_buffer (b[2])->l2_hdr_offset);
354  u32x8_scatter_one ((u32x8) r, 6, &vnet_buffer (b[3])->l2_hdr_offset);
355 
356  if (is_l3)
357  {
358  ASSERT (b[0]->current_data == vnet_buffer (b[0])->l3_hdr_offset);
359  ASSERT (b[1]->current_data == vnet_buffer (b[1])->l3_hdr_offset);
360  ASSERT (b[2]->current_data == vnet_buffer (b[2])->l3_hdr_offset);
361  ASSERT (b[3]->current_data == vnet_buffer (b[3])->l3_hdr_offset);
362 
363  ASSERT (b[0]->current_data - vnet_buffer (b[0])->l2_hdr_offset == adv);
364  ASSERT (b[1]->current_data - vnet_buffer (b[1])->l2_hdr_offset == adv);
365  ASSERT (b[2]->current_data - vnet_buffer (b[2])->l2_hdr_offset == adv);
366  ASSERT (b[3]->current_data - vnet_buffer (b[3])->l2_hdr_offset == adv);
367  }
368  else
369  {
370  ASSERT (b[0]->current_data == vnet_buffer (b[0])->l2_hdr_offset);
371  ASSERT (b[1]->current_data == vnet_buffer (b[1])->l2_hdr_offset);
372  ASSERT (b[2]->current_data == vnet_buffer (b[2])->l2_hdr_offset);
373  ASSERT (b[3]->current_data == vnet_buffer (b[3])->l2_hdr_offset);
374 
375  ASSERT (b[0]->current_data - vnet_buffer (b[0])->l3_hdr_offset == -adv);
376  ASSERT (b[1]->current_data - vnet_buffer (b[1])->l3_hdr_offset == -adv);
377  ASSERT (b[2]->current_data - vnet_buffer (b[2])->l3_hdr_offset == -adv);
378  ASSERT (b[3]->current_data - vnet_buffer (b[3])->l3_hdr_offset == -adv);
379  }
380 
381 #else
382  vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
383  vnet_buffer (b[1])->l2_hdr_offset = b[1]->current_data;
384  vnet_buffer (b[2])->l2_hdr_offset = b[2]->current_data;
385  vnet_buffer (b[3])->l2_hdr_offset = b[3]->current_data;
386  vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
387  vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data + adv;
388  vnet_buffer (b[2])->l3_hdr_offset = b[2]->current_data + adv;
389  vnet_buffer (b[3])->l3_hdr_offset = b[3]->current_data + adv;
390 
391  if (is_l3)
392  {
393  vlib_buffer_advance (b[0], adv);
394  vlib_buffer_advance (b[1], adv);
395  vlib_buffer_advance (b[2], adv);
396  vlib_buffer_advance (b[3], adv);
397  }
398 
399  b[0]->flags |= flags;
400  b[1]->flags |= flags;
401  b[2]->flags |= flags;
402  b[3]->flags |= flags;
403 #endif
404 
405  if (!is_l3)
406  {
407  vnet_buffer (b[0])->l2.l2_len = adv;
408  vnet_buffer (b[1])->l2.l2_len = adv;
409  vnet_buffer (b[2])->l2.l2_len = adv;
410  vnet_buffer (b[3])->l2.l2_len = adv;
411  }
412 }
413 
416 {
417  i16 adv = sizeof (ethernet_header_t);
418  u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
419  VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
420 
421  vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
422  vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
423 
424  if (is_l3)
425  vlib_buffer_advance (b[0], adv);
426  b[0]->flags |= flags;
427  if (!is_l3)
428  vnet_buffer (b[0])->l2.l2_len = adv;
429 }
430 
431 
434  u64 * dmacs, int offset, int dmac_check)
435 {
437  e = vlib_buffer_get_current (b[offset]);
438 #ifdef CLIB_HAVE_VEC128
439  u64x2 r = u64x2_load_unaligned (((u8 *) & e->type) - 6);
440  etype[offset] = ((u16x8) r)[3];
441  tags[offset] = r[1];
442 #else
443  etype[offset] = e->type;
444  tags[offset] = *(u64 *) (e + 1);
445 #endif
446 
447  if (dmac_check)
448  dmacs[offset] = *(u64 *) e;
449 }
450 
453 {
455 
456  return (etype < 0x600) ? ETHERNET_INPUT_NEXT_LLC :
459 }
460 
461 typedef struct
462 {
463  u64 tag, mask;
465  u16 type, len, next;
467  u8 err, n_tags;
468  u64 n_packets, n_bytes;
470 
474 {
475  if (l->n_packets == 0 || l->sw_if_index == ~0)
476  return;
477 
478  if (l->adv > 0)
479  l->n_bytes += l->n_packets * l->len;
480 
484  l->n_packets, l->n_bytes);
485 }
486 
490  u64 tag, u16 * next, vlib_buffer_t * b,
491  eth_input_tag_lookup_t * l, u8 dmac_bad, int is_dot1ad,
492  int main_is_l3, int check_dmac)
493 {
495 
496  if ((tag ^ l->tag) & l->mask)
497  {
499  vlan_intf_t *vif;
500  qinq_intf_t *qif;
501  vlan_table_t *vlan_table;
502  qinq_table_t *qinq_table;
503  u16 *t = (u16 *) & tag;
504  u16 vlan1 = clib_net_to_host_u16 (t[0]) & 0xFFF;
505  u16 vlan2 = clib_net_to_host_u16 (t[2]) & 0xFFF;
506  u32 matched, is_l2, new_sw_if_index;
507 
508  vlan_table = vec_elt_at_index (em->vlan_pool, is_dot1ad ?
509  mif->dot1ad_vlans : mif->dot1q_vlans);
510  vif = &vlan_table->vlans[vlan1];
511  qinq_table = vec_elt_at_index (em->qinq_pool, vif->qinqs);
512  qif = &qinq_table->vlans[vlan2];
513  l->err = ETHERNET_ERROR_NONE;
514  l->type = clib_net_to_host_u16 (t[1]);
515 
516  if (l->type == ETHERNET_TYPE_VLAN)
517  {
518  l->type = clib_net_to_host_u16 (t[3]);
519  l->n_tags = 2;
521  SUBINT_CONFIG_MATCH_2_TAG, mif, vif,
522  qif, &new_sw_if_index, &l->err,
523  &is_l2);
524  }
525  else
526  {
527  l->n_tags = 1;
528  if (vlan1 == 0)
529  {
530  new_sw_if_index = hi->sw_if_index;
531  l->err = ETHERNET_ERROR_NONE;
532  matched = 1;
533  is_l2 = main_is_l3 == 0;
534  }
535  else
538  vif, qif, &new_sw_if_index,
539  &l->err, &is_l2);
540  }
541 
542  if (l->sw_if_index != new_sw_if_index)
543  {
544  eth_input_update_if_counters (vm, vnm, l);
545  l->n_packets = 0;
546  l->n_bytes = 0;
547  l->sw_if_index = new_sw_if_index;
548  }
549  l->tag = tag;
550  l->mask = (l->n_tags == 2) ?
551  clib_net_to_host_u64 (0xffffffffffffffff) :
552  clib_net_to_host_u64 (0xffffffff00000000);
553 
554  if (matched && l->sw_if_index == ~0)
555  l->err = ETHERNET_ERROR_DOWN;
556 
557  l->len = sizeof (ethernet_header_t) +
558  l->n_tags * sizeof (ethernet_vlan_header_t);
559  if (main_is_l3)
560  l->adv = is_l2 ? -(int) sizeof (ethernet_header_t) :
561  l->n_tags * sizeof (ethernet_vlan_header_t);
562  else
563  l->adv = is_l2 ? 0 : l->len;
564 
565  if (PREDICT_FALSE (l->err != ETHERNET_ERROR_NONE))
566  l->next = ETHERNET_INPUT_NEXT_DROP;
567  else if (is_l2)
568  l->next = em->l2_next;
569  else if (l->type == ETHERNET_TYPE_IP4)
570  l->next = em->l3_next.input_next_ip4;
571  else if (l->type == ETHERNET_TYPE_IP6)
572  l->next = em->l3_next.input_next_ip6;
573  else if (l->type == ETHERNET_TYPE_MPLS)
574  l->next = em->l3_next.input_next_mpls;
575  else if (em->redirect_l3)
576  l->next = em->redirect_l3_next;
577  else
578  {
579  l->next = eth_input_next_by_type (l->type);
580  if (l->next == ETHERNET_INPUT_NEXT_PUNT)
581  l->err = ETHERNET_ERROR_UNKNOWN_TYPE;
582  }
583  }
584 
585  if (check_dmac && l->adv > 0 && dmac_bad)
586  {
587  l->err = ETHERNET_ERROR_L3_MAC_MISMATCH;
588  next[0] = ETHERNET_INPUT_NEXT_PUNT;
589  }
590  else
591  next[0] = l->next;
592 
593  vlib_buffer_advance (b, l->adv);
594  vnet_buffer (b)->l2.l2_len = l->len;
595  vnet_buffer (b)->l3_hdr_offset = vnet_buffer (b)->l2_hdr_offset + l->len;
596 
597  if (l->err == ETHERNET_ERROR_NONE)
598  {
599  vnet_buffer (b)->sw_if_index[VLIB_RX] = l->sw_if_index;
601  }
602  else
603  b->error = node->errors[l->err];
604 
605  /* update counters */
606  l->n_packets += 1;
607  l->n_bytes += vlib_buffer_length_in_chain (vm, b);
608 }
609 
610 /* process frame of buffers, store ethertype into array and update
611  buffer metadata fields depending on interface being l2 or l3 assuming that
612  packets are untagged. For tagged packets those fields are updated later.
613  Optionally store Destionation MAC address and tag data into arrays
614  for further processing */
615 
617  "VLIB_FRAME_SIZE must be power of 8");
621  u32 * buffer_indices, u32 n_packets, int main_is_l3,
622  int ip4_cksum_ok, int dmac_check)
623 {
625  u16 nexts[VLIB_FRAME_SIZE], *next;
626  u16 etypes[VLIB_FRAME_SIZE], *etype = etypes;
627  u64 dmacs[VLIB_FRAME_SIZE], *dmac = dmacs;
628  u8 dmacs_bad[VLIB_FRAME_SIZE];
629  u64 tags[VLIB_FRAME_SIZE], *tag = tags;
630  u16 slowpath_indices[VLIB_FRAME_SIZE];
631  u16 n_slowpath, i;
632  u16 next_ip4, next_ip6, next_mpls, next_l2;
633  u16 et_ip4 = clib_host_to_net_u16 (ETHERNET_TYPE_IP4);
634  u16 et_ip6 = clib_host_to_net_u16 (ETHERNET_TYPE_IP6);
635  u16 et_mpls = clib_host_to_net_u16 (ETHERNET_TYPE_MPLS);
636  u16 et_vlan = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN);
637  u16 et_dot1ad = clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD);
638  i32 n_left = n_packets;
639  vlib_buffer_t *b[20];
640  u32 *from;
641 
642  from = buffer_indices;
643 
644  while (n_left >= 20)
645  {
646  vlib_buffer_t **ph = b + 16, **pd = b + 8;
647  vlib_get_buffers (vm, from, b, 4);
648  vlib_get_buffers (vm, from + 8, pd, 4);
649  vlib_get_buffers (vm, from + 16, ph, 4);
650 
651  vlib_prefetch_buffer_header (ph[0], LOAD);
652  vlib_prefetch_buffer_data (pd[0], LOAD);
653  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
654 
655  vlib_prefetch_buffer_header (ph[1], LOAD);
656  vlib_prefetch_buffer_data (pd[1], LOAD);
657  eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
658 
659  vlib_prefetch_buffer_header (ph[2], LOAD);
660  vlib_prefetch_buffer_data (pd[2], LOAD);
661  eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
662 
663  vlib_prefetch_buffer_header (ph[3], LOAD);
664  vlib_prefetch_buffer_data (pd[3], LOAD);
665  eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
666 
667  eth_input_adv_and_flags_x4 (b, main_is_l3);
668 
669  /* next */
670  n_left -= 4;
671  etype += 4;
672  tag += 4;
673  dmac += 4;
674  from += 4;
675  }
676  while (n_left >= 4)
677  {
678  vlib_get_buffers (vm, from, b, 4);
679  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
680  eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
681  eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
682  eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
683  eth_input_adv_and_flags_x4 (b, main_is_l3);
684 
685  /* next */
686  n_left -= 4;
687  etype += 4;
688  tag += 4;
689  dmac += 4;
690  from += 4;
691  }
692  while (n_left)
693  {
694  vlib_get_buffers (vm, from, b, 1);
695  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
696  eth_input_adv_and_flags_x1 (b, main_is_l3);
697 
698  /* next */
699  n_left -= 1;
700  etype += 1;
701  tag += 1;
702  dmac += 4;
703  from += 1;
704  }
705 
706  if (dmac_check)
707  {
708  u64 mask = clib_net_to_host_u64 (0xFFFFFFFFFFFF0000);
709  u64 igbit = clib_net_to_host_u64 (0x0100000000000000);
710  u64 hwaddr = (*(u64 *) hi->hw_address) & mask;
711  u64 *dmac = dmacs;
712  u8 *dmac_bad = dmacs_bad;
713 
714  n_left = n_packets;
715 
716 #ifdef CLIB_HAVE_VEC256
717  u64x4 igbit4 = u64x4_splat (igbit);
718  u64x4 mask4 = u64x4_splat (mask);
719  u64x4 hwaddr4 = u64x4_splat (hwaddr);
720  while (n_left > 0)
721  {
722  u64x4 r0, r1;
723  r0 = u64x4_load_unaligned (dmac + 0) & mask4;
724  r1 = u64x4_load_unaligned (dmac + 4) & mask4;
725 
726  r0 = (r0 != hwaddr4) & ((r0 & igbit4) == 0);
727  r1 = (r1 != hwaddr4) & ((r1 & igbit4) == 0);
728 
729  *(u32 *) (dmac_bad + 0) = u8x32_msb_mask ((u8x32) (r0));
730  *(u32 *) (dmac_bad + 4) = u8x32_msb_mask ((u8x32) (r1));
731 
732  /* next */
733  dmac += 8;
734  dmac_bad += 8;
735  n_left -= 8;
736  }
737 #else
738  while (n_left > 0)
739  {
740  u64 r0, r1, r2, r3;
741  r0 = dmac[0] & mask;
742  r1 = dmac[1] & mask;
743  r2 = dmac[2] & mask;
744  r3 = dmac[3] & mask;
745 
746  r0 = (r0 != hwaddr) && ((r0 & igbit) == 0);
747  r1 = (r1 != hwaddr) && ((r1 & igbit) == 0);
748  r2 = (r2 != hwaddr) && ((r2 & igbit) == 0);
749  r3 = (r3 != hwaddr) && ((r3 & igbit) == 0);
750 
751  dmac_bad[0] = r0;
752  dmac_bad[1] = r1;
753  dmac_bad[2] = r2;
754  dmac_bad[3] = r3;
755 
756  /* next */
757  dmac += 4;
758  dmac_bad += 4;
759  n_left -= 4;
760  }
761 #endif
762  }
763 
764  next_ip4 = em->l3_next.input_next_ip4;
765  next_ip6 = em->l3_next.input_next_ip6;
766  next_mpls = em->l3_next.input_next_mpls;
767  next_l2 = em->l2_next;
768 
769  if (next_ip4 == ETHERNET_INPUT_NEXT_IP4_INPUT && ip4_cksum_ok)
770  next_ip4 = ETHERNET_INPUT_NEXT_IP4_INPUT_NCS;
771 
772 #ifdef CLIB_HAVE_VEC256
773  u16x16 et16_ip4 = u16x16_splat (et_ip4);
774  u16x16 et16_ip6 = u16x16_splat (et_ip6);
775  u16x16 et16_mpls = u16x16_splat (et_mpls);
776  u16x16 et16_vlan = u16x16_splat (et_vlan);
777  u16x16 et16_dot1ad = u16x16_splat (et_dot1ad);
778  u16x16 next16_ip4 = u16x16_splat (next_ip4);
779  u16x16 next16_ip6 = u16x16_splat (next_ip6);
780  u16x16 next16_mpls = u16x16_splat (next_mpls);
781  u16x16 next16_l2 = u16x16_splat (next_l2);
782  u16x16 zero = { 0 };
783  u16x16 stairs = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
784 #endif
785 
786  etype = etypes;
787  n_left = n_packets;
788  next = nexts;
789  n_slowpath = 0;
790  i = 0;
791 
792  /* fastpath - in l3 mode hadles ip4, ip6 and mpls packets, other packets
793  are considered as slowpath, in l2 mode all untagged packets are
794  considered as fastpath */
795  while (n_left > 0)
796  {
797 #ifdef CLIB_HAVE_VEC256
798  if (n_left >= 16)
799  {
800  u16x16 r = zero;
801  u16x16 e16 = u16x16_load_unaligned (etype);
802  if (main_is_l3)
803  {
804  r += (e16 == et16_ip4) & next16_ip4;
805  r += (e16 == et16_ip6) & next16_ip6;
806  r += (e16 == et16_mpls) & next16_mpls;
807  }
808  else
809  r = ((e16 != et16_vlan) & (e16 != et16_dot1ad)) & next16_l2;
810  u16x16_store_unaligned (r, next);
811 
812  if (!u16x16_is_all_zero (r == zero))
813  {
814  if (u16x16_is_all_zero (r))
815  {
816  u16x16_store_unaligned (u16x16_splat (i) + stairs,
817  slowpath_indices + n_slowpath);
818  n_slowpath += 16;
819  }
820  else
821  {
822  for (int j = 0; j < 16; j++)
823  if (next[j] == 0)
824  slowpath_indices[n_slowpath++] = i + j;
825  }
826  }
827 
828  etype += 16;
829  next += 16;
830  n_left -= 16;
831  i += 16;
832  continue;
833  }
834 #endif
835  if (main_is_l3 && etype[0] == et_ip4)
836  next[0] = next_ip4;
837  else if (main_is_l3 && etype[0] == et_ip6)
838  next[0] = next_ip6;
839  else if (main_is_l3 && etype[0] == et_mpls)
840  next[0] = next_mpls;
841  else if (main_is_l3 == 0 &&
842  etype[0] != et_vlan && etype[0] != et_dot1ad)
843  next[0] = next_l2;
844  else
845  {
846  next[0] = 0;
847  slowpath_indices[n_slowpath++] = i;
848  }
849 
850  etype += 1;
851  next += 1;
852  n_left -= 1;
853  i += 1;
854  }
855 
856  if (n_slowpath)
857  {
858  vnet_main_t *vnm = vnet_get_main ();
859  n_left = n_slowpath;
860  u16 *si = slowpath_indices;
861  u32 last_unknown_etype = ~0;
862  u32 last_unknown_next = ~0;
863  eth_input_tag_lookup_t dot1ad_lookup, dot1q_lookup = {
864  .mask = -1LL,
865  .tag = tags[si[0]] ^ -1LL,
866  .sw_if_index = ~0
867  };
868 
869  clib_memcpy_fast (&dot1ad_lookup, &dot1q_lookup, sizeof (dot1q_lookup));
870 
871  while (n_left)
872  {
873  i = si[0];
874  u16 etype = etypes[i];
875 
876  if (etype == et_vlan)
877  {
878  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
879  eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
880  &dot1q_lookup, dmacs_bad[i], 0,
881  main_is_l3, dmac_check);
882 
883  }
884  else if (etype == et_dot1ad)
885  {
886  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
887  eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
888  &dot1ad_lookup, dmacs_bad[i], 1,
889  main_is_l3, dmac_check);
890  }
891  else
892  {
893  /* untagged packet with not well known etyertype */
894  if (last_unknown_etype != etype)
895  {
896  last_unknown_etype = etype;
897  etype = clib_host_to_net_u16 (etype);
898  last_unknown_next = eth_input_next_by_type (etype);
899  }
900  if (dmac_check && main_is_l3 && dmacs_bad[i])
901  {
902  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
903  b->error = node->errors[ETHERNET_ERROR_L3_MAC_MISMATCH];
904  nexts[i] = ETHERNET_INPUT_NEXT_PUNT;
905  }
906  else
907  nexts[i] = last_unknown_next;
908  }
909 
910  /* next */
911  n_left--;
912  si++;
913  }
914 
915  eth_input_update_if_counters (vm, vnm, &dot1q_lookup);
916  eth_input_update_if_counters (vm, vnm, &dot1ad_lookup);
917  }
918 
919  vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts, n_packets);
920 }
921 
924  vnet_hw_interface_t * hi, u32 * from, u32 n_pkts,
925  int ip4_cksum_ok)
926 {
929  ei = pool_elt_at_index (em->interfaces, hi->hw_instance);
931  subint_config_t *subint0 = &intf0->untagged_subint;
932 
933  int main_is_l3 = (subint0->flags & SUBINT_CONFIG_L2) == 0;
934  int promisc = (ei->flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL) != 0;
935 
936  if (main_is_l3)
937  {
938  /* main interface is L3, we dont expect tagged packets and interface
939  is not in promisc node, so we dont't need to check DMAC */
940  int is_l3 = 1;
941 
942  if (promisc == 0)
943  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
944  ip4_cksum_ok, 0);
945  else
946  /* subinterfaces and promisc mode so DMAC check is needed */
947  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
948  ip4_cksum_ok, 1);
949  return;
950  }
951  else
952  {
953  /* untagged packets are treated as L2 */
954  int is_l3 = 0;
955  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
956  ip4_cksum_ok, 1);
957  return;
958  }
959 }
960 
963  vlib_frame_t * from_frame)
964 {
965  u32 *from, n_left;
966  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
967  {
968  from = vlib_frame_vector_args (from_frame);
969  n_left = from_frame->n_vectors;
970 
971  while (n_left)
972  {
974  vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]);
975 
976  if (b0->flags & VLIB_BUFFER_IS_TRACED)
977  {
978  t0 = vlib_add_trace (vm, node, b0,
979  sizeof (ethernet_input_trace_t));
981  sizeof (t0->packet_data));
982  t0->frame_flags = from_frame->flags;
984  vlib_frame_scalar_args (from_frame),
985  sizeof (ethernet_input_frame_t));
986  }
987  from += 1;
988  n_left -= 1;
989  }
990  }
991 
992  /* rx pcap capture if enabled */
994  {
995  u32 bi0;
996 
997  from = vlib_frame_vector_args (from_frame);
998  n_left = from_frame->n_vectors;
999  while (n_left > 0)
1000  {
1001  vlib_buffer_t *b0;
1002  bi0 = from[0];
1003  from++;
1004  b0 = vlib_get_buffer (vm, bi0);
1005 
1006  if (vm->pcap[VLIB_RX].pcap_sw_if_index == 0 ||
1008  == vnet_buffer (b0)->sw_if_index[VLIB_RX])
1009  {
1010  pcap_add_buffer (&vm->pcap[VLIB_RX].pcap_main, vm, bi0, 512);
1011  }
1012  n_left--;
1013  }
1014  }
1015 }
1016 
1019  vlib_node_runtime_t * node,
1020  u32 * from, u32 n_packets,
1021  ethernet_input_variant_t variant)
1022 {
1023  vnet_main_t *vnm = vnet_get_main ();
1025  vlib_node_runtime_t *error_node;
1026  u32 n_left_from, next_index, *to_next;
1027  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
1028  u32 thread_index = vm->thread_index;
1029  u32 cached_sw_if_index = ~0;
1030  u32 cached_is_l2 = 0; /* shut up gcc */
1031  vnet_hw_interface_t *hi = NULL; /* used for main interface only */
1032 
1033  if (variant != ETHERNET_INPUT_VARIANT_ETHERNET)
1034  error_node = vlib_node_get_runtime (vm, ethernet_input_node.index);
1035  else
1036  error_node = node;
1037 
1038  n_left_from = n_packets;
1039 
1040  next_index = node->cached_next_index;
1041  stats_sw_if_index = node->runtime_data[0];
1042  stats_n_packets = stats_n_bytes = 0;
1043 
1044  while (n_left_from > 0)
1045  {
1046  u32 n_left_to_next;
1047 
1048  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1049 
1050  while (n_left_from >= 4 && n_left_to_next >= 2)
1051  {
1052  u32 bi0, bi1;
1053  vlib_buffer_t *b0, *b1;
1054  u8 next0, next1, error0, error1;
1055  u16 type0, orig_type0, type1, orig_type1;
1056  u16 outer_id0, inner_id0, outer_id1, inner_id1;
1057  u32 match_flags0, match_flags1;
1058  u32 old_sw_if_index0, new_sw_if_index0, len0, old_sw_if_index1,
1059  new_sw_if_index1, len1;
1060  vnet_hw_interface_t *hi0, *hi1;
1061  main_intf_t *main_intf0, *main_intf1;
1062  vlan_intf_t *vlan_intf0, *vlan_intf1;
1063  qinq_intf_t *qinq_intf0, *qinq_intf1;
1064  u32 is_l20, is_l21;
1065  ethernet_header_t *e0, *e1;
1066 
1067  /* Prefetch next iteration. */
1068  {
1069  vlib_buffer_t *b2, *b3;
1070 
1071  b2 = vlib_get_buffer (vm, from[2]);
1072  b3 = vlib_get_buffer (vm, from[3]);
1073 
1074  vlib_prefetch_buffer_header (b2, STORE);
1075  vlib_prefetch_buffer_header (b3, STORE);
1076 
1077  CLIB_PREFETCH (b2->data, sizeof (ethernet_header_t), LOAD);
1078  CLIB_PREFETCH (b3->data, sizeof (ethernet_header_t), LOAD);
1079  }
1080 
1081  bi0 = from[0];
1082  bi1 = from[1];
1083  to_next[0] = bi0;
1084  to_next[1] = bi1;
1085  from += 2;
1086  to_next += 2;
1087  n_left_to_next -= 2;
1088  n_left_from -= 2;
1089 
1090  b0 = vlib_get_buffer (vm, bi0);
1091  b1 = vlib_get_buffer (vm, bi1);
1092 
1093  error0 = error1 = ETHERNET_ERROR_NONE;
1094  e0 = vlib_buffer_get_current (b0);
1095  type0 = clib_net_to_host_u16 (e0->type);
1096  e1 = vlib_buffer_get_current (b1);
1097  type1 = clib_net_to_host_u16 (e1->type);
1098 
1099  /* Set the L2 header offset for all packets */
1100  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1101  vnet_buffer (b1)->l2_hdr_offset = b1->current_data;
1102  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1103  b1->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1104 
1105  /* Speed-path for the untagged case */
1108  type1)))
1109  {
1110  main_intf_t *intf0;
1111  subint_config_t *subint0;
1112  u32 sw_if_index0, sw_if_index1;
1113 
1114  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1115  sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1116  is_l20 = cached_is_l2;
1117 
1118  /* This is probably wholly unnecessary */
1119  if (PREDICT_FALSE (sw_if_index0 != sw_if_index1))
1120  goto slowpath;
1121 
1122  /* Now sw_if_index0 == sw_if_index1 */
1123  if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1124  {
1125  cached_sw_if_index = sw_if_index0;
1126  hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1127  intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1128  subint0 = &intf0->untagged_subint;
1129  cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1130  }
1131 
1132  if (PREDICT_TRUE (is_l20 != 0))
1133  {
1134  vnet_buffer (b0)->l3_hdr_offset =
1135  vnet_buffer (b0)->l2_hdr_offset +
1136  sizeof (ethernet_header_t);
1137  vnet_buffer (b1)->l3_hdr_offset =
1138  vnet_buffer (b1)->l2_hdr_offset +
1139  sizeof (ethernet_header_t);
1140  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1141  b1->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1142  next0 = em->l2_next;
1143  vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1144  next1 = em->l2_next;
1145  vnet_buffer (b1)->l2.l2_len = sizeof (ethernet_header_t);
1146  }
1147  else
1148  {
1149  if (!ethernet_address_cast (e0->dst_address) &&
1150  (hi->hw_address != 0) &&
1151  !ethernet_mac_address_equal ((u8 *) e0, hi->hw_address))
1152  error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1153  if (!ethernet_address_cast (e1->dst_address) &&
1154  (hi->hw_address != 0) &&
1155  !ethernet_mac_address_equal ((u8 *) e1, hi->hw_address))
1156  error1 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1157  vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1158  determine_next_node (em, variant, 0, type0, b0,
1159  &error0, &next0);
1160  vlib_buffer_advance (b1, sizeof (ethernet_header_t));
1161  determine_next_node (em, variant, 0, type1, b1,
1162  &error1, &next1);
1163  }
1164  goto ship_it01;
1165  }
1166 
1167  /* Slow-path for the tagged case */
1168  slowpath:
1169  parse_header (variant,
1170  b0,
1171  &type0,
1172  &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1173 
1174  parse_header (variant,
1175  b1,
1176  &type1,
1177  &orig_type1, &outer_id1, &inner_id1, &match_flags1);
1178 
1179  old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1180  old_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1181 
1183  vnm,
1184  old_sw_if_index0,
1185  orig_type0,
1186  outer_id0,
1187  inner_id0,
1188  &hi0,
1189  &main_intf0, &vlan_intf0, &qinq_intf0);
1190 
1192  vnm,
1193  old_sw_if_index1,
1194  orig_type1,
1195  outer_id1,
1196  inner_id1,
1197  &hi1,
1198  &main_intf1, &vlan_intf1, &qinq_intf1);
1199 
1200  identify_subint (hi0,
1201  b0,
1202  match_flags0,
1203  main_intf0,
1204  vlan_intf0,
1205  qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1206 
1207  identify_subint (hi1,
1208  b1,
1209  match_flags1,
1210  main_intf1,
1211  vlan_intf1,
1212  qinq_intf1, &new_sw_if_index1, &error1, &is_l21);
1213 
1214  // Save RX sw_if_index for later nodes
1215  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1216  error0 !=
1217  ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1218  vnet_buffer (b1)->sw_if_index[VLIB_RX] =
1219  error1 !=
1220  ETHERNET_ERROR_NONE ? old_sw_if_index1 : new_sw_if_index1;
1221 
1222  // Check if there is a stat to take (valid and non-main sw_if_index for pkt 0 or pkt 1)
1223  if (((new_sw_if_index0 != ~0)
1224  && (new_sw_if_index0 != old_sw_if_index0))
1225  || ((new_sw_if_index1 != ~0)
1226  && (new_sw_if_index1 != old_sw_if_index1)))
1227  {
1228 
1229  len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1230  - vnet_buffer (b0)->l2_hdr_offset;
1231  len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data
1232  - vnet_buffer (b1)->l2_hdr_offset;
1233 
1234  stats_n_packets += 2;
1235  stats_n_bytes += len0 + len1;
1236 
1237  if (PREDICT_FALSE
1238  (!(new_sw_if_index0 == stats_sw_if_index
1239  && new_sw_if_index1 == stats_sw_if_index)))
1240  {
1241  stats_n_packets -= 2;
1242  stats_n_bytes -= len0 + len1;
1243 
1244  if (new_sw_if_index0 != old_sw_if_index0
1245  && new_sw_if_index0 != ~0)
1247  interface_main.combined_sw_if_counters
1248  +
1250  thread_index,
1251  new_sw_if_index0, 1,
1252  len0);
1253  if (new_sw_if_index1 != old_sw_if_index1
1254  && new_sw_if_index1 != ~0)
1256  interface_main.combined_sw_if_counters
1257  +
1259  thread_index,
1260  new_sw_if_index1, 1,
1261  len1);
1262 
1263  if (new_sw_if_index0 == new_sw_if_index1)
1264  {
1265  if (stats_n_packets > 0)
1266  {
1270  thread_index,
1271  stats_sw_if_index,
1272  stats_n_packets, stats_n_bytes);
1273  stats_n_packets = stats_n_bytes = 0;
1274  }
1275  stats_sw_if_index = new_sw_if_index0;
1276  }
1277  }
1278  }
1279 
1280  if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1281  is_l20 = is_l21 = 0;
1282 
1283  determine_next_node (em, variant, is_l20, type0, b0, &error0,
1284  &next0);
1285  determine_next_node (em, variant, is_l21, type1, b1, &error1,
1286  &next1);
1287 
1288  ship_it01:
1289  b0->error = error_node->errors[error0];
1290  b1->error = error_node->errors[error1];
1291 
1292  // verify speculative enqueue
1293  vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1294  n_left_to_next, bi0, bi1, next0,
1295  next1);
1296  }
1297 
1298  while (n_left_from > 0 && n_left_to_next > 0)
1299  {
1300  u32 bi0;
1301  vlib_buffer_t *b0;
1302  u8 error0, next0;
1303  u16 type0, orig_type0;
1304  u16 outer_id0, inner_id0;
1305  u32 match_flags0;
1306  u32 old_sw_if_index0, new_sw_if_index0, len0;
1307  vnet_hw_interface_t *hi0;
1308  main_intf_t *main_intf0;
1309  vlan_intf_t *vlan_intf0;
1310  qinq_intf_t *qinq_intf0;
1311  ethernet_header_t *e0;
1312  u32 is_l20;
1313 
1314  // Prefetch next iteration
1315  if (n_left_from > 1)
1316  {
1317  vlib_buffer_t *p2;
1318 
1319  p2 = vlib_get_buffer (vm, from[1]);
1320  vlib_prefetch_buffer_header (p2, STORE);
1322  }
1323 
1324  bi0 = from[0];
1325  to_next[0] = bi0;
1326  from += 1;
1327  to_next += 1;
1328  n_left_from -= 1;
1329  n_left_to_next -= 1;
1330 
1331  b0 = vlib_get_buffer (vm, bi0);
1332 
1333  error0 = ETHERNET_ERROR_NONE;
1334  e0 = vlib_buffer_get_current (b0);
1335  type0 = clib_net_to_host_u16 (e0->type);
1336 
1337  /* Set the L2 header offset for all packets */
1338  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1339  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1340 
1341  /* Speed-path for the untagged case */
1343  && !ethernet_frame_is_tagged (type0)))
1344  {
1345  main_intf_t *intf0;
1346  subint_config_t *subint0;
1347  u32 sw_if_index0;
1348 
1349  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1350  is_l20 = cached_is_l2;
1351 
1352  if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1353  {
1354  cached_sw_if_index = sw_if_index0;
1355  hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1356  intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1357  subint0 = &intf0->untagged_subint;
1358  cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1359  }
1360 
1361 
1362  if (PREDICT_TRUE (is_l20 != 0))
1363  {
1364  vnet_buffer (b0)->l3_hdr_offset =
1365  vnet_buffer (b0)->l2_hdr_offset +
1366  sizeof (ethernet_header_t);
1367  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1368  next0 = em->l2_next;
1369  vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1370  }
1371  else
1372  {
1373  if (!ethernet_address_cast (e0->dst_address) &&
1374  (hi->hw_address != 0) &&
1375  !ethernet_mac_address_equal ((u8 *) e0, hi->hw_address))
1376  error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1377  vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1378  determine_next_node (em, variant, 0, type0, b0,
1379  &error0, &next0);
1380  }
1381  goto ship_it0;
1382  }
1383 
1384  /* Slow-path for the tagged case */
1385  parse_header (variant,
1386  b0,
1387  &type0,
1388  &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1389 
1390  old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1391 
1393  vnm,
1394  old_sw_if_index0,
1395  orig_type0,
1396  outer_id0,
1397  inner_id0,
1398  &hi0,
1399  &main_intf0, &vlan_intf0, &qinq_intf0);
1400 
1401  identify_subint (hi0,
1402  b0,
1403  match_flags0,
1404  main_intf0,
1405  vlan_intf0,
1406  qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1407 
1408  // Save RX sw_if_index for later nodes
1409  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1410  error0 !=
1411  ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1412 
1413  // Increment subinterface stats
1414  // Note that interface-level counters have already been incremented
1415  // prior to calling this function. Thus only subinterface counters
1416  // are incremented here.
1417  //
1418  // Interface level counters include packets received on the main
1419  // interface and all subinterfaces. Subinterface level counters
1420  // include only those packets received on that subinterface
1421  // Increment stats if the subint is valid and it is not the main intf
1422  if ((new_sw_if_index0 != ~0)
1423  && (new_sw_if_index0 != old_sw_if_index0))
1424  {
1425 
1426  len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1427  - vnet_buffer (b0)->l2_hdr_offset;
1428 
1429  stats_n_packets += 1;
1430  stats_n_bytes += len0;
1431 
1432  // Batch stat increments from the same subinterface so counters
1433  // don't need to be incremented for every packet.
1434  if (PREDICT_FALSE (new_sw_if_index0 != stats_sw_if_index))
1435  {
1436  stats_n_packets -= 1;
1437  stats_n_bytes -= len0;
1438 
1439  if (new_sw_if_index0 != ~0)
1443  thread_index, new_sw_if_index0, 1, len0);
1444  if (stats_n_packets > 0)
1445  {
1449  thread_index,
1450  stats_sw_if_index, stats_n_packets, stats_n_bytes);
1451  stats_n_packets = stats_n_bytes = 0;
1452  }
1453  stats_sw_if_index = new_sw_if_index0;
1454  }
1455  }
1456 
1457  if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1458  is_l20 = 0;
1459 
1460  determine_next_node (em, variant, is_l20, type0, b0, &error0,
1461  &next0);
1462 
1463  ship_it0:
1464  b0->error = error_node->errors[error0];
1465 
1466  // verify speculative enqueue
1467  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1468  to_next, n_left_to_next,
1469  bi0, next0);
1470  }
1471 
1472  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1473  }
1474 
1475  // Increment any remaining batched stats
1476  if (stats_n_packets > 0)
1477  {
1481  thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
1482  node->runtime_data[0] = stats_sw_if_index;
1483  }
1484 }
1485 
1487  vlib_node_runtime_t * node,
1488  vlib_frame_t * frame)
1489 {
1490  vnet_main_t *vnm = vnet_get_main ();
1491  u32 *from = vlib_frame_vector_args (frame);
1492  u32 n_packets = frame->n_vectors;
1493 
1494  ethernet_input_trace (vm, node, frame);
1495 
1496  if (frame->flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX)
1497  {
1499  int ip4_cksum_ok = (frame->flags & ETH_INPUT_FRAME_F_IP4_CKSUM_OK) != 0;
1501  eth_input_single_int (vm, node, hi, from, n_packets, ip4_cksum_ok);
1502  }
1503  else
1504  ethernet_input_inline (vm, node, from, n_packets,
1506  return n_packets;
1507 }
1508 
1510  vlib_node_runtime_t * node,
1511  vlib_frame_t * from_frame)
1512 {
1513  u32 *from = vlib_frame_vector_args (from_frame);
1514  u32 n_packets = from_frame->n_vectors;
1515  ethernet_input_trace (vm, node, from_frame);
1516  ethernet_input_inline (vm, node, from, n_packets,
1518  return n_packets;
1519 }
1520 
1522  vlib_node_runtime_t * node,
1523  vlib_frame_t * from_frame)
1524 {
1525  u32 *from = vlib_frame_vector_args (from_frame);
1526  u32 n_packets = from_frame->n_vectors;
1527  ethernet_input_trace (vm, node, from_frame);
1528  ethernet_input_inline (vm, node, from, n_packets,
1530  return n_packets;
1531 }
1532 
1533 
1534 // Return the subinterface config struct for the given sw_if_index
1535 // Also return via parameter the appropriate match flags for the
1536 // configured number of tags.
1537 // On error (unsupported or not ethernet) return 0.
1538 static subint_config_t *
1540  u32 sw_if_index,
1541  u32 * flags, u32 * unsupported)
1542 {
1545  vnet_sw_interface_t *si;
1546  main_intf_t *main_intf;
1547  vlan_table_t *vlan_table;
1548  qinq_table_t *qinq_table;
1549  subint_config_t *subint = 0;
1550 
1551  hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1552 
1553  if (!hi || (hi->hw_class_index != ethernet_hw_interface_class.index))
1554  {
1555  *unsupported = 0;
1556  goto done; // non-ethernet interface
1557  }
1558 
1559  // ensure there's an entry for the main intf (shouldn't really be necessary)
1560  vec_validate (em->main_intfs, hi->hw_if_index);
1561  main_intf = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1562 
1563  // Locate the subint for the given ethernet config
1564  si = vnet_get_sw_interface (vnm, sw_if_index);
1565 
1566  if (si->type == VNET_SW_INTERFACE_TYPE_P2P)
1567  {
1568  p2p_ethernet_main_t *p2pm = &p2p_main;
1569  u32 p2pe_sw_if_index =
1571  if (p2pe_sw_if_index == ~0)
1572  {
1573  pool_get (p2pm->p2p_subif_pool, subint);
1574  si->p2p.pool_index = subint - p2pm->p2p_subif_pool;
1575  }
1576  else
1577  subint = vec_elt_at_index (p2pm->p2p_subif_pool, si->p2p.pool_index);
1578  *flags = SUBINT_CONFIG_P2P;
1579  }
1580  else if (si->type == VNET_SW_INTERFACE_TYPE_PIPE)
1581  {
1582  pipe_t *pipe;
1583 
1584  pipe = pipe_get (sw_if_index);
1585  subint = &pipe->subint;
1586  *flags = SUBINT_CONFIG_P2P;
1587  }
1588  else if (si->sub.eth.flags.default_sub)
1589  {
1590  subint = &main_intf->default_subint;
1591  *flags = SUBINT_CONFIG_MATCH_1_TAG |
1593  }
1594  else if ((si->sub.eth.flags.no_tags) || (si->sub.eth.raw_flags == 0))
1595  {
1596  // if no flags are set then this is a main interface
1597  // so treat as untagged
1598  subint = &main_intf->untagged_subint;
1599  *flags = SUBINT_CONFIG_MATCH_0_TAG;
1600  }
1601  else
1602  {
1603  // one or two tags
1604  // first get the vlan table
1605  if (si->sub.eth.flags.dot1ad)
1606  {
1607  if (main_intf->dot1ad_vlans == 0)
1608  {
1609  // Allocate a vlan table from the pool
1610  pool_get (em->vlan_pool, vlan_table);
1611  main_intf->dot1ad_vlans = vlan_table - em->vlan_pool;
1612  }
1613  else
1614  {
1615  // Get ptr to existing vlan table
1616  vlan_table =
1617  vec_elt_at_index (em->vlan_pool, main_intf->dot1ad_vlans);
1618  }
1619  }
1620  else
1621  { // dot1q
1622  if (main_intf->dot1q_vlans == 0)
1623  {
1624  // Allocate a vlan table from the pool
1625  pool_get (em->vlan_pool, vlan_table);
1626  main_intf->dot1q_vlans = vlan_table - em->vlan_pool;
1627  }
1628  else
1629  {
1630  // Get ptr to existing vlan table
1631  vlan_table =
1632  vec_elt_at_index (em->vlan_pool, main_intf->dot1q_vlans);
1633  }
1634  }
1635 
1636  if (si->sub.eth.flags.one_tag)
1637  {
1638  *flags = si->sub.eth.flags.exact_match ?
1642 
1643  if (si->sub.eth.flags.outer_vlan_id_any)
1644  {
1645  // not implemented yet
1646  *unsupported = 1;
1647  goto done;
1648  }
1649  else
1650  {
1651  // a single vlan, a common case
1652  subint =
1653  &vlan_table->vlans[si->sub.eth.
1654  outer_vlan_id].single_tag_subint;
1655  }
1656 
1657  }
1658  else
1659  {
1660  // Two tags
1661  *flags = si->sub.eth.flags.exact_match ?
1664 
1665  if (si->sub.eth.flags.outer_vlan_id_any
1666  && si->sub.eth.flags.inner_vlan_id_any)
1667  {
1668  // not implemented yet
1669  *unsupported = 1;
1670  goto done;
1671  }
1672 
1673  if (si->sub.eth.flags.inner_vlan_id_any)
1674  {
1675  // a specific outer and "any" inner
1676  // don't need a qinq table for this
1677  subint =
1678  &vlan_table->vlans[si->sub.eth.
1679  outer_vlan_id].inner_any_subint;
1680  if (si->sub.eth.flags.exact_match)
1681  {
1682  *flags = SUBINT_CONFIG_MATCH_2_TAG;
1683  }
1684  else
1685  {
1686  *flags = SUBINT_CONFIG_MATCH_2_TAG |
1688  }
1689  }
1690  else
1691  {
1692  // a specific outer + specifc innner vlan id, a common case
1693 
1694  // get the qinq table
1695  if (vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs == 0)
1696  {
1697  // Allocate a qinq table from the pool
1698  pool_get (em->qinq_pool, qinq_table);
1699  vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs =
1700  qinq_table - em->qinq_pool;
1701  }
1702  else
1703  {
1704  // Get ptr to existing qinq table
1705  qinq_table =
1707  vlan_table->vlans[si->sub.
1708  eth.outer_vlan_id].
1709  qinqs);
1710  }
1711  subint = &qinq_table->vlans[si->sub.eth.inner_vlan_id].subint;
1712  }
1713  }
1714  }
1715 
1716 done:
1717  return subint;
1718 }
1719 
1720 static clib_error_t *
1722 {
1723  subint_config_t *subint;
1724  u32 dummy_flags;
1725  u32 dummy_unsup;
1726  clib_error_t *error = 0;
1727 
1728  // Find the config for this subinterface
1729  subint =
1730  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1731  &dummy_unsup);
1732 
1733  if (subint == 0)
1734  {
1735  // not implemented yet or not ethernet
1736  goto done;
1737  }
1738 
1739  subint->sw_if_index =
1740  ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? sw_if_index : ~0);
1741 
1742 done:
1743  return error;
1744 }
1745 
1747 
1748 
1749 #ifndef CLIB_MARCH_VARIANT
1750 // Set the L2/L3 mode for the subinterface
1751 void
1753 {
1754  subint_config_t *subint;
1755  u32 dummy_flags;
1756  u32 dummy_unsup;
1757  int is_port;
1758  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index);
1759 
1760  is_port = !(sw->type == VNET_SW_INTERFACE_TYPE_SUB);
1761 
1762  // Find the config for this subinterface
1763  subint =
1764  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1765  &dummy_unsup);
1766 
1767  if (subint == 0)
1768  {
1769  // unimplemented or not ethernet
1770  goto done;
1771  }
1772 
1773  // Double check that the config we found is for our interface (or the interface is down)
1774  ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1775 
1776  if (l2)
1777  {
1778  subint->flags |= SUBINT_CONFIG_L2;
1779  if (is_port)
1780  subint->flags |=
1783  }
1784  else
1785  {
1786  subint->flags &= ~SUBINT_CONFIG_L2;
1787  if (is_port)
1788  subint->flags &=
1791  }
1792 
1793 done:
1794  return;
1795 }
1796 
1797 /*
1798  * Set the L2/L3 mode for the subinterface regardless of port
1799  */
1800 void
1802  u32 sw_if_index, u32 l2)
1803 {
1804  subint_config_t *subint;
1805  u32 dummy_flags;
1806  u32 dummy_unsup;
1807 
1808  /* Find the config for this subinterface */
1809  subint =
1810  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1811  &dummy_unsup);
1812 
1813  if (subint == 0)
1814  {
1815  /* unimplemented or not ethernet */
1816  goto done;
1817  }
1818 
1819  /*
1820  * Double check that the config we found is for our interface (or the
1821  * interface is down)
1822  */
1823  ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1824 
1825  if (l2)
1826  {
1827  subint->flags |= SUBINT_CONFIG_L2;
1828  }
1829  else
1830  {
1831  subint->flags &= ~SUBINT_CONFIG_L2;
1832  }
1833 
1834 done:
1835  return;
1836 }
1837 #endif
1838 
1839 static clib_error_t *
1841  u32 sw_if_index, u32 is_create)
1842 {
1843  clib_error_t *error = 0;
1844  subint_config_t *subint;
1845  u32 match_flags;
1846  u32 unsupported = 0;
1847 
1848  // Find the config for this subinterface
1849  subint =
1850  ethernet_sw_interface_get_config (vnm, sw_if_index, &match_flags,
1851  &unsupported);
1852 
1853  if (subint == 0)
1854  {
1855  // not implemented yet or not ethernet
1856  if (unsupported)
1857  {
1858  // this is the NYI case
1859  error = clib_error_return (0, "not implemented yet");
1860  }
1861  goto done;
1862  }
1863 
1864  if (!is_create)
1865  {
1866  subint->flags = 0;
1867  return error;
1868  }
1869 
1870  // Initialize the subint
1871  if (subint->flags & SUBINT_CONFIG_VALID)
1872  {
1873  // Error vlan already in use
1874  error = clib_error_return (0, "vlan is already in use");
1875  }
1876  else
1877  {
1878  // Note that config is L3 by default
1879  subint->flags = SUBINT_CONFIG_VALID | match_flags;
1880  subint->sw_if_index = ~0; // because interfaces are initially down
1881  }
1882 
1883 done:
1884  return error;
1885 }
1886 
1888 
1889 static char *ethernet_error_strings[] = {
1890 #define ethernet_error(n,c,s) s,
1891 #include "error.def"
1892 #undef ethernet_error
1893 };
1894 
1895 /* *INDENT-OFF* */
1897  .name = "ethernet-input",
1898  /* Takes a vector of packets. */
1899  .vector_size = sizeof (u32),
1900  .scalar_size = sizeof (ethernet_input_frame_t),
1901  .n_errors = ETHERNET_N_ERROR,
1902  .error_strings = ethernet_error_strings,
1903  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1904  .next_nodes = {
1905 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1907 #undef _
1908  },
1909  .format_buffer = format_ethernet_header_with_length,
1910  .format_trace = format_ethernet_input_trace,
1911  .unformat_buffer = unformat_ethernet_header,
1912 };
1913 
1915  .name = "ethernet-input-type",
1916  /* Takes a vector of packets. */
1917  .vector_size = sizeof (u32),
1918  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1919  .next_nodes = {
1920 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1922 #undef _
1923  },
1924 };
1925 
1927  .name = "ethernet-input-not-l2",
1928  /* Takes a vector of packets. */
1929  .vector_size = sizeof (u32),
1930  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1931  .next_nodes = {
1932 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1934 #undef _
1935  },
1936 };
1937 /* *INDENT-ON* */
1938 
1939 #ifndef CLIB_MARCH_VARIANT
1940 void
1942  vnet_hw_interface_t * hi, u32 enable)
1943 {
1944  // Insure all packets go to ethernet-input (i.e. untagged ipv4 packets
1945  // don't go directly to ip4-input)
1947  (vnm, hi->hw_if_index, enable ? ethernet_input_node.index : ~0);
1948 }
1949 
1950 
1951 /*
1952  * Initialization and registration for the next_by_ethernet structure
1953  */
1954 
1955 clib_error_t *
1957 {
1959  ( /* elt bytes */ sizeof (l3_next->input_next_by_type[0]),
1960  /* bits in index */ BITS (((ethernet_header_t *) 0)->type));
1961 
1963  ETHERNET_INPUT_NEXT_DROP);
1965  ETHERNET_INPUT_NEXT_PUNT);
1966  l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_DROP] =
1968  l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_PUNT] =
1970 
1971  /*
1972  * Make sure we don't wipe out an ethernet registration by mistake
1973  * Can happen if init function ordering constraints are missing.
1974  */
1975  if (CLIB_DEBUG > 0)
1976  {
1979  }
1980 
1981  return 0;
1982 }
1983 
1984 // Add an ethertype -> next index mapping to the structure
1985 clib_error_t *
1987  u32 ethertype, u32 next_index)
1988 {
1989  u32 i;
1990  u16 *n;
1992 
1993  if (CLIB_DEBUG > 0)
1994  {
1997  }
1998 
1999  /* Setup ethernet type -> next index sparse vector mapping. */
2000  n = sparse_vec_validate (l3_next->input_next_by_type, ethertype);
2001  n[0] = next_index;
2002 
2003  /* Rebuild next index -> sparse index inverse mapping when sparse vector
2004  is updated. */
2005  vec_validate (l3_next->sparse_index_by_input_next_index, next_index);
2006  for (i = 1; i < vec_len (l3_next->input_next_by_type); i++)
2007  l3_next->
2008  sparse_index_by_input_next_index[l3_next->input_next_by_type[i]] = i;
2009 
2010  // do not allow the cached next index's to be updated if L3
2011  // redirect is enabled, as it will have overwritten them
2012  if (!em->redirect_l3)
2013  {
2014  // Cache common ethertypes directly
2015  if (ethertype == ETHERNET_TYPE_IP4)
2016  {
2017  l3_next->input_next_ip4 = next_index;
2018  }
2019  else if (ethertype == ETHERNET_TYPE_IP6)
2020  {
2021  l3_next->input_next_ip6 = next_index;
2022  }
2023  else if (ethertype == ETHERNET_TYPE_MPLS)
2024  {
2025  l3_next->input_next_mpls = next_index;
2026  }
2027  }
2028  return 0;
2029 }
2030 
2031 
2032 static clib_error_t *
2034 {
2036  __attribute__ ((unused)) vlan_table_t *invalid_vlan_table;
2037  __attribute__ ((unused)) qinq_table_t *invalid_qinq_table;
2038 
2042 
2044 
2045  // Initialize pools and vector for vlan parsing
2046  vec_validate (em->main_intfs, 10); // 10 main interfaces
2047  pool_alloc (em->vlan_pool, 10);
2048  pool_alloc (em->qinq_pool, 1);
2049 
2050  // The first vlan pool will always be reserved for an invalid table
2051  pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0
2052  // The first qinq pool will always be reserved for an invalid table
2053  pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0
2054 
2055  return 0;
2056 }
2057 
2059 
2060 void
2062  ethernet_type_t type, u32 node_index)
2063 {
2066  u32 i;
2067 
2068  {
2070  if (error)
2071  clib_error_report (error);
2072  }
2073 
2074  ti = ethernet_get_type_info (em, type);
2075  ti->node_index = node_index;
2076  ti->next_index = vlib_node_add_next (vm,
2077  ethernet_input_node.index, node_index);
2078  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2079  ASSERT (i == ti->next_index);
2080 
2081  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2082  ASSERT (i == ti->next_index);
2083 
2084  // Add the L3 node for this ethertype to the next nodes structure
2085  next_by_ethertype_register (&em->l3_next, type, ti->next_index);
2086 
2087  // Call the registration functions for other nodes that want a mapping
2088  l2bvi_register_input_type (vm, type, node_index);
2089 }
2090 
2091 void
2093 {
2095  u32 i;
2096 
2097  em->l2_next =
2098  vlib_node_add_next (vm, ethernet_input_node.index, node_index);
2099 
2100  /*
2101  * Even if we never use these arcs, we have to align the next indices...
2102  */
2103  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2104 
2105  ASSERT (i == em->l2_next);
2106 
2107  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2108  ASSERT (i == em->l2_next);
2109 }
2110 
2111 // Register a next node for L3 redirect, and enable L3 redirect
2112 void
2114 {
2116  u32 i;
2117 
2118  em->redirect_l3 = 1;
2120  ethernet_input_node.index,
2121  node_index);
2122  /*
2123  * Change the cached next nodes to the redirect node
2124  */
2128 
2129  /*
2130  * Even if we never use these arcs, we have to align the next indices...
2131  */
2132  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2133 
2134  ASSERT (i == em->redirect_l3_next);
2135 
2136  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2137 
2138  ASSERT (i == em->redirect_l3_next);
2139 }
2140 #endif
2141 
2142 /*
2143  * fd.io coding-style-patch-verification: ON
2144  *
2145  * Local Variables:
2146  * eval: (c-set-style "gnu")
2147  * End:
2148  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
u32 sw_if_index
Definition: ipsec_gre.api:37
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
subint_config_t subint
Definition: ethernet.h:233
vmrglw vmrglh hi
pcap_main_t pcap_main
Definition: main.h:65
VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(bond_sw_interface_up_down)
static_always_inline void identify_subint(vnet_hw_interface_t *hi, vlib_buffer_t *b0, u32 match_flags, main_intf_t *main_intf, vlan_intf_t *vlan_intf, qinq_intf_t *qinq_intf, u32 *new_sw_if_index, u8 *error0, u32 *is_l2)
Definition: node.c:200
u32 flags
Definition: vhost_user.h:115
#define CLIB_UNUSED(x)
Definition: clib.h:82
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, current_data, 0)
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:521
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
#define pool_alloc(P, N)
Allocate N more free elements to pool (unspecified alignment).
Definition: pool.h:341
vnet_interface_main_t interface_main
Definition: vnet.h:56
vnet_p2p_sub_interface_t p2p
Definition: interface.h:707
#define PREDICT_TRUE(x)
Definition: clib.h:112
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
int vnet_hw_interface_rx_redirect_to_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: interface.c:1177
u16x16 u64x4 static_always_inline u32 u8x32_msb_mask(u8x32 v)
Definition: vector_avx2.h:108
void ethernet_register_l2_input(vlib_main_t *vm, u32 node_index)
Definition: node.c:2092
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static_always_inline void eth_input_tag_lookup(vlib_main_t *vm, vnet_main_t *vnm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u64 tag, u16 *next, vlib_buffer_t *b, eth_input_tag_lookup_t *l, u8 dmac_bad, int is_dot1ad, int main_is_l3, int check_dmac)
Definition: node.c:488
#define NULL
Definition: clib.h:58
ethernet_type_t
Definition: packet.h:45
#define SUBINT_CONFIG_MATCH_0_TAG
Definition: ethernet.h:194
#define ethernet_buffer_header_size(b)
Determine the size of the Ethernet headers of the current frame in the buffer.
Definition: ethernet.h:420
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:197
u8 data[0]
Packet data.
Definition: buffer.h:181
static subint_config_t * ethernet_sw_interface_get_config(vnet_main_t *vnm, u32 sw_if_index, u32 *flags, u32 *unsupported)
Definition: node.c:1539
ethernet_input_variant_t
Definition: node.c:93
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
main_intf_t * main_intfs
Definition: ethernet.h:281
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:65
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
subint_config_t inner_any_subint
Definition: ethernet.h:222
u8 packet_data[32]
Definition: node.c:65
#define VLIB_NODE_FN(node)
Definition: node.h:201
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:468
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:366
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static_always_inline void u64x4_scatter(u64x4 r, void *p0, void *p1, void *p2, void *p3)
Definition: vector_avx2.h:216
static clib_error_t * ethernet_sw_interface_add_del(vnet_main_t *vnm, u32 sw_if_index, u32 is_create)
Definition: node.c:1840
epu16_epi64 epu8_epi64 epu8_epi64 epi16_epi64 i16x16
Definition: vector_avx2.h:129
vlib_node_registration_t ethernet_input_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_node)
Definition: node.c:1896
ethernet_main_t ethernet_main
Definition: init.c:45
represenation of a pipe interface
Definition: pipe.h:24
#define static_always_inline
Definition: clib.h:99
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
static uword ethernet_address_cast(u8 *a)
Definition: packet.h:67
static_always_inline u16 eth_input_next_by_type(u16 etype)
Definition: node.c:452
void ethernet_set_rx_redirect(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 enable)
Definition: node.c:1941
clib_error_t * next_by_ethertype_init(next_by_ethertype_t *l3_next)
Definition: node.c:1956
#define ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX
Definition: ethernet.h:52
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:824
u8 dst_address[6]
Definition: packet.h:55
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
#define sparse_vec_validate(v, i)
Definition: sparse_vec.h:231
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
subint_config_t default_subint
Definition: ethernet.h:214
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
unsigned int u32
Definition: types.h:88
#define vlib_call_init_function(vm, x)
Definition: init.h:260
#define VLIB_FRAME_SIZE
Definition: node.h:376
static char * ethernet_error_strings[]
Definition: node.c:1889
ethernet_input_next_t
Definition: node.c:55
epu8_epi32 epu16_epi32 u64x2
Definition: vector_sse42.h:665
struct vnet_sub_interface_t::@208::@209::@211 flags
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
static clib_error_t * ethernet_input_init(vlib_main_t *vm)
Definition: node.c:2033
int pcap_enable
Definition: main.h:63
u16 dot1q_vlans
Definition: ethernet.h:215
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
subint_config_t single_tag_subint
Definition: ethernet.h:221
vnet_sub_interface_t sub
Definition: interface.h:704
u32 * sparse_index_by_input_next_index
Definition: ethernet.h:249
static ethernet_type_info_t * ethernet_get_type_info(ethernet_main_t *em, ethernet_type_t type)
Definition: ethernet.h:305
unsigned short u16
Definition: types.h:57
static_always_inline u64x4 u64x4_gather(void *p0, void *p1, void *p2, void *p3)
Definition: vector_avx2.h:195
#define ETH_INPUT_FRAME_F_IP4_CKSUM_OK
Definition: ethernet.h:55
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
static_always_inline void eth_input_process_frame(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u32 *buffer_indices, u32 n_packets, int main_is_l3, int ip4_cksum_ok, int dmac_check)
Definition: node.c:619
vlib_node_registration_t ethernet_input_type_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_type_node)
Definition: node.c:1914
#define PREDICT_FALSE(x)
Definition: clib.h:111
#define ethernet_buffer_set_vlan_count(b, v)
Sets the number of VLAN headers in the current Ethernet frame in the buffer.
Definition: ethernet.h:393
u16 dot1ad_vlans
Definition: ethernet.h:216
u32 pcap_sw_if_index
Definition: main.h:64
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:338
#define SUBINT_CONFIG_MATCH_3_TAG
Definition: ethernet.h:197
vnet_hw_interface_class_t ethernet_hw_interface_class
u8 next_by_ethertype_register_called
Definition: ethernet.h:293
subint_config_t untagged_subint
Definition: ethernet.h:213
u8 len
Definition: ip_types.api:49
static_always_inline void u32x8_scatter_one(u32x8 r, int index, void *p)
Definition: vector_avx2.h:245
u32 p2p_ethernet_lookup(u32 parent_if_index, u8 *client_mac)
Definition: p2p_ethernet.c:35
u8 * format_ethernet_header_with_length(u8 *s, va_list *args)
Definition: format.c:97
#define SUBINT_CONFIG_VALID
Definition: ethernet.h:198
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
static_always_inline void eth_input_single_int(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u32 *from, u32 n_pkts, int ip4_cksum_ok)
Definition: node.c:923
qinq_intf_t vlans[ETHERNET_N_VLAN]
Definition: ethernet.h:238
u16 n_vectors
Definition: node.h:395
VNET_SW_INTERFACE_ADD_DEL_FUNCTION(ethernet_sw_interface_add_del)
struct vnet_sub_interface_t::@208 eth
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
vlib_main_t * vm
Definition: buffer.c:312
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
static int ethernet_mac_address_equal(const u8 *a, const u8 *b)
Definition: mac_address.h:85
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:141
#define vlib_prefetch_buffer_data(b, type)
Definition: buffer.h:204
static u32 eth_identify_subint(vnet_hw_interface_t *hi, u32 match_flags, main_intf_t *main_intf, vlan_intf_t *vlan_intf, qinq_intf_t *qinq_intf, u32 *new_sw_if_index, u8 *error0, u32 *is_l2)
Definition: ethernet.h:486
static void * vlib_frame_scalar_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:258
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:458
#define SUBINT_CONFIG_MATCH_1_TAG
Definition: ethernet.h:195
static_always_inline void parse_header(ethernet_input_variant_t variant, vlib_buffer_t *b0, u16 *type, u16 *orig_type, u16 *outer_id, u16 *inner_id, u32 *match_flags)
Definition: node.c:103
static_always_inline void eth_input_adv_and_flags_x1(vlib_buffer_t **b, int is_l3)
Definition: node.c:415
static void eth_vlan_table_lookups(ethernet_main_t *em, vnet_main_t *vnm, u32 port_sw_if_index0, u16 first_ethertype, u16 outer_id, u16 inner_id, vnet_hw_interface_t **hi, main_intf_t **main_intf, vlan_intf_t **vlan_intf, qinq_intf_t **qinq_intf)
Definition: ethernet.h:447
signed int i32
Definition: types.h:77
static uword sparse_vec_index(void *v, uword sparse_index)
Definition: sparse_vec.h:161
static_always_inline void ethernet_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *from, u32 n_packets, ethernet_input_variant_t variant)
Definition: node.c:1018
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:513
#define SUBINT_CONFIG_L2
Definition: ethernet.h:199
vlan_table_t * vlan_pool
Definition: ethernet.h:284
#define ASSERT(truth)
u8 * format_ethernet_header(u8 *s, va_list *args)
Definition: format.c:178
u32 redirect_l3_next
Definition: ethernet.h:269
void ethernet_register_l3_redirect(vlib_main_t *vm, u32 node_index)
Definition: node.c:2113
u16 flags
Definition: node.h:386
void ethernet_sw_interface_set_l2_mode_noport(vnet_main_t *vnm, u32 sw_if_index, u32 l2)
Definition: node.c:1801
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:78
static_always_inline void determine_next_node(ethernet_main_t *em, ethernet_input_variant_t variant, u32 is_l20, u32 type0, vlib_buffer_t *b0, u8 *error0, u8 *next0)
Definition: node.c:239
void ethernet_register_input_type(vlib_main_t *vm, ethernet_type_t type, u32 node_index)
Definition: node.c:2061
#define clib_error_report(e)
Definition: error.h:113
static u8 * format_ethernet_input_trace(u8 *s, va_list *va)
Definition: node.c:71
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
STATIC_ASSERT(STRUCT_OFFSET_OF(vnet_buffer_opaque_t, l2_hdr_offset)==STRUCT_OFFSET_OF(vnet_buffer_opaque_t, l3_hdr_offset) - 2, "l3_hdr_offset must follow l2_hdr_offset")
vlan_intf_t vlans[ETHERNET_N_VLAN]
Definition: ethernet.h:228
#define SUBINT_CONFIG_MATCH_2_TAG
Definition: ethernet.h:196
clib_error_t * next_by_ethertype_register(next_by_ethertype_t *l3_next, u32 ethertype, u32 next_index)
Definition: node.c:1986
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:57
#define vec_elt(v, i)
Get vector value at index i.
subint_config_t subint
Sub-interface config.
Definition: pipe.h:30
vlib_node_registration_t ethernet_input_not_l2_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_not_l2_node)
Definition: node.c:1926
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
#define foreach_ethernet_input_next
Definition: node.c:48
#define SUBINT_CONFIG_P2P
Definition: ethernet.h:200
void l2bvi_register_input_type(vlib_main_t *vm, ethernet_type_t type, u32 node_index)
Definition: l2_bvi.c:29
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
pipe_t * pipe_get(u32 sw_if_index)
Get the pipe instnace based on one end.
Definition: pipe.c:95
static_always_inline int ethernet_frame_is_any_tagged_x2(u16 type0, u16 type1)
Definition: ethernet.h:92
VLIB buffer representation.
Definition: buffer.h:102
static_always_inline void eth_input_adv_and_flags_x4(vlib_buffer_t **b, int is_l3)
Definition: node.c:313
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
static clib_error_t * ethernet_init(vlib_main_t *vm)
Definition: init.c:83
void ethernet_sw_interface_set_l2_mode(vnet_main_t *vnm, u32 sw_if_index, u32 l2)
Definition: node.c:1752
struct clib_bihash_value offset
template key/value backing page structure
u64x4
Definition: vector_avx2.h:121
vnet_pcap_t pcap[VLIB_N_RX_TX]
Definition: main.h:149
#define vnet_buffer(b)
Definition: buffer.h:369
p2p_ethernet_main_t p2p_main
Definition: p2p_ethernet.c:23
vnet_sw_interface_type_t type
Definition: interface.h:682
qinq_table_t * qinq_pool
Definition: ethernet.h:287
#define u16x16_blend(v1, v2, mask)
Definition: vector_avx2.h:191
u16 flags
Copy of main node flags.
Definition: node.h:507
next_by_ethertype_t l3_next
Definition: ethernet.h:262
subint_config_t * p2p_subif_pool
Definition: p2p_ethernet.h:31
static clib_error_t * ethernet_sw_interface_up_down(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: node.c:1721
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:301
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static void * sparse_vec_new(uword elt_bytes, uword sparse_index_bits)
Definition: sparse_vec.h:71
ethernet_interface_t * interfaces
Definition: ethernet.h:272
static_always_inline void eth_input_update_if_counters(vlib_main_t *vm, vnet_main_t *vnm, eth_input_tag_lookup_t *l)
Definition: node.c:472
epu16_epi64 u16x16
Definition: vector_avx2.h:123
ethernet_input_frame_t frame_data
Definition: node.c:67
#define BITS(x)
Definition: clib.h:61
static_always_inline void eth_input_get_etype_and_tags(vlib_buffer_t **b, u16 *etype, u64 *tags, u64 *dmacs, int offset, int dmac_check)
Definition: node.c:433
static void pcap_add_buffer(pcap_main_t *pm, struct vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
Definition: pcap_funcs.h:63
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static_always_inline void ethernet_input_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: node.c:962
static void ethernet_setup_node(vlib_main_t *vm, u32 node_index)
Definition: ethernet.h:365
Definition: defs.h:46
uword unformat_ethernet_header(unformat_input_t *input, va_list *args)
Definition: format.c:290
u16 * input_next_by_type
Definition: ethernet.h:248
#define SPARSE_VEC_INVALID_INDEX
Definition: sparse_vec.h:68
signed short i16
Definition: types.h:46