FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * ethernet_node.c: ethernet packet processing
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <vlib/vlib.h>
41 #include <vnet/pg/pg.h>
42 #include <vnet/ethernet/ethernet.h>
44 #include <vnet/devices/pipe/pipe.h>
45 #include <vppinfra/sparse_vec.h>
46 #include <vnet/l2/l2_bvi.h>
48 
49 #define foreach_ethernet_input_next \
50  _ (PUNT, "error-punt") \
51  _ (DROP, "error-drop") \
52  _ (LLC, "llc-input") \
53  _ (IP4_INPUT, "ip4-input") \
54  _ (IP4_INPUT_NCS, "ip4-input-no-checksum")
55 
56 typedef enum
57 {
58 #define _(s,n) ETHERNET_INPUT_NEXT_##s,
60 #undef _
63 
64 typedef struct
65 {
66  u8 packet_data[32];
70 
71 static u8 *
72 format_ethernet_input_trace (u8 * s, va_list * va)
73 {
74  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
75  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
77  u32 indent = format_get_indent (s);
78 
79  if (t->frame_flags)
80  {
81  s = format (s, "frame: flags 0x%x", t->frame_flags);
83  s = format (s, ", hw-if-index %u, sw-if-index %u",
85  s = format (s, "\n%U", format_white_space, indent);
86  }
87  s = format (s, "%U", format_ethernet_header, t->packet_data);
88 
89  return s;
90 }
91 
93 
94 typedef enum
95 {
100 
101 
102 // Parse the ethernet header to extract vlan tags and innermost ethertype
105  vlib_buffer_t * b0,
106  u16 * type,
107  u16 * orig_type,
108  u16 * outer_id, u16 * inner_id, u32 * match_flags)
109 {
110  u8 vlan_count;
111 
112  if (variant == ETHERNET_INPUT_VARIANT_ETHERNET
113  || variant == ETHERNET_INPUT_VARIANT_NOT_L2)
114  {
115  ethernet_header_t *e0;
116 
117  e0 = (void *) (b0->data + b0->current_data);
118 
119  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
120  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
121 
122  vlib_buffer_advance (b0, sizeof (e0[0]));
123 
124  *type = clib_net_to_host_u16 (e0->type);
125  }
126  else if (variant == ETHERNET_INPUT_VARIANT_ETHERNET_TYPE)
127  {
128  // here when prior node was LLC/SNAP processing
129  u16 *e0;
130 
131  e0 = (void *) (b0->data + b0->current_data);
132 
133  vlib_buffer_advance (b0, sizeof (e0[0]));
134 
135  *type = clib_net_to_host_u16 (e0[0]);
136  }
137 
138  // save for distinguishing between dot1q and dot1ad later
139  *orig_type = *type;
140 
141  // default the tags to 0 (used if there is no corresponding tag)
142  *outer_id = 0;
143  *inner_id = 0;
144 
146  vlan_count = 0;
147 
148  // check for vlan encaps
149  if (ethernet_frame_is_tagged (*type))
150  {
152  u16 tag;
153 
155 
156  h0 = (void *) (b0->data + b0->current_data);
157 
158  tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
159 
160  *outer_id = tag & 0xfff;
161  if (0 == *outer_id)
162  *match_flags &= ~SUBINT_CONFIG_MATCH_1_TAG;
163 
164  *type = clib_net_to_host_u16 (h0->type);
165 
166  vlib_buffer_advance (b0, sizeof (h0[0]));
167  vlan_count = 1;
168 
169  if (*type == ETHERNET_TYPE_VLAN)
170  {
171  // Double tagged packet
173 
174  h0 = (void *) (b0->data + b0->current_data);
175 
176  tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
177 
178  *inner_id = tag & 0xfff;
179 
180  *type = clib_net_to_host_u16 (h0->type);
181 
182  vlib_buffer_advance (b0, sizeof (h0[0]));
183  vlan_count = 2;
184  if (*type == ETHERNET_TYPE_VLAN)
185  {
186  // More than double tagged packet
188 
189  vlib_buffer_advance (b0, sizeof (h0[0]));
190  vlan_count = 3; // "unknown" number, aka, 3-or-more
191  }
192  }
193  }
194  ethernet_buffer_set_vlan_count (b0, vlan_count);
195 }
196 
197 // Determine the subinterface for this packet, given the result of the
198 // vlan table lookups and vlan header parsing. Check the most specific
199 // matches first.
202  vlib_buffer_t * b0,
203  u32 match_flags,
204  main_intf_t * main_intf,
205  vlan_intf_t * vlan_intf,
206  qinq_intf_t * qinq_intf,
207  u32 * new_sw_if_index, u8 * error0, u32 * is_l2)
208 {
209  u32 matched;
210 
211  matched = eth_identify_subint (hi, match_flags, main_intf, vlan_intf,
212  qinq_intf, new_sw_if_index, error0, is_l2);
213 
214  if (matched)
215  {
216 
217  // Perform L3 my-mac filter
218  // A unicast packet arriving on an L3 interface must have a dmac matching the interface mac.
219  // This is required for promiscuous mode, else we will forward packets we aren't supposed to.
220  if (!(*is_l2))
221  {
222  ethernet_header_t *e0;
223  e0 = (void *) (b0->data + vnet_buffer (b0)->l2_hdr_offset);
224 
225  if (!(ethernet_address_cast (e0->dst_address)))
226  {
227  if (!ethernet_mac_address_equal ((u8 *) e0, hi->hw_address))
228  {
229  *error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
230  }
231  }
232  }
233 
234  // Check for down subinterface
235  *error0 = (*new_sw_if_index) != ~0 ? (*error0) : ETHERNET_ERROR_DOWN;
236  }
237 }
238 
241  ethernet_input_variant_t variant,
242  u32 is_l20,
243  u32 type0, vlib_buffer_t * b0, u8 * error0, u8 * next0)
244 {
245  vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
246  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
247 
248  if (PREDICT_FALSE (*error0 != ETHERNET_ERROR_NONE))
249  {
250  // some error occurred
251  *next0 = ETHERNET_INPUT_NEXT_DROP;
252  }
253  else if (is_l20)
254  {
255  // record the L2 len and reset the buffer so the L2 header is preserved
256  u32 eth_start = vnet_buffer (b0)->l2_hdr_offset;
257  vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start;
258  *next0 = em->l2_next;
259  ASSERT (vnet_buffer (b0)->l2.l2_len ==
261  vlib_buffer_advance (b0, -(vnet_buffer (b0)->l2.l2_len));
262 
263  // check for common IP/MPLS ethertypes
264  }
265  else if (type0 == ETHERNET_TYPE_IP4)
266  {
267  *next0 = em->l3_next.input_next_ip4;
268  }
269  else if (type0 == ETHERNET_TYPE_IP6)
270  {
271  *next0 = em->l3_next.input_next_ip6;
272  }
273  else if (type0 == ETHERNET_TYPE_MPLS)
274  {
275  *next0 = em->l3_next.input_next_mpls;
276 
277  }
278  else if (em->redirect_l3)
279  {
280  // L3 Redirect is on, the cached common next nodes will be
281  // pointing to the redirect node, catch the uncommon types here
282  *next0 = em->redirect_l3_next;
283  }
284  else
285  {
286  // uncommon ethertype, check table
287  u32 i0;
288  i0 = sparse_vec_index (em->l3_next.input_next_by_type, type0);
289  *next0 = vec_elt (em->l3_next.input_next_by_type, i0);
290  *error0 =
291  i0 ==
292  SPARSE_VEC_INVALID_INDEX ? ETHERNET_ERROR_UNKNOWN_TYPE : *error0;
293 
294  // The table is not populated with LLC values, so check that now.
295  // If variant is variant_ethernet then we came from LLC processing. Don't
296  // go back there; drop instead using by keeping the drop/bad table result.
297  if ((type0 < 0x600) && (variant == ETHERNET_INPUT_VARIANT_ETHERNET))
298  {
299  *next0 = ETHERNET_INPUT_NEXT_LLC;
300  }
301  }
302 }
303 
304 
305 /* following vector code relies on following assumptions */
306 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_data, 0);
307 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_length, 2);
310  STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l3_hdr_offset) - 2,
311  "l3_hdr_offset must follow l2_hdr_offset");
312 
315 {
316  i16 adv = sizeof (ethernet_header_t);
317  u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
318  VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
319 
320 #ifdef CLIB_HAVE_VEC256
321  /* to reduce number of small loads/stores we are loading first 64 bits
322  of each buffer metadata into 256-bit register so we can advance
323  current_data, current_length and flags.
324  Observed saving of this code is ~2 clocks per packet */
325  u64x4 r, radv;
326 
327  /* vector if signed 16 bit integers used in signed vector add operation
328  to advnce current_data and current_length */
329  u32x8 flags4 = { 0, flags, 0, flags, 0, flags, 0, flags };
330  i16x16 adv4 = {
331  adv, -adv, 0, 0, adv, -adv, 0, 0,
332  adv, -adv, 0, 0, adv, -adv, 0, 0
333  };
334 
335  /* load 4 x 64 bits */
336  r = u64x4_gather (b[0], b[1], b[2], b[3]);
337 
338  /* set flags */
339  r |= (u64x4) flags4;
340 
341  /* advance buffer */
342  radv = (u64x4) ((i16x16) r + adv4);
343 
344  /* write 4 x 64 bits */
345  u64x4_scatter (is_l3 ? radv : r, b[0], b[1], b[2], b[3]);
346 
347  /* use old current_data as l2_hdr_offset and new current_data as
348  l3_hdr_offset */
349  r = (u64x4) u16x16_blend (r, radv << 16, 0xaa);
350 
351  /* store both l2_hdr_offset and l3_hdr_offset in single store operation */
352  u32x8_scatter_one ((u32x8) r, 0, &vnet_buffer (b[0])->l2_hdr_offset);
353  u32x8_scatter_one ((u32x8) r, 2, &vnet_buffer (b[1])->l2_hdr_offset);
354  u32x8_scatter_one ((u32x8) r, 4, &vnet_buffer (b[2])->l2_hdr_offset);
355  u32x8_scatter_one ((u32x8) r, 6, &vnet_buffer (b[3])->l2_hdr_offset);
356 
357  if (is_l3)
358  {
359  ASSERT (b[0]->current_data == vnet_buffer (b[0])->l3_hdr_offset);
360  ASSERT (b[1]->current_data == vnet_buffer (b[1])->l3_hdr_offset);
361  ASSERT (b[2]->current_data == vnet_buffer (b[2])->l3_hdr_offset);
362  ASSERT (b[3]->current_data == vnet_buffer (b[3])->l3_hdr_offset);
363 
364  ASSERT (b[0]->current_data - vnet_buffer (b[0])->l2_hdr_offset == adv);
365  ASSERT (b[1]->current_data - vnet_buffer (b[1])->l2_hdr_offset == adv);
366  ASSERT (b[2]->current_data - vnet_buffer (b[2])->l2_hdr_offset == adv);
367  ASSERT (b[3]->current_data - vnet_buffer (b[3])->l2_hdr_offset == adv);
368  }
369  else
370  {
371  ASSERT (b[0]->current_data == vnet_buffer (b[0])->l2_hdr_offset);
372  ASSERT (b[1]->current_data == vnet_buffer (b[1])->l2_hdr_offset);
373  ASSERT (b[2]->current_data == vnet_buffer (b[2])->l2_hdr_offset);
374  ASSERT (b[3]->current_data == vnet_buffer (b[3])->l2_hdr_offset);
375 
376  ASSERT (b[0]->current_data - vnet_buffer (b[0])->l3_hdr_offset == -adv);
377  ASSERT (b[1]->current_data - vnet_buffer (b[1])->l3_hdr_offset == -adv);
378  ASSERT (b[2]->current_data - vnet_buffer (b[2])->l3_hdr_offset == -adv);
379  ASSERT (b[3]->current_data - vnet_buffer (b[3])->l3_hdr_offset == -adv);
380  }
381 
382 #else
383  vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
384  vnet_buffer (b[1])->l2_hdr_offset = b[1]->current_data;
385  vnet_buffer (b[2])->l2_hdr_offset = b[2]->current_data;
386  vnet_buffer (b[3])->l2_hdr_offset = b[3]->current_data;
387  vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
388  vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data + adv;
389  vnet_buffer (b[2])->l3_hdr_offset = b[2]->current_data + adv;
390  vnet_buffer (b[3])->l3_hdr_offset = b[3]->current_data + adv;
391 
392  if (is_l3)
393  {
394  vlib_buffer_advance (b[0], adv);
395  vlib_buffer_advance (b[1], adv);
396  vlib_buffer_advance (b[2], adv);
397  vlib_buffer_advance (b[3], adv);
398  }
399 
400  b[0]->flags |= flags;
401  b[1]->flags |= flags;
402  b[2]->flags |= flags;
403  b[3]->flags |= flags;
404 #endif
405 
406  if (!is_l3)
407  {
408  vnet_buffer (b[0])->l2.l2_len = adv;
409  vnet_buffer (b[1])->l2.l2_len = adv;
410  vnet_buffer (b[2])->l2.l2_len = adv;
411  vnet_buffer (b[3])->l2.l2_len = adv;
412  }
413 }
414 
417 {
418  i16 adv = sizeof (ethernet_header_t);
419  u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
420  VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
421 
422  vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
423  vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
424 
425  if (is_l3)
426  vlib_buffer_advance (b[0], adv);
427  b[0]->flags |= flags;
428  if (!is_l3)
429  vnet_buffer (b[0])->l2.l2_len = adv;
430 }
431 
432 
435  u64 * dmacs, int offset, int dmac_check)
436 {
438  e = vlib_buffer_get_current (b[offset]);
439 #ifdef CLIB_HAVE_VEC128
440  u64x2 r = u64x2_load_unaligned (((u8 *) & e->type) - 6);
441  etype[offset] = ((u16x8) r)[3];
442  tags[offset] = r[1];
443 #else
444  etype[offset] = e->type;
445  tags[offset] = *(u64 *) (e + 1);
446 #endif
447 
448  if (dmac_check)
449  dmacs[offset] = *(u64 *) e;
450 }
451 
454 {
456 
457  return (etype < 0x600) ? ETHERNET_INPUT_NEXT_LLC :
460 }
461 
462 typedef struct
463 {
464  u64 tag, mask;
466  u16 type, len, next;
468  u8 err, n_tags;
469  u64 n_packets, n_bytes;
471 
475 {
476  if (l->n_packets == 0 || l->sw_if_index == ~0)
477  return;
478 
479  if (l->adv > 0)
480  l->n_bytes += l->n_packets * l->len;
481 
485  l->n_packets, l->n_bytes);
486 }
487 
491  u64 tag, u16 * next, vlib_buffer_t * b,
492  eth_input_tag_lookup_t * l, u8 dmac_bad, int is_dot1ad,
493  int main_is_l3, int check_dmac)
494 {
496 
497  if ((tag ^ l->tag) & l->mask)
498  {
500  vlan_intf_t *vif;
501  qinq_intf_t *qif;
502  vlan_table_t *vlan_table;
503  qinq_table_t *qinq_table;
504  u16 *t = (u16 *) & tag;
505  u16 vlan1 = clib_net_to_host_u16 (t[0]) & 0xFFF;
506  u16 vlan2 = clib_net_to_host_u16 (t[2]) & 0xFFF;
507  u32 matched, is_l2, new_sw_if_index;
508 
509  vlan_table = vec_elt_at_index (em->vlan_pool, is_dot1ad ?
510  mif->dot1ad_vlans : mif->dot1q_vlans);
511  vif = &vlan_table->vlans[vlan1];
512  qinq_table = vec_elt_at_index (em->qinq_pool, vif->qinqs);
513  qif = &qinq_table->vlans[vlan2];
514  l->err = ETHERNET_ERROR_NONE;
515  l->type = clib_net_to_host_u16 (t[1]);
516 
517  if (l->type == ETHERNET_TYPE_VLAN)
518  {
519  l->type = clib_net_to_host_u16 (t[3]);
520  l->n_tags = 2;
522  SUBINT_CONFIG_MATCH_2_TAG, mif, vif,
523  qif, &new_sw_if_index, &l->err,
524  &is_l2);
525  }
526  else
527  {
528  l->n_tags = 1;
529  if (vlan1 == 0)
530  {
531  new_sw_if_index = hi->sw_if_index;
532  l->err = ETHERNET_ERROR_NONE;
533  matched = 1;
534  is_l2 = main_is_l3 == 0;
535  }
536  else
539  vif, qif, &new_sw_if_index,
540  &l->err, &is_l2);
541  }
542 
543  if (l->sw_if_index != new_sw_if_index)
544  {
545  eth_input_update_if_counters (vm, vnm, l);
546  l->n_packets = 0;
547  l->n_bytes = 0;
548  l->sw_if_index = new_sw_if_index;
549  }
550  l->tag = tag;
551  l->mask = (l->n_tags == 2) ?
552  clib_net_to_host_u64 (0xffffffffffffffff) :
553  clib_net_to_host_u64 (0xffffffff00000000);
554 
555  if (matched && l->sw_if_index == ~0)
556  l->err = ETHERNET_ERROR_DOWN;
557 
558  l->len = sizeof (ethernet_header_t) +
559  l->n_tags * sizeof (ethernet_vlan_header_t);
560  if (main_is_l3)
561  l->adv = is_l2 ? -(int) sizeof (ethernet_header_t) :
562  l->n_tags * sizeof (ethernet_vlan_header_t);
563  else
564  l->adv = is_l2 ? 0 : l->len;
565 
566  if (PREDICT_FALSE (l->err != ETHERNET_ERROR_NONE))
567  l->next = ETHERNET_INPUT_NEXT_DROP;
568  else if (is_l2)
569  l->next = em->l2_next;
570  else if (l->type == ETHERNET_TYPE_IP4)
571  l->next = em->l3_next.input_next_ip4;
572  else if (l->type == ETHERNET_TYPE_IP6)
573  l->next = em->l3_next.input_next_ip6;
574  else if (l->type == ETHERNET_TYPE_MPLS)
575  l->next = em->l3_next.input_next_mpls;
576  else if (em->redirect_l3)
577  l->next = em->redirect_l3_next;
578  else
579  {
580  l->next = eth_input_next_by_type (l->type);
581  if (l->next == ETHERNET_INPUT_NEXT_PUNT)
582  l->err = ETHERNET_ERROR_UNKNOWN_TYPE;
583  }
584  }
585 
586  if (check_dmac && l->adv > 0 && dmac_bad)
587  {
588  l->err = ETHERNET_ERROR_L3_MAC_MISMATCH;
589  next[0] = ETHERNET_INPUT_NEXT_PUNT;
590  }
591  else
592  next[0] = l->next;
593 
594  vlib_buffer_advance (b, l->adv);
595  vnet_buffer (b)->l2.l2_len = l->len;
596  vnet_buffer (b)->l3_hdr_offset = vnet_buffer (b)->l2_hdr_offset + l->len;
597 
598  if (l->err == ETHERNET_ERROR_NONE)
599  {
600  vnet_buffer (b)->sw_if_index[VLIB_RX] = l->sw_if_index;
602  }
603  else
604  b->error = node->errors[l->err];
605 
606  /* update counters */
607  l->n_packets += 1;
608  l->n_bytes += vlib_buffer_length_in_chain (vm, b);
609 }
610 
613  u64 * dmacs, u8 * dmacs_bad,
614  u32 n_packets)
615 {
616  u64 mask = clib_net_to_host_u64 (0xFFFFFFFFFFFF0000);
617  u64 igbit = clib_net_to_host_u64 (0x0100000000000000);
618  u64 hwaddr = (*(u64 *) hi->hw_address) & mask;
619  u64 *dmac = dmacs;
620  u8 *dmac_bad = dmacs_bad;
621 
622  i32 n_left = n_packets;
623 
624 #ifdef CLIB_HAVE_VEC256
625  u64x4 igbit4 = u64x4_splat (igbit);
626  u64x4 mask4 = u64x4_splat (mask);
627  u64x4 hwaddr4 = u64x4_splat (hwaddr);
628  while (n_left > 0)
629  {
630  u64x4 r0, r1;
631  r0 = u64x4_load_unaligned (dmac + 0) & mask4;
632  r1 = u64x4_load_unaligned (dmac + 4) & mask4;
633 
634  r0 = (r0 != hwaddr4) & ((r0 & igbit4) == 0);
635  r1 = (r1 != hwaddr4) & ((r1 & igbit4) == 0);
636 
637  *(u32 *) (dmac_bad + 0) = u8x32_msb_mask ((u8x32) (r0));
638  *(u32 *) (dmac_bad + 4) = u8x32_msb_mask ((u8x32) (r1));
639 
640  /* next */
641  dmac += 8;
642  dmac_bad += 8;
643  n_left -= 8;
644  }
645 #else
646  while (n_left > 0)
647  {
648  u64 r0, r1, r2, r3;
649 
650  r0 = dmac[0] & mask;
651  r1 = dmac[1] & mask;
652  r2 = dmac[2] & mask;
653  r3 = dmac[3] & mask;
654 
655  r0 = (r0 != hwaddr) && ((r0 & igbit) == 0);
656  r1 = (r1 != hwaddr) && ((r1 & igbit) == 0);
657  r2 = (r2 != hwaddr) && ((r2 & igbit) == 0);
658  r3 = (r3 != hwaddr) && ((r3 & igbit) == 0);
659 
660  dmac_bad[0] = r0;
661  dmac_bad[1] = r1;
662  dmac_bad[2] = r2;
663  dmac_bad[3] = r3;
664 
665  /* next */
666  dmac += 4;
667  dmac_bad += 4;
668  n_left -= 4;
669  }
670 #endif
671 }
672 
673 /* process frame of buffers, store ethertype into array and update
674  buffer metadata fields depending on interface being l2 or l3 assuming that
675  packets are untagged. For tagged packets those fields are updated later.
676  Optionally store Destionation MAC address and tag data into arrays
677  for further processing */
678 
680  "VLIB_FRAME_SIZE must be power of 8");
684  u32 * buffer_indices, u32 n_packets, int main_is_l3,
685  int ip4_cksum_ok, int dmac_check)
686 {
688  u16 nexts[VLIB_FRAME_SIZE], *next;
689  u16 etypes[VLIB_FRAME_SIZE], *etype = etypes;
690  u64 dmacs[VLIB_FRAME_SIZE], *dmac = dmacs;
691  u8 dmacs_bad[VLIB_FRAME_SIZE];
692  u64 tags[VLIB_FRAME_SIZE], *tag = tags;
693  u16 slowpath_indices[VLIB_FRAME_SIZE];
694  u16 n_slowpath, i;
695  u16 next_ip4, next_ip6, next_mpls, next_l2;
696  u16 et_ip4 = clib_host_to_net_u16 (ETHERNET_TYPE_IP4);
697  u16 et_ip6 = clib_host_to_net_u16 (ETHERNET_TYPE_IP6);
698  u16 et_mpls = clib_host_to_net_u16 (ETHERNET_TYPE_MPLS);
699  u16 et_vlan = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN);
700  u16 et_dot1ad = clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD);
701  i32 n_left = n_packets;
702  vlib_buffer_t *b[20];
703  u32 *from;
704 
705  from = buffer_indices;
706 
707  while (n_left >= 20)
708  {
709  vlib_buffer_t **ph = b + 16, **pd = b + 8;
710  vlib_get_buffers (vm, from, b, 4);
711  vlib_get_buffers (vm, from + 8, pd, 4);
712  vlib_get_buffers (vm, from + 16, ph, 4);
713 
714  vlib_prefetch_buffer_header (ph[0], LOAD);
715  vlib_prefetch_buffer_data (pd[0], LOAD);
716  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
717 
718  vlib_prefetch_buffer_header (ph[1], LOAD);
719  vlib_prefetch_buffer_data (pd[1], LOAD);
720  eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
721 
722  vlib_prefetch_buffer_header (ph[2], LOAD);
723  vlib_prefetch_buffer_data (pd[2], LOAD);
724  eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
725 
726  vlib_prefetch_buffer_header (ph[3], LOAD);
727  vlib_prefetch_buffer_data (pd[3], LOAD);
728  eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
729 
730  eth_input_adv_and_flags_x4 (b, main_is_l3);
731 
732  /* next */
733  n_left -= 4;
734  etype += 4;
735  tag += 4;
736  dmac += 4;
737  from += 4;
738  }
739  while (n_left >= 4)
740  {
741  vlib_get_buffers (vm, from, b, 4);
742  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
743  eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
744  eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
745  eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
746  eth_input_adv_and_flags_x4 (b, main_is_l3);
747 
748  /* next */
749  n_left -= 4;
750  etype += 4;
751  tag += 4;
752  dmac += 4;
753  from += 4;
754  }
755  while (n_left)
756  {
757  vlib_get_buffers (vm, from, b, 1);
758  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
759  eth_input_adv_and_flags_x1 (b, main_is_l3);
760 
761  /* next */
762  n_left -= 1;
763  etype += 1;
764  tag += 1;
765  dmac += 1;
766  from += 1;
767  }
768 
769  if (dmac_check)
770  eth_input_process_frame_dmac_check (hi, dmacs, dmacs_bad, n_packets);
771 
772  next_ip4 = em->l3_next.input_next_ip4;
773  next_ip6 = em->l3_next.input_next_ip6;
774  next_mpls = em->l3_next.input_next_mpls;
775  next_l2 = em->l2_next;
776 
777  if (next_ip4 == ETHERNET_INPUT_NEXT_IP4_INPUT && ip4_cksum_ok)
778  next_ip4 = ETHERNET_INPUT_NEXT_IP4_INPUT_NCS;
779 
780 #ifdef CLIB_HAVE_VEC256
781  u16x16 et16_ip4 = u16x16_splat (et_ip4);
782  u16x16 et16_ip6 = u16x16_splat (et_ip6);
783  u16x16 et16_mpls = u16x16_splat (et_mpls);
784  u16x16 et16_vlan = u16x16_splat (et_vlan);
785  u16x16 et16_dot1ad = u16x16_splat (et_dot1ad);
786  u16x16 next16_ip4 = u16x16_splat (next_ip4);
787  u16x16 next16_ip6 = u16x16_splat (next_ip6);
788  u16x16 next16_mpls = u16x16_splat (next_mpls);
789  u16x16 next16_l2 = u16x16_splat (next_l2);
790  u16x16 zero = { 0 };
791  u16x16 stairs = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
792 #endif
793 
794  etype = etypes;
795  n_left = n_packets;
796  next = nexts;
797  n_slowpath = 0;
798  i = 0;
799 
800  /* fastpath - in l3 mode hadles ip4, ip6 and mpls packets, other packets
801  are considered as slowpath, in l2 mode all untagged packets are
802  considered as fastpath */
803  while (n_left > 0)
804  {
805 #ifdef CLIB_HAVE_VEC256
806  if (n_left >= 16)
807  {
808  u16x16 r = zero;
809  u16x16 e16 = u16x16_load_unaligned (etype);
810  if (main_is_l3)
811  {
812  r += (e16 == et16_ip4) & next16_ip4;
813  r += (e16 == et16_ip6) & next16_ip6;
814  r += (e16 == et16_mpls) & next16_mpls;
815  }
816  else
817  r = ((e16 != et16_vlan) & (e16 != et16_dot1ad)) & next16_l2;
818  u16x16_store_unaligned (r, next);
819 
820  if (!u16x16_is_all_zero (r == zero))
821  {
822  if (u16x16_is_all_zero (r))
823  {
824  u16x16_store_unaligned (u16x16_splat (i) + stairs,
825  slowpath_indices + n_slowpath);
826  n_slowpath += 16;
827  }
828  else
829  {
830  for (int j = 0; j < 16; j++)
831  if (next[j] == 0)
832  slowpath_indices[n_slowpath++] = i + j;
833  }
834  }
835 
836  etype += 16;
837  next += 16;
838  n_left -= 16;
839  i += 16;
840  continue;
841  }
842 #endif
843  if (main_is_l3 && etype[0] == et_ip4)
844  next[0] = next_ip4;
845  else if (main_is_l3 && etype[0] == et_ip6)
846  next[0] = next_ip6;
847  else if (main_is_l3 && etype[0] == et_mpls)
848  next[0] = next_mpls;
849  else if (main_is_l3 == 0 &&
850  etype[0] != et_vlan && etype[0] != et_dot1ad)
851  next[0] = next_l2;
852  else
853  {
854  next[0] = 0;
855  slowpath_indices[n_slowpath++] = i;
856  }
857 
858  etype += 1;
859  next += 1;
860  n_left -= 1;
861  i += 1;
862  }
863 
864  if (n_slowpath)
865  {
866  vnet_main_t *vnm = vnet_get_main ();
867  n_left = n_slowpath;
868  u16 *si = slowpath_indices;
869  u32 last_unknown_etype = ~0;
870  u32 last_unknown_next = ~0;
871  eth_input_tag_lookup_t dot1ad_lookup, dot1q_lookup = {
872  .mask = -1LL,
873  .tag = tags[si[0]] ^ -1LL,
874  .sw_if_index = ~0
875  };
876 
877  clib_memcpy_fast (&dot1ad_lookup, &dot1q_lookup, sizeof (dot1q_lookup));
878 
879  while (n_left)
880  {
881  i = si[0];
882  u16 etype = etypes[i];
883 
884  if (etype == et_vlan)
885  {
886  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
887  eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
888  &dot1q_lookup, dmacs_bad[i], 0,
889  main_is_l3, dmac_check);
890 
891  }
892  else if (etype == et_dot1ad)
893  {
894  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
895  eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
896  &dot1ad_lookup, dmacs_bad[i], 1,
897  main_is_l3, dmac_check);
898  }
899  else
900  {
901  /* untagged packet with not well known etyertype */
902  if (last_unknown_etype != etype)
903  {
904  last_unknown_etype = etype;
905  etype = clib_host_to_net_u16 (etype);
906  last_unknown_next = eth_input_next_by_type (etype);
907  }
908  if (dmac_check && main_is_l3 && dmacs_bad[i])
909  {
910  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
911  b->error = node->errors[ETHERNET_ERROR_L3_MAC_MISMATCH];
912  nexts[i] = ETHERNET_INPUT_NEXT_PUNT;
913  }
914  else
915  nexts[i] = last_unknown_next;
916  }
917 
918  /* next */
919  n_left--;
920  si++;
921  }
922 
923  eth_input_update_if_counters (vm, vnm, &dot1q_lookup);
924  eth_input_update_if_counters (vm, vnm, &dot1ad_lookup);
925  }
926 
927  vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts, n_packets);
928 }
929 
932  vnet_hw_interface_t * hi, u32 * from, u32 n_pkts,
933  int ip4_cksum_ok)
934 {
937  ei = pool_elt_at_index (em->interfaces, hi->hw_instance);
939  subint_config_t *subint0 = &intf0->untagged_subint;
940 
941  int main_is_l3 = (subint0->flags & SUBINT_CONFIG_L2) == 0;
942  int promisc = (ei->flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL) != 0;
943 
944  if (main_is_l3)
945  {
946  /* main interface is L3, we dont expect tagged packets and interface
947  is not in promisc node, so we dont't need to check DMAC */
948  int is_l3 = 1;
949 
950  if (promisc == 0)
951  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
952  ip4_cksum_ok, 0);
953  else
954  /* subinterfaces and promisc mode so DMAC check is needed */
955  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
956  ip4_cksum_ok, 1);
957  return;
958  }
959  else
960  {
961  /* untagged packets are treated as L2 */
962  int is_l3 = 0;
963  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
964  ip4_cksum_ok, 1);
965  return;
966  }
967 }
968 
971  vlib_frame_t * from_frame)
972 {
973  u32 *from, n_left;
974  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
975  {
976  from = vlib_frame_vector_args (from_frame);
977  n_left = from_frame->n_vectors;
978 
979  while (n_left)
980  {
982  vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]);
983 
984  if (b0->flags & VLIB_BUFFER_IS_TRACED)
985  {
986  t0 = vlib_add_trace (vm, node, b0,
987  sizeof (ethernet_input_trace_t));
989  sizeof (t0->packet_data));
990  t0->frame_flags = from_frame->flags;
992  vlib_frame_scalar_args (from_frame),
993  sizeof (ethernet_input_frame_t));
994  }
995  from += 1;
996  n_left -= 1;
997  }
998  }
999 
1000  /* rx pcap capture if enabled */
1002  {
1003  u32 bi0;
1005 
1006  from = vlib_frame_vector_args (from_frame);
1007  n_left = from_frame->n_vectors;
1008  while (n_left > 0)
1009  {
1010  int classify_filter_result;
1011  vlib_buffer_t *b0;
1012  bi0 = from[0];
1013  from++;
1014  n_left--;
1015  b0 = vlib_get_buffer (vm, bi0);
1016  if (pp->filter_classify_table_index != ~0)
1017  {
1018  classify_filter_result =
1020  (b0, pp->filter_classify_table_index, 0 /* full classify */ );
1021  if (classify_filter_result)
1022  pcap_add_buffer (&pp->pcap_main, vm, bi0,
1023  pp->max_bytes_per_pkt);
1024  continue;
1025  }
1026 
1027  if (pp->pcap_sw_if_index == 0 ||
1029  {
1030  pcap_add_buffer (&pp->pcap_main, vm, bi0,
1031  pp->max_bytes_per_pkt);
1032  }
1033  }
1034  }
1035 }
1036 
1039  vlib_node_runtime_t * node,
1040  u32 * from, u32 n_packets,
1041  ethernet_input_variant_t variant)
1042 {
1043  vnet_main_t *vnm = vnet_get_main ();
1045  vlib_node_runtime_t *error_node;
1046  u32 n_left_from, next_index, *to_next;
1047  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
1048  u32 thread_index = vm->thread_index;
1049  u32 cached_sw_if_index = ~0;
1050  u32 cached_is_l2 = 0; /* shut up gcc */
1051  vnet_hw_interface_t *hi = NULL; /* used for main interface only */
1053  vlib_buffer_t **b = bufs;
1054 
1055  if (variant != ETHERNET_INPUT_VARIANT_ETHERNET)
1056  error_node = vlib_node_get_runtime (vm, ethernet_input_node.index);
1057  else
1058  error_node = node;
1059 
1060  n_left_from = n_packets;
1061 
1062  next_index = node->cached_next_index;
1063  stats_sw_if_index = node->runtime_data[0];
1064  stats_n_packets = stats_n_bytes = 0;
1065  vlib_get_buffers (vm, from, bufs, n_left_from);
1066 
1067  while (n_left_from > 0)
1068  {
1069  u32 n_left_to_next;
1070 
1071  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1072 
1073  while (n_left_from >= 4 && n_left_to_next >= 2)
1074  {
1075  u32 bi0, bi1;
1076  vlib_buffer_t *b0, *b1;
1077  u8 next0, next1, error0, error1;
1078  u16 type0, orig_type0, type1, orig_type1;
1079  u16 outer_id0, inner_id0, outer_id1, inner_id1;
1080  u32 match_flags0, match_flags1;
1081  u32 old_sw_if_index0, new_sw_if_index0, len0, old_sw_if_index1,
1082  new_sw_if_index1, len1;
1083  vnet_hw_interface_t *hi0, *hi1;
1084  main_intf_t *main_intf0, *main_intf1;
1085  vlan_intf_t *vlan_intf0, *vlan_intf1;
1086  qinq_intf_t *qinq_intf0, *qinq_intf1;
1087  u32 is_l20, is_l21;
1088  ethernet_header_t *e0, *e1;
1089 
1090  /* Prefetch next iteration. */
1091  {
1092  vlib_prefetch_buffer_header (b[2], STORE);
1093  vlib_prefetch_buffer_header (b[3], STORE);
1094 
1095  CLIB_PREFETCH (b[2]->data, sizeof (ethernet_header_t), LOAD);
1096  CLIB_PREFETCH (b[3]->data, sizeof (ethernet_header_t), LOAD);
1097  }
1098 
1099  bi0 = from[0];
1100  bi1 = from[1];
1101  to_next[0] = bi0;
1102  to_next[1] = bi1;
1103  from += 2;
1104  to_next += 2;
1105  n_left_to_next -= 2;
1106  n_left_from -= 2;
1107 
1108  b0 = b[0];
1109  b1 = b[1];
1110  b += 2;
1111 
1112  error0 = error1 = ETHERNET_ERROR_NONE;
1113  e0 = vlib_buffer_get_current (b0);
1114  type0 = clib_net_to_host_u16 (e0->type);
1115  e1 = vlib_buffer_get_current (b1);
1116  type1 = clib_net_to_host_u16 (e1->type);
1117 
1118  /* Set the L2 header offset for all packets */
1119  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1120  vnet_buffer (b1)->l2_hdr_offset = b1->current_data;
1121  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1122  b1->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1123 
1124  /* Speed-path for the untagged case */
1127  type1)))
1128  {
1129  main_intf_t *intf0;
1130  subint_config_t *subint0;
1131  u32 sw_if_index0, sw_if_index1;
1132 
1133  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1134  sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1135  is_l20 = cached_is_l2;
1136 
1137  /* This is probably wholly unnecessary */
1138  if (PREDICT_FALSE (sw_if_index0 != sw_if_index1))
1139  goto slowpath;
1140 
1141  /* Now sw_if_index0 == sw_if_index1 */
1142  if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1143  {
1144  cached_sw_if_index = sw_if_index0;
1145  hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1146  intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1147  subint0 = &intf0->untagged_subint;
1148  cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1149  }
1150 
1151  if (PREDICT_TRUE (is_l20 != 0))
1152  {
1153  vnet_buffer (b0)->l3_hdr_offset =
1154  vnet_buffer (b0)->l2_hdr_offset +
1155  sizeof (ethernet_header_t);
1156  vnet_buffer (b1)->l3_hdr_offset =
1157  vnet_buffer (b1)->l2_hdr_offset +
1158  sizeof (ethernet_header_t);
1159  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1160  b1->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1161  next0 = em->l2_next;
1162  vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1163  next1 = em->l2_next;
1164  vnet_buffer (b1)->l2.l2_len = sizeof (ethernet_header_t);
1165  }
1166  else
1167  {
1168  if (!ethernet_address_cast (e0->dst_address) &&
1169  (hi->hw_address != 0) &&
1170  !ethernet_mac_address_equal ((u8 *) e0, hi->hw_address))
1171  error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1172  if (!ethernet_address_cast (e1->dst_address) &&
1173  (hi->hw_address != 0) &&
1174  !ethernet_mac_address_equal ((u8 *) e1, hi->hw_address))
1175  error1 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1176  vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1177  determine_next_node (em, variant, 0, type0, b0,
1178  &error0, &next0);
1179  vlib_buffer_advance (b1, sizeof (ethernet_header_t));
1180  determine_next_node (em, variant, 0, type1, b1,
1181  &error1, &next1);
1182  }
1183  goto ship_it01;
1184  }
1185 
1186  /* Slow-path for the tagged case */
1187  slowpath:
1188  parse_header (variant,
1189  b0,
1190  &type0,
1191  &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1192 
1193  parse_header (variant,
1194  b1,
1195  &type1,
1196  &orig_type1, &outer_id1, &inner_id1, &match_flags1);
1197 
1198  old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1199  old_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1200 
1202  vnm,
1203  old_sw_if_index0,
1204  orig_type0,
1205  outer_id0,
1206  inner_id0,
1207  &hi0,
1208  &main_intf0, &vlan_intf0, &qinq_intf0);
1209 
1211  vnm,
1212  old_sw_if_index1,
1213  orig_type1,
1214  outer_id1,
1215  inner_id1,
1216  &hi1,
1217  &main_intf1, &vlan_intf1, &qinq_intf1);
1218 
1219  identify_subint (hi0,
1220  b0,
1221  match_flags0,
1222  main_intf0,
1223  vlan_intf0,
1224  qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1225 
1226  identify_subint (hi1,
1227  b1,
1228  match_flags1,
1229  main_intf1,
1230  vlan_intf1,
1231  qinq_intf1, &new_sw_if_index1, &error1, &is_l21);
1232 
1233  // Save RX sw_if_index for later nodes
1234  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1235  error0 !=
1236  ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1237  vnet_buffer (b1)->sw_if_index[VLIB_RX] =
1238  error1 !=
1239  ETHERNET_ERROR_NONE ? old_sw_if_index1 : new_sw_if_index1;
1240 
1241  // Check if there is a stat to take (valid and non-main sw_if_index for pkt 0 or pkt 1)
1242  if (((new_sw_if_index0 != ~0)
1243  && (new_sw_if_index0 != old_sw_if_index0))
1244  || ((new_sw_if_index1 != ~0)
1245  && (new_sw_if_index1 != old_sw_if_index1)))
1246  {
1247 
1248  len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1249  - vnet_buffer (b0)->l2_hdr_offset;
1250  len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data
1251  - vnet_buffer (b1)->l2_hdr_offset;
1252 
1253  stats_n_packets += 2;
1254  stats_n_bytes += len0 + len1;
1255 
1256  if (PREDICT_FALSE
1257  (!(new_sw_if_index0 == stats_sw_if_index
1258  && new_sw_if_index1 == stats_sw_if_index)))
1259  {
1260  stats_n_packets -= 2;
1261  stats_n_bytes -= len0 + len1;
1262 
1263  if (new_sw_if_index0 != old_sw_if_index0
1264  && new_sw_if_index0 != ~0)
1266  interface_main.combined_sw_if_counters
1267  +
1269  thread_index,
1270  new_sw_if_index0, 1,
1271  len0);
1272  if (new_sw_if_index1 != old_sw_if_index1
1273  && new_sw_if_index1 != ~0)
1275  interface_main.combined_sw_if_counters
1276  +
1278  thread_index,
1279  new_sw_if_index1, 1,
1280  len1);
1281 
1282  if (new_sw_if_index0 == new_sw_if_index1)
1283  {
1284  if (stats_n_packets > 0)
1285  {
1289  thread_index,
1290  stats_sw_if_index,
1291  stats_n_packets, stats_n_bytes);
1292  stats_n_packets = stats_n_bytes = 0;
1293  }
1294  stats_sw_if_index = new_sw_if_index0;
1295  }
1296  }
1297  }
1298 
1299  if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1300  is_l20 = is_l21 = 0;
1301 
1302  determine_next_node (em, variant, is_l20, type0, b0, &error0,
1303  &next0);
1304  determine_next_node (em, variant, is_l21, type1, b1, &error1,
1305  &next1);
1306 
1307  ship_it01:
1308  b0->error = error_node->errors[error0];
1309  b1->error = error_node->errors[error1];
1310 
1311  // verify speculative enqueue
1312  vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1313  n_left_to_next, bi0, bi1, next0,
1314  next1);
1315  }
1316 
1317  while (n_left_from > 0 && n_left_to_next > 0)
1318  {
1319  u32 bi0;
1320  vlib_buffer_t *b0;
1321  u8 error0, next0;
1322  u16 type0, orig_type0;
1323  u16 outer_id0, inner_id0;
1324  u32 match_flags0;
1325  u32 old_sw_if_index0, new_sw_if_index0, len0;
1326  vnet_hw_interface_t *hi0;
1327  main_intf_t *main_intf0;
1328  vlan_intf_t *vlan_intf0;
1329  qinq_intf_t *qinq_intf0;
1330  ethernet_header_t *e0;
1331  u32 is_l20;
1332 
1333  // Prefetch next iteration
1334  if (n_left_from > 1)
1335  {
1336  vlib_prefetch_buffer_header (b[1], STORE);
1337  CLIB_PREFETCH (b[1]->data, CLIB_CACHE_LINE_BYTES, LOAD);
1338  }
1339 
1340  bi0 = from[0];
1341  to_next[0] = bi0;
1342  from += 1;
1343  to_next += 1;
1344  n_left_from -= 1;
1345  n_left_to_next -= 1;
1346 
1347  b0 = b[0];
1348  b += 1;
1349 
1350  error0 = ETHERNET_ERROR_NONE;
1351  e0 = vlib_buffer_get_current (b0);
1352  type0 = clib_net_to_host_u16 (e0->type);
1353 
1354  /* Set the L2 header offset for all packets */
1355  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1356  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1357 
1358  /* Speed-path for the untagged case */
1360  && !ethernet_frame_is_tagged (type0)))
1361  {
1362  main_intf_t *intf0;
1363  subint_config_t *subint0;
1364  u32 sw_if_index0;
1365 
1366  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1367  is_l20 = cached_is_l2;
1368 
1369  if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1370  {
1371  cached_sw_if_index = sw_if_index0;
1372  hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1373  intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1374  subint0 = &intf0->untagged_subint;
1375  cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1376  }
1377 
1378 
1379  if (PREDICT_TRUE (is_l20 != 0))
1380  {
1381  vnet_buffer (b0)->l3_hdr_offset =
1382  vnet_buffer (b0)->l2_hdr_offset +
1383  sizeof (ethernet_header_t);
1384  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1385  next0 = em->l2_next;
1386  vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1387  }
1388  else
1389  {
1390  if (!ethernet_address_cast (e0->dst_address) &&
1391  (hi->hw_address != 0) &&
1392  !ethernet_mac_address_equal ((u8 *) e0, hi->hw_address))
1393  error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1394  vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1395  determine_next_node (em, variant, 0, type0, b0,
1396  &error0, &next0);
1397  }
1398  goto ship_it0;
1399  }
1400 
1401  /* Slow-path for the tagged case */
1402  parse_header (variant,
1403  b0,
1404  &type0,
1405  &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1406 
1407  old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1408 
1410  vnm,
1411  old_sw_if_index0,
1412  orig_type0,
1413  outer_id0,
1414  inner_id0,
1415  &hi0,
1416  &main_intf0, &vlan_intf0, &qinq_intf0);
1417 
1418  identify_subint (hi0,
1419  b0,
1420  match_flags0,
1421  main_intf0,
1422  vlan_intf0,
1423  qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1424 
1425  // Save RX sw_if_index for later nodes
1426  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1427  error0 !=
1428  ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1429 
1430  // Increment subinterface stats
1431  // Note that interface-level counters have already been incremented
1432  // prior to calling this function. Thus only subinterface counters
1433  // are incremented here.
1434  //
1435  // Interface level counters include packets received on the main
1436  // interface and all subinterfaces. Subinterface level counters
1437  // include only those packets received on that subinterface
1438  // Increment stats if the subint is valid and it is not the main intf
1439  if ((new_sw_if_index0 != ~0)
1440  && (new_sw_if_index0 != old_sw_if_index0))
1441  {
1442 
1443  len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1444  - vnet_buffer (b0)->l2_hdr_offset;
1445 
1446  stats_n_packets += 1;
1447  stats_n_bytes += len0;
1448 
1449  // Batch stat increments from the same subinterface so counters
1450  // don't need to be incremented for every packet.
1451  if (PREDICT_FALSE (new_sw_if_index0 != stats_sw_if_index))
1452  {
1453  stats_n_packets -= 1;
1454  stats_n_bytes -= len0;
1455 
1456  if (new_sw_if_index0 != ~0)
1460  thread_index, new_sw_if_index0, 1, len0);
1461  if (stats_n_packets > 0)
1462  {
1466  thread_index,
1467  stats_sw_if_index, stats_n_packets, stats_n_bytes);
1468  stats_n_packets = stats_n_bytes = 0;
1469  }
1470  stats_sw_if_index = new_sw_if_index0;
1471  }
1472  }
1473 
1474  if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1475  is_l20 = 0;
1476 
1477  determine_next_node (em, variant, is_l20, type0, b0, &error0,
1478  &next0);
1479 
1480  ship_it0:
1481  b0->error = error_node->errors[error0];
1482 
1483  // verify speculative enqueue
1484  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1485  to_next, n_left_to_next,
1486  bi0, next0);
1487  }
1488 
1489  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1490  }
1491 
1492  // Increment any remaining batched stats
1493  if (stats_n_packets > 0)
1494  {
1498  thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
1499  node->runtime_data[0] = stats_sw_if_index;
1500  }
1501 }
1502 
1504  vlib_node_runtime_t * node,
1505  vlib_frame_t * frame)
1506 {
1507  vnet_main_t *vnm = vnet_get_main ();
1508  u32 *from = vlib_frame_vector_args (frame);
1509  u32 n_packets = frame->n_vectors;
1510 
1511  ethernet_input_trace (vm, node, frame);
1512 
1513  if (frame->flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX)
1514  {
1516  int ip4_cksum_ok = (frame->flags & ETH_INPUT_FRAME_F_IP4_CKSUM_OK) != 0;
1518  eth_input_single_int (vm, node, hi, from, n_packets, ip4_cksum_ok);
1519  }
1520  else
1521  ethernet_input_inline (vm, node, from, n_packets,
1523  return n_packets;
1524 }
1525 
1527  vlib_node_runtime_t * node,
1528  vlib_frame_t * from_frame)
1529 {
1530  u32 *from = vlib_frame_vector_args (from_frame);
1531  u32 n_packets = from_frame->n_vectors;
1532  ethernet_input_trace (vm, node, from_frame);
1533  ethernet_input_inline (vm, node, from, n_packets,
1535  return n_packets;
1536 }
1537 
1539  vlib_node_runtime_t * node,
1540  vlib_frame_t * from_frame)
1541 {
1542  u32 *from = vlib_frame_vector_args (from_frame);
1543  u32 n_packets = from_frame->n_vectors;
1544  ethernet_input_trace (vm, node, from_frame);
1545  ethernet_input_inline (vm, node, from, n_packets,
1547  return n_packets;
1548 }
1549 
1550 
1551 // Return the subinterface config struct for the given sw_if_index
1552 // Also return via parameter the appropriate match flags for the
1553 // configured number of tags.
1554 // On error (unsupported or not ethernet) return 0.
1555 static subint_config_t *
1557  u32 sw_if_index,
1558  u32 * flags, u32 * unsupported)
1559 {
1562  vnet_sw_interface_t *si;
1563  main_intf_t *main_intf;
1564  vlan_table_t *vlan_table;
1565  qinq_table_t *qinq_table;
1566  subint_config_t *subint = 0;
1567 
1568  hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1569 
1570  if (!hi || (hi->hw_class_index != ethernet_hw_interface_class.index))
1571  {
1572  *unsupported = 0;
1573  goto done; // non-ethernet interface
1574  }
1575 
1576  // ensure there's an entry for the main intf (shouldn't really be necessary)
1577  vec_validate (em->main_intfs, hi->hw_if_index);
1578  main_intf = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1579 
1580  // Locate the subint for the given ethernet config
1581  si = vnet_get_sw_interface (vnm, sw_if_index);
1582 
1583  if (si->type == VNET_SW_INTERFACE_TYPE_P2P)
1584  {
1585  p2p_ethernet_main_t *p2pm = &p2p_main;
1586  u32 p2pe_sw_if_index =
1588  if (p2pe_sw_if_index == ~0)
1589  {
1590  pool_get (p2pm->p2p_subif_pool, subint);
1591  si->p2p.pool_index = subint - p2pm->p2p_subif_pool;
1592  }
1593  else
1594  subint = vec_elt_at_index (p2pm->p2p_subif_pool, si->p2p.pool_index);
1595  *flags = SUBINT_CONFIG_P2P;
1596  }
1597  else if (si->type == VNET_SW_INTERFACE_TYPE_PIPE)
1598  {
1599  pipe_t *pipe;
1600 
1601  pipe = pipe_get (sw_if_index);
1602  subint = &pipe->subint;
1603  *flags = SUBINT_CONFIG_P2P;
1604  }
1605  else if (si->sub.eth.flags.default_sub)
1606  {
1607  subint = &main_intf->default_subint;
1608  *flags = SUBINT_CONFIG_MATCH_1_TAG |
1610  }
1611  else if ((si->sub.eth.flags.no_tags) || (si->sub.eth.raw_flags == 0))
1612  {
1613  // if no flags are set then this is a main interface
1614  // so treat as untagged
1615  subint = &main_intf->untagged_subint;
1616  *flags = SUBINT_CONFIG_MATCH_0_TAG;
1617  }
1618  else
1619  {
1620  // one or two tags
1621  // first get the vlan table
1622  if (si->sub.eth.flags.dot1ad)
1623  {
1624  if (main_intf->dot1ad_vlans == 0)
1625  {
1626  // Allocate a vlan table from the pool
1627  pool_get (em->vlan_pool, vlan_table);
1628  main_intf->dot1ad_vlans = vlan_table - em->vlan_pool;
1629  }
1630  else
1631  {
1632  // Get ptr to existing vlan table
1633  vlan_table =
1634  vec_elt_at_index (em->vlan_pool, main_intf->dot1ad_vlans);
1635  }
1636  }
1637  else
1638  { // dot1q
1639  if (main_intf->dot1q_vlans == 0)
1640  {
1641  // Allocate a vlan table from the pool
1642  pool_get (em->vlan_pool, vlan_table);
1643  main_intf->dot1q_vlans = vlan_table - em->vlan_pool;
1644  }
1645  else
1646  {
1647  // Get ptr to existing vlan table
1648  vlan_table =
1649  vec_elt_at_index (em->vlan_pool, main_intf->dot1q_vlans);
1650  }
1651  }
1652 
1653  if (si->sub.eth.flags.one_tag)
1654  {
1655  *flags = si->sub.eth.flags.exact_match ?
1659 
1660  if (si->sub.eth.flags.outer_vlan_id_any)
1661  {
1662  // not implemented yet
1663  *unsupported = 1;
1664  goto done;
1665  }
1666  else
1667  {
1668  // a single vlan, a common case
1669  subint =
1670  &vlan_table->vlans[si->sub.eth.
1671  outer_vlan_id].single_tag_subint;
1672  }
1673 
1674  }
1675  else
1676  {
1677  // Two tags
1678  *flags = si->sub.eth.flags.exact_match ?
1681 
1682  if (si->sub.eth.flags.outer_vlan_id_any
1683  && si->sub.eth.flags.inner_vlan_id_any)
1684  {
1685  // not implemented yet
1686  *unsupported = 1;
1687  goto done;
1688  }
1689 
1690  if (si->sub.eth.flags.inner_vlan_id_any)
1691  {
1692  // a specific outer and "any" inner
1693  // don't need a qinq table for this
1694  subint =
1695  &vlan_table->vlans[si->sub.eth.
1696  outer_vlan_id].inner_any_subint;
1697  if (si->sub.eth.flags.exact_match)
1698  {
1699  *flags = SUBINT_CONFIG_MATCH_2_TAG;
1700  }
1701  else
1702  {
1703  *flags = SUBINT_CONFIG_MATCH_2_TAG |
1705  }
1706  }
1707  else
1708  {
1709  // a specific outer + specifc innner vlan id, a common case
1710 
1711  // get the qinq table
1712  if (vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs == 0)
1713  {
1714  // Allocate a qinq table from the pool
1715  pool_get (em->qinq_pool, qinq_table);
1716  vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs =
1717  qinq_table - em->qinq_pool;
1718  }
1719  else
1720  {
1721  // Get ptr to existing qinq table
1722  qinq_table =
1724  vlan_table->vlans[si->sub.
1725  eth.outer_vlan_id].
1726  qinqs);
1727  }
1728  subint = &qinq_table->vlans[si->sub.eth.inner_vlan_id].subint;
1729  }
1730  }
1731  }
1732 
1733 done:
1734  return subint;
1735 }
1736 
1737 static clib_error_t *
1739 {
1740  subint_config_t *subint;
1741  u32 dummy_flags;
1742  u32 dummy_unsup;
1743  clib_error_t *error = 0;
1744 
1745  // Find the config for this subinterface
1746  subint =
1747  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1748  &dummy_unsup);
1749 
1750  if (subint == 0)
1751  {
1752  // not implemented yet or not ethernet
1753  goto done;
1754  }
1755 
1756  subint->sw_if_index =
1757  ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? sw_if_index : ~0);
1758 
1759 done:
1760  return error;
1761 }
1762 
1764 
1765 
1766 #ifndef CLIB_MARCH_VARIANT
1767 // Set the L2/L3 mode for the subinterface
1768 void
1770 {
1771  subint_config_t *subint;
1772  u32 dummy_flags;
1773  u32 dummy_unsup;
1774  int is_port;
1775  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index);
1776 
1777  is_port = !(sw->type == VNET_SW_INTERFACE_TYPE_SUB);
1778 
1779  // Find the config for this subinterface
1780  subint =
1781  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1782  &dummy_unsup);
1783 
1784  if (subint == 0)
1785  {
1786  // unimplemented or not ethernet
1787  goto done;
1788  }
1789 
1790  // Double check that the config we found is for our interface (or the interface is down)
1791  ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1792 
1793  if (l2)
1794  {
1795  subint->flags |= SUBINT_CONFIG_L2;
1796  if (is_port)
1797  subint->flags |=
1800  }
1801  else
1802  {
1803  subint->flags &= ~SUBINT_CONFIG_L2;
1804  if (is_port)
1805  subint->flags &=
1808  }
1809 
1810 done:
1811  return;
1812 }
1813 
1814 /*
1815  * Set the L2/L3 mode for the subinterface regardless of port
1816  */
1817 void
1819  u32 sw_if_index, u32 l2)
1820 {
1821  subint_config_t *subint;
1822  u32 dummy_flags;
1823  u32 dummy_unsup;
1824 
1825  /* Find the config for this subinterface */
1826  subint =
1827  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1828  &dummy_unsup);
1829 
1830  if (subint == 0)
1831  {
1832  /* unimplemented or not ethernet */
1833  goto done;
1834  }
1835 
1836  /*
1837  * Double check that the config we found is for our interface (or the
1838  * interface is down)
1839  */
1840  ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1841 
1842  if (l2)
1843  {
1844  subint->flags |= SUBINT_CONFIG_L2;
1845  }
1846  else
1847  {
1848  subint->flags &= ~SUBINT_CONFIG_L2;
1849  }
1850 
1851 done:
1852  return;
1853 }
1854 #endif
1855 
1856 static clib_error_t *
1858  u32 sw_if_index, u32 is_create)
1859 {
1860  clib_error_t *error = 0;
1861  subint_config_t *subint;
1862  u32 match_flags;
1863  u32 unsupported = 0;
1864 
1865  // Find the config for this subinterface
1866  subint =
1867  ethernet_sw_interface_get_config (vnm, sw_if_index, &match_flags,
1868  &unsupported);
1869 
1870  if (subint == 0)
1871  {
1872  // not implemented yet or not ethernet
1873  if (unsupported)
1874  {
1875  // this is the NYI case
1876  error = clib_error_return (0, "not implemented yet");
1877  }
1878  goto done;
1879  }
1880 
1881  if (!is_create)
1882  {
1883  subint->flags = 0;
1884  return error;
1885  }
1886 
1887  // Initialize the subint
1888  if (subint->flags & SUBINT_CONFIG_VALID)
1889  {
1890  // Error vlan already in use
1891  error = clib_error_return (0, "vlan is already in use");
1892  }
1893  else
1894  {
1895  // Note that config is L3 by default
1896  subint->flags = SUBINT_CONFIG_VALID | match_flags;
1897  subint->sw_if_index = ~0; // because interfaces are initially down
1898  }
1899 
1900 done:
1901  return error;
1902 }
1903 
1905 
1906 static char *ethernet_error_strings[] = {
1907 #define ethernet_error(n,c,s) s,
1908 #include "error.def"
1909 #undef ethernet_error
1910 };
1911 
1912 /* *INDENT-OFF* */
1914  .name = "ethernet-input",
1915  /* Takes a vector of packets. */
1916  .vector_size = sizeof (u32),
1917  .scalar_size = sizeof (ethernet_input_frame_t),
1918  .n_errors = ETHERNET_N_ERROR,
1919  .error_strings = ethernet_error_strings,
1920  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1921  .next_nodes = {
1922 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1924 #undef _
1925  },
1926  .format_buffer = format_ethernet_header_with_length,
1927  .format_trace = format_ethernet_input_trace,
1928  .unformat_buffer = unformat_ethernet_header,
1929 };
1930 
1932  .name = "ethernet-input-type",
1933  /* Takes a vector of packets. */
1934  .vector_size = sizeof (u32),
1935  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1936  .next_nodes = {
1937 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1939 #undef _
1940  },
1941 };
1942 
1944  .name = "ethernet-input-not-l2",
1945  /* Takes a vector of packets. */
1946  .vector_size = sizeof (u32),
1947  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1948  .next_nodes = {
1949 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1951 #undef _
1952  },
1953 };
1954 /* *INDENT-ON* */
1955 
1956 #ifndef CLIB_MARCH_VARIANT
1957 void
1959  vnet_hw_interface_t * hi, u32 enable)
1960 {
1961  // Insure all packets go to ethernet-input (i.e. untagged ipv4 packets
1962  // don't go directly to ip4-input)
1964  (vnm, hi->hw_if_index, enable ? ethernet_input_node.index : ~0);
1965 }
1966 
1967 
1968 /*
1969  * Initialization and registration for the next_by_ethernet structure
1970  */
1971 
1972 clib_error_t *
1974 {
1976  ( /* elt bytes */ sizeof (l3_next->input_next_by_type[0]),
1977  /* bits in index */ BITS (((ethernet_header_t *) 0)->type));
1978 
1980  ETHERNET_INPUT_NEXT_DROP);
1982  ETHERNET_INPUT_NEXT_PUNT);
1983  l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_DROP] =
1985  l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_PUNT] =
1987 
1988  /*
1989  * Make sure we don't wipe out an ethernet registration by mistake
1990  * Can happen if init function ordering constraints are missing.
1991  */
1992  if (CLIB_DEBUG > 0)
1993  {
1996  }
1997 
1998  return 0;
1999 }
2000 
2001 // Add an ethertype -> next index mapping to the structure
2002 clib_error_t *
2004  u32 ethertype, u32 next_index)
2005 {
2006  u32 i;
2007  u16 *n;
2009 
2010  if (CLIB_DEBUG > 0)
2011  {
2014  }
2015 
2016  /* Setup ethernet type -> next index sparse vector mapping. */
2017  n = sparse_vec_validate (l3_next->input_next_by_type, ethertype);
2018  n[0] = next_index;
2019 
2020  /* Rebuild next index -> sparse index inverse mapping when sparse vector
2021  is updated. */
2022  vec_validate (l3_next->sparse_index_by_input_next_index, next_index);
2023  for (i = 1; i < vec_len (l3_next->input_next_by_type); i++)
2024  l3_next->
2025  sparse_index_by_input_next_index[l3_next->input_next_by_type[i]] = i;
2026 
2027  // do not allow the cached next index's to be updated if L3
2028  // redirect is enabled, as it will have overwritten them
2029  if (!em->redirect_l3)
2030  {
2031  // Cache common ethertypes directly
2032  if (ethertype == ETHERNET_TYPE_IP4)
2033  {
2034  l3_next->input_next_ip4 = next_index;
2035  }
2036  else if (ethertype == ETHERNET_TYPE_IP6)
2037  {
2038  l3_next->input_next_ip6 = next_index;
2039  }
2040  else if (ethertype == ETHERNET_TYPE_MPLS)
2041  {
2042  l3_next->input_next_mpls = next_index;
2043  }
2044  }
2045  return 0;
2046 }
2047 
2048 void
2050 {
2051  __attribute__ ((unused)) vlan_table_t *invalid_vlan_table;
2052  __attribute__ ((unused)) qinq_table_t *invalid_qinq_table;
2053 
2057 
2059 
2060  // Initialize pools and vector for vlan parsing
2061  vec_validate (em->main_intfs, 10); // 10 main interfaces
2062  pool_alloc (em->vlan_pool, 10);
2063  pool_alloc (em->qinq_pool, 1);
2064 
2065  // The first vlan pool will always be reserved for an invalid table
2066  pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0
2067  // The first qinq pool will always be reserved for an invalid table
2068  pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0
2069 }
2070 
2071 void
2073  ethernet_type_t type, u32 node_index)
2074 {
2077  u32 i;
2078 
2079  {
2081  if (error)
2082  clib_error_report (error);
2083  }
2084 
2085  ti = ethernet_get_type_info (em, type);
2086  if (ti == 0)
2087  {
2088  clib_warning ("type_info NULL for type %d", type);
2089  return;
2090  }
2091  ti->node_index = node_index;
2092  ti->next_index = vlib_node_add_next (vm,
2093  ethernet_input_node.index, node_index);
2094  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2095  ASSERT (i == ti->next_index);
2096 
2097  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2098  ASSERT (i == ti->next_index);
2099 
2100  // Add the L3 node for this ethertype to the next nodes structure
2101  next_by_ethertype_register (&em->l3_next, type, ti->next_index);
2102 
2103  // Call the registration functions for other nodes that want a mapping
2104  l2bvi_register_input_type (vm, type, node_index);
2105 }
2106 
2107 void
2109 {
2111  u32 i;
2112 
2113  em->l2_next =
2114  vlib_node_add_next (vm, ethernet_input_node.index, node_index);
2115 
2116  /*
2117  * Even if we never use these arcs, we have to align the next indices...
2118  */
2119  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2120 
2121  ASSERT (i == em->l2_next);
2122 
2123  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2124  ASSERT (i == em->l2_next);
2125 }
2126 
2127 // Register a next node for L3 redirect, and enable L3 redirect
2128 void
2130 {
2132  u32 i;
2133 
2134  em->redirect_l3 = 1;
2136  ethernet_input_node.index,
2137  node_index);
2138  /*
2139  * Change the cached next nodes to the redirect node
2140  */
2144 
2145  /*
2146  * Even if we never use these arcs, we have to align the next indices...
2147  */
2148  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2149 
2150  ASSERT (i == em->redirect_l3_next);
2151 
2152  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2153 
2154  ASSERT (i == em->redirect_l3_next);
2155 }
2156 #endif
2157 
2158 /*
2159  * fd.io coding-style-patch-verification: ON
2160  *
2161  * Local Variables:
2162  * eval: (c-set-style "gnu")
2163  * End:
2164  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
subint_config_t subint
Definition: ethernet.h:233
vmrglw vmrglh hi
pcap_main_t pcap_main
Definition: main.h:72
VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(bond_sw_interface_up_down)
static_always_inline void identify_subint(vnet_hw_interface_t *hi, vlib_buffer_t *b0, u32 match_flags, main_intf_t *main_intf, vlan_intf_t *vlan_intf, qinq_intf_t *qinq_intf, u32 *new_sw_if_index, u8 *error0, u32 *is_l2)
Definition: node.c:201
vlib_main_t vlib_global_main
Definition: main.c:1940
u32 flags
Definition: vhost_user.h:141
#define CLIB_UNUSED(x)
Definition: clib.h:83
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, current_data, 0)
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:523
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
u32 max_bytes_per_pkt
Definition: main.h:70
#define pool_alloc(P, N)
Allocate N more free elements to pool (unspecified alignment).
Definition: pool.h:341
vnet_interface_main_t interface_main
Definition: vnet.h:56
vnet_p2p_sub_interface_t p2p
Definition: interface.h:726
#define PREDICT_TRUE(x)
Definition: clib.h:113
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
int vnet_hw_interface_rx_redirect_to_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: interface.c:1199
u16x16 u64x4 static_always_inline u32 u8x32_msb_mask(u8x32 v)
Definition: vector_avx2.h:108
void ethernet_register_l2_input(vlib_main_t *vm, u32 node_index)
Definition: node.c:2108
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static_always_inline void eth_input_tag_lookup(vlib_main_t *vm, vnet_main_t *vnm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u64 tag, u16 *next, vlib_buffer_t *b, eth_input_tag_lookup_t *l, u8 dmac_bad, int is_dot1ad, int main_is_l3, int check_dmac)
Definition: node.c:489
ethernet_type_t
Definition: packet.h:45
#define SUBINT_CONFIG_MATCH_0_TAG
Definition: ethernet.h:194
#define ethernet_buffer_header_size(b)
Determine the size of the Ethernet headers of the current frame in the buffer.
Definition: ethernet.h:420
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:218
static subint_config_t * ethernet_sw_interface_get_config(vnet_main_t *vnm, u32 sw_if_index, u32 *flags, u32 *unsupported)
Definition: node.c:1556
ethernet_input_variant_t
Definition: node.c:94
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
main_intf_t * main_intfs
Definition: ethernet.h:281
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:66
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
subint_config_t inner_any_subint
Definition: ethernet.h:222
u8 data[128]
Definition: ipsec.api:251
u8 packet_data[32]
Definition: node.c:66
#define VLIB_NODE_FN(node)
Definition: node.h:202
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:470
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:366
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static_always_inline void u64x4_scatter(u64x4 r, void *p0, void *p1, void *p2, void *p3)
Definition: vector_avx2.h:229
static clib_error_t * ethernet_sw_interface_add_del(vnet_main_t *vnm, u32 sw_if_index, u32 is_create)
Definition: node.c:1857
epu16_epi64 epu8_epi64 epu8_epi64 epi16_epi64 i16x16
Definition: vector_avx2.h:129
vlib_node_registration_t ethernet_input_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_node)
Definition: node.c:1913
ethernet_main_t ethernet_main
Definition: init.c:45
representation of a pipe interface
Definition: pipe.h:24
#define static_always_inline
Definition: clib.h:100
vl_api_interface_index_t sw_if_index
Definition: gre.api:50
static uword ethernet_address_cast(u8 *a)
Definition: packet.h:67
static_always_inline u16 eth_input_next_by_type(u16 etype)
Definition: node.c:453
void ethernet_set_rx_redirect(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 enable)
Definition: node.c:1958
clib_error_t * next_by_ethertype_init(next_by_ethertype_t *l3_next)
Definition: node.c:1973
#define ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX
Definition: ethernet.h:52
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:846
u8 dst_address[6]
Definition: packet.h:55
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
#define sparse_vec_validate(v, i)
Definition: sparse_vec.h:231
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
subint_config_t default_subint
Definition: ethernet.h:214
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u32 filter_classify_table_index
Definition: main.h:73
#define clib_error_return(e, args...)
Definition: error.h:99
u8 pcap_rx_enable
Definition: main.h:64
unsigned int u32
Definition: types.h:88
#define vlib_call_init_function(vm, x)
Definition: init.h:270
#define VLIB_FRAME_SIZE
Definition: node.h:378
static char * ethernet_error_strings[]
Definition: node.c:1906
ethernet_input_next_t
Definition: node.c:56
Use the vpp classifier to decide whether to trace packets.
epu8_epi32 epu16_epi32 u64x2
Definition: vector_sse42.h:674
vl_api_fib_path_type_t type
Definition: fib_types.api:123
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
u16 dot1q_vlans
Definition: ethernet.h:215
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
subint_config_t single_tag_subint
Definition: ethernet.h:221
vnet_sub_interface_t sub
Definition: interface.h:723
u32 * sparse_index_by_input_next_index
Definition: ethernet.h:249
static ethernet_type_info_t * ethernet_get_type_info(ethernet_main_t *em, ethernet_type_t type)
Definition: ethernet.h:305
static const __m128i zero
Definition: aes_gcm.c:39
unsigned short u16
Definition: types.h:57
static_always_inline u64x4 u64x4_gather(void *p0, void *p1, void *p2, void *p3)
Definition: vector_avx2.h:208
#define ETH_INPUT_FRAME_F_IP4_CKSUM_OK
Definition: ethernet.h:55
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
static_always_inline void eth_input_process_frame(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u32 *buffer_indices, u32 n_packets, int main_is_l3, int ip4_cksum_ok, int dmac_check)
Definition: node.c:682
vlib_node_registration_t ethernet_input_type_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_type_node)
Definition: node.c:1931
#define PREDICT_FALSE(x)
Definition: clib.h:112
#define ethernet_buffer_set_vlan_count(b, v)
Sets the number of VLAN headers in the current Ethernet frame in the buffer.
Definition: ethernet.h:393
u16 dot1ad_vlans
Definition: ethernet.h:216
u32 pcap_sw_if_index
Definition: main.h:71
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:338
#define SUBINT_CONFIG_MATCH_3_TAG
Definition: ethernet.h:197
vnet_hw_interface_class_t ethernet_hw_interface_class
u8 next_by_ethertype_register_called
Definition: ethernet.h:293
subint_config_t untagged_subint
Definition: ethernet.h:213
u8 len
Definition: ip_types.api:90
static_always_inline void u32x8_scatter_one(u32x8 r, int index, void *p)
Definition: vector_avx2.h:258
u32 p2p_ethernet_lookup(u32 parent_if_index, u8 *client_mac)
Definition: p2p_ethernet.c:35
u8 * format_ethernet_header_with_length(u8 *s, va_list *args)
Definition: format.c:97
#define SUBINT_CONFIG_VALID
Definition: ethernet.h:198
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
static_always_inline void eth_input_single_int(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u32 *from, u32 n_pkts, int ip4_cksum_ok)
Definition: node.c:931
qinq_intf_t vlans[ETHERNET_N_VLAN]
Definition: ethernet.h:238
u16 n_vectors
Definition: node.h:397
VNET_SW_INTERFACE_ADD_DEL_FUNCTION(ethernet_sw_interface_add_del)
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
vlib_main_t * vm
Definition: buffer.c:323
void ethernet_input_init(vlib_main_t *vm, ethernet_main_t *em)
Definition: node.c:2049
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
static int ethernet_mac_address_equal(const u8 *a, const u8 *b)
Definition: mac_address.h:85
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
u8 data[]
Packet data.
Definition: buffer.h:181
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:141
#define vlib_prefetch_buffer_data(b, type)
Definition: buffer.h:204
static u32 eth_identify_subint(vnet_hw_interface_t *hi, u32 match_flags, main_intf_t *main_intf, vlan_intf_t *vlan_intf, qinq_intf_t *qinq_intf, u32 *new_sw_if_index, u8 *error0, u32 *is_l2)
Definition: ethernet.h:486
static void * vlib_frame_scalar_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:258
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:456
static int vnet_is_packet_traced_inline(vlib_buffer_t *b, u32 classify_table_index, int func)
vnet_is_packet_traced
#define SUBINT_CONFIG_MATCH_1_TAG
Definition: ethernet.h:195
static_always_inline void parse_header(ethernet_input_variant_t variant, vlib_buffer_t *b0, u16 *type, u16 *orig_type, u16 *outer_id, u16 *inner_id, u32 *match_flags)
Definition: node.c:104
static_always_inline void eth_input_adv_and_flags_x1(vlib_buffer_t **b, int is_l3)
Definition: node.c:416
static void eth_vlan_table_lookups(ethernet_main_t *em, vnet_main_t *vnm, u32 port_sw_if_index0, u16 first_ethertype, u16 outer_id, u16 inner_id, vnet_hw_interface_t **hi, main_intf_t **main_intf, vlan_intf_t **vlan_intf, qinq_intf_t **qinq_intf)
Definition: ethernet.h:447
signed int i32
Definition: types.h:77
static uword sparse_vec_index(void *v, uword sparse_index)
Definition: sparse_vec.h:161
static_always_inline void ethernet_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *from, u32 n_packets, ethernet_input_variant_t variant)
Definition: node.c:1038
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:515
#define SUBINT_CONFIG_L2
Definition: ethernet.h:199
vlan_table_t * vlan_pool
Definition: ethernet.h:284
#define ASSERT(truth)
u8 * format_ethernet_header(u8 *s, va_list *args)
Definition: format.c:178
u32 redirect_l3_next
Definition: ethernet.h:269
void ethernet_register_l3_redirect(vlib_main_t *vm, u32 node_index)
Definition: node.c:2129
u16 flags
Definition: node.h:388
void ethernet_sw_interface_set_l2_mode_noport(vnet_main_t *vnm, u32 sw_if_index, u32 l2)
Definition: node.c:1818
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:78
static_always_inline void determine_next_node(ethernet_main_t *em, ethernet_input_variant_t variant, u32 is_l20, u32 type0, vlib_buffer_t *b0, u8 *error0, u8 *next0)
Definition: node.c:240
void ethernet_register_input_type(vlib_main_t *vm, ethernet_type_t type, u32 node_index)
Definition: node.c:2072
#define clib_error_report(e)
Definition: error.h:113
static u8 * format_ethernet_input_trace(u8 *s, va_list *va)
Definition: node.c:72
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
STATIC_ASSERT(STRUCT_OFFSET_OF(vnet_buffer_opaque_t, l2_hdr_offset)==STRUCT_OFFSET_OF(vnet_buffer_opaque_t, l3_hdr_offset) - 2, "l3_hdr_offset must follow l2_hdr_offset")
vlan_intf_t vlans[ETHERNET_N_VLAN]
Definition: ethernet.h:228
#define SUBINT_CONFIG_MATCH_2_TAG
Definition: ethernet.h:196
clib_error_t * next_by_ethertype_register(next_by_ethertype_t *l3_next, u32 ethertype, u32 next_index)
Definition: node.c:2003
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
#define vec_elt(v, i)
Get vector value at index i.
subint_config_t subint
Sub-interface config.
Definition: pipe.h:30
vlib_node_registration_t ethernet_input_not_l2_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_not_l2_node)
Definition: node.c:1943
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
#define foreach_ethernet_input_next
Definition: node.c:49
#define SUBINT_CONFIG_P2P
Definition: ethernet.h:200
struct vnet_sub_interface_t::@210::@211::@213 flags
void l2bvi_register_input_type(vlib_main_t *vm, ethernet_type_t type, u32 node_index)
Definition: l2_bvi.c:29
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
pipe_t * pipe_get(u32 sw_if_index)
Get the pipe instnace based on one end.
Definition: pipe.c:95
static_always_inline int ethernet_frame_is_any_tagged_x2(u16 type0, u16 type1)
Definition: ethernet.h:92
VLIB buffer representation.
Definition: buffer.h:102
static_always_inline void eth_input_adv_and_flags_x4(vlib_buffer_t **b, int is_l3)
Definition: node.c:314
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
static clib_error_t * ethernet_init(vlib_main_t *vm)
Definition: init.c:83
void ethernet_sw_interface_set_l2_mode(vnet_main_t *vnm, u32 sw_if_index, u32 l2)
Definition: node.c:1769
struct vnet_sub_interface_t::@210 eth
struct clib_bihash_value offset
template key/value backing page structure
u64x4
Definition: vector_avx2.h:121
#define vnet_buffer(b)
Definition: buffer.h:365
p2p_ethernet_main_t p2p_main
Definition: p2p_ethernet.c:23
vnet_sw_interface_type_t type
Definition: interface.h:701
qinq_table_t * qinq_pool
Definition: ethernet.h:287
static_always_inline void eth_input_process_frame_dmac_check(vnet_hw_interface_t *hi, u64 *dmacs, u8 *dmacs_bad, u32 n_packets)
Definition: node.c:612
#define u16x16_blend(v1, v2, mask)
Definition: vector_avx2.h:204
u16 flags
Copy of main node flags.
Definition: node.h:509
next_by_ethertype_t l3_next
Definition: ethernet.h:262
subint_config_t * p2p_subif_pool
Definition: p2p_ethernet.h:31
vnet_pcap_t pcap
Definition: main.h:173
static clib_error_t * ethernet_sw_interface_up_down(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: node.c:1738
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:302
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static void * sparse_vec_new(uword elt_bytes, uword sparse_index_bits)
Definition: sparse_vec.h:71
ethernet_interface_t * interfaces
Definition: ethernet.h:272
static_always_inline void eth_input_update_if_counters(vlib_main_t *vm, vnet_main_t *vnm, eth_input_tag_lookup_t *l)
Definition: node.c:473
epu16_epi64 u16x16
Definition: vector_avx2.h:123
ethernet_input_frame_t frame_data
Definition: node.c:68
#define BITS(x)
Definition: clib.h:62
static_always_inline void eth_input_get_etype_and_tags(vlib_buffer_t **b, u16 *etype, u64 *tags, u64 *dmacs, int offset, int dmac_check)
Definition: node.c:434
static void pcap_add_buffer(pcap_main_t *pm, struct vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
Definition: pcap_funcs.h:67
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static_always_inline void ethernet_input_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: node.c:970
static void ethernet_setup_node(vlib_main_t *vm, u32 node_index)
Definition: ethernet.h:365
Definition: defs.h:46
uword unformat_ethernet_header(unformat_input_t *input, va_list *args)
Definition: format.c:290
u16 * input_next_by_type
Definition: ethernet.h:248
#define SPARSE_VEC_INVALID_INDEX
Definition: sparse_vec.h:68
signed short i16
Definition: types.h:46