FD.io VPP  v18.07-rc0-415-g6c78436
Vector Packet Processing
decap.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 SUSE LLC.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vlib/vlib.h>
17 #include <vnet/pg/pg.h>
18 #include <vnet/geneve/geneve.h>
19 
22 
23 typedef struct
24 {
30 
31 static u8 *
32 format_geneve_rx_trace (u8 * s, va_list * args)
33 {
34  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
35  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
36  geneve_rx_trace_t *t = va_arg (*args, geneve_rx_trace_t *);
37 
38  if (t->tunnel_index != ~0)
39  {
40  s =
41  format (s,
42  "GENEVE decap from geneve_tunnel%d vni %d next %d error %d",
43  t->tunnel_index, t->vni_rsvd, t->next_index, t->error);
44  }
45  else
46  {
47  s = format (s, "GENEVE decap error - tunnel for vni %d does not exist",
48  t->vni_rsvd);
49  }
50  return s;
51 }
52 
55 {
56  u32 fib_index, sw_if_index;
57 
58  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
59 
60  if (is_ip4)
61  fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
63  vnet_buffer (b)->sw_if_index[VLIB_TX];
64  else
65  fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
67  vnet_buffer (b)->sw_if_index[VLIB_TX];
68 
69  return (fib_index == t->encap_fib_index);
70 }
71 
74  vlib_node_runtime_t * node,
75  vlib_frame_t * from_frame, u32 is_ip4)
76 {
77  u32 n_left_from, next_index, *from, *to_next;
78  geneve_main_t *vxm = &geneve_main;
79  vnet_main_t *vnm = vxm->vnet_main;
81  u32 last_tunnel_index = ~0;
82  geneve4_tunnel_key_t last_key4;
83  geneve6_tunnel_key_t last_key6;
84  u32 pkts_decapsulated = 0;
85  u32 thread_index = vlib_get_thread_index ();
86  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
87 
88  if (is_ip4)
89  last_key4.as_u64 = ~0;
90  else
91  memset (&last_key6, 0xff, sizeof (last_key6));
92 
93  from = vlib_frame_vector_args (from_frame);
94  n_left_from = from_frame->n_vectors;
95 
96  next_index = node->cached_next_index;
97  stats_sw_if_index = node->runtime_data[0];
98  stats_n_packets = stats_n_bytes = 0;
99 
100  while (n_left_from > 0)
101  {
102  u32 n_left_to_next;
103 
104  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
105  while (n_left_from >= 4 && n_left_to_next >= 2)
106  {
107  u32 bi0, bi1;
108  vlib_buffer_t *b0, *b1;
109  u32 next0, next1;
110  ip4_header_t *ip4_0, *ip4_1;
111  ip6_header_t *ip6_0, *ip6_1;
112  geneve_header_t *geneve0, *geneve1;
113  uword *p0, *p1;
114  u32 tunnel_index0, tunnel_index1;
115  geneve_tunnel_t *t0, *t1, *mt0 = NULL, *mt1 = NULL;
116  geneve4_tunnel_key_t key4_0, key4_1;
117  geneve6_tunnel_key_t key6_0, key6_1;
118  u32 error0, error1;
119  u32 sw_if_index0, sw_if_index1, len0, len1;
120 
121  /* Prefetch next iteration. */
122  {
123  vlib_buffer_t *p2, *p3;
124 
125  p2 = vlib_get_buffer (vm, from[2]);
126  p3 = vlib_get_buffer (vm, from[3]);
127 
128  vlib_prefetch_buffer_header (p2, LOAD);
129  vlib_prefetch_buffer_header (p3, LOAD);
130 
131  CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
132  CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
133  }
134 
135  bi0 = from[0];
136  bi1 = from[1];
137  to_next[0] = bi0;
138  to_next[1] = bi1;
139  from += 2;
140  to_next += 2;
141  n_left_to_next -= 2;
142  n_left_from -= 2;
143 
144  b0 = vlib_get_buffer (vm, bi0);
145  b1 = vlib_get_buffer (vm, bi1);
146 
147  /* udp leaves current_data pointing at the geneve header */
148  geneve0 = vlib_buffer_get_current (b0);
149  geneve1 = vlib_buffer_get_current (b1);
150 
151  vnet_geneve_hdr_1word_ntoh (geneve0);
152  vnet_geneve_hdr_1word_ntoh (geneve1);
153 
154  if (is_ip4)
155  {
157  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
159  (b1, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
160  ip4_0 = vlib_buffer_get_current (b0);
161  ip4_1 = vlib_buffer_get_current (b1);
162  }
163  else
164  {
166  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
168  (b1, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
169  ip6_0 = vlib_buffer_get_current (b0);
170  ip6_1 = vlib_buffer_get_current (b1);
171  }
172 
173  /* pop (ip, udp, geneve) */
174  if (is_ip4)
175  {
177  (b0,
178  sizeof (*ip4_0) + sizeof (udp_header_t) +
180  vnet_get_geneve_options_len (geneve0));
182  sizeof (*ip4_1) + sizeof (udp_header_t) +
184  vnet_get_geneve_options_len (geneve1));
185  }
186  else
187  {
189  (b0,
190  sizeof (*ip6_0) + sizeof (udp_header_t) +
192  vnet_get_geneve_options_len (geneve0));
194  sizeof (*ip6_0) + sizeof (udp_header_t) +
196  vnet_get_geneve_options_len (geneve1));
197  }
198 
199  tunnel_index0 = ~0;
200  error0 = 0;
201 
202  tunnel_index1 = ~0;
203  error1 = 0;
204 
205  if (PREDICT_FALSE
207  {
208  error0 = GENEVE_ERROR_BAD_FLAGS;
209  next0 = GENEVE_INPUT_NEXT_DROP;
210  goto trace0;
211  }
212 #if SUPPORT_OPTIONS_HEADER==1
213  if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve0) == 1))
214  {
215  error0 = GENEVE_ERROR_BAD_FLAGS;
216  next0 = GENEVE_INPUT_NEXT_DROP;
217  goto trace0;
218  }
219 #endif
220  if (is_ip4)
221  {
222  key4_0.remote = ip4_0->src_address.as_u32;
223  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
224 
225  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
226  if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
227  {
228  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
229  if (PREDICT_FALSE (p0 == NULL))
230  {
231  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
232  next0 = GENEVE_INPUT_NEXT_DROP;
233  goto trace0;
234  }
235  last_key4.as_u64 = key4_0.as_u64;
236  tunnel_index0 = last_tunnel_index = p0[0];
237  }
238  else
239  tunnel_index0 = last_tunnel_index;
240  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
241 
242  /* Validate GENEVE tunnel encap-fib index agaist packet */
243  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
244  {
245  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
246  next0 = GENEVE_INPUT_NEXT_DROP;
247  goto trace0;
248  }
249 
250  /* Validate GENEVE tunnel SIP against packet DIP */
251  if (PREDICT_TRUE
252  (ip4_0->dst_address.as_u32 == t0->local.ip4.as_u32))
253  goto next0; /* valid packet */
254  if (PREDICT_FALSE
256  {
257  key4_0.remote = ip4_0->dst_address.as_u32;
258  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
259  /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
260  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
261  if (PREDICT_TRUE (p0 != NULL))
262  {
263  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
264  goto next0; /* valid packet */
265  }
266  }
267  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
268  next0 = GENEVE_INPUT_NEXT_DROP;
269  goto trace0;
270 
271  }
272  else /* !is_ip4 */
273  {
274  key6_0.remote.as_u64[0] = ip6_0->src_address.as_u64[0];
275  key6_0.remote.as_u64[1] = ip6_0->src_address.as_u64[1];
276  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
277 
278  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
279  if (PREDICT_FALSE
280  (memcmp (&key6_0, &last_key6, sizeof (last_key6)) != 0))
281  {
282  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
283  if (PREDICT_FALSE (p0 == NULL))
284  {
285  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
286  next0 = GENEVE_INPUT_NEXT_DROP;
287  goto trace0;
288  }
289  clib_memcpy (&last_key6, &key6_0, sizeof (key6_0));
290  tunnel_index0 = last_tunnel_index = p0[0];
291  }
292  else
293  tunnel_index0 = last_tunnel_index;
294  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
295 
296  /* Validate GENEVE tunnel encap-fib index agaist packet */
297  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
298  {
299  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
300  next0 = GENEVE_INPUT_NEXT_DROP;
301  goto trace0;
302  }
303 
304  /* Validate GENEVE tunnel SIP against packet DIP */
306  &t0->local.ip6)))
307  goto next0; /* valid packet */
308  if (PREDICT_FALSE
310  {
311  key6_0.remote.as_u64[0] = ip6_0->dst_address.as_u64[0];
312  key6_0.remote.as_u64[1] = ip6_0->dst_address.as_u64[1];
313  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
314  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
315  if (PREDICT_TRUE (p0 != NULL))
316  {
317  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
318  goto next0; /* valid packet */
319  }
320  }
321  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
322  next0 = GENEVE_INPUT_NEXT_DROP;
323  goto trace0;
324  }
325 
326  next0:
327  next0 = t0->decap_next_index;
328  sw_if_index0 = t0->sw_if_index;
329  len0 = vlib_buffer_length_in_chain (vm, b0);
330 
331  /* Required to make the l2 tag push / pop code work on l2 subifs */
332  if (PREDICT_TRUE (next0 == GENEVE_INPUT_NEXT_L2_INPUT))
333  vnet_update_l2_len (b0);
334 
335  /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
336  vnet_buffer (b0)->sw_if_index[VLIB_RX] = sw_if_index0;
337  sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
338 
339  pkts_decapsulated++;
340  stats_n_packets += 1;
341  stats_n_bytes += len0;
342 
343  /* Batch stats increment on the same geneve tunnel so counter
344  is not incremented per packet */
345  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
346  {
347  stats_n_packets -= 1;
348  stats_n_bytes -= len0;
349  if (stats_n_packets)
352  thread_index, stats_sw_if_index,
353  stats_n_packets, stats_n_bytes);
354  stats_n_packets = 1;
355  stats_n_bytes = len0;
356  stats_sw_if_index = sw_if_index0;
357  }
358 
359  trace0:
360  b0->error = error0 ? node->errors[error0] : 0;
361 
362  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
363  {
365  = vlib_add_trace (vm, node, b0, sizeof (*tr));
366  tr->next_index = next0;
367  tr->error = error0;
368  tr->tunnel_index = tunnel_index0;
369  tr->vni_rsvd = vnet_get_geneve_vni (geneve0);
370  }
371 
372  if (PREDICT_FALSE
374  {
375  error1 = GENEVE_ERROR_BAD_FLAGS;
376  next1 = GENEVE_INPUT_NEXT_DROP;
377  goto trace1;
378  }
379 #if SUPPORT_OPTIONS_HEADER==1
380  if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve1) == 1))
381  {
382  error1 = GENEVE_ERROR_BAD_FLAGS;
383  next1 = GENEVE_INPUT_NEXT_DROP;
384  goto trace1;
385  }
386 #endif
387  if (is_ip4)
388  {
389  key4_1.remote = ip4_1->src_address.as_u32;
390  key4_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
391 
392  /* Make sure unicast GENEVE tunnel exist by packet SIP and VNI */
393  if (PREDICT_FALSE (key4_1.as_u64 != last_key4.as_u64))
394  {
395  p1 = hash_get (vxm->geneve4_tunnel_by_key, key4_1.as_u64);
396  if (PREDICT_FALSE (p1 == NULL))
397  {
398  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
399  next1 = GENEVE_INPUT_NEXT_DROP;
400  goto trace1;
401  }
402  last_key4.as_u64 = key4_1.as_u64;
403  tunnel_index1 = last_tunnel_index = p1[0];
404  }
405  else
406  tunnel_index1 = last_tunnel_index;
407  t1 = pool_elt_at_index (vxm->tunnels, tunnel_index1);
408 
409  /* Validate GENEVE tunnel encap-fib index agaist packet */
410  if (PREDICT_FALSE (validate_geneve_fib (b1, t1, is_ip4) == 0))
411  {
412  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
413  next1 = GENEVE_INPUT_NEXT_DROP;
414  goto trace1;
415  }
416 
417  /* Validate GENEVE tunnel SIP against packet DIP */
418  if (PREDICT_TRUE
419  (ip4_1->dst_address.as_u32 == t1->local.ip4.as_u32))
420  goto next1; /* valid packet */
421  if (PREDICT_FALSE
423  {
424  key4_1.remote = ip4_1->dst_address.as_u32;
425  key4_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
426  /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
427  p1 = hash_get (vxm->geneve4_tunnel_by_key, key4_1.as_u64);
428  if (PREDICT_TRUE (p1 != NULL))
429  {
430  mt1 = pool_elt_at_index (vxm->tunnels, p1[0]);
431  goto next1; /* valid packet */
432  }
433  }
434  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
435  next1 = GENEVE_INPUT_NEXT_DROP;
436  goto trace1;
437 
438  }
439  else /* !is_ip4 */
440  {
441  key6_1.remote.as_u64[0] = ip6_1->src_address.as_u64[0];
442  key6_1.remote.as_u64[1] = ip6_1->src_address.as_u64[1];
443  key6_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
444 
445  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
446  if (PREDICT_FALSE
447  (memcmp (&key6_1, &last_key6, sizeof (last_key6)) != 0))
448  {
449  p1 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_1);
450 
451  if (PREDICT_FALSE (p1 == NULL))
452  {
453  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
454  next1 = GENEVE_INPUT_NEXT_DROP;
455  goto trace1;
456  }
457 
458  clib_memcpy (&last_key6, &key6_1, sizeof (key6_1));
459  tunnel_index1 = last_tunnel_index = p1[0];
460  }
461  else
462  tunnel_index1 = last_tunnel_index;
463  t1 = pool_elt_at_index (vxm->tunnels, tunnel_index1);
464 
465  /* Validate GENEVE tunnel encap-fib index agaist packet */
466  if (PREDICT_FALSE (validate_geneve_fib (b1, t1, is_ip4) == 0))
467  {
468  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
469  next1 = GENEVE_INPUT_NEXT_DROP;
470  goto trace1;
471  }
472 
473  /* Validate GENEVE tunnel SIP against packet DIP */
475  &t1->local.ip6)))
476  goto next1; /* valid packet */
477  if (PREDICT_FALSE
479  {
480  key6_1.remote.as_u64[0] = ip6_1->dst_address.as_u64[0];
481  key6_1.remote.as_u64[1] = ip6_1->dst_address.as_u64[1];
482  key6_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
483  p1 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_1);
484  if (PREDICT_TRUE (p1 != NULL))
485  {
486  mt1 = pool_elt_at_index (vxm->tunnels, p1[0]);
487  goto next1; /* valid packet */
488  }
489  }
490  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
491  next1 = GENEVE_INPUT_NEXT_DROP;
492  goto trace1;
493  }
494 
495  next1:
496  next1 = t1->decap_next_index;
497  sw_if_index1 = t1->sw_if_index;
498  len1 = vlib_buffer_length_in_chain (vm, b1);
499 
500  /* Required to make the l2 tag push / pop code work on l2 subifs */
501  if (PREDICT_TRUE (next1 == GENEVE_INPUT_NEXT_L2_INPUT))
502  vnet_update_l2_len (b1);
503 
504  /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
505  vnet_buffer (b1)->sw_if_index[VLIB_RX] = sw_if_index1;
506  sw_if_index1 = (mt1) ? mt1->sw_if_index : sw_if_index1;
507 
508  pkts_decapsulated++;
509  stats_n_packets += 1;
510  stats_n_bytes += len1;
511 
512  /* Batch stats increment on the same geneve tunnel so counter
513  is not incremented per packet */
514  if (PREDICT_FALSE (sw_if_index1 != stats_sw_if_index))
515  {
516  stats_n_packets -= 1;
517  stats_n_bytes -= len1;
518  if (stats_n_packets)
521  thread_index, stats_sw_if_index,
522  stats_n_packets, stats_n_bytes);
523  stats_n_packets = 1;
524  stats_n_bytes = len1;
525  stats_sw_if_index = sw_if_index1;
526  }
527 
528  trace1:
529  b1->error = error1 ? node->errors[error1] : 0;
530 
531  if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
532  {
534  = vlib_add_trace (vm, node, b1, sizeof (*tr));
535  tr->next_index = next1;
536  tr->error = error1;
537  tr->tunnel_index = tunnel_index1;
538  tr->vni_rsvd = vnet_get_geneve_vni (geneve1);
539  }
540 
541  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
542  to_next, n_left_to_next,
543  bi0, bi1, next0, next1);
544  }
545 
546  while (n_left_from > 0 && n_left_to_next > 0)
547  {
548  u32 bi0;
549  vlib_buffer_t *b0;
550  u32 next0;
551  ip4_header_t *ip4_0;
552  ip6_header_t *ip6_0;
553  geneve_header_t *geneve0;
554  uword *p0;
555  u32 tunnel_index0;
556  geneve_tunnel_t *t0, *mt0 = NULL;
557  geneve4_tunnel_key_t key4_0;
558  geneve6_tunnel_key_t key6_0;
559  u32 error0;
560  u32 sw_if_index0, len0;
561 
562  bi0 = from[0];
563  to_next[0] = bi0;
564  from += 1;
565  to_next += 1;
566  n_left_from -= 1;
567  n_left_to_next -= 1;
568 
569  b0 = vlib_get_buffer (vm, bi0);
570 
571  /* udp leaves current_data pointing at the geneve header */
572  geneve0 = vlib_buffer_get_current (b0);
573  vnet_geneve_hdr_1word_ntoh (geneve0);
574 
575  if (is_ip4)
576  {
578  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
579  ip4_0 = vlib_buffer_get_current (b0);
580  }
581  else
582  {
584  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
585  ip6_0 = vlib_buffer_get_current (b0);
586  }
587 
588  /* pop (ip, udp, geneve) */
589  if (is_ip4)
590  {
592  (b0,
593  sizeof (*ip4_0) + sizeof (udp_header_t) +
595  vnet_get_geneve_options_len (geneve0));
596  }
597  else
598  {
600  (b0,
601  sizeof (*ip6_0) + sizeof (udp_header_t) +
603  vnet_get_geneve_options_len (geneve0));
604  }
605 
606  tunnel_index0 = ~0;
607  error0 = 0;
608 
609  if (PREDICT_FALSE
611  {
612  error0 = GENEVE_ERROR_BAD_FLAGS;
613  next0 = GENEVE_INPUT_NEXT_DROP;
614  goto trace00;
615  }
616 #if SUPPORT_OPTIONS_HEADER==1
617  if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve0) == 1))
618  {
619  error0 = GENEVE_ERROR_BAD_FLAGS;
620  next0 = GENEVE_INPUT_NEXT_DROP;
621  goto trace00;
622  }
623 #endif
624  if (is_ip4)
625  {
626  key4_0.remote = ip4_0->src_address.as_u32;
627  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
628 
629  /* Make sure unicast GENEVE tunnel exist by packet SIP and VNI */
630  if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
631  {
632  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
633  if (PREDICT_FALSE (p0 == NULL))
634  {
635  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
636  next0 = GENEVE_INPUT_NEXT_DROP;
637  goto trace00;
638  }
639  last_key4.as_u64 = key4_0.as_u64;
640  tunnel_index0 = last_tunnel_index = p0[0];
641  }
642  else
643  tunnel_index0 = last_tunnel_index;
644  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
645 
646  /* Validate GENEVE tunnel encap-fib index agaist packet */
647  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
648  {
649  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
650  next0 = GENEVE_INPUT_NEXT_DROP;
651  goto trace00;
652  }
653 
654  /* Validate GENEVE tunnel SIP against packet DIP */
655  if (PREDICT_TRUE
656  (ip4_0->dst_address.as_u32 == t0->local.ip4.as_u32))
657  goto next00; /* valid packet */
658  if (PREDICT_FALSE
660  {
661  key4_0.remote = ip4_0->dst_address.as_u32;
662  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
663  /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
664  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
665  if (PREDICT_TRUE (p0 != NULL))
666  {
667  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
668  goto next00; /* valid packet */
669  }
670  }
671  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
672  next0 = GENEVE_INPUT_NEXT_DROP;
673  goto trace00;
674 
675  }
676  else /* !is_ip4 */
677  {
678  key6_0.remote.as_u64[0] = ip6_0->src_address.as_u64[0];
679  key6_0.remote.as_u64[1] = ip6_0->src_address.as_u64[1];
680  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
681 
682  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
683  if (PREDICT_FALSE
684  (memcmp (&key6_0, &last_key6, sizeof (last_key6)) != 0))
685  {
686  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
687  if (PREDICT_FALSE (p0 == NULL))
688  {
689  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
690  next0 = GENEVE_INPUT_NEXT_DROP;
691  goto trace00;
692  }
693  clib_memcpy (&last_key6, &key6_0, sizeof (key6_0));
694  tunnel_index0 = last_tunnel_index = p0[0];
695  }
696  else
697  tunnel_index0 = last_tunnel_index;
698  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
699 
700  /* Validate GENEVE tunnel encap-fib index agaist packet */
701  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
702  {
703  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
704  next0 = GENEVE_INPUT_NEXT_DROP;
705  goto trace00;
706  }
707 
708  /* Validate GENEVE tunnel SIP against packet DIP */
710  &t0->local.ip6)))
711  goto next00; /* valid packet */
712  if (PREDICT_FALSE
714  {
715  key6_0.remote.as_u64[0] = ip6_0->dst_address.as_u64[0];
716  key6_0.remote.as_u64[1] = ip6_0->dst_address.as_u64[1];
717  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
718  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
719  if (PREDICT_TRUE (p0 != NULL))
720  {
721  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
722  goto next00; /* valid packet */
723  }
724  }
725  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
726  next0 = GENEVE_INPUT_NEXT_DROP;
727  goto trace00;
728  }
729 
730  next00:
731  next0 = t0->decap_next_index;
732  sw_if_index0 = t0->sw_if_index;
733  len0 = vlib_buffer_length_in_chain (vm, b0);
734 
735  /* Required to make the l2 tag push / pop code work on l2 subifs */
736  if (PREDICT_TRUE (next0 == GENEVE_INPUT_NEXT_L2_INPUT))
737  vnet_update_l2_len (b0);
738 
739  /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
740  vnet_buffer (b0)->sw_if_index[VLIB_RX] = sw_if_index0;
741  sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
742 
743  pkts_decapsulated++;
744  stats_n_packets += 1;
745  stats_n_bytes += len0;
746 
747  /* Batch stats increment on the same geneve tunnel so counter
748  is not incremented per packet */
749  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
750  {
751  stats_n_packets -= 1;
752  stats_n_bytes -= len0;
753  if (stats_n_packets)
756  thread_index, stats_sw_if_index,
757  stats_n_packets, stats_n_bytes);
758  stats_n_packets = 1;
759  stats_n_bytes = len0;
760  stats_sw_if_index = sw_if_index0;
761  }
762 
763  trace00:
764  b0->error = error0 ? node->errors[error0] : 0;
765 
766  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
767  {
769  = vlib_add_trace (vm, node, b0, sizeof (*tr));
770  tr->next_index = next0;
771  tr->error = error0;
772  tr->tunnel_index = tunnel_index0;
773  tr->vni_rsvd = vnet_get_geneve_vni (geneve0);
774  }
775  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
776  to_next, n_left_to_next,
777  bi0, next0);
778  }
779 
780  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
781  }
782  /* Do we still need this now that tunnel tx stats is kept? */
783  vlib_node_increment_counter (vm, is_ip4 ?
785  index : geneve6_input_node.index,
786  GENEVE_ERROR_DECAPSULATED, pkts_decapsulated);
787 
788  /* Increment any remaining batch stats */
789  if (stats_n_packets)
790  {
793  thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
794  node->runtime_data[0] = stats_sw_if_index;
795  }
796 
797  return from_frame->n_vectors;
798 }
799 
800 static uword
802  vlib_node_runtime_t * node, vlib_frame_t * from_frame)
803 {
804  return geneve_input (vm, node, from_frame, /* is_ip4 */ 1);
805 }
806 
807 static uword
809  vlib_node_runtime_t * node, vlib_frame_t * from_frame)
810 {
811  return geneve_input (vm, node, from_frame, /* is_ip4 */ 0);
812 }
813 
814 static char *geneve_error_strings[] = {
815 #define geneve_error(n,s) s,
817 #undef geneve_error
818 #undef _
819 };
820 
821 /* *INDENT-OFF* */
823  .function = geneve4_input,
824  .name = "geneve4-input",
825  /* Takes a vector of packets. */
826  .vector_size = sizeof (u32),
827  .n_errors = GENEVE_N_ERROR,
828  .error_strings = geneve_error_strings,
829  .n_next_nodes = GENEVE_INPUT_N_NEXT,
830  .next_nodes = {
831 #define _(s,n) [GENEVE_INPUT_NEXT_##s] = n,
833 #undef _
834  },
835 
836 //temp .format_buffer = format_geneve_header,
837  .format_trace = format_geneve_rx_trace,
838  // $$$$ .unformat_buffer = unformat_geneve_header,
839 };
840 
842 
844  .function = geneve6_input,
845  .name = "geneve6-input",
846  /* Takes a vector of packets. */
847  .vector_size = sizeof (u32),
848  .n_errors = GENEVE_N_ERROR,
849  .error_strings = geneve_error_strings,
850  .n_next_nodes = GENEVE_INPUT_N_NEXT,
851  .next_nodes = {
852 #define _(s,n) [GENEVE_INPUT_NEXT_##s] = n,
854 #undef _
855  },
856 //temp .format_buffer = format_geneve_header,
857  .format_trace = format_geneve_rx_trace,
858  // $$$$ .unformat_buffer = unformat_geneve_header,
859 };
860 
862 /* *INDENT-ON* */
863 
864 typedef enum
865 {
870 
873  vlib_node_runtime_t * node,
874  vlib_frame_t * frame, u32 is_ip4)
875 {
876  geneve_main_t *vxm = &geneve_main;
877  u32 *from, *to_next, n_left_from, n_left_to_next, next_index;
878  vlib_node_runtime_t *error_node =
880  ip4_address_t addr4; /* last IPv4 address matching a local VTEP address */
881  ip6_address_t addr6; /* last IPv6 address matching a local VTEP address */
882 
883  from = vlib_frame_vector_args (frame);
884  n_left_from = frame->n_vectors;
885  next_index = node->cached_next_index;
886 
887  if (node->flags & VLIB_NODE_FLAG_TRACE)
888  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
889 
890  if (is_ip4)
891  addr4.data_u32 = ~0;
892  else
893  ip6_address_set_zero (&addr6);
894 
895  while (n_left_from > 0)
896  {
897  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
898 
899  while (n_left_from >= 4 && n_left_to_next >= 2)
900  {
901  vlib_buffer_t *b0, *b1;
902  ip4_header_t *ip40, *ip41;
903  ip6_header_t *ip60, *ip61;
904  udp_header_t *udp0, *udp1;
905  u32 bi0, ip_len0, udp_len0, flags0, next0;
906  u32 bi1, ip_len1, udp_len1, flags1, next1;
907  i32 len_diff0, len_diff1;
908  u8 error0, good_udp0, proto0;
909  u8 error1, good_udp1, proto1;
910 
911  /* Prefetch next iteration. */
912  {
913  vlib_buffer_t *p2, *p3;
914 
915  p2 = vlib_get_buffer (vm, from[2]);
916  p3 = vlib_get_buffer (vm, from[3]);
917 
918  vlib_prefetch_buffer_header (p2, LOAD);
919  vlib_prefetch_buffer_header (p3, LOAD);
920 
921  CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
922  CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
923  }
924 
925  bi0 = to_next[0] = from[0];
926  bi1 = to_next[1] = from[1];
927  from += 2;
928  n_left_from -= 2;
929  to_next += 2;
930  n_left_to_next -= 2;
931 
932  b0 = vlib_get_buffer (vm, bi0);
933  b1 = vlib_get_buffer (vm, bi1);
934  if (is_ip4)
935  {
936  ip40 = vlib_buffer_get_current (b0);
937  ip41 = vlib_buffer_get_current (b1);
938  }
939  else
940  {
941  ip60 = vlib_buffer_get_current (b0);
942  ip61 = vlib_buffer_get_current (b1);
943  }
944 
945  /* Setup packet for next IP feature */
946  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0,
947  b0);
948  vnet_feature_next (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1,
949  b1);
950 
951  if (is_ip4)
952  {
953  /* Treat IP frag packets as "experimental" protocol for now
954  until support of IP frag reassembly is implemented */
955  proto0 = ip4_is_fragment (ip40) ? 0xfe : ip40->protocol;
956  proto1 = ip4_is_fragment (ip41) ? 0xfe : ip41->protocol;
957  }
958  else
959  {
960  proto0 = ip60->protocol;
961  proto1 = ip61->protocol;
962  }
963 
964  /* Process packet 0 */
965  if (proto0 != IP_PROTOCOL_UDP)
966  goto exit0; /* not UDP packet */
967 
968  if (is_ip4)
969  udp0 = ip4_next_header (ip40);
970  else
971  udp0 = ip6_next_header (ip60);
972 
973  if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
974  goto exit0; /* not GENEVE packet */
975 
976  /* Validate DIP against VTEPs */
977  if (is_ip4)
978  {
979  if (addr4.as_u32 != ip40->dst_address.as_u32)
980  {
981  if (!hash_get (vxm->vtep4, ip40->dst_address.as_u32))
982  goto exit0; /* no local VTEP for GENEVE packet */
983  addr4 = ip40->dst_address;
984  }
985  }
986  else
987  {
988  if (!ip6_address_is_equal (&addr6, &ip60->dst_address))
989  {
990  if (!hash_get_mem (vxm->vtep6, &ip60->dst_address))
991  goto exit0; /* no local VTEP for GENEVE packet */
992  addr6 = ip60->dst_address;
993  }
994  }
995 
996  flags0 = b0->flags;
997  good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
998 
999  /* Don't verify UDP checksum for packets with explicit zero checksum. */
1000  good_udp0 |= udp0->checksum == 0;
1001 
1002  /* Verify UDP length */
1003  if (is_ip4)
1004  ip_len0 = clib_net_to_host_u16 (ip40->length);
1005  else
1006  ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
1007  udp_len0 = clib_net_to_host_u16 (udp0->length);
1008  len_diff0 = ip_len0 - udp_len0;
1009 
1010  /* Verify UDP checksum */
1011  if (PREDICT_FALSE (!good_udp0))
1012  {
1013  if ((flags0 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1014  {
1015  if (is_ip4)
1016  flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
1017  else
1018  flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
1019  good_udp0 =
1020  (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1021  }
1022  }
1023 
1024  if (is_ip4)
1025  {
1026  error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1027  error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
1028  }
1029  else
1030  {
1031  error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1032  error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
1033  }
1034 
1035  next0 = error0 ?
1037  b0->error = error0 ? error_node->errors[error0] : 0;
1038 
1039  /* geneve-input node expect current at GENEVE header */
1040  if (is_ip4)
1041  vlib_buffer_advance (b0,
1042  sizeof (ip4_header_t) +
1043  sizeof (udp_header_t));
1044  else
1045  vlib_buffer_advance (b0,
1046  sizeof (ip6_header_t) +
1047  sizeof (udp_header_t));
1048 
1049  exit0:
1050  /* Process packet 1 */
1051  if (proto1 != IP_PROTOCOL_UDP)
1052  goto exit1; /* not UDP packet */
1053 
1054  if (is_ip4)
1055  udp1 = ip4_next_header (ip41);
1056  else
1057  udp1 = ip6_next_header (ip61);
1058 
1059  if (udp1->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
1060  goto exit1; /* not GENEVE packet */
1061 
1062  /* Validate DIP against VTEPs */
1063  if (is_ip4)
1064  {
1065  if (addr4.as_u32 != ip41->dst_address.as_u32)
1066  {
1067  if (!hash_get (vxm->vtep4, ip41->dst_address.as_u32))
1068  goto exit1; /* no local VTEP for GENEVE packet */
1069  addr4 = ip41->dst_address;
1070  }
1071  }
1072  else
1073  {
1074  if (!ip6_address_is_equal (&addr6, &ip61->dst_address))
1075  {
1076  if (!hash_get_mem (vxm->vtep6, &ip61->dst_address))
1077  goto exit1; /* no local VTEP for GENEVE packet */
1078  addr6 = ip61->dst_address;
1079  }
1080  }
1081 
1082  flags1 = b1->flags;
1083  good_udp1 = (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1084 
1085  /* Don't verify UDP checksum for packets with explicit zero checksum. */
1086  good_udp1 |= udp1->checksum == 0;
1087 
1088  /* Verify UDP length */
1089  if (is_ip4)
1090  ip_len1 = clib_net_to_host_u16 (ip41->length);
1091  else
1092  ip_len1 = clib_net_to_host_u16 (ip61->payload_length);
1093  udp_len1 = clib_net_to_host_u16 (udp1->length);
1094  len_diff1 = ip_len1 - udp_len1;
1095 
1096  /* Verify UDP checksum */
1097  if (PREDICT_FALSE (!good_udp1))
1098  {
1099  if ((flags1 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1100  {
1101  if (is_ip4)
1102  flags1 = ip4_tcp_udp_validate_checksum (vm, b1);
1103  else
1104  flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, b1);
1105  good_udp1 =
1106  (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1107  }
1108  }
1109 
1110  if (is_ip4)
1111  {
1112  error1 = good_udp1 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1113  error1 = (len_diff1 >= 0) ? error1 : IP4_ERROR_UDP_LENGTH;
1114  }
1115  else
1116  {
1117  error1 = good_udp1 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1118  error1 = (len_diff1 >= 0) ? error1 : IP6_ERROR_UDP_LENGTH;
1119  }
1120 
1121  next1 = error1 ?
1123  b1->error = error1 ? error_node->errors[error1] : 0;
1124 
1125  /* geneve-input node expect current at GENEVE header */
1126  if (is_ip4)
1127  vlib_buffer_advance (b1,
1128  sizeof (ip4_header_t) +
1129  sizeof (udp_header_t));
1130  else
1131  vlib_buffer_advance (b1,
1132  sizeof (ip6_header_t) +
1133  sizeof (udp_header_t));
1134 
1135  exit1:
1136  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1137  to_next, n_left_to_next,
1138  bi0, bi1, next0, next1);
1139  }
1140 
1141  while (n_left_from > 0 && n_left_to_next > 0)
1142  {
1143  vlib_buffer_t *b0;
1144  ip4_header_t *ip40;
1145  ip6_header_t *ip60;
1146  udp_header_t *udp0;
1147  u32 bi0, ip_len0, udp_len0, flags0, next0;
1148  i32 len_diff0;
1149  u8 error0, good_udp0, proto0;
1150 
1151  bi0 = to_next[0] = from[0];
1152  from += 1;
1153  n_left_from -= 1;
1154  to_next += 1;
1155  n_left_to_next -= 1;
1156 
1157  b0 = vlib_get_buffer (vm, bi0);
1158  if (is_ip4)
1159  ip40 = vlib_buffer_get_current (b0);
1160  else
1161  ip60 = vlib_buffer_get_current (b0);
1162 
1163  /* Setup packet for next IP feature */
1164  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0,
1165  b0);
1166 
1167  if (is_ip4)
1168  /* Treat IP4 frag packets as "experimental" protocol for now
1169  until support of IP frag reassembly is implemented */
1170  proto0 = ip4_is_fragment (ip40) ? 0xfe : ip40->protocol;
1171  else
1172  proto0 = ip60->protocol;
1173 
1174  if (proto0 != IP_PROTOCOL_UDP)
1175  goto exit; /* not UDP packet */
1176 
1177  if (is_ip4)
1178  udp0 = ip4_next_header (ip40);
1179  else
1180  udp0 = ip6_next_header (ip60);
1181 
1182  if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
1183  goto exit; /* not GENEVE packet */
1184 
1185  /* Validate DIP against VTEPs */
1186  if (is_ip4)
1187  {
1188  if (addr4.as_u32 != ip40->dst_address.as_u32)
1189  {
1190  if (!hash_get (vxm->vtep4, ip40->dst_address.as_u32))
1191  goto exit; /* no local VTEP for GENEVE packet */
1192  addr4 = ip40->dst_address;
1193  }
1194  }
1195  else
1196  {
1197  if (!ip6_address_is_equal (&addr6, &ip60->dst_address))
1198  {
1199  if (!hash_get_mem (vxm->vtep6, &ip60->dst_address))
1200  goto exit; /* no local VTEP for GENEVE packet */
1201  addr6 = ip60->dst_address;
1202  }
1203  }
1204 
1205  flags0 = b0->flags;
1206  good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1207 
1208  /* Don't verify UDP checksum for packets with explicit zero checksum. */
1209  good_udp0 |= udp0->checksum == 0;
1210 
1211  /* Verify UDP length */
1212  if (is_ip4)
1213  ip_len0 = clib_net_to_host_u16 (ip40->length);
1214  else
1215  ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
1216  udp_len0 = clib_net_to_host_u16 (udp0->length);
1217  len_diff0 = ip_len0 - udp_len0;
1218 
1219  /* Verify UDP checksum */
1220  if (PREDICT_FALSE (!good_udp0))
1221  {
1222  if ((flags0 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1223  {
1224  if (is_ip4)
1225  flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
1226  else
1227  flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
1228  good_udp0 =
1229  (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1230  }
1231  }
1232 
1233  if (is_ip4)
1234  {
1235  error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1236  error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
1237  }
1238  else
1239  {
1240  error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1241  error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
1242  }
1243 
1244  next0 = error0 ?
1246  b0->error = error0 ? error_node->errors[error0] : 0;
1247 
1248  /* geneve-input node expect current at GENEVE header */
1249  if (is_ip4)
1250  vlib_buffer_advance (b0,
1251  sizeof (ip4_header_t) +
1252  sizeof (udp_header_t));
1253  else
1254  vlib_buffer_advance (b0,
1255  sizeof (ip6_header_t) +
1256  sizeof (udp_header_t));
1257 
1258  exit:
1259  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1260  to_next, n_left_to_next,
1261  bi0, next0);
1262  }
1263 
1264  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1265  }
1266 
1267  return frame->n_vectors;
1268 }
1269 
1270 static uword
1272  vlib_node_runtime_t * node, vlib_frame_t * frame)
1273 {
1274  return ip_geneve_bypass_inline (vm, node, frame, /* is_ip4 */ 1);
1275 }
1276 
1277 /* *INDENT-OFF* */
1279 {
1280  .function = ip4_geneve_bypass,.name = "ip4-geneve-bypass",.vector_size =
1281  sizeof (u32),.n_next_nodes = IP_GENEVE_BYPASS_N_NEXT,.next_nodes =
1282  {
1283  [IP_GENEVE_BYPASS_NEXT_DROP] = "error-drop",
1284  [IP_GENEVE_BYPASS_NEXT_GENEVE] = "geneve4-input",}
1285 ,.format_buffer = format_ip4_header,.format_trace =
1287 
1289 /* Dummy init function to get us linked in. */
1291 {
1292  return 0;
1293 }
1294 
1296 /* *INDENT-ON* */
1297 
1298 static uword
1300  vlib_node_runtime_t * node, vlib_frame_t * frame)
1301 {
1302  return ip_geneve_bypass_inline (vm, node, frame, /* is_ip4 */ 0);
1303 }
1304 
1305 /* *INDENT-OFF* */
1307 {
1308  .function = ip6_geneve_bypass,.name = "ip6-geneve-bypass",.vector_size =
1309  sizeof (u32),.n_next_nodes = IP_GENEVE_BYPASS_N_NEXT,.next_nodes =
1310  {
1311  [IP_GENEVE_BYPASS_NEXT_DROP] = "error-drop",
1312  [IP_GENEVE_BYPASS_NEXT_GENEVE] = "geneve6-input",}
1313 ,.format_buffer = format_ip6_header,.format_trace =
1315 /* *INDENT-ON* */
1316 
1318 /* Dummy init function to get us linked in. */
1320 {
1321  return 0;
1322 }
1323 
1325 
1326 /*
1327  * fd.io coding-style-patch-verification: ON
1328  *
1329  * Local Variables:
1330  * eval: (c-set-style "gnu")
1331  * End:
1332  */
static u8 vnet_get_geneve_critical_bit(geneve_header_t *h)
static u32 vnet_get_geneve_vni(geneve_header_t *h)
#define CLIB_UNUSED(x)
Definition: clib.h:79
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:213
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:500
ip4_address_t src_address
Definition: ip4_packet.h:169
vnet_interface_main_t interface_main
Definition: vnet.h:56
format_function_t format_ip4_header
Definition: format.h:89
u32 tunnel_index
Definition: decap.c:26
#define PREDICT_TRUE(x)
Definition: clib.h:106
u64 as_u64[2]
Definition: ip6_packet.h:51
#define NULL
Definition: clib.h:55
u32 decap_next_index
Definition: geneve.h:103
uword * vtep6
Definition: geneve.h:165
u32 * fib_index_by_sw_if_index
Table index indexed by software interface.
Definition: ip4.h:111
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:451
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:250
ip6_address_t src_address
Definition: ip6_packet.h:347
unsigned char u8
Definition: types.h:56
static char * geneve_error_strings[]
Definition: decap.c:814
#define GENEVE_BASE_HEADER_LENGTH
Definition: geneve_packet.h:95
vlib_node_registration_t ip4_geneve_bypass_node
(constructor) VLIB_REGISTER_NODE (ip4_geneve_bypass_node)
Definition: decap.c:1278
geneve_tunnel_t * tunnels
Definition: geneve.h:156
static u8 vnet_get_geneve_options_len(geneve_header_t *h)
i64 word
Definition: types.h:111
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:156
#define always_inline
Definition: clib.h:92
ip4_address_t dst_address
Definition: ip4_packet.h:169
#define foreach_geneve_input_next
Definition: geneve.h:133
static u32 vnet_get_geneve_vni_bigendian(geneve_header_t *h)
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:810
static uword ip6_address_is_equal(ip6_address_t *a, ip6_address_t *b)
Definition: ip6_packet.h:214
static uword ip_geneve_bypass_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, u32 is_ip4)
Definition: decap.c:872
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:184
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:238
unsigned int u32
Definition: types.h:88
static int ip4_is_fragment(ip4_header_t *i)
Definition: ip4_packet.h:210
clib_error_t * ip6_geneve_bypass_init(vlib_main_t *vm)
Definition: decap.c:1319
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:461
static void vnet_geneve_hdr_1word_ntoh(geneve_header_t *h)
vlib_node_registration_t ip4_input_node
Global ip4 input node.
Definition: ip4_input.c:316
u32 encap_fib_index
Definition: geneve.h:106
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:202
static uword ip4_address_is_multicast(ip4_address_t *a)
Definition: ip4_packet.h:315
#define PREDICT_FALSE(x)
Definition: clib.h:105
static u32 validate_geneve_fib(vlib_buffer_t *b, geneve_tunnel_t *t, u32 is_ip4)
Definition: decap.c:54
u32 ip4_tcp_udp_validate_checksum(vlib_main_t *vm, vlib_buffer_t *p0)
Definition: ip4_forward.c:1113
vlib_node_registration_t ip6_geneve_bypass_node
(constructor) VLIB_REGISTER_NODE (ip6_geneve_bypass_node)
Definition: decap.c:1306
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
static_always_inline void vnet_feature_next(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:237
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:135
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1168
vlib_node_registration_t geneve4_input_node
(constructor) VLIB_REGISTER_NODE (geneve4_input_node)
Definition: decap.c:20
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
uword * vtep4
Definition: geneve.h:164
u16 n_vectors
Definition: node.h:380
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:77
vlib_main_t * vm
Definition: buffer.c:294
static void ip6_address_set_zero(ip6_address_t *a)
Definition: ip6_packet.h:249
ip46_address_t local
Definition: geneve.h:96
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
#define clib_memcpy(a, b, c)
Definition: string.h:75
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
uword * geneve4_tunnel_by_key
Definition: geneve.h:159
static void * ip6_next_header(ip6_header_t *i)
Definition: ip6_packet.h:374
ip_vxan_bypass_next_t
Definition: decap.c:864
signed int i32
Definition: types.h:81
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:492
u32 sw_if_index
Definition: geneve.h:109
static uword geneve6_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: decap.c:808
static u8 * format_geneve_rx_trace(u8 *s, va_list *args)
Definition: decap.c:32
ip6_main_t ip6_main
Definition: ip6_forward.c:2574
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:215
static u8 vnet_get_geneve_version(geneve_header_t *h)
format_function_t format_ip6_header
Definition: format.h:103
u32 next_index
Definition: decap.c:25
vnet_main_t * vnet_main
Definition: geneve.h:178
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
#define vec_elt(v, i)
Get vector value at index i.
struct _vlib_node_registration vlib_node_registration_t
Definition: defs.h:47
static uword ip6_address_is_multicast(ip6_address_t *a)
Definition: ip6_packet.h:151
u16 payload_length
Definition: ip6_packet.h:338
static uword ip4_geneve_bypass(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: decap.c:1271
u32 ip6_tcp_udp_icmp_validate_checksum(vlib_main_t *vm, vlib_buffer_t *p0)
Definition: ip6_forward.c:986
static void vnet_update_l2_len(vlib_buffer_t *b)
Definition: l2_input.h:221
void ip4_forward_next_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vlib_rx_or_tx_t which_adj_index)
Definition: ip4_forward.c:967
u64 uword
Definition: types.h:112
uword * geneve6_tunnel_by_key
Definition: geneve.h:160
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
u8 * format_ip4_forward_next_trace(u8 *s, va_list *args)
Definition: ip4_forward.c:919
#define GENEVE_VERSION
Definition: geneve_packet.h:98
#define hash_get_mem(h, key)
Definition: hash.h:269
static uword ip6_geneve_bypass(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: decap.c:1299
vlib_node_registration_t geneve6_input_node
(constructor) VLIB_REGISTER_NODE (geneve6_input_node)
Definition: decap.c:21
#define vnet_buffer(b)
Definition: buffer.h:360
VLIB_NODE_FUNCTION_MULTIARCH(l2t_decap_node, l2t_decap_node_fn)
ip4_main_t ip4_main
Global ip4 main structure.
Definition: ip4_forward.c:832
geneve_main_t geneve_main
Definition: geneve.c:38
u8 data[0]
Packet data.
Definition: buffer.h:172
u16 flags
Copy of main node flags.
Definition: node.h:486
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:295
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
clib_error_t * ip4_geneve_bypass_init(vlib_main_t *vm)
Definition: decap.c:1290
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
u32 * fib_index_by_sw_if_index
Definition: ip6.h:176
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
static uword geneve4_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: decap.c:801
Definition: defs.h:46
ip6_address_t dst_address
Definition: ip6_packet.h:347
static uword geneve_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, u32 is_ip4)
Definition: decap.c:73
u8 * format_ip6_forward_next_trace(u8 *s, va_list *args)
Definition: ip6_forward.c:768