FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
in2out.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /**
16  * @file
17  * @brief NAT44 inside to outside network translation
18  */
19 
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 
24 #include <vnet/ip/ip.h>
25 #include <vnet/ethernet/ethernet.h>
26 #include <vnet/fib/ip4_fib.h>
27 #include <nat/nat.h>
28 #include <nat/nat_ipfix_logging.h>
29 #include <nat/nat_reass.h>
30 #include <nat/nat_inlines.h>
31 
32 #include <vppinfra/hash.h>
33 #include <vppinfra/error.h>
34 #include <vppinfra/elog.h>
35 
36 typedef struct
37 {
43 
44 /* packet trace format function */
45 static u8 *
46 format_snat_in2out_trace (u8 * s, va_list * args)
47 {
48  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50  snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
51  char *tag;
52 
53  tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
54 
55  s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
57 
58  return s;
59 }
60 
61 static u8 *
62 format_snat_in2out_fast_trace (u8 * s, va_list * args)
63 {
64  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
65  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
66  snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
67 
68  s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
69  t->sw_if_index, t->next_index);
70 
71  return s;
72 }
73 
80 
81 #define foreach_snat_in2out_error \
82 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
83 _(IN2OUT_PACKETS, "Good in2out packets processed") \
84 _(OUT_OF_PORTS, "Out of ports") \
85 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
86 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
87 _(NO_TRANSLATION, "No translation") \
88 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
89 _(DROP_FRAGMENT, "Drop fragment") \
90 _(MAX_REASS, "Maximum reassemblies exceeded") \
91 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
92 
93 typedef enum
94 {
95 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
97 #undef _
100 
101 static char *snat_in2out_error_strings[] = {
102 #define _(sym,string) string,
104 #undef _
105 };
106 
107 typedef enum
108 {
116 
117 static inline int
119  u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
120  u32 rx_fib_index0, u32 thread_index)
121 {
122  udp_header_t *udp0 = ip4_next_header (ip0);
123  snat_session_key_t key0, sm0;
124  clib_bihash_kv_8_8_t kv0, value0;
125 
126  key0.addr = ip0->dst_address;
127  key0.port = udp0->dst_port;
128  key0.protocol = proto0;
129  key0.fib_index = sm->outside_fib_index;
130  kv0.key = key0.as_u64;
131 
132  /* NAT packet aimed at external address if */
133  /* has active sessions */
134  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
135  &value0))
136  {
137  /* or is static mappings */
138  if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
139  return 0;
140  }
141  else
142  return 0;
143 
144  if (sm->forwarding_enabled)
145  return 1;
146 
147  return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
148  rx_fib_index0);
149 }
150 
151 static inline int
153  u32 proto0, u16 src_port, u16 dst_port,
154  u32 thread_index, u32 sw_if_index)
155 {
156  snat_session_key_t key0;
157  clib_bihash_kv_8_8_t kv0, value0;
159 
160  /* src NAT check */
161  key0.addr = ip0->src_address;
162  key0.port = src_port;
163  key0.protocol = proto0;
165  kv0.key = key0.as_u64;
166 
167  if (!clib_bihash_search_8_8
168  (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
169  return 1;
170 
171  /* dst NAT check */
172  key0.addr = ip0->dst_address;
173  key0.port = dst_port;
174  key0.protocol = proto0;
175  kv0.key = key0.as_u64;
176  if (!clib_bihash_search_8_8
177  (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
178  {
179  /* hairpinning */
180  /* *INDENT-OFF* */
182  ({
183  if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
184  return 0;
185  }));
186  /* *INDENT-ON* */
187  return 1;
188  }
189 
190  return 0;
191 }
192 
193 int
195 {
196  snat_main_t *sm = &snat_main;
198  snat_session_t *s;
199  u64 sess_timeout_time;
201  ctx->thread_index);
203 
204  s = pool_elt_at_index (tsm->sessions, kv->value);
205  sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
206  if (ctx->now >= sess_timeout_time)
207  {
208  s_kv.key = s->out2in.as_u64;
209  if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
210  nat_log_warn ("out2in key del failed");
211 
212  snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
213  s->out2in.addr.as_u32,
214  s->in2out.protocol,
215  s->in2out.port,
216  s->out2in.port,
217  s->in2out.fib_index);
218 
219  if (!snat_is_session_static (s))
221  &s->out2in);
222 
223  nat44_delete_session (sm, s, ctx->thread_index);
224  return 1;
225  }
226 
227  return 0;
228 }
229 
230 static u32
232  ip4_header_t * ip0,
233  u32 rx_fib_index0,
234  snat_session_key_t * key0,
235  snat_session_t ** sessionp,
236  vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
237 {
238  snat_user_t *u;
239  snat_session_t *s = 0;
241  snat_session_key_t key1;
242  udp_header_t *udp0 = ip4_next_header (ip0);
243  u8 is_sm = 0;
244  nat_outside_fib_t *outside_fib;
246  u8 identity_nat;
247  fib_prefix_t pfx = {
249  .fp_len = 32,
250  .fp_addr = {
251  .ip4.as_u32 = ip0->dst_address.as_u32,
252  },
253  };
255 
256  if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
257  {
258  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
260  nat_log_notice ("maximum sessions exceeded");
261  return SNAT_IN2OUT_NEXT_DROP;
262  }
263 
264  key1.protocol = key0->protocol;
265 
266  /* First try to match static mapping by local address and port */
268  (sm, *key0, &key1, 0, 0, 0, 0, 0, &identity_nat))
269  {
270  /* Try to create dynamic translation */
271  if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
272  thread_index, &key1,
273  sm->port_per_thread,
274  sm->per_thread_data
275  [thread_index].snat_thread_index))
276  {
277  b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
278  return SNAT_IN2OUT_NEXT_DROP;
279  }
280  }
281  else
282  {
283  if (PREDICT_FALSE (identity_nat))
284  {
285  *sessionp = s;
286  return next0;
287  }
288 
289  is_sm = 1;
290  }
291 
292  u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
293  thread_index);
294  if (!u)
295  {
296  nat_log_warn ("create NAT user failed");
297  return SNAT_IN2OUT_NEXT_DROP;
298  }
299 
300  s = nat_session_alloc_or_recycle (sm, u, thread_index);
301  if (!s)
302  {
303  nat44_delete_user_with_no_session (sm, u, thread_index);
304  nat_log_warn ("create NAT session failed");
305  return SNAT_IN2OUT_NEXT_DROP;
306  }
307 
308  if (is_sm)
310  user_session_increment (sm, u, is_sm);
311  s->in2out = *key0;
312  s->out2in = key1;
313  s->out2in.protocol = key0->protocol;
314  s->out2in.fib_index = sm->outside_fib_index;
315  switch (vec_len (sm->outside_fibs))
316  {
317  case 0:
318  s->out2in.fib_index = sm->outside_fib_index;
319  break;
320  case 1:
321  s->out2in.fib_index = sm->outside_fibs[0].fib_index;
322  break;
323  default:
324  /* *INDENT-OFF* */
325  vec_foreach (outside_fib, sm->outside_fibs)
326  {
327  fei = fib_table_lookup (outside_fib->fib_index, &pfx);
328  if (FIB_NODE_INDEX_INVALID != fei)
329  {
330  if (fib_entry_get_resolving_interface (fei) != ~0)
331  {
332  s->out2in.fib_index = outside_fib->fib_index;
333  break;
334  }
335  }
336  }
337  /* *INDENT-ON* */
338  break;
339  }
340  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
341  s->ext_host_port = udp0->dst_port;
342  *sessionp = s;
343 
344  /* Add to translation hashes */
345  ctx0.now = now;
346  ctx0.thread_index = thread_index;
347  kv0.key = s->in2out.as_u64;
348  kv0.value = s - sm->per_thread_data[thread_index].sessions;
349  if (clib_bihash_add_or_overwrite_stale_8_8
350  (&sm->per_thread_data[thread_index].in2out, &kv0,
352  nat_log_notice ("in2out key add failed");
353 
354  kv0.key = s->out2in.as_u64;
355  kv0.value = s - sm->per_thread_data[thread_index].sessions;
356 
357  if (clib_bihash_add_or_overwrite_stale_8_8
358  (&sm->per_thread_data[thread_index].out2in, &kv0,
360  nat_log_notice ("out2in key add failed");
361 
362  /* log NAT event */
363  snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
364  s->out2in.addr.as_u32,
365  s->in2out.protocol,
366  s->in2out.port,
367  s->out2in.port, s->in2out.fib_index);
368  return next0;
369 }
370 
373  snat_session_key_t * p_key0)
374 {
375  icmp46_header_t *icmp0;
376  snat_session_key_t key0;
377  icmp_echo_header_t *echo0, *inner_echo0 = 0;
378  ip4_header_t *inner_ip0 = 0;
379  void *l4_header = 0;
380  icmp46_header_t *inner_icmp0;
381 
382  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
383  echo0 = (icmp_echo_header_t *) (icmp0 + 1);
384 
385  if (!icmp_is_error_message (icmp0))
386  {
387  key0.protocol = SNAT_PROTOCOL_ICMP;
388  key0.addr = ip0->src_address;
389  key0.port = echo0->identifier;
390  }
391  else
392  {
393  inner_ip0 = (ip4_header_t *) (echo0 + 1);
394  l4_header = ip4_next_header (inner_ip0);
395  key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
396  key0.addr = inner_ip0->dst_address;
397  switch (key0.protocol)
398  {
399  case SNAT_PROTOCOL_ICMP:
400  inner_icmp0 = (icmp46_header_t *) l4_header;
401  inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
402  key0.port = inner_echo0->identifier;
403  break;
404  case SNAT_PROTOCOL_UDP:
405  case SNAT_PROTOCOL_TCP:
406  key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
407  break;
408  default:
409  return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
410  }
411  }
412  *p_key0 = key0;
413  return -1; /* success */
414 }
415 
416 /**
417  * Get address and port values to be used for ICMP packet translation
418  * and create session if needed
419  *
420  * @param[in,out] sm NAT main
421  * @param[in,out] node NAT node runtime
422  * @param[in] thread_index thread index
423  * @param[in,out] b0 buffer containing packet to be translated
424  * @param[out] p_proto protocol used for matching
425  * @param[out] p_value address and port after NAT translation
426  * @param[out] p_dont_translate if packet should not be translated
427  * @param d optional parameter
428  * @param e optional parameter
429  */
430 u32
432  u32 thread_index, vlib_buffer_t * b0,
433  ip4_header_t * ip0, u8 * p_proto,
434  snat_session_key_t * p_value,
435  u8 * p_dont_translate, void *d, void *e)
436 {
437  icmp46_header_t *icmp0;
438  u32 sw_if_index0;
439  u32 rx_fib_index0;
440  snat_session_key_t key0;
441  snat_session_t *s0 = 0;
442  u8 dont_translate = 0;
443  clib_bihash_kv_8_8_t kv0, value0;
444  u32 next0 = ~0;
445  int err;
446 
447  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
448  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
449  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
450 
451  err = icmp_get_key (ip0, &key0);
452  if (err != -1)
453  {
454  b0->error = node->errors[err];
455  next0 = SNAT_IN2OUT_NEXT_DROP;
456  goto out;
457  }
458  key0.fib_index = rx_fib_index0;
459 
460  kv0.key = key0.as_u64;
461 
462  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
463  &value0))
464  {
465  if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
466  {
468  key0.protocol,
469  key0.port,
470  key0.port,
471  thread_index,
472  sw_if_index0)))
473  {
474  dont_translate = 1;
475  goto out;
476  }
477  }
478  else
479  {
480  if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
481  ip0, SNAT_PROTOCOL_ICMP,
482  rx_fib_index0,
483  thread_index)))
484  {
485  dont_translate = 1;
486  goto out;
487  }
488  }
489 
490  if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
491  {
492  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
493  next0 = SNAT_IN2OUT_NEXT_DROP;
494  goto out;
495  }
496 
497  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
498  thread_index, vlib_time_now (sm->vlib_main));
499 
500  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
501  goto out;
502 
503  if (!s0)
504  {
505  dont_translate = 1;
506  goto out;
507  }
508  }
509  else
510  {
511  if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
512  icmp0->type != ICMP4_echo_reply &&
513  !icmp_is_error_message (icmp0)))
514  {
515  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
516  next0 = SNAT_IN2OUT_NEXT_DROP;
517  goto out;
518  }
519 
520  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
521  value0.value);
522  }
523 
524 out:
525  *p_proto = key0.protocol;
526  if (s0)
527  *p_value = s0->out2in;
528  *p_dont_translate = dont_translate;
529  if (d)
530  *(snat_session_t **) d = s0;
531  return next0;
532 }
533 
534 /**
535  * Get address and port values to be used for ICMP packet translation
536  *
537  * @param[in] sm NAT main
538  * @param[in,out] node NAT node runtime
539  * @param[in] thread_index thread index
540  * @param[in,out] b0 buffer containing packet to be translated
541  * @param[out] p_proto protocol used for matching
542  * @param[out] p_value address and port after NAT translation
543  * @param[out] p_dont_translate if packet should not be translated
544  * @param d optional parameter
545  * @param e optional parameter
546  */
547 u32
549  u32 thread_index, vlib_buffer_t * b0,
550  ip4_header_t * ip0, u8 * p_proto,
551  snat_session_key_t * p_value,
552  u8 * p_dont_translate, void *d, void *e)
553 {
554  icmp46_header_t *icmp0;
555  u32 sw_if_index0;
556  u32 rx_fib_index0;
557  snat_session_key_t key0;
558  snat_session_key_t sm0;
559  u8 dont_translate = 0;
560  u8 is_addr_only;
561  u32 next0 = ~0;
562  int err;
563 
564  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
565  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
566  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
567 
568  err = icmp_get_key (ip0, &key0);
569  if (err != -1)
570  {
571  b0->error = node->errors[err];
572  next0 = SNAT_IN2OUT_NEXT_DROP;
573  goto out2;
574  }
575  key0.fib_index = rx_fib_index0;
576 
578  (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0, 0))
579  {
580  if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
581  IP_PROTOCOL_ICMP,
582  rx_fib_index0)))
583  {
584  dont_translate = 1;
585  goto out;
586  }
587 
588  if (icmp_is_error_message (icmp0))
589  {
590  next0 = SNAT_IN2OUT_NEXT_DROP;
591  goto out;
592  }
593 
594  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
595  next0 = SNAT_IN2OUT_NEXT_DROP;
596  goto out;
597  }
598 
599  if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
600  (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
601  !icmp_is_error_message (icmp0)))
602  {
603  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
604  next0 = SNAT_IN2OUT_NEXT_DROP;
605  goto out;
606  }
607 
608 out:
609  *p_value = sm0;
610 out2:
611  *p_proto = key0.protocol;
612  *p_dont_translate = dont_translate;
613  return next0;
614 }
615 
616 u32
618  vlib_buffer_t * b0,
619  ip4_header_t * ip0,
620  icmp46_header_t * icmp0,
621  u32 sw_if_index0,
622  u32 rx_fib_index0,
623  vlib_node_runtime_t * node,
624  u32 next0, u32 thread_index, void *d, void *e)
625 {
626  snat_session_key_t sm0;
627  u8 protocol;
628  icmp_echo_header_t *echo0, *inner_echo0 = 0;
629  ip4_header_t *inner_ip0;
630  void *l4_header = 0;
631  icmp46_header_t *inner_icmp0;
632  u8 dont_translate;
633  u32 new_addr0, old_addr0;
634  u16 old_id0, new_id0;
635  u16 old_checksum0, new_checksum0;
636  ip_csum_t sum0;
637  u16 checksum0;
638  u32 next0_tmp;
639 
640  echo0 = (icmp_echo_header_t *) (icmp0 + 1);
641 
642  next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
643  &protocol, &sm0, &dont_translate, d,
644  e);
645  if (next0_tmp != ~0)
646  next0 = next0_tmp;
647  if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
648  goto out;
649 
650  if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
651  {
652  sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
653  (u8 *)
655  ntohs (ip0->length) -
656  ip4_header_bytes (ip0), 0);
657  checksum0 = ~ip_csum_fold (sum0);
658  if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
659  {
660  next0 = SNAT_IN2OUT_NEXT_DROP;
661  goto out;
662  }
663  }
664 
665  old_addr0 = ip0->src_address.as_u32;
666  new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
667 
668  sum0 = ip0->checksum;
669  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
670  src_address /* changed member */ );
671  ip0->checksum = ip_csum_fold (sum0);
672 
673  if (icmp0->checksum == 0)
674  icmp0->checksum = 0xffff;
675 
676  if (!icmp_is_error_message (icmp0))
677  {
678  new_id0 = sm0.port;
679  if (PREDICT_FALSE (new_id0 != echo0->identifier))
680  {
681  old_id0 = echo0->identifier;
682  new_id0 = sm0.port;
683  echo0->identifier = new_id0;
684 
685  sum0 = icmp0->checksum;
686  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
687  identifier);
688  icmp0->checksum = ip_csum_fold (sum0);
689  }
690  }
691  else
692  {
693  inner_ip0 = (ip4_header_t *) (echo0 + 1);
694  l4_header = ip4_next_header (inner_ip0);
695 
696  if (!ip4_header_checksum_is_valid (inner_ip0))
697  {
698  next0 = SNAT_IN2OUT_NEXT_DROP;
699  goto out;
700  }
701 
702  /* update inner destination IP address */
703  old_addr0 = inner_ip0->dst_address.as_u32;
704  inner_ip0->dst_address = sm0.addr;
705  new_addr0 = inner_ip0->dst_address.as_u32;
706  sum0 = icmp0->checksum;
707  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
708  dst_address /* changed member */ );
709  icmp0->checksum = ip_csum_fold (sum0);
710 
711  /* update inner IP header checksum */
712  old_checksum0 = inner_ip0->checksum;
713  sum0 = inner_ip0->checksum;
714  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
715  dst_address /* changed member */ );
716  inner_ip0->checksum = ip_csum_fold (sum0);
717  new_checksum0 = inner_ip0->checksum;
718  sum0 = icmp0->checksum;
719  sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
720  checksum);
721  icmp0->checksum = ip_csum_fold (sum0);
722 
723  switch (protocol)
724  {
725  case SNAT_PROTOCOL_ICMP:
726  inner_icmp0 = (icmp46_header_t *) l4_header;
727  inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
728 
729  old_id0 = inner_echo0->identifier;
730  new_id0 = sm0.port;
731  inner_echo0->identifier = new_id0;
732 
733  sum0 = icmp0->checksum;
734  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
735  identifier);
736  icmp0->checksum = ip_csum_fold (sum0);
737  break;
738  case SNAT_PROTOCOL_UDP:
739  case SNAT_PROTOCOL_TCP:
740  old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
741  new_id0 = sm0.port;
742  ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
743 
744  sum0 = icmp0->checksum;
745  sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
746  dst_port);
747  icmp0->checksum = ip_csum_fold (sum0);
748  break;
749  default:
750  ASSERT (0);
751  }
752  }
753 
754  if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
755  {
756  if (sm->deterministic ||
757  0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
758  sm->endpoint_dependent))
759  vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
760  }
761 
762 out:
763  return next0;
764 }
765 
766 static inline u32
768  vlib_buffer_t * b0,
769  ip4_header_t * ip0,
770  icmp46_header_t * icmp0,
771  u32 sw_if_index0,
772  u32 rx_fib_index0,
773  vlib_node_runtime_t * node,
774  u32 next0,
775  f64 now, u32 thread_index, snat_session_t ** p_s0)
776 {
777  next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
778  next0, thread_index, p_s0, 0);
779  snat_session_t *s0 = *p_s0;
780  if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
781  {
782  /* Accounting */
785  (sm->vlib_main, b0));
786  /* Per-user LRU list maintenance */
787  nat44_session_update_lru (sm, s0, thread_index);
788  }
789  return next0;
790 }
791 
792 static int
794  vlib_buffer_t * b,
795  ip4_header_t * ip, u32 rx_fib_index)
796 {
797  clib_bihash_kv_8_8_t kv, value;
799  snat_session_key_t m_key;
800  u32 old_addr, new_addr;
801  ip_csum_t sum;
802 
803  m_key.addr = ip->src_address;
804  m_key.port = 0;
805  m_key.protocol = 0;
806  m_key.fib_index = rx_fib_index;
807  kv.key = m_key.as_u64;
808  if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
809  return 1;
810 
811  m = pool_elt_at_index (sm->static_mappings, value.value);
812 
813  old_addr = ip->src_address.as_u32;
814  new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
815  sum = ip->checksum;
816  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
817  ip->checksum = ip_csum_fold (sum);
818 
819 
820  /* Hairpinning */
821  if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
822  {
823  vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
825  }
826 
827  return 0;
828 }
829 
830 static inline uword
832  vlib_node_runtime_t * node,
833  vlib_frame_t * frame, int is_slow_path,
834  int is_output_feature)
835 {
836  u32 n_left_from, *from, *to_next;
837  snat_in2out_next_t next_index;
838  u32 pkts_processed = 0;
839  snat_main_t *sm = &snat_main;
840  f64 now = vlib_time_now (vm);
841  u32 stats_node_index;
842  u32 thread_index = vm->thread_index;
843 
844  stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
845  snat_in2out_node.index;
846 
847  from = vlib_frame_vector_args (frame);
848  n_left_from = frame->n_vectors;
849  next_index = node->cached_next_index;
850 
851  while (n_left_from > 0)
852  {
853  u32 n_left_to_next;
854 
855  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
856 
857  while (n_left_from >= 4 && n_left_to_next >= 2)
858  {
859  u32 bi0, bi1;
860  vlib_buffer_t *b0, *b1;
861  u32 next0, next1;
862  u32 sw_if_index0, sw_if_index1;
863  ip4_header_t *ip0, *ip1;
864  ip_csum_t sum0, sum1;
865  u32 new_addr0, old_addr0, new_addr1, old_addr1;
866  u16 old_port0, new_port0, old_port1, new_port1;
867  udp_header_t *udp0, *udp1;
868  tcp_header_t *tcp0, *tcp1;
869  icmp46_header_t *icmp0, *icmp1;
870  snat_session_key_t key0, key1;
871  u32 rx_fib_index0, rx_fib_index1;
872  u32 proto0, proto1;
873  snat_session_t *s0 = 0, *s1 = 0;
874  clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
875  u32 iph_offset0 = 0, iph_offset1 = 0;
876 
877  /* Prefetch next iteration. */
878  {
879  vlib_buffer_t *p2, *p3;
880 
881  p2 = vlib_get_buffer (vm, from[2]);
882  p3 = vlib_get_buffer (vm, from[3]);
883 
884  vlib_prefetch_buffer_header (p2, LOAD);
885  vlib_prefetch_buffer_header (p3, LOAD);
886 
889  }
890 
891  /* speculatively enqueue b0 and b1 to the current next frame */
892  to_next[0] = bi0 = from[0];
893  to_next[1] = bi1 = from[1];
894  from += 2;
895  to_next += 2;
896  n_left_from -= 2;
897  n_left_to_next -= 2;
898 
899  b0 = vlib_get_buffer (vm, bi0);
900  b1 = vlib_get_buffer (vm, bi1);
901 
902  if (is_output_feature)
903  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
904 
905  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
906  iph_offset0);
907 
908  udp0 = ip4_next_header (ip0);
909  tcp0 = (tcp_header_t *) udp0;
910  icmp0 = (icmp46_header_t *) udp0;
911 
912  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
913  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
914  sw_if_index0);
915 
916  next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
917 
918  if (PREDICT_FALSE (ip0->ttl == 1))
919  {
920  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
921  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
922  ICMP4_time_exceeded_ttl_exceeded_in_transit,
923  0);
925  goto trace00;
926  }
927 
928  proto0 = ip_proto_to_snat_proto (ip0->protocol);
929 
930  /* Next configured feature, probably ip4-lookup */
931  if (is_slow_path)
932  {
933  if (PREDICT_FALSE (proto0 == ~0))
934  {
936  (sm, b0, ip0, rx_fib_index0))
937  {
938  next0 = SNAT_IN2OUT_NEXT_DROP;
939  b0->error =
940  node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
941  }
942  goto trace00;
943  }
944 
945  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
946  {
947  next0 = icmp_in2out_slow_path
948  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
949  node, next0, now, thread_index, &s0);
950  goto trace00;
951  }
952  }
953  else
954  {
955  if (PREDICT_FALSE (proto0 == ~0))
956  {
958  goto trace00;
959  }
960 
961  if (ip4_is_fragment (ip0))
962  {
963  next0 = SNAT_IN2OUT_NEXT_REASS;
964  goto trace00;
965  }
966 
967  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
968  {
970  goto trace00;
971  }
972  }
973 
974  key0.addr = ip0->src_address;
975  key0.port = udp0->src_port;
976  key0.protocol = proto0;
977  key0.fib_index = rx_fib_index0;
978 
979  kv0.key = key0.as_u64;
980 
981  if (PREDICT_FALSE
982  (clib_bihash_search_8_8
983  (&sm->per_thread_data[thread_index].in2out, &kv0,
984  &value0) != 0))
985  {
986  if (is_slow_path)
987  {
988  if (is_output_feature)
989  {
991  ip0,
992  proto0,
993  udp0->src_port,
994  udp0->dst_port,
995  thread_index,
996  sw_if_index0)))
997  goto trace00;
998  }
999  else
1000  {
1001  if (PREDICT_FALSE
1003  (sm, node, sw_if_index0, ip0, proto0,
1004  rx_fib_index0, thread_index)))
1005  goto trace00;
1006  }
1007 
1008  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1009  &s0, node, next0, thread_index, now);
1010  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1011  goto trace00;
1012 
1013  if (PREDICT_FALSE (!s0))
1014  goto trace00;
1015  }
1016  else
1017  {
1019  goto trace00;
1020  }
1021  }
1022  else
1023  s0 =
1024  pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1025  value0.value);
1026 
1027  b0->flags |= VNET_BUFFER_F_IS_NATED;
1028 
1029  old_addr0 = ip0->src_address.as_u32;
1030  ip0->src_address = s0->out2in.addr;
1031  new_addr0 = ip0->src_address.as_u32;
1032  if (!is_output_feature)
1033  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1034 
1035  sum0 = ip0->checksum;
1036  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1037  ip4_header_t,
1038  src_address /* changed member */ );
1039  ip0->checksum = ip_csum_fold (sum0);
1040 
1041  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1042  {
1043  old_port0 = tcp0->src_port;
1044  tcp0->src_port = s0->out2in.port;
1045  new_port0 = tcp0->src_port;
1046 
1047  sum0 = tcp0->checksum;
1048  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1049  ip4_header_t,
1050  dst_address /* changed member */ );
1051  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1052  ip4_header_t /* cheat */ ,
1053  length /* changed member */ );
1054  mss_clamping (sm, tcp0, &sum0);
1055  tcp0->checksum = ip_csum_fold (sum0);
1056  }
1057  else
1058  {
1059  old_port0 = udp0->src_port;
1060  udp0->src_port = s0->out2in.port;
1061  udp0->checksum = 0;
1062  }
1063 
1064  /* Accounting */
1067  b0));
1068  /* Per-user LRU list maintenance */
1069  nat44_session_update_lru (sm, s0, thread_index);
1070  trace00:
1071 
1073  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1074  {
1075  snat_in2out_trace_t *t =
1076  vlib_add_trace (vm, node, b0, sizeof (*t));
1077  t->is_slow_path = is_slow_path;
1078  t->sw_if_index = sw_if_index0;
1079  t->next_index = next0;
1080  t->session_index = ~0;
1081  if (s0)
1082  t->session_index =
1083  s0 - sm->per_thread_data[thread_index].sessions;
1084  }
1085 
1086  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1087 
1088  if (is_output_feature)
1089  iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1090 
1091  ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1092  iph_offset1);
1093 
1094  udp1 = ip4_next_header (ip1);
1095  tcp1 = (tcp_header_t *) udp1;
1096  icmp1 = (icmp46_header_t *) udp1;
1097 
1098  sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1099  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1100  sw_if_index1);
1101 
1102  if (PREDICT_FALSE (ip1->ttl == 1))
1103  {
1104  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1105  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1106  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1107  0);
1109  goto trace01;
1110  }
1111 
1112  proto1 = ip_proto_to_snat_proto (ip1->protocol);
1113 
1114  /* Next configured feature, probably ip4-lookup */
1115  if (is_slow_path)
1116  {
1117  if (PREDICT_FALSE (proto1 == ~0))
1118  {
1120  (sm, b1, ip1, rx_fib_index1))
1121  {
1122  next1 = SNAT_IN2OUT_NEXT_DROP;
1123  b1->error =
1124  node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1125  }
1126  goto trace01;
1127  }
1128 
1129  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1130  {
1131  next1 = icmp_in2out_slow_path
1132  (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1133  next1, now, thread_index, &s1);
1134  goto trace01;
1135  }
1136  }
1137  else
1138  {
1139  if (PREDICT_FALSE (proto1 == ~0))
1140  {
1142  goto trace01;
1143  }
1144 
1145  if (ip4_is_fragment (ip1))
1146  {
1147  next1 = SNAT_IN2OUT_NEXT_REASS;
1148  goto trace01;
1149  }
1150 
1151  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1152  {
1154  goto trace01;
1155  }
1156  }
1157 
1158  key1.addr = ip1->src_address;
1159  key1.port = udp1->src_port;
1160  key1.protocol = proto1;
1161  key1.fib_index = rx_fib_index1;
1162 
1163  kv1.key = key1.as_u64;
1164 
1165  if (PREDICT_FALSE
1166  (clib_bihash_search_8_8
1167  (&sm->per_thread_data[thread_index].in2out, &kv1,
1168  &value1) != 0))
1169  {
1170  if (is_slow_path)
1171  {
1172  if (is_output_feature)
1173  {
1175  ip1,
1176  proto1,
1177  udp1->src_port,
1178  udp1->dst_port,
1179  thread_index,
1180  sw_if_index1)))
1181  goto trace01;
1182  }
1183  else
1184  {
1185  if (PREDICT_FALSE
1187  (sm, node, sw_if_index1, ip1, proto1,
1188  rx_fib_index1, thread_index)))
1189  goto trace01;
1190  }
1191 
1192  next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1193  &s1, node, next1, thread_index, now);
1194  if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1195  goto trace01;
1196 
1197  if (PREDICT_FALSE (!s1))
1198  goto trace01;
1199  }
1200  else
1201  {
1203  goto trace01;
1204  }
1205  }
1206  else
1207  s1 =
1208  pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1209  value1.value);
1210 
1211  b1->flags |= VNET_BUFFER_F_IS_NATED;
1212 
1213  old_addr1 = ip1->src_address.as_u32;
1214  ip1->src_address = s1->out2in.addr;
1215  new_addr1 = ip1->src_address.as_u32;
1216  if (!is_output_feature)
1217  vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1218 
1219  sum1 = ip1->checksum;
1220  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1221  ip4_header_t,
1222  src_address /* changed member */ );
1223  ip1->checksum = ip_csum_fold (sum1);
1224 
1225  if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1226  {
1227  old_port1 = tcp1->src_port;
1228  tcp1->src_port = s1->out2in.port;
1229  new_port1 = tcp1->src_port;
1230 
1231  sum1 = tcp1->checksum;
1232  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1233  ip4_header_t,
1234  dst_address /* changed member */ );
1235  sum1 = ip_csum_update (sum1, old_port1, new_port1,
1236  ip4_header_t /* cheat */ ,
1237  length /* changed member */ );
1238  mss_clamping (sm, tcp1, &sum1);
1239  tcp1->checksum = ip_csum_fold (sum1);
1240  }
1241  else
1242  {
1243  old_port1 = udp1->src_port;
1244  udp1->src_port = s1->out2in.port;
1245  udp1->checksum = 0;
1246  }
1247 
1248  /* Accounting */
1251  b1));
1252  /* Per-user LRU list maintenance */
1253  nat44_session_update_lru (sm, s1, thread_index);
1254  trace01:
1255 
1257  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1258  {
1259  snat_in2out_trace_t *t =
1260  vlib_add_trace (vm, node, b1, sizeof (*t));
1261  t->sw_if_index = sw_if_index1;
1262  t->next_index = next1;
1263  t->session_index = ~0;
1264  if (s1)
1265  t->session_index =
1266  s1 - sm->per_thread_data[thread_index].sessions;
1267  }
1268 
1269  pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1270 
1271  /* verify speculative enqueues, maybe switch current next frame */
1272  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1273  to_next, n_left_to_next,
1274  bi0, bi1, next0, next1);
1275  }
1276 
1277  while (n_left_from > 0 && n_left_to_next > 0)
1278  {
1279  u32 bi0;
1280  vlib_buffer_t *b0;
1281  u32 next0;
1282  u32 sw_if_index0;
1283  ip4_header_t *ip0;
1284  ip_csum_t sum0;
1285  u32 new_addr0, old_addr0;
1286  u16 old_port0, new_port0;
1287  udp_header_t *udp0;
1288  tcp_header_t *tcp0;
1289  icmp46_header_t *icmp0;
1290  snat_session_key_t key0;
1291  u32 rx_fib_index0;
1292  u32 proto0;
1293  snat_session_t *s0 = 0;
1294  clib_bihash_kv_8_8_t kv0, value0;
1295  u32 iph_offset0 = 0;
1296 
1297  /* speculatively enqueue b0 to the current next frame */
1298  bi0 = from[0];
1299  to_next[0] = bi0;
1300  from += 1;
1301  to_next += 1;
1302  n_left_from -= 1;
1303  n_left_to_next -= 1;
1304 
1305  b0 = vlib_get_buffer (vm, bi0);
1306  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1307 
1308  if (is_output_feature)
1309  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1310 
1311  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1312  iph_offset0);
1313 
1314  udp0 = ip4_next_header (ip0);
1315  tcp0 = (tcp_header_t *) udp0;
1316  icmp0 = (icmp46_header_t *) udp0;
1317 
1318  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1319  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1320  sw_if_index0);
1321 
1322  if (PREDICT_FALSE (ip0->ttl == 1))
1323  {
1324  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1325  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1326  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1327  0);
1329  goto trace0;
1330  }
1331 
1332  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1333 
1334  /* Next configured feature, probably ip4-lookup */
1335  if (is_slow_path)
1336  {
1337  if (PREDICT_FALSE (proto0 == ~0))
1338  {
1340  (sm, b0, ip0, rx_fib_index0))
1341  {
1342  next0 = SNAT_IN2OUT_NEXT_DROP;
1343  b0->error =
1344  node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1345  }
1346  goto trace0;
1347  }
1348 
1349  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1350  {
1351  next0 = icmp_in2out_slow_path
1352  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1353  next0, now, thread_index, &s0);
1354  goto trace0;
1355  }
1356  }
1357  else
1358  {
1359  if (PREDICT_FALSE (proto0 == ~0))
1360  {
1362  goto trace0;
1363  }
1364 
1365  if (ip4_is_fragment (ip0))
1366  {
1367  next0 = SNAT_IN2OUT_NEXT_REASS;
1368  goto trace0;
1369  }
1370 
1371  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1372  {
1374  goto trace0;
1375  }
1376  }
1377 
1378  key0.addr = ip0->src_address;
1379  key0.port = udp0->src_port;
1380  key0.protocol = proto0;
1381  key0.fib_index = rx_fib_index0;
1382 
1383  kv0.key = key0.as_u64;
1384 
1385  if (clib_bihash_search_8_8
1386  (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1387  {
1388  if (is_slow_path)
1389  {
1390  if (is_output_feature)
1391  {
1393  ip0,
1394  proto0,
1395  udp0->src_port,
1396  udp0->dst_port,
1397  thread_index,
1398  sw_if_index0)))
1399  goto trace0;
1400  }
1401  else
1402  {
1403  if (PREDICT_FALSE
1405  (sm, node, sw_if_index0, ip0, proto0,
1406  rx_fib_index0, thread_index)))
1407  goto trace0;
1408  }
1409 
1410  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1411  &s0, node, next0, thread_index, now);
1412 
1413  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1414  goto trace0;
1415 
1416  if (PREDICT_FALSE (!s0))
1417  goto trace0;
1418  }
1419  else
1420  {
1422  goto trace0;
1423  }
1424  }
1425  else
1426  s0 =
1427  pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1428  value0.value);
1429 
1430  b0->flags |= VNET_BUFFER_F_IS_NATED;
1431 
1432  old_addr0 = ip0->src_address.as_u32;
1433  ip0->src_address = s0->out2in.addr;
1434  new_addr0 = ip0->src_address.as_u32;
1435  if (!is_output_feature)
1436  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1437 
1438  sum0 = ip0->checksum;
1439  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1440  ip4_header_t,
1441  src_address /* changed member */ );
1442  ip0->checksum = ip_csum_fold (sum0);
1443 
1444  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1445  {
1446  old_port0 = tcp0->src_port;
1447  tcp0->src_port = s0->out2in.port;
1448  new_port0 = tcp0->src_port;
1449 
1450  sum0 = tcp0->checksum;
1451  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1452  ip4_header_t,
1453  dst_address /* changed member */ );
1454  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1455  ip4_header_t /* cheat */ ,
1456  length /* changed member */ );
1457  mss_clamping (sm, tcp0, &sum0);
1458  tcp0->checksum = ip_csum_fold (sum0);
1459  }
1460  else
1461  {
1462  old_port0 = udp0->src_port;
1463  udp0->src_port = s0->out2in.port;
1464  udp0->checksum = 0;
1465  }
1466 
1467  /* Accounting */
1470  b0));
1471  /* Per-user LRU list maintenance */
1472  nat44_session_update_lru (sm, s0, thread_index);
1473 
1474  trace0:
1476  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1477  {
1478  snat_in2out_trace_t *t =
1479  vlib_add_trace (vm, node, b0, sizeof (*t));
1480  t->is_slow_path = is_slow_path;
1481  t->sw_if_index = sw_if_index0;
1482  t->next_index = next0;
1483  t->session_index = ~0;
1484  if (s0)
1485  t->session_index =
1486  s0 - sm->per_thread_data[thread_index].sessions;
1487  }
1488 
1489  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1490 
1491  /* verify speculative enqueue, maybe switch current next frame */
1492  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1493  to_next, n_left_to_next,
1494  bi0, next0);
1495  }
1496 
1497  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1498  }
1499 
1500  vlib_node_increment_counter (vm, stats_node_index,
1501  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1502  pkts_processed);
1503  return frame->n_vectors;
1504 }
1505 
1506 static uword
1508  vlib_node_runtime_t * node, vlib_frame_t * frame)
1509 {
1510  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1511  0);
1512 }
1513 
1514 /* *INDENT-OFF* */
1516  .function = snat_in2out_fast_path_fn,
1517  .name = "nat44-in2out",
1518  .vector_size = sizeof (u32),
1519  .format_trace = format_snat_in2out_trace,
1520  .type = VLIB_NODE_TYPE_INTERNAL,
1521 
1522  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1523  .error_strings = snat_in2out_error_strings,
1524 
1525  .runtime_data_bytes = sizeof (snat_runtime_t),
1526 
1527  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1528 
1529  /* edit / add dispositions here */
1530  .next_nodes = {
1531  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1532  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1533  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1534  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1535  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1536  },
1537 };
1538 /* *INDENT-ON* */
1539 
1541 
1542 static uword
1544  vlib_node_runtime_t * node,
1545  vlib_frame_t * frame)
1546 {
1547  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1548  1);
1549 }
1550 
1551 /* *INDENT-OFF* */
1553  .function = snat_in2out_output_fast_path_fn,
1554  .name = "nat44-in2out-output",
1555  .vector_size = sizeof (u32),
1556  .format_trace = format_snat_in2out_trace,
1557  .type = VLIB_NODE_TYPE_INTERNAL,
1558 
1559  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1560  .error_strings = snat_in2out_error_strings,
1561 
1562  .runtime_data_bytes = sizeof (snat_runtime_t),
1563 
1564  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1565 
1566  /* edit / add dispositions here */
1567  .next_nodes = {
1568  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1569  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1570  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1571  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1572  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1573  },
1574 };
1575 /* *INDENT-ON* */
1576 
1579 
1580 static uword
1582  vlib_node_runtime_t * node, vlib_frame_t * frame)
1583 {
1584  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1585  0);
1586 }
1587 
1588 /* *INDENT-OFF* */
1590  .function = snat_in2out_slow_path_fn,
1591  .name = "nat44-in2out-slowpath",
1592  .vector_size = sizeof (u32),
1593  .format_trace = format_snat_in2out_trace,
1594  .type = VLIB_NODE_TYPE_INTERNAL,
1595 
1596  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1597  .error_strings = snat_in2out_error_strings,
1598 
1599  .runtime_data_bytes = sizeof (snat_runtime_t),
1600 
1601  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1602 
1603  /* edit / add dispositions here */
1604  .next_nodes = {
1605  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1606  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1607  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1608  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1609  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1610  },
1611 };
1612 /* *INDENT-ON* */
1613 
1616 
1617 static uword
1619  vlib_node_runtime_t * node,
1620  vlib_frame_t * frame)
1621 {
1622  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1623  1);
1624 }
1625 
1626 /* *INDENT-OFF* */
1628  .function = snat_in2out_output_slow_path_fn,
1629  .name = "nat44-in2out-output-slowpath",
1630  .vector_size = sizeof (u32),
1631  .format_trace = format_snat_in2out_trace,
1632  .type = VLIB_NODE_TYPE_INTERNAL,
1633 
1634  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1635  .error_strings = snat_in2out_error_strings,
1636 
1637  .runtime_data_bytes = sizeof (snat_runtime_t),
1638 
1639  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1640 
1641  /* edit / add dispositions here */
1642  .next_nodes = {
1643  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1644  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1645  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1646  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1647  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1648  },
1649 };
1650 /* *INDENT-ON* */
1651 
1654 
1655 static uword
1657  vlib_node_runtime_t * node, vlib_frame_t * frame)
1658 {
1659  u32 n_left_from, *from, *to_next;
1660  snat_in2out_next_t next_index;
1661  u32 pkts_processed = 0;
1662  snat_main_t *sm = &snat_main;
1663  f64 now = vlib_time_now (vm);
1664  u32 thread_index = vm->thread_index;
1665  snat_main_per_thread_data_t *per_thread_data =
1666  &sm->per_thread_data[thread_index];
1667  u32 *fragments_to_drop = 0;
1668  u32 *fragments_to_loopback = 0;
1669 
1670  from = vlib_frame_vector_args (frame);
1671  n_left_from = frame->n_vectors;
1672  next_index = node->cached_next_index;
1673 
1674  while (n_left_from > 0)
1675  {
1676  u32 n_left_to_next;
1677 
1678  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1679 
1680  while (n_left_from > 0 && n_left_to_next > 0)
1681  {
1682  u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1683  vlib_buffer_t *b0;
1684  u32 next0;
1685  u8 cached0 = 0;
1686  ip4_header_t *ip0;
1687  nat_reass_ip4_t *reass0;
1688  udp_header_t *udp0;
1689  tcp_header_t *tcp0;
1690  icmp46_header_t *icmp0;
1691  snat_session_key_t key0;
1692  clib_bihash_kv_8_8_t kv0, value0;
1693  snat_session_t *s0 = 0;
1694  u16 old_port0, new_port0;
1695  ip_csum_t sum0;
1696 
1697  /* speculatively enqueue b0 to the current next frame */
1698  bi0 = from[0];
1699  to_next[0] = bi0;
1700  from += 1;
1701  to_next += 1;
1702  n_left_from -= 1;
1703  n_left_to_next -= 1;
1704 
1705  b0 = vlib_get_buffer (vm, bi0);
1706  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1707 
1708  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1709  rx_fib_index0 =
1711  sw_if_index0);
1712 
1714  {
1715  next0 = SNAT_IN2OUT_NEXT_DROP;
1716  b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1717  goto trace0;
1718  }
1719 
1720  ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1721  udp0 = ip4_next_header (ip0);
1722  tcp0 = (tcp_header_t *) udp0;
1723  icmp0 = (icmp46_header_t *) udp0;
1724  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1725 
1727  ip0->dst_address,
1728  ip0->fragment_id,
1729  ip0->protocol,
1730  1, &fragments_to_drop);
1731 
1732  if (PREDICT_FALSE (!reass0))
1733  {
1734  next0 = SNAT_IN2OUT_NEXT_DROP;
1735  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1736  nat_log_notice ("maximum reassemblies exceeded");
1737  goto trace0;
1738  }
1739 
1741  {
1742  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1743  {
1744  next0 = icmp_in2out_slow_path
1745  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1746  next0, now, thread_index, &s0);
1747 
1748  if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP))
1749  {
1750  if (s0)
1751  reass0->sess_index = s0 - per_thread_data->sessions;
1752  else
1753  reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1754  nat_ip4_reass_get_frags (reass0,
1755  &fragments_to_loopback);
1756  }
1757 
1758  goto trace0;
1759  }
1760 
1761  key0.addr = ip0->src_address;
1762  key0.port = udp0->src_port;
1763  key0.protocol = proto0;
1764  key0.fib_index = rx_fib_index0;
1765  kv0.key = key0.as_u64;
1766 
1767  if (clib_bihash_search_8_8
1768  (&per_thread_data->in2out, &kv0, &value0))
1769  {
1770  if (PREDICT_FALSE
1772  (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1773  thread_index)))
1774  goto trace0;
1775 
1776  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1777  &s0, node, next0, thread_index, now);
1778 
1779  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1780  goto trace0;
1781 
1782  if (PREDICT_FALSE (!s0))
1783  goto trace0;
1784 
1785  reass0->sess_index = s0 - per_thread_data->sessions;
1786  }
1787  else
1788  {
1789  s0 = pool_elt_at_index (per_thread_data->sessions,
1790  value0.value);
1791  reass0->sess_index = value0.value;
1792  }
1793  nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1794  }
1795  else
1796  {
1797  if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1798  {
1800  (reass0, bi0, &fragments_to_drop))
1801  {
1802  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1804  ("maximum fragments per reassembly exceeded");
1805  next0 = SNAT_IN2OUT_NEXT_DROP;
1806  goto trace0;
1807  }
1808  cached0 = 1;
1809  goto trace0;
1810  }
1811  s0 = pool_elt_at_index (per_thread_data->sessions,
1812  reass0->sess_index);
1813  }
1814 
1815  old_addr0 = ip0->src_address.as_u32;
1816  ip0->src_address = s0->out2in.addr;
1817  new_addr0 = ip0->src_address.as_u32;
1818  vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1819 
1820  sum0 = ip0->checksum;
1821  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1822  ip4_header_t,
1823  src_address /* changed member */ );
1824  ip0->checksum = ip_csum_fold (sum0);
1825 
1827  {
1828  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1829  {
1830  old_port0 = tcp0->src_port;
1831  tcp0->src_port = s0->out2in.port;
1832  new_port0 = tcp0->src_port;
1833 
1834  sum0 = tcp0->checksum;
1835  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1836  ip4_header_t,
1837  dst_address /* changed member */ );
1838  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1839  ip4_header_t /* cheat */ ,
1840  length /* changed member */ );
1841  tcp0->checksum = ip_csum_fold (sum0);
1842  }
1843  else
1844  {
1845  old_port0 = udp0->src_port;
1846  udp0->src_port = s0->out2in.port;
1847  udp0->checksum = 0;
1848  }
1849  }
1850 
1851  /* Hairpinning */
1852  nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1853  s0->ext_host_port, proto0, 0);
1854 
1855  /* Accounting */
1858  b0));
1859  /* Per-user LRU list maintenance */
1860  nat44_session_update_lru (sm, s0, thread_index);
1861 
1862  trace0:
1864  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1865  {
1866  nat44_reass_trace_t *t =
1867  vlib_add_trace (vm, node, b0, sizeof (*t));
1868  t->cached = cached0;
1869  t->sw_if_index = sw_if_index0;
1870  t->next_index = next0;
1871  }
1872 
1873  if (cached0)
1874  {
1875  n_left_to_next++;
1876  to_next--;
1877  }
1878  else
1879  {
1880  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1881 
1882  /* verify speculative enqueue, maybe switch current next frame */
1883  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1884  to_next, n_left_to_next,
1885  bi0, next0);
1886  }
1887 
1888  if (n_left_from == 0 && vec_len (fragments_to_loopback))
1889  {
1890  from = vlib_frame_vector_args (frame);
1891  u32 len = vec_len (fragments_to_loopback);
1892  if (len <= VLIB_FRAME_SIZE)
1893  {
1894  clib_memcpy (from, fragments_to_loopback,
1895  sizeof (u32) * len);
1896  n_left_from = len;
1897  vec_reset_length (fragments_to_loopback);
1898  }
1899  else
1900  {
1901  clib_memcpy (from,
1902  fragments_to_loopback + (len -
1903  VLIB_FRAME_SIZE),
1904  sizeof (u32) * VLIB_FRAME_SIZE);
1905  n_left_from = VLIB_FRAME_SIZE;
1906  _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1907  }
1908  }
1909  }
1910 
1911  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1912  }
1913 
1915  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1916  pkts_processed);
1917 
1918  nat_send_all_to_node (vm, fragments_to_drop, node,
1919  &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
1921 
1922  vec_free (fragments_to_drop);
1923  vec_free (fragments_to_loopback);
1924  return frame->n_vectors;
1925 }
1926 
1927 /* *INDENT-OFF* */
1929  .function = nat44_in2out_reass_node_fn,
1930  .name = "nat44-in2out-reass",
1931  .vector_size = sizeof (u32),
1932  .format_trace = format_nat44_reass_trace,
1933  .type = VLIB_NODE_TYPE_INTERNAL,
1934 
1935  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1936  .error_strings = snat_in2out_error_strings,
1937 
1938  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1939  .next_nodes = {
1940  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1941  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1942  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1943  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1944  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1945  },
1946 };
1947 /* *INDENT-ON* */
1948 
1951 
1952 static uword
1954  vlib_node_runtime_t * node,
1955  vlib_frame_t * frame)
1956 {
1957  u32 n_left_from, *from, *to_next;
1958  snat_in2out_next_t next_index;
1959  u32 pkts_processed = 0;
1960  snat_main_t *sm = &snat_main;
1961  u32 stats_node_index;
1962 
1963  stats_node_index = snat_in2out_fast_node.index;
1964 
1965  from = vlib_frame_vector_args (frame);
1966  n_left_from = frame->n_vectors;
1967  next_index = node->cached_next_index;
1968 
1969  while (n_left_from > 0)
1970  {
1971  u32 n_left_to_next;
1972 
1973  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1974 
1975  while (n_left_from > 0 && n_left_to_next > 0)
1976  {
1977  u32 bi0;
1978  vlib_buffer_t *b0;
1979  u32 next0;
1980  u32 sw_if_index0;
1981  ip4_header_t *ip0;
1982  ip_csum_t sum0;
1983  u32 new_addr0, old_addr0;
1984  u16 old_port0, new_port0;
1985  udp_header_t *udp0;
1986  tcp_header_t *tcp0;
1987  icmp46_header_t *icmp0;
1988  snat_session_key_t key0, sm0;
1989  u32 proto0;
1990  u32 rx_fib_index0;
1991 
1992  /* speculatively enqueue b0 to the current next frame */
1993  bi0 = from[0];
1994  to_next[0] = bi0;
1995  from += 1;
1996  to_next += 1;
1997  n_left_from -= 1;
1998  n_left_to_next -= 1;
1999 
2000  b0 = vlib_get_buffer (vm, bi0);
2001  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2002 
2003  ip0 = vlib_buffer_get_current (b0);
2004  udp0 = ip4_next_header (ip0);
2005  tcp0 = (tcp_header_t *) udp0;
2006  icmp0 = (icmp46_header_t *) udp0;
2007 
2008  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2009  rx_fib_index0 =
2011 
2012  if (PREDICT_FALSE (ip0->ttl == 1))
2013  {
2014  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2015  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2016  ICMP4_time_exceeded_ttl_exceeded_in_transit,
2017  0);
2019  goto trace0;
2020  }
2021 
2022  proto0 = ip_proto_to_snat_proto (ip0->protocol);
2023 
2024  if (PREDICT_FALSE (proto0 == ~0))
2025  goto trace0;
2026 
2027  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2028  {
2029  next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
2030  rx_fib_index0, node, next0, ~0, 0, 0);
2031  goto trace0;
2032  }
2033 
2034  key0.addr = ip0->src_address;
2035  key0.protocol = proto0;
2036  key0.port = udp0->src_port;
2037  key0.fib_index = rx_fib_index0;
2038 
2039  if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0, 0))
2040  {
2041  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2042  next0 = SNAT_IN2OUT_NEXT_DROP;
2043  goto trace0;
2044  }
2045 
2046  new_addr0 = sm0.addr.as_u32;
2047  new_port0 = sm0.port;
2048  vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2049  old_addr0 = ip0->src_address.as_u32;
2050  ip0->src_address.as_u32 = new_addr0;
2051 
2052  sum0 = ip0->checksum;
2053  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2054  ip4_header_t,
2055  src_address /* changed member */ );
2056  ip0->checksum = ip_csum_fold (sum0);
2057 
2058  if (PREDICT_FALSE (new_port0 != udp0->dst_port))
2059  {
2060  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2061  {
2062  old_port0 = tcp0->src_port;
2063  tcp0->src_port = new_port0;
2064 
2065  sum0 = tcp0->checksum;
2066  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2067  ip4_header_t,
2068  dst_address /* changed member */ );
2069  sum0 = ip_csum_update (sum0, old_port0, new_port0,
2070  ip4_header_t /* cheat */ ,
2071  length /* changed member */ );
2072  mss_clamping (sm, tcp0, &sum0);
2073  tcp0->checksum = ip_csum_fold (sum0);
2074  }
2075  else
2076  {
2077  old_port0 = udp0->src_port;
2078  udp0->src_port = new_port0;
2079  udp0->checksum = 0;
2080  }
2081  }
2082  else
2083  {
2084  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2085  {
2086  sum0 = tcp0->checksum;
2087  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2088  ip4_header_t,
2089  dst_address /* changed member */ );
2090  mss_clamping (sm, tcp0, &sum0);
2091  tcp0->checksum = ip_csum_fold (sum0);
2092  }
2093  }
2094 
2095  /* Hairpinning */
2096  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2097 
2098  trace0:
2100  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2101  {
2102  snat_in2out_trace_t *t =
2103  vlib_add_trace (vm, node, b0, sizeof (*t));
2104  t->sw_if_index = sw_if_index0;
2105  t->next_index = next0;
2106  }
2107 
2108  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2109 
2110  /* verify speculative enqueue, maybe switch current next frame */
2111  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2112  to_next, n_left_to_next,
2113  bi0, next0);
2114  }
2115 
2116  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2117  }
2118 
2119  vlib_node_increment_counter (vm, stats_node_index,
2120  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2121  pkts_processed);
2122  return frame->n_vectors;
2123 }
2124 
2125 
2126 /* *INDENT-OFF* */
2128  .function = snat_in2out_fast_static_map_fn,
2129  .name = "nat44-in2out-fast",
2130  .vector_size = sizeof (u32),
2131  .format_trace = format_snat_in2out_fast_trace,
2132  .type = VLIB_NODE_TYPE_INTERNAL,
2133 
2134  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2135  .error_strings = snat_in2out_error_strings,
2136 
2137  .runtime_data_bytes = sizeof (snat_runtime_t),
2138 
2139  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2140 
2141  /* edit / add dispositions here */
2142  .next_nodes = {
2143  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2144  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2145  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2146  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2147  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2148  },
2149 };
2150 /* *INDENT-ON* */
2151 
2154 
2155 /*
2156  * fd.io coding-style-patch-verification: ON
2157  *
2158  * Local Variables:
2159  * eval: (c-set-style "gnu")
2160  * End:
2161  */
ip4_address_t external_addr
Definition: nat.h:327
void nat_ipfix_logging_max_sessions(u32 limit)
Generate maximum session entries exceeded event.
fib_protocol_t fp_proto
protocol type
Definition: fib_types.h:212
static ip_csum_t ip_incremental_checksum_buffer(vlib_main_t *vm, vlib_buffer_t *first_buffer, u32 first_buffer_offset, u32 n_bytes_to_checksum, ip_csum_t sum)
Definition: ip.h:154
nat_outside_fib_t * outside_fibs
Definition: nat.h:473
static u32 slow_path(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, u32 rx_fib_index0, snat_session_key_t *key0, snat_session_t **sessionp, vlib_node_runtime_t *node, u32 next0, u32 thread_index, f64 now)
Definition: in2out.c:231
#define snat_is_session_static(s)
Check if SNAT session is created from static mapping.
Definition: nat.h:598
static u8 * format_snat_in2out_trace(u8 *s, va_list *args)
Definition: in2out.c:46
#define CLIB_UNUSED(x)
Definition: clib.h:81
snat_in2out_error_t
Definition: in2out.c:93
static uword snat_in2out_slow_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1581
ip4_address_t src_address
Definition: ip4_packet.h:169
static int nat_not_translate_output_feature(snat_main_t *sm, ip4_header_t *ip0, u32 proto0, u16 src_port, u16 dst_port, u32 thread_index, u32 sw_if_index)
Definition: in2out.c:152
static u32 nat44_session_get_timeout(snat_main_t *sm, snat_session_t *s)
Definition: nat_inlines.h:269
static u8 * format_snat_in2out_fast_trace(u8 *s, va_list *args)
Definition: in2out.c:62
static uword snat_in2out_node_fn_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_slow_path, int is_output_feature)
Definition: in2out.c:831
#define PREDICT_TRUE(x)
Definition: clib.h:108
static_always_inline u8 icmp_is_error_message(icmp46_header_t *icmp)
Definition: nat_inlines.h:53
unsigned long u64
Definition: types.h:89
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:227
static char * snat_in2out_error_strings[]
Definition: in2out.c:101
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
Definition: fib_table.c:956
u16 port_per_thread
Definition: nat.h:439
u32 thread_index
Definition: main.h:179
#define nat_log_warn(...)
Definition: nat.h:687
int i
uword ip_csum_t
Definition: ip_packet.h:181
snat_in2out_next_t
Definition: in2out.c:107
u32 * fib_index_by_sw_if_index
Table index indexed by software interface.
Definition: ip4.h:112
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:472
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:263
struct _tcp_header tcp_header_t
int snat_hairpinning(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, udp_header_t *udp0, tcp_header_t *tcp0, u32 proto0, int is_ed)
VLIB_NODE_FUNCTION_MULTIARCH(snat_in2out_node, snat_in2out_fast_path_fn)
u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, ip4_header_t *ip0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for ICMP packet translation.
Definition: in2out.c:548
unsigned char u8
Definition: types.h:56
u8 deterministic
Definition: nat.h:508
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
static int ip4_is_fragment(const ip4_header_t *i)
Definition: ip4_packet.h:212
static int snat_not_translate_fast(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0)
Check if packet should be translated.
Definition: nat_inlines.h:406
u16 src_port
Definition: udp.api:41
static uword snat_in2out_output_slow_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1618
clib_bihash_8_8_t in2out
Definition: nat.h:380
vlib_node_registration_t snat_in2out_output_slowpath_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node)
Definition: in2out.c:78
u32 ip4_fib_table_get_index_for_sw_if_index(u32 sw_if_index)
Definition: ip4_fib.c:224
#define static_always_inline
Definition: clib.h:95
static uword ip4_header_checksum_is_valid(ip4_header_t *i)
Definition: ip4_packet.h:267
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
vlib_node_registration_t snat_in2out_fast_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_fast_node)
Definition: in2out.c:76
u32 sw_if_index
Definition: vxlan_gbp.api:39
static int snat_not_translate(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0, u32 thread_index)
Definition: in2out.c:118
ip4_address_t dst_address
Definition: ip4_packet.h:169
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:187
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Aggregrate type for a prefix.
Definition: fib_types.h:203
u32 icmp_in2out(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0, u32 thread_index, void *d, void *e)
Definition: in2out.c:617
ip4_main_t * ip4_main
Definition: nat.h:541
#define NAT_REASS_FLAG_ED_DONT_TRANSLATE
Definition: nat_reass.h:35
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:240
unsigned int u32
Definition: types.h:88
vlib_node_registration_t snat_in2out_slowpath_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_slowpath_node)
Definition: in2out.c:75
#define VLIB_FRAME_SIZE
Definition: node.h:382
fib_node_index_t fib_table_lookup(u32 fib_index, const fib_prefix_t *prefix)
Perfom a longest prefix match in the non-forwarding table.
Definition: fib_table.c:66
static u8 maximum_sessions_exceeded(snat_main_t *sm, u32 thread_index)
Definition: nat_inlines.h:94
int snat_static_mapping_match(snat_main_t *sm, snat_session_key_t match, snat_session_key_t *mapping, u8 by_external, u8 *is_addr_only, twice_nat_type_t *twice_nat, lb_nat_type_t *lb, ip4_address_t *ext_host_addr, u8 *is_identity_nat)
Match NAT44 static mapping.
Definition: nat.c:2070
void snat_free_outside_address_and_port(snat_address_t *addresses, u32 thread_index, snat_session_key_t *k)
Free outside address and port pair.
Definition: nat.c:2032
static void nat44_delete_session(snat_main_t *sm, snat_session_t *ses, u32 thread_index)
Definition: nat_inlines.h:166
#define foreach_snat_in2out_error
Definition: in2out.c:81
u32 max_translations
Definition: nat.h:513
static void mss_clamping(snat_main_t *sm, tcp_header_t *tcp, ip_csum_t *sum)
Definition: nat_inlines.h:343
int snat_alloc_outside_address_and_port(snat_address_t *addresses, u32 fib_index, u32 thread_index, snat_session_key_t *k, u16 port_per_thread, u32 snat_thread_index)
Alloc outside address and port.
Definition: nat.c:2204
snat_user_t * nat_user_get_or_create(snat_main_t *sm, ip4_address_t *addr, u32 fib_index, u32 thread_index)
Find or create NAT user.
Definition: nat.c:278
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
u32 fib_index
Definition: nat.h:254
static uword nat44_in2out_reass_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1656
u64 key
the key
Definition: bihash_8_8.h:33
static u32 icmp_in2out_slow_path(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0, f64 now, u32 thread_index, snat_session_t **p_s0)
Definition: in2out.c:767
long ctx[MAX_CONNS]
Definition: main.c:144
vlib_main_t * vlib_main
Definition: nat.h:539
unsigned short u16
Definition: types.h:57
u16 protocol
Definition: nat.h:54
snat_static_mapping_t * static_mappings
Definition: nat.h:452
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:205
void snat_ipfix_logging_nat44_ses_delete(u32 src_ip, u32 nat_src_ip, snat_protocol_t snat_proto, u16 src_port, u16 nat_src_port, u32 vrf_id)
Generate NAT44 session delete event.
#define PREDICT_FALSE(x)
Definition: clib.h:107
void snat_ipfix_logging_nat44_ses_create(u32 src_ip, u32 nat_src_ip, snat_protocol_t snat_proto, u16 src_port, u16 nat_src_port, u32 vrf_id)
Generate NAT44 session create event.
vl_api_address_union_t src_address
Definition: ip_types.api:49
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
#define nat_log_notice(...)
Definition: nat.h:689
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:138
u32 snat_icmp_hairpinning(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, int is_ed)
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1176
int nat44_o2i_is_idle_session_cb(clib_bihash_kv_8_8_t *kv, void *arg)
Definition: out2in.c:110
u32 fib_entry_get_resolving_interface(fib_node_index_t entry_index)
Definition: fib_entry.c:1409
snat_interface_t * output_feature_interfaces
Definition: nat.h:456
snat_main_t snat_main
Definition: nat.c:37
The fine-grained event logger allows lightweight, thread-safe event logging at minimum cost...
u64 value
the value
Definition: bihash_8_8.h:34
static int nat_in2out_sm_unknown_proto(snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip, u32 rx_fib_index)
Definition: in2out.c:793
vlib_node_registration_t nat44_in2out_reass_node
(constructor) VLIB_REGISTER_NODE (nat44_in2out_reass_node)
Definition: in2out.c:79
static void nat44_delete_user_with_no_session(snat_main_t *sm, snat_user_t *u, u32 thread_index)
Definition: nat_inlines.h:146
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:155
u16 n_vectors
Definition: node.h:401
clib_bihash_8_8_t out2in
Definition: nat.h:379
u8 nat_reass_is_drop_frag(u8 is_ip6)
Get status of virtual fragmentation reassembly.
Definition: nat_reass.c:168
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:79
vlib_main_t * vm
Definition: buffer.c:294
static uword snat_in2out_output_fast_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1543
vlib_node_registration_t snat_in2out_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_node)
Definition: in2out.c:74
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
void icmp4_error_set_vnet_buffer(vlib_buffer_t *b, u8 type, u8 code, u32 data)
Definition: icmp4.c:431
#define clib_memcpy(a, b, c)
Definition: string.h:75
static void nat44_session_update_counters(snat_session_t *s, f64 now, uword bytes)
Definition: nat_inlines.h:292
u32 outside_fib_index
Definition: nat.h:518
static uword snat_in2out_fast_static_map_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1953
u32 fib_node_index_t
A typedef of a node index.
Definition: fib_types.h:30
8 octet key, 8 octet key value pair
Definition: bihash_8_8.h:31
#define ARRAY_LEN(x)
Definition: clib.h:61
ip4_address_t addr
Definition: nat.h:52
static_always_inline snat_in2out_error_t icmp_get_key(ip4_header_t *ip0, snat_session_key_t *p_key0)
Definition: in2out.c:372
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:455
u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, ip4_header_t *ip0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for ICMP packet translation and create session if needed...
Definition: in2out.c:431
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:513
#define ASSERT(truth)
static void nat44_session_update_lru(snat_main_t *sm, snat_session_t *s, u32 thread_index)
Per-user LRU list maintenance.
Definition: nat_inlines.h:301
snat_session_t * nat_session_alloc_or_recycle(snat_main_t *sm, snat_user_t *u, u32 thread_index)
Allocate new NAT session or recycle last used.
Definition: nat.c:322
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:57
#define vec_elt(v, i)
Get vector value at index i.
struct _vlib_node_registration vlib_node_registration_t
Definition: defs.h:47
format_function_t format_nat44_reass_trace
Definition: nat.h:590
static int ip4_is_first_fragment(const ip4_header_t *i)
Definition: ip4_packet.h:219
void nat_hairpinning_sm_unknown_proto(snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip)
static u32 ip_proto_to_snat_proto(u8 ip_proto)
The NAT inline functions.
Definition: nat_inlines.h:26
#define FIB_NODE_INDEX_INVALID
Definition: fib_types.h:31
static void user_session_increment(snat_main_t *sm, snat_user_t *u, u8 is_static)
Definition: nat_inlines.h:134
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u64 uword
Definition: types.h:112
snat_main_per_thread_data_t * per_thread_data
Definition: nat.h:443
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
#define ip_csum_update(sum, old, new, type, field)
Definition: ip_packet.h:231
nat_reass_ip4_t * nat_ip4_reass_find_or_create(ip4_address_t src, ip4_address_t dst, u16 frag_id, u8 proto, u8 reset_timeout, u32 **bi_to_drop)
Find or create reassembly.
Definition: nat_reass.c:220
snat_address_t * addresses
Definition: nat.h:459
static uword snat_in2out_fast_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1507
#define vnet_buffer(b)
Definition: buffer.h:344
#define SNAT_SESSION_FLAG_STATIC_MAPPING
Definition: nat.h:171
int nat_ip4_reass_add_fragment(nat_reass_ip4_t *reass, u32 bi, u32 **bi_to_drop)
Cache fragment.
Definition: nat_reass.c:338
u8 data[0]
Packet data.
Definition: buffer.h:175
u8 forwarding_enabled
Definition: nat.h:503
#define vec_foreach(var, vec)
Vector iterator.
vlib_node_registration_t snat_in2out_output_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_output_node)
Definition: in2out.c:77
u16 flags
Copy of main node flags.
Definition: node.h:507
static void nat_send_all_to_node(vlib_main_t *vm, u32 *bi_vector, vlib_node_runtime_t *node, vlib_error_t *error, u32 next)
Definition: nat_inlines.h:104
u8 endpoint_dependent
Definition: nat.h:510
u16 dst_port
Definition: udp.api:42
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:234
void nat_ip4_reass_get_frags(nat_reass_ip4_t *reass, u32 **bi)
Get cached fragments.
Definition: nat_reass.c:370
NAT plugin virtual fragmentation reassembly.
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:310
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:116
int nat44_i2o_is_idle_session_cb(clib_bihash_kv_8_8_t *kv, void *arg)
Definition: in2out.c:194
snat_session_t * sessions
Definition: nat.h:393
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:58
snat_icmp_match_function_t * icmp_match_in2out_cb
Definition: nat.h:430
clib_bihash_8_8_t static_mapping_by_local
Definition: nat.h:446
static u16 ip_csum_fold(ip_csum_t c)
Definition: ip_packet.h:237
void nat44_reass_hairpinning(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, u16 sport, u16 dport, u32 proto0, int is_ed)
Definition: defs.h:46
u16 fib_index
Definition: nat.h:54