FD.io VPP  v17.07.01-10-g3be13f0
Vector Packet Processing
out2in.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
20 
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
25 #include <snat/snat.h>
27 #include <snat/snat_det.h>
28 
29 #include <vppinfra/hash.h>
30 #include <vppinfra/error.h>
31 #include <vppinfra/elog.h>
32 
33 typedef struct {
38 
39 typedef struct {
43 
44 /* packet trace format function */
45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
46 {
47  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
48  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
49  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
50 
51  s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
53  return s;
54 }
55 
56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
57 {
58  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
59  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
60  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
61 
62  s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
63  t->sw_if_index, t->next_index);
64  return s;
65 }
66 
67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
68 {
69  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
70  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
72  va_arg (*args, snat_out2in_worker_handoff_trace_t *);
73  char * m;
74 
75  m = t->do_handoff ? "next worker" : "same worker";
76  s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
77 
78  return s;
79 }
80 
85 
86 #define foreach_snat_out2in_error \
87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
88 _(OUT2IN_PACKETS, "Good out2in packets processed") \
89 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
90 _(NO_TRANSLATION, "No translation")
91 
92 typedef enum {
93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
95 #undef _
98 
99 static char * snat_out2in_error_strings[] = {
100 #define _(sym,string) string,
102 #undef _
103 };
104 
105 typedef enum {
111 
112 /**
113  * @brief Create session for static mapping.
114  *
115  * Create NAT session initiated by host from external network with static
116  * mapping.
117  *
118  * @param sm SNAT main.
119  * @param b0 Vlib buffer.
120  * @param in2out In2out SNAT session key.
121  * @param out2in Out2in SNAT session key.
122  * @param node Vlib node.
123  *
124  * @returns SNAT session if successfully created otherwise 0.
125  */
126 static inline snat_session_t *
128  vlib_buffer_t *b0,
129  snat_session_key_t in2out,
130  snat_session_key_t out2in,
131  vlib_node_runtime_t * node,
132  u32 thread_index)
133 {
134  snat_user_t *u;
135  snat_user_key_t user_key;
136  snat_session_t *s;
137  clib_bihash_kv_8_8_t kv0, value0;
138  dlist_elt_t * per_user_translation_list_elt;
139  dlist_elt_t * per_user_list_head_elt;
140 
141  user_key.addr = in2out.addr;
142  user_key.fib_index = in2out.fib_index;
143  kv0.key = user_key.as_u64;
144 
145  /* Ever heard of the "user" = inside ip4 address before? */
146  if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
147  {
148  /* no, make a new one */
149  pool_get (sm->per_thread_data[thread_index].users, u);
150  memset (u, 0, sizeof (*u));
151  u->addr = in2out.addr;
152  u->fib_index = in2out.fib_index;
153 
154  pool_get (sm->per_thread_data[thread_index].list_pool,
155  per_user_list_head_elt);
156 
157  u->sessions_per_user_list_head_index = per_user_list_head_elt -
158  sm->per_thread_data[thread_index].list_pool;
159 
160  clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
162 
163  kv0.value = u - sm->per_thread_data[thread_index].users;
164 
165  /* add user */
166  clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
167 
168  /* add non-traslated packets worker lookup */
169  kv0.value = thread_index;
170  clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
171  }
172  else
173  {
174  u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
175  value0.value);
176  }
177 
178  pool_get (sm->per_thread_data[thread_index].sessions, s);
179  memset (s, 0, sizeof (*s));
180 
181  s->outside_address_index = ~0;
183  u->nstaticsessions++;
184 
185  /* Create list elts */
186  pool_get (sm->per_thread_data[thread_index].list_pool,
187  per_user_translation_list_elt);
188  clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
189  per_user_translation_list_elt -
190  sm->per_thread_data[thread_index].list_pool);
191 
192  per_user_translation_list_elt->value =
193  s - sm->per_thread_data[thread_index].sessions;
194  s->per_user_index =
195  per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
196  s->per_user_list_head_index = u->sessions_per_user_list_head_index;
197 
198  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
199  s->per_user_list_head_index,
200  per_user_translation_list_elt -
201  sm->per_thread_data[thread_index].list_pool);
202 
203  s->in2out = in2out;
204  s->out2in = out2in;
205  s->in2out.protocol = out2in.protocol;
206 
207  /* Add to translation hashes */
208  kv0.key = s->in2out.as_u64;
209  kv0.value = s - sm->per_thread_data[thread_index].sessions;
210  if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
211  clib_warning ("in2out key add failed");
212 
213  kv0.key = s->out2in.as_u64;
214  kv0.value = s - sm->per_thread_data[thread_index].sessions;
215 
216  if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
217  clib_warning ("out2in key add failed");
218 
219  /* log NAT event */
220  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
221  s->out2in.addr.as_u32,
222  s->in2out.protocol,
223  s->in2out.port,
224  s->out2in.port,
225  s->in2out.fib_index);
226  return s;
227 }
228 
231  snat_session_key_t *p_key0)
232 {
233  icmp46_header_t *icmp0;
234  snat_session_key_t key0;
235  icmp_echo_header_t *echo0, *inner_echo0 = 0;
236  ip4_header_t *inner_ip0;
237  void *l4_header = 0;
238  icmp46_header_t *inner_icmp0;
239 
240  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
241  echo0 = (icmp_echo_header_t *)(icmp0+1);
242 
243  if (!icmp_is_error_message (icmp0))
244  {
245  key0.protocol = SNAT_PROTOCOL_ICMP;
246  key0.addr = ip0->dst_address;
247  key0.port = echo0->identifier;
248  }
249  else
250  {
251  inner_ip0 = (ip4_header_t *)(echo0+1);
252  l4_header = ip4_next_header (inner_ip0);
253  key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
254  key0.addr = inner_ip0->src_address;
255  switch (key0.protocol)
256  {
257  case SNAT_PROTOCOL_ICMP:
258  inner_icmp0 = (icmp46_header_t*)l4_header;
259  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
260  key0.port = inner_echo0->identifier;
261  break;
262  case SNAT_PROTOCOL_UDP:
263  case SNAT_PROTOCOL_TCP:
264  key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
265  break;
266  default:
267  return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
268  }
269  }
270  *p_key0 = key0;
271  return -1; /* success */
272 }
273 
274 /**
275  * Get address and port values to be used for packet SNAT translation
276  * and create session if needed
277  *
278  * @param[in,out] sm SNAT main
279  * @param[in,out] node SNAT node runtime
280  * @param[in] thread_index thread index
281  * @param[in,out] b0 buffer containing packet to be translated
282  * @param[out] p_proto protocol used for matching
283  * @param[out] p_value address and port after NAT translation
284  * @param[out] p_dont_translate if packet should not be translated
285  * @param d optional parameter
286  * @param e optional parameter
287  */
289  u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
290  snat_session_key_t *p_value,
291  u8 *p_dont_translate, void *d, void *e)
292 {
293  ip4_header_t *ip0;
294  icmp46_header_t *icmp0;
295  u32 sw_if_index0;
296  u32 rx_fib_index0;
297  snat_session_key_t key0;
298  snat_session_key_t sm0;
299  snat_session_t *s0 = 0;
300  u8 dont_translate = 0;
301  clib_bihash_kv_8_8_t kv0, value0;
302  u8 is_addr_only;
303  u32 next0 = ~0;
304  int err;
305 
306  ip0 = vlib_buffer_get_current (b0);
307  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
308  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
309  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
310 
311  key0.protocol = 0;
312 
313  err = icmp_get_key (ip0, &key0);
314  if (err != -1)
315  {
316  b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
317  next0 = SNAT_OUT2IN_NEXT_DROP;
318  goto out;
319  }
320  key0.fib_index = rx_fib_index0;
321 
322  kv0.key = key0.as_u64;
323 
324  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
325  {
326  /* Try to match static mapping by external address and port,
327  destination address and port in packet */
328  if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
329  {
330  /* Don't NAT packet aimed at the intfc address */
331  if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
332  ip0->dst_address.as_u32)))
333  {
334  dont_translate = 1;
335  goto out;
336  }
337  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
338  next0 = SNAT_OUT2IN_NEXT_DROP;
339  goto out;
340  }
341 
342  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
343  (icmp0->type != ICMP4_echo_request || !is_addr_only)))
344  {
345  b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
346  next0 = SNAT_OUT2IN_NEXT_DROP;
347  goto out;
348  }
349 
350  /* Create session initiated by host from external network */
351  s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
352  node, thread_index);
353 
354  if (!s0)
355  {
356  next0 = SNAT_OUT2IN_NEXT_DROP;
357  goto out;
358  }
359  }
360  else
361  {
362  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
363  icmp0->type != ICMP4_echo_request &&
364  !icmp_is_error_message (icmp0)))
365  {
366  b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
367  next0 = SNAT_OUT2IN_NEXT_DROP;
368  goto out;
369  }
370 
371  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
372  value0.value);
373  }
374 
375 out:
376  *p_proto = key0.protocol;
377  if (s0)
378  *p_value = s0->in2out;
379  *p_dont_translate = dont_translate;
380  if (d)
381  *(snat_session_t**)d = s0;
382  return next0;
383 }
384 
385 /**
386  * Get address and port values to be used for packet SNAT translation
387  *
388  * @param[in] sm SNAT main
389  * @param[in,out] node SNAT node runtime
390  * @param[in] thread_index thread index
391  * @param[in,out] b0 buffer containing packet to be translated
392  * @param[out] p_proto protocol used for matching
393  * @param[out] p_value address and port after NAT translation
394  * @param[out] p_dont_translate if packet should not be translated
395  * @param d optional parameter
396  * @param e optional parameter
397  */
399  u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
400  snat_session_key_t *p_value,
401  u8 *p_dont_translate, void *d, void *e)
402 {
403  ip4_header_t *ip0;
404  icmp46_header_t *icmp0;
405  u32 sw_if_index0;
406  u32 rx_fib_index0;
407  snat_session_key_t key0;
408  snat_session_key_t sm0;
409  u8 dont_translate = 0;
410  u8 is_addr_only;
411  u32 next0 = ~0;
412  int err;
413 
414  ip0 = vlib_buffer_get_current (b0);
415  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
416  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
417  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
418 
419  err = icmp_get_key (ip0, &key0);
420  if (err != -1)
421  {
422  b0->error = node->errors[err];
423  next0 = SNAT_OUT2IN_NEXT_DROP;
424  goto out2;
425  }
426  key0.fib_index = rx_fib_index0;
427 
428  if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
429  {
430  /* Don't NAT packet aimed at the intfc address */
431  if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
432  {
433  dont_translate = 1;
434  goto out;
435  }
436  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
437  next0 = SNAT_OUT2IN_NEXT_DROP;
438  goto out;
439  }
440 
441  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
442  (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
443  !icmp_is_error_message (icmp0)))
444  {
445  b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
446  next0 = SNAT_OUT2IN_NEXT_DROP;
447  goto out;
448  }
449 
450 out:
451  *p_value = sm0;
452 out2:
453  *p_proto = key0.protocol;
454  *p_dont_translate = dont_translate;
455  return next0;
456 }
457 
458 static inline u32 icmp_out2in (snat_main_t *sm,
459  vlib_buffer_t * b0,
460  ip4_header_t * ip0,
461  icmp46_header_t * icmp0,
462  u32 sw_if_index0,
463  u32 rx_fib_index0,
464  vlib_node_runtime_t * node,
465  u32 next0,
466  u32 thread_index,
467  void *d,
468  void *e)
469 {
470  snat_session_key_t sm0;
471  u8 protocol;
472  icmp_echo_header_t *echo0, *inner_echo0 = 0;
473  ip4_header_t *inner_ip0 = 0;
474  void *l4_header = 0;
475  icmp46_header_t *inner_icmp0;
476  u8 dont_translate;
477  u32 new_addr0, old_addr0;
478  u16 old_id0, new_id0;
479  ip_csum_t sum0;
480  u16 checksum0;
481  u32 next0_tmp;
482 
483  echo0 = (icmp_echo_header_t *)(icmp0+1);
484 
485  next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
486  &protocol, &sm0, &dont_translate, d, e);
487  if (next0_tmp != ~0)
488  next0 = next0_tmp;
489  if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
490  goto out;
491 
492  sum0 = ip_incremental_checksum (0, icmp0,
493  ntohs(ip0->length) - ip4_header_bytes (ip0));
494  checksum0 = ~ip_csum_fold (sum0);
495  if (checksum0 != 0 && checksum0 != 0xffff)
496  {
497  next0 = SNAT_OUT2IN_NEXT_DROP;
498  goto out;
499  }
500 
501  old_addr0 = ip0->dst_address.as_u32;
502  new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
503  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
504 
505  sum0 = ip0->checksum;
506  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
507  dst_address /* changed member */);
508  ip0->checksum = ip_csum_fold (sum0);
509 
510  if (!icmp_is_error_message (icmp0))
511  {
512  new_id0 = sm0.port;
513  if (PREDICT_FALSE(new_id0 != echo0->identifier))
514  {
515  old_id0 = echo0->identifier;
516  new_id0 = sm0.port;
517  echo0->identifier = new_id0;
518 
519  sum0 = icmp0->checksum;
520  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
521  identifier /* changed member */);
522  icmp0->checksum = ip_csum_fold (sum0);
523  }
524  }
525  else
526  {
527  inner_ip0 = (ip4_header_t *)(echo0+1);
528  l4_header = ip4_next_header (inner_ip0);
529 
530  if (!ip4_header_checksum_is_valid (inner_ip0))
531  {
532  next0 = SNAT_OUT2IN_NEXT_DROP;
533  goto out;
534  }
535 
536  old_addr0 = inner_ip0->src_address.as_u32;
537  inner_ip0->src_address = sm0.addr;
538  new_addr0 = inner_ip0->src_address.as_u32;
539 
540  sum0 = icmp0->checksum;
541  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
542  src_address /* changed member */);
543  icmp0->checksum = ip_csum_fold (sum0);
544 
545  switch (protocol)
546  {
547  case SNAT_PROTOCOL_ICMP:
548  inner_icmp0 = (icmp46_header_t*)l4_header;
549  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
550 
551  old_id0 = inner_echo0->identifier;
552  new_id0 = sm0.port;
553  inner_echo0->identifier = new_id0;
554 
555  sum0 = icmp0->checksum;
556  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
557  identifier);
558  icmp0->checksum = ip_csum_fold (sum0);
559  break;
560  case SNAT_PROTOCOL_UDP:
561  case SNAT_PROTOCOL_TCP:
562  old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
563  new_id0 = sm0.port;
564  ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
565 
566  sum0 = icmp0->checksum;
567  sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
568  src_port);
569  icmp0->checksum = ip_csum_fold (sum0);
570  break;
571  default:
572  ASSERT(0);
573  }
574  }
575 
576 out:
577  return next0;
578 }
579 
580 
582  vlib_buffer_t * b0,
583  ip4_header_t * ip0,
584  icmp46_header_t * icmp0,
585  u32 sw_if_index0,
586  u32 rx_fib_index0,
587  vlib_node_runtime_t * node,
588  u32 next0, f64 now,
589  u32 thread_index,
590  snat_session_t ** p_s0)
591 {
592  next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
593  next0, thread_index, p_s0, 0);
594  snat_session_t * s0 = *p_s0;
595  if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
596  {
597  /* Accounting */
598  s0->last_heard = now;
599  s0->total_pkts++;
600  s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
601  /* Per-user LRU list maintenance for dynamic translation */
602  if (!snat_is_session_static (s0))
603  {
604  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
605  s0->per_user_index);
606  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
607  s0->per_user_list_head_index,
608  s0->per_user_index);
609  }
610  }
611  return next0;
612 }
613 
614 static void
616  vlib_buffer_t * b,
617  ip4_header_t * ip,
618  u32 rx_fib_index)
619 {
620  clib_bihash_kv_8_8_t kv, value;
622  snat_session_key_t m_key;
623  u32 old_addr, new_addr;
624  ip_csum_t sum;
625 
626  m_key.addr = ip->dst_address;
627  m_key.port = 0;
628  m_key.protocol = 0;
629  m_key.fib_index = rx_fib_index;
630  kv.key = m_key.as_u64;
631  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
632  return;
633 
634  m = pool_elt_at_index (sm->static_mappings, value.value);
635 
636  old_addr = ip->dst_address.as_u32;
637  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
638  sum = ip->checksum;
639  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
640  ip->checksum = ip_csum_fold (sum);
641 
642  vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
643 }
644 
645 static uword
647  vlib_node_runtime_t * node,
648  vlib_frame_t * frame)
649 {
650  u32 n_left_from, * from, * to_next;
651  snat_out2in_next_t next_index;
652  u32 pkts_processed = 0;
653  snat_main_t * sm = &snat_main;
654  f64 now = vlib_time_now (vm);
655  u32 thread_index = vlib_get_thread_index ();
656 
657  from = vlib_frame_vector_args (frame);
658  n_left_from = frame->n_vectors;
659  next_index = node->cached_next_index;
660 
661  while (n_left_from > 0)
662  {
663  u32 n_left_to_next;
664 
665  vlib_get_next_frame (vm, node, next_index,
666  to_next, n_left_to_next);
667 
668  while (n_left_from >= 4 && n_left_to_next >= 2)
669  {
670  u32 bi0, bi1;
671  vlib_buffer_t * b0, * b1;
674  u32 sw_if_index0, sw_if_index1;
675  ip4_header_t * ip0, *ip1;
676  ip_csum_t sum0, sum1;
677  u32 new_addr0, old_addr0;
678  u16 new_port0, old_port0;
679  u32 new_addr1, old_addr1;
680  u16 new_port1, old_port1;
681  udp_header_t * udp0, * udp1;
682  tcp_header_t * tcp0, * tcp1;
683  icmp46_header_t * icmp0, * icmp1;
684  snat_session_key_t key0, key1, sm0, sm1;
685  u32 rx_fib_index0, rx_fib_index1;
686  u32 proto0, proto1;
687  snat_session_t * s0 = 0, * s1 = 0;
688  clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
689 
690  /* Prefetch next iteration. */
691  {
692  vlib_buffer_t * p2, * p3;
693 
694  p2 = vlib_get_buffer (vm, from[2]);
695  p3 = vlib_get_buffer (vm, from[3]);
696 
697  vlib_prefetch_buffer_header (p2, LOAD);
698  vlib_prefetch_buffer_header (p3, LOAD);
699 
702  }
703 
704  /* speculatively enqueue b0 and b1 to the current next frame */
705  to_next[0] = bi0 = from[0];
706  to_next[1] = bi1 = from[1];
707  from += 2;
708  to_next += 2;
709  n_left_from -= 2;
710  n_left_to_next -= 2;
711 
712  b0 = vlib_get_buffer (vm, bi0);
713  b1 = vlib_get_buffer (vm, bi1);
714 
715  ip0 = vlib_buffer_get_current (b0);
716  udp0 = ip4_next_header (ip0);
717  tcp0 = (tcp_header_t *) udp0;
718  icmp0 = (icmp46_header_t *) udp0;
719 
720  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
721  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
722  sw_if_index0);
723 
724  if (PREDICT_FALSE(ip0->ttl == 1))
725  {
726  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
727  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
728  ICMP4_time_exceeded_ttl_exceeded_in_transit,
729  0);
731  goto trace0;
732  }
733 
734  proto0 = ip_proto_to_snat_proto (ip0->protocol);
735 
736  if (PREDICT_FALSE (proto0 == ~0))
737  {
738  snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0);
739  goto trace0;
740  }
741 
742  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
743  {
744  next0 = icmp_out2in_slow_path
745  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
746  next0, now, thread_index, &s0);
747  goto trace0;
748  }
749 
750  key0.addr = ip0->dst_address;
751  key0.port = udp0->dst_port;
752  key0.protocol = proto0;
753  key0.fib_index = rx_fib_index0;
754 
755  kv0.key = key0.as_u64;
756 
757  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
758  {
759  /* Try to match static mapping by external address and port,
760  destination address and port in packet */
761  if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
762  {
763  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
764  /*
765  * Send DHCP packets to the ipv4 stack, or we won't
766  * be able to use dhcp client on the outside interface
767  */
768  if (proto0 != SNAT_PROTOCOL_UDP
769  || (udp0->dst_port
770  != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
771  next0 = SNAT_OUT2IN_NEXT_DROP;
772  goto trace0;
773  }
774 
775  /* Create session initiated by host from external network */
776  s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
777  thread_index);
778  if (!s0)
779  {
780  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
781  next0 = SNAT_OUT2IN_NEXT_DROP;
782  goto trace0;
783  }
784  }
785  else
786  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
787  value0.value);
788 
789  old_addr0 = ip0->dst_address.as_u32;
790  ip0->dst_address = s0->in2out.addr;
791  new_addr0 = ip0->dst_address.as_u32;
792  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
793 
794  sum0 = ip0->checksum;
795  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
796  ip4_header_t,
797  dst_address /* changed member */);
798  ip0->checksum = ip_csum_fold (sum0);
799 
800  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
801  {
802  old_port0 = tcp0->dst_port;
803  tcp0->dst_port = s0->in2out.port;
804  new_port0 = tcp0->dst_port;
805 
806  sum0 = tcp0->checksum;
807  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
808  ip4_header_t,
809  dst_address /* changed member */);
810 
811  sum0 = ip_csum_update (sum0, old_port0, new_port0,
812  ip4_header_t /* cheat */,
813  length /* changed member */);
814  tcp0->checksum = ip_csum_fold(sum0);
815  }
816  else
817  {
818  old_port0 = udp0->dst_port;
819  udp0->dst_port = s0->in2out.port;
820  udp0->checksum = 0;
821  }
822 
823  /* Accounting */
824  s0->last_heard = now;
825  s0->total_pkts++;
826  s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
827  /* Per-user LRU list maintenance for dynamic translation */
828  if (!snat_is_session_static (s0))
829  {
830  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
831  s0->per_user_index);
832  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
833  s0->per_user_list_head_index,
834  s0->per_user_index);
835  }
836  trace0:
837 
839  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
840  {
841  snat_out2in_trace_t *t =
842  vlib_add_trace (vm, node, b0, sizeof (*t));
843  t->sw_if_index = sw_if_index0;
844  t->next_index = next0;
845  t->session_index = ~0;
846  if (s0)
847  t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
848  }
849 
850  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
851 
852 
853  ip1 = vlib_buffer_get_current (b1);
854  udp1 = ip4_next_header (ip1);
855  tcp1 = (tcp_header_t *) udp1;
856  icmp1 = (icmp46_header_t *) udp1;
857 
858  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
859  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
860  sw_if_index1);
861 
862  if (PREDICT_FALSE(ip1->ttl == 1))
863  {
864  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
865  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
866  ICMP4_time_exceeded_ttl_exceeded_in_transit,
867  0);
869  goto trace1;
870  }
871 
872  proto1 = ip_proto_to_snat_proto (ip1->protocol);
873 
874  if (PREDICT_FALSE (proto1 == ~0))
875  {
876  snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1);
877  goto trace1;
878  }
879 
880  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
881  {
882  next1 = icmp_out2in_slow_path
883  (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
884  next1, now, thread_index, &s1);
885  goto trace1;
886  }
887 
888  key1.addr = ip1->dst_address;
889  key1.port = udp1->dst_port;
890  key1.protocol = proto1;
891  key1.fib_index = rx_fib_index1;
892 
893  kv1.key = key1.as_u64;
894 
895  if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
896  {
897  /* Try to match static mapping by external address and port,
898  destination address and port in packet */
899  if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
900  {
901  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
902  /*
903  * Send DHCP packets to the ipv4 stack, or we won't
904  * be able to use dhcp client on the outside interface
905  */
906  if (proto1 != SNAT_PROTOCOL_UDP
907  || (udp1->dst_port
908  != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
909  next1 = SNAT_OUT2IN_NEXT_DROP;
910  goto trace1;
911  }
912 
913  /* Create session initiated by host from external network */
914  s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
915  thread_index);
916  if (!s1)
917  {
918  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
919  next1 = SNAT_OUT2IN_NEXT_DROP;
920  goto trace1;
921  }
922  }
923  else
924  s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
925  value1.value);
926 
927  old_addr1 = ip1->dst_address.as_u32;
928  ip1->dst_address = s1->in2out.addr;
929  new_addr1 = ip1->dst_address.as_u32;
930  vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
931 
932  sum1 = ip1->checksum;
933  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
934  ip4_header_t,
935  dst_address /* changed member */);
936  ip1->checksum = ip_csum_fold (sum1);
937 
938  if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
939  {
940  old_port1 = tcp1->dst_port;
941  tcp1->dst_port = s1->in2out.port;
942  new_port1 = tcp1->dst_port;
943 
944  sum1 = tcp1->checksum;
945  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
946  ip4_header_t,
947  dst_address /* changed member */);
948 
949  sum1 = ip_csum_update (sum1, old_port1, new_port1,
950  ip4_header_t /* cheat */,
951  length /* changed member */);
952  tcp1->checksum = ip_csum_fold(sum1);
953  }
954  else
955  {
956  old_port1 = udp1->dst_port;
957  udp1->dst_port = s1->in2out.port;
958  udp1->checksum = 0;
959  }
960 
961  /* Accounting */
962  s1->last_heard = now;
963  s1->total_pkts++;
964  s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
965  /* Per-user LRU list maintenance for dynamic translation */
966  if (!snat_is_session_static (s1))
967  {
968  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
969  s1->per_user_index);
970  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
971  s1->per_user_list_head_index,
972  s1->per_user_index);
973  }
974  trace1:
975 
977  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
978  {
979  snat_out2in_trace_t *t =
980  vlib_add_trace (vm, node, b1, sizeof (*t));
981  t->sw_if_index = sw_if_index1;
982  t->next_index = next1;
983  t->session_index = ~0;
984  if (s1)
985  t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
986  }
987 
988  pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
989 
990  /* verify speculative enqueues, maybe switch current next frame */
991  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
992  to_next, n_left_to_next,
993  bi0, bi1, next0, next1);
994  }
995 
996  while (n_left_from > 0 && n_left_to_next > 0)
997  {
998  u32 bi0;
999  vlib_buffer_t * b0;
1000  u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1001  u32 sw_if_index0;
1002  ip4_header_t * ip0;
1003  ip_csum_t sum0;
1004  u32 new_addr0, old_addr0;
1005  u16 new_port0, old_port0;
1006  udp_header_t * udp0;
1007  tcp_header_t * tcp0;
1008  icmp46_header_t * icmp0;
1009  snat_session_key_t key0, sm0;
1010  u32 rx_fib_index0;
1011  u32 proto0;
1012  snat_session_t * s0 = 0;
1013  clib_bihash_kv_8_8_t kv0, value0;
1014 
1015  /* speculatively enqueue b0 to the current next frame */
1016  bi0 = from[0];
1017  to_next[0] = bi0;
1018  from += 1;
1019  to_next += 1;
1020  n_left_from -= 1;
1021  n_left_to_next -= 1;
1022 
1023  b0 = vlib_get_buffer (vm, bi0);
1024 
1025  ip0 = vlib_buffer_get_current (b0);
1026  udp0 = ip4_next_header (ip0);
1027  tcp0 = (tcp_header_t *) udp0;
1028  icmp0 = (icmp46_header_t *) udp0;
1029 
1030  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1031  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1032  sw_if_index0);
1033 
1034  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1035 
1036  if (PREDICT_FALSE (proto0 == ~0))
1037  {
1038  snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0);
1039  goto trace00;
1040  }
1041 
1042  if (PREDICT_FALSE(ip0->ttl == 1))
1043  {
1044  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1045  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1046  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1047  0);
1049  goto trace00;
1050  }
1051 
1052  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1053  {
1054  next0 = icmp_out2in_slow_path
1055  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1056  next0, now, thread_index, &s0);
1057  goto trace00;
1058  }
1059 
1060  key0.addr = ip0->dst_address;
1061  key0.port = udp0->dst_port;
1062  key0.protocol = proto0;
1063  key0.fib_index = rx_fib_index0;
1064 
1065  kv0.key = key0.as_u64;
1066 
1067  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1068  {
1069  /* Try to match static mapping by external address and port,
1070  destination address and port in packet */
1071  if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1072  {
1073  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1074  /*
1075  * Send DHCP packets to the ipv4 stack, or we won't
1076  * be able to use dhcp client on the outside interface
1077  */
1078  if (proto0 != SNAT_PROTOCOL_UDP
1079  || (udp0->dst_port
1080  != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1081 
1082  next0 = SNAT_OUT2IN_NEXT_DROP;
1083  goto trace00;
1084  }
1085 
1086  /* Create session initiated by host from external network */
1087  s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1088  thread_index);
1089  if (!s0)
1090  {
1091  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1092  next0 = SNAT_OUT2IN_NEXT_DROP;
1093  goto trace00;
1094  }
1095  }
1096  else
1097  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1098  value0.value);
1099 
1100  old_addr0 = ip0->dst_address.as_u32;
1101  ip0->dst_address = s0->in2out.addr;
1102  new_addr0 = ip0->dst_address.as_u32;
1103  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1104 
1105  sum0 = ip0->checksum;
1106  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1107  ip4_header_t,
1108  dst_address /* changed member */);
1109  ip0->checksum = ip_csum_fold (sum0);
1110 
1111  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1112  {
1113  old_port0 = tcp0->dst_port;
1114  tcp0->dst_port = s0->in2out.port;
1115  new_port0 = tcp0->dst_port;
1116 
1117  sum0 = tcp0->checksum;
1118  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1119  ip4_header_t,
1120  dst_address /* changed member */);
1121 
1122  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1123  ip4_header_t /* cheat */,
1124  length /* changed member */);
1125  tcp0->checksum = ip_csum_fold(sum0);
1126  }
1127  else
1128  {
1129  old_port0 = udp0->dst_port;
1130  udp0->dst_port = s0->in2out.port;
1131  udp0->checksum = 0;
1132  }
1133 
1134  /* Accounting */
1135  s0->last_heard = now;
1136  s0->total_pkts++;
1137  s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1138  /* Per-user LRU list maintenance for dynamic translation */
1139  if (!snat_is_session_static (s0))
1140  {
1141  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1142  s0->per_user_index);
1143  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1144  s0->per_user_list_head_index,
1145  s0->per_user_index);
1146  }
1147  trace00:
1148 
1150  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1151  {
1152  snat_out2in_trace_t *t =
1153  vlib_add_trace (vm, node, b0, sizeof (*t));
1154  t->sw_if_index = sw_if_index0;
1155  t->next_index = next0;
1156  t->session_index = ~0;
1157  if (s0)
1158  t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1159  }
1160 
1161  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1162 
1163  /* verify speculative enqueue, maybe switch current next frame */
1164  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1165  to_next, n_left_to_next,
1166  bi0, next0);
1167  }
1168 
1169  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1170  }
1171 
1173  SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1174  pkts_processed);
1175  return frame->n_vectors;
1176 }
1177 
1179  .function = snat_out2in_node_fn,
1180  .name = "snat-out2in",
1181  .vector_size = sizeof (u32),
1182  .format_trace = format_snat_out2in_trace,
1183  .type = VLIB_NODE_TYPE_INTERNAL,
1184 
1185  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1186  .error_strings = snat_out2in_error_strings,
1187 
1188  .runtime_data_bytes = sizeof (snat_runtime_t),
1189 
1190  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1191 
1192  /* edit / add dispositions here */
1193  .next_nodes = {
1194  [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1195  [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1196  [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1197  },
1198 };
1200 
1201 /**************************/
1202 /*** deterministic mode ***/
1203 /**************************/
1204 static uword
1206  vlib_node_runtime_t * node,
1207  vlib_frame_t * frame)
1208 {
1209  u32 n_left_from, * from, * to_next;
1210  snat_out2in_next_t next_index;
1211  u32 pkts_processed = 0;
1212  snat_main_t * sm = &snat_main;
1213  u32 thread_index = vlib_get_thread_index ();
1214 
1215  from = vlib_frame_vector_args (frame);
1216  n_left_from = frame->n_vectors;
1217  next_index = node->cached_next_index;
1218 
1219  while (n_left_from > 0)
1220  {
1221  u32 n_left_to_next;
1222 
1223  vlib_get_next_frame (vm, node, next_index,
1224  to_next, n_left_to_next);
1225 
1226  while (n_left_from >= 4 && n_left_to_next >= 2)
1227  {
1228  u32 bi0, bi1;
1229  vlib_buffer_t * b0, * b1;
1230  u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1231  u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1232  u32 sw_if_index0, sw_if_index1;
1233  ip4_header_t * ip0, * ip1;
1234  ip_csum_t sum0, sum1;
1235  ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1236  u16 new_port0, old_port0, old_port1, new_port1;
1237  udp_header_t * udp0, * udp1;
1238  tcp_header_t * tcp0, * tcp1;
1239  u32 proto0, proto1;
1240  snat_det_out_key_t key0, key1;
1241  snat_det_map_t * dm0, * dm1;
1242  snat_det_session_t * ses0 = 0, * ses1 = 0;
1243  u32 rx_fib_index0, rx_fib_index1;
1244  icmp46_header_t * icmp0, * icmp1;
1245 
1246  /* Prefetch next iteration. */
1247  {
1248  vlib_buffer_t * p2, * p3;
1249 
1250  p2 = vlib_get_buffer (vm, from[2]);
1251  p3 = vlib_get_buffer (vm, from[3]);
1252 
1253  vlib_prefetch_buffer_header (p2, LOAD);
1254  vlib_prefetch_buffer_header (p3, LOAD);
1255 
1258  }
1259 
1260  /* speculatively enqueue b0 and b1 to the current next frame */
1261  to_next[0] = bi0 = from[0];
1262  to_next[1] = bi1 = from[1];
1263  from += 2;
1264  to_next += 2;
1265  n_left_from -= 2;
1266  n_left_to_next -= 2;
1267 
1268  b0 = vlib_get_buffer (vm, bi0);
1269  b1 = vlib_get_buffer (vm, bi1);
1270 
1271  ip0 = vlib_buffer_get_current (b0);
1272  udp0 = ip4_next_header (ip0);
1273  tcp0 = (tcp_header_t *) udp0;
1274 
1275  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1276 
1277  if (PREDICT_FALSE(ip0->ttl == 1))
1278  {
1279  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1280  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1281  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1282  0);
1284  goto trace0;
1285  }
1286 
1287  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1288 
1289  if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1290  {
1291  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1292  icmp0 = (icmp46_header_t *) udp0;
1293 
1294  next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1295  rx_fib_index0, node, next0, thread_index,
1296  &ses0, &dm0);
1297  goto trace0;
1298  }
1299 
1300  key0.ext_host_addr = ip0->src_address;
1301  key0.ext_host_port = tcp0->src;
1302  key0.out_port = tcp0->dst;
1303 
1304  dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1305  if (PREDICT_FALSE(!dm0))
1306  {
1307  clib_warning("unknown dst address: %U",
1309  next0 = SNAT_OUT2IN_NEXT_DROP;
1310  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1311  goto trace0;
1312  }
1313 
1314  snat_det_reverse(dm0, &ip0->dst_address,
1315  clib_net_to_host_u16(tcp0->dst), &new_addr0);
1316 
1317  ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1318  if (PREDICT_FALSE(!ses0))
1319  {
1320  clib_warning("no match src %U:%d dst %U:%d for user %U",
1322  clib_net_to_host_u16 (tcp0->src),
1324  clib_net_to_host_u16 (tcp0->dst),
1325  format_ip4_address, &new_addr0);
1326  next0 = SNAT_OUT2IN_NEXT_DROP;
1327  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1328  goto trace0;
1329  }
1330  new_port0 = ses0->in_port;
1331 
1332  old_addr0 = ip0->dst_address;
1333  ip0->dst_address = new_addr0;
1334  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1335 
1336  sum0 = ip0->checksum;
1337  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1338  ip4_header_t,
1339  dst_address /* changed member */);
1340  ip0->checksum = ip_csum_fold (sum0);
1341 
1342  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1343  {
1344  if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1345  ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1346  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1347  snat_det_ses_close(dm0, ses0);
1348 
1349  old_port0 = tcp0->dst;
1350  tcp0->dst = new_port0;
1351 
1352  sum0 = tcp0->checksum;
1353  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1354  ip4_header_t,
1355  dst_address /* changed member */);
1356 
1357  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1358  ip4_header_t /* cheat */,
1359  length /* changed member */);
1360  tcp0->checksum = ip_csum_fold(sum0);
1361  }
1362  else
1363  {
1364  old_port0 = udp0->dst_port;
1365  udp0->dst_port = new_port0;
1366  udp0->checksum = 0;
1367  }
1368 
1369  trace0:
1370 
1372  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1373  {
1374  snat_out2in_trace_t *t =
1375  vlib_add_trace (vm, node, b0, sizeof (*t));
1376  t->sw_if_index = sw_if_index0;
1377  t->next_index = next0;
1378  t->session_index = ~0;
1379  if (ses0)
1380  t->session_index = ses0 - dm0->sessions;
1381  }
1382 
1383  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1384 
1385  b1 = vlib_get_buffer (vm, bi1);
1386 
1387  ip1 = vlib_buffer_get_current (b1);
1388  udp1 = ip4_next_header (ip1);
1389  tcp1 = (tcp_header_t *) udp1;
1390 
1391  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1392 
1393  if (PREDICT_FALSE(ip1->ttl == 1))
1394  {
1395  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1396  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1397  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1398  0);
1400  goto trace1;
1401  }
1402 
1403  proto1 = ip_proto_to_snat_proto (ip1->protocol);
1404 
1405  if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1406  {
1407  rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1408  icmp1 = (icmp46_header_t *) udp1;
1409 
1410  next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1411  rx_fib_index1, node, next1, thread_index,
1412  &ses1, &dm1);
1413  goto trace1;
1414  }
1415 
1416  key1.ext_host_addr = ip1->src_address;
1417  key1.ext_host_port = tcp1->src;
1418  key1.out_port = tcp1->dst;
1419 
1420  dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1421  if (PREDICT_FALSE(!dm1))
1422  {
1423  clib_warning("unknown dst address: %U",
1425  next1 = SNAT_OUT2IN_NEXT_DROP;
1426  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1427  goto trace1;
1428  }
1429 
1430  snat_det_reverse(dm1, &ip1->dst_address,
1431  clib_net_to_host_u16(tcp1->dst), &new_addr1);
1432 
1433  ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1434  if (PREDICT_FALSE(!ses1))
1435  {
1436  clib_warning("no match src %U:%d dst %U:%d for user %U",
1438  clib_net_to_host_u16 (tcp1->src),
1440  clib_net_to_host_u16 (tcp1->dst),
1441  format_ip4_address, &new_addr1);
1442  next1 = SNAT_OUT2IN_NEXT_DROP;
1443  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1444  goto trace1;
1445  }
1446  new_port1 = ses1->in_port;
1447 
1448  old_addr1 = ip1->dst_address;
1449  ip1->dst_address = new_addr1;
1450  vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1451 
1452  sum1 = ip1->checksum;
1453  sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1454  ip4_header_t,
1455  dst_address /* changed member */);
1456  ip1->checksum = ip_csum_fold (sum1);
1457 
1458  if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1459  {
1460  if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1461  ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1462  else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1463  snat_det_ses_close(dm1, ses1);
1464 
1465  old_port1 = tcp1->dst;
1466  tcp1->dst = new_port1;
1467 
1468  sum1 = tcp1->checksum;
1469  sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1470  ip4_header_t,
1471  dst_address /* changed member */);
1472 
1473  sum1 = ip_csum_update (sum1, old_port1, new_port1,
1474  ip4_header_t /* cheat */,
1475  length /* changed member */);
1476  tcp1->checksum = ip_csum_fold(sum1);
1477  }
1478  else
1479  {
1480  old_port1 = udp1->dst_port;
1481  udp1->dst_port = new_port1;
1482  udp1->checksum = 0;
1483  }
1484 
1485  trace1:
1486 
1488  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1489  {
1490  snat_out2in_trace_t *t =
1491  vlib_add_trace (vm, node, b1, sizeof (*t));
1492  t->sw_if_index = sw_if_index1;
1493  t->next_index = next1;
1494  t->session_index = ~0;
1495  if (ses1)
1496  t->session_index = ses1 - dm1->sessions;
1497  }
1498 
1499  pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1500 
1501  /* verify speculative enqueues, maybe switch current next frame */
1502  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1503  to_next, n_left_to_next,
1504  bi0, bi1, next0, next1);
1505  }
1506 
1507  while (n_left_from > 0 && n_left_to_next > 0)
1508  {
1509  u32 bi0;
1510  vlib_buffer_t * b0;
1511  u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1512  u32 sw_if_index0;
1513  ip4_header_t * ip0;
1514  ip_csum_t sum0;
1515  ip4_address_t new_addr0, old_addr0;
1516  u16 new_port0, old_port0;
1517  udp_header_t * udp0;
1518  tcp_header_t * tcp0;
1519  u32 proto0;
1520  snat_det_out_key_t key0;
1521  snat_det_map_t * dm0;
1522  snat_det_session_t * ses0 = 0;
1523  u32 rx_fib_index0;
1524  icmp46_header_t * icmp0;
1525 
1526  /* speculatively enqueue b0 to the current next frame */
1527  bi0 = from[0];
1528  to_next[0] = bi0;
1529  from += 1;
1530  to_next += 1;
1531  n_left_from -= 1;
1532  n_left_to_next -= 1;
1533 
1534  b0 = vlib_get_buffer (vm, bi0);
1535 
1536  ip0 = vlib_buffer_get_current (b0);
1537  udp0 = ip4_next_header (ip0);
1538  tcp0 = (tcp_header_t *) udp0;
1539 
1540  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1541 
1542  if (PREDICT_FALSE(ip0->ttl == 1))
1543  {
1544  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1545  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1546  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1547  0);
1549  goto trace00;
1550  }
1551 
1552  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1553 
1554  if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1555  {
1556  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1557  icmp0 = (icmp46_header_t *) udp0;
1558 
1559  next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1560  rx_fib_index0, node, next0, thread_index,
1561  &ses0, &dm0);
1562  goto trace00;
1563  }
1564 
1565  key0.ext_host_addr = ip0->src_address;
1566  key0.ext_host_port = tcp0->src;
1567  key0.out_port = tcp0->dst;
1568 
1569  dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1570  if (PREDICT_FALSE(!dm0))
1571  {
1572  clib_warning("unknown dst address: %U",
1574  next0 = SNAT_OUT2IN_NEXT_DROP;
1575  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1576  goto trace00;
1577  }
1578 
1579  snat_det_reverse(dm0, &ip0->dst_address,
1580  clib_net_to_host_u16(tcp0->dst), &new_addr0);
1581 
1582  ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1583  if (PREDICT_FALSE(!ses0))
1584  {
1585  clib_warning("no match src %U:%d dst %U:%d for user %U",
1587  clib_net_to_host_u16 (tcp0->src),
1589  clib_net_to_host_u16 (tcp0->dst),
1590  format_ip4_address, &new_addr0);
1591  next0 = SNAT_OUT2IN_NEXT_DROP;
1592  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1593  goto trace00;
1594  }
1595  new_port0 = ses0->in_port;
1596 
1597  old_addr0 = ip0->dst_address;
1598  ip0->dst_address = new_addr0;
1599  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1600 
1601  sum0 = ip0->checksum;
1602  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1603  ip4_header_t,
1604  dst_address /* changed member */);
1605  ip0->checksum = ip_csum_fold (sum0);
1606 
1607  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1608  {
1609  if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1610  ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1611  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1612  snat_det_ses_close(dm0, ses0);
1613 
1614  old_port0 = tcp0->dst;
1615  tcp0->dst = new_port0;
1616 
1617  sum0 = tcp0->checksum;
1618  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1619  ip4_header_t,
1620  dst_address /* changed member */);
1621 
1622  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1623  ip4_header_t /* cheat */,
1624  length /* changed member */);
1625  tcp0->checksum = ip_csum_fold(sum0);
1626  }
1627  else
1628  {
1629  old_port0 = udp0->dst_port;
1630  udp0->dst_port = new_port0;
1631  udp0->checksum = 0;
1632  }
1633 
1634  trace00:
1635 
1637  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1638  {
1639  snat_out2in_trace_t *t =
1640  vlib_add_trace (vm, node, b0, sizeof (*t));
1641  t->sw_if_index = sw_if_index0;
1642  t->next_index = next0;
1643  t->session_index = ~0;
1644  if (ses0)
1645  t->session_index = ses0 - dm0->sessions;
1646  }
1647 
1648  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1649 
1650  /* verify speculative enqueue, maybe switch current next frame */
1651  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1652  to_next, n_left_to_next,
1653  bi0, next0);
1654  }
1655 
1656  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1657  }
1658 
1660  SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1661  pkts_processed);
1662  return frame->n_vectors;
1663 }
1664 
1666  .function = snat_det_out2in_node_fn,
1667  .name = "snat-det-out2in",
1668  .vector_size = sizeof (u32),
1669  .format_trace = format_snat_out2in_trace,
1670  .type = VLIB_NODE_TYPE_INTERNAL,
1671 
1672  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1673  .error_strings = snat_out2in_error_strings,
1674 
1675  .runtime_data_bytes = sizeof (snat_runtime_t),
1676 
1677  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1678 
1679  /* edit / add dispositions here */
1680  .next_nodes = {
1681  [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1682  [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1683  [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1684  },
1685 };
1687 
1688 /**
1689  * Get address and port values to be used for packet SNAT translation
1690  * and create session if needed
1691  *
1692  * @param[in,out] sm SNAT main
1693  * @param[in,out] node SNAT node runtime
1694  * @param[in] thread_index thread index
1695  * @param[in,out] b0 buffer containing packet to be translated
1696  * @param[out] p_proto protocol used for matching
1697  * @param[out] p_value address and port after NAT translation
1698  * @param[out] p_dont_translate if packet should not be translated
1699  * @param d optional parameter
1700  * @param e optional parameter
1701  */
1703  u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
1704  snat_session_key_t *p_value,
1705  u8 *p_dont_translate, void *d, void *e)
1706 {
1707  ip4_header_t *ip0;
1708  icmp46_header_t *icmp0;
1709  u32 sw_if_index0;
1710  u8 protocol;
1711  snat_det_out_key_t key0;
1712  u8 dont_translate = 0;
1713  u32 next0 = ~0;
1714  icmp_echo_header_t *echo0, *inner_echo0 = 0;
1715  ip4_header_t *inner_ip0;
1716  void *l4_header = 0;
1717  icmp46_header_t *inner_icmp0;
1718  snat_det_map_t * dm0 = 0;
1719  ip4_address_t new_addr0 = {{0}};
1720  snat_det_session_t * ses0 = 0;
1721  ip4_address_t out_addr;
1722 
1723  ip0 = vlib_buffer_get_current (b0);
1724  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1725  echo0 = (icmp_echo_header_t *)(icmp0+1);
1726  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1727 
1728  if (!icmp_is_error_message (icmp0))
1729  {
1730  protocol = SNAT_PROTOCOL_ICMP;
1731  key0.ext_host_addr = ip0->src_address;
1732  key0.ext_host_port = 0;
1733  key0.out_port = echo0->identifier;
1734  out_addr = ip0->dst_address;
1735  }
1736  else
1737  {
1738  inner_ip0 = (ip4_header_t *)(echo0+1);
1739  l4_header = ip4_next_header (inner_ip0);
1740  protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
1741  key0.ext_host_addr = inner_ip0->dst_address;
1742  out_addr = inner_ip0->src_address;
1743  switch (protocol)
1744  {
1745  case SNAT_PROTOCOL_ICMP:
1746  inner_icmp0 = (icmp46_header_t*)l4_header;
1747  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1748  key0.ext_host_port = 0;
1749  key0.out_port = inner_echo0->identifier;
1750  break;
1751  case SNAT_PROTOCOL_UDP:
1752  case SNAT_PROTOCOL_TCP:
1753  key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1754  key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
1755  break;
1756  default:
1757  b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1758  next0 = SNAT_OUT2IN_NEXT_DROP;
1759  goto out;
1760  }
1761  }
1762 
1763  dm0 = snat_det_map_by_out(sm, &out_addr);
1764  if (PREDICT_FALSE(!dm0))
1765  {
1766  /* Don't NAT packet aimed at the intfc address */
1767  if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1768  ip0->dst_address.as_u32)))
1769  {
1770  dont_translate = 1;
1771  goto out;
1772  }
1773  clib_warning("unknown dst address: %U",
1775  goto out;
1776  }
1777 
1778  snat_det_reverse(dm0, &ip0->dst_address,
1779  clib_net_to_host_u16(key0.out_port), &new_addr0);
1780 
1781  ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1782  if (PREDICT_FALSE(!ses0))
1783  {
1784  /* Don't NAT packet aimed at the intfc address */
1785  if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1786  ip0->dst_address.as_u32)))
1787  {
1788  dont_translate = 1;
1789  goto out;
1790  }
1791  clib_warning("no match src %U:%d dst %U:%d for user %U",
1793  clib_net_to_host_u16 (key0.ext_host_port),
1794  format_ip4_address, &out_addr,
1795  clib_net_to_host_u16 (key0.out_port),
1796  format_ip4_address, &new_addr0);
1797  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1798  next0 = SNAT_OUT2IN_NEXT_DROP;
1799  goto out;
1800  }
1801 
1802  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
1803  !icmp_is_error_message (icmp0)))
1804  {
1805  b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1806  next0 = SNAT_OUT2IN_NEXT_DROP;
1807  goto out;
1808  }
1809 
1810  goto out;
1811 
1812 out:
1813  *p_proto = protocol;
1814  if (ses0)
1815  {
1816  p_value->addr = new_addr0;
1817  p_value->fib_index = sm->inside_fib_index;
1818  p_value->port = ses0->in_port;
1819  }
1820  *p_dont_translate = dont_translate;
1821  if (d)
1822  *(snat_det_session_t**)d = ses0;
1823  if (e)
1824  *(snat_det_map_t**)e = dm0;
1825  return next0;
1826 }
1827 
1828 /**********************/
1829 /*** worker handoff ***/
1830 /**********************/
1831 static uword
1833  vlib_node_runtime_t * node,
1834  vlib_frame_t * frame)
1835 {
1836  snat_main_t *sm = &snat_main;
1838  u32 n_left_from, *from, *to_next = 0;
1839  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1840  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1841  = 0;
1842  vlib_frame_queue_elt_t *hf = 0;
1843  vlib_frame_t *f = 0;
1844  int i;
1845  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1846  u32 next_worker_index = 0;
1847  u32 current_worker_index = ~0;
1848  u32 thread_index = vlib_get_thread_index ();
1849 
1850  ASSERT (vec_len (sm->workers));
1851 
1852  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1853  {
1854  vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1855 
1856  vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1857  sm->first_worker_index + sm->num_workers - 1,
1858  (vlib_frame_queue_t *) (~0));
1859  }
1860 
1861  from = vlib_frame_vector_args (frame);
1862  n_left_from = frame->n_vectors;
1863 
1864  while (n_left_from > 0)
1865  {
1866  u32 bi0;
1867  vlib_buffer_t *b0;
1868  u32 sw_if_index0;
1869  u32 rx_fib_index0;
1870  ip4_header_t * ip0;
1871  u8 do_handoff;
1872 
1873  bi0 = from[0];
1874  from += 1;
1875  n_left_from -= 1;
1876 
1877  b0 = vlib_get_buffer (vm, bi0);
1878 
1879  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1880  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1881 
1882  ip0 = vlib_buffer_get_current (b0);
1883 
1884  next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1885 
1886  if (PREDICT_FALSE (next_worker_index != thread_index))
1887  {
1888  do_handoff = 1;
1889 
1890  if (next_worker_index != current_worker_index)
1891  {
1892  if (hf)
1893  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1894 
1895  hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1896  next_worker_index,
1897  handoff_queue_elt_by_worker_index);
1898 
1899  n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1900  to_next_worker = &hf->buffer_index[hf->n_vectors];
1901  current_worker_index = next_worker_index;
1902  }
1903 
1904  /* enqueue to correct worker thread */
1905  to_next_worker[0] = bi0;
1906  to_next_worker++;
1907  n_left_to_next_worker--;
1908 
1909  if (n_left_to_next_worker == 0)
1910  {
1911  hf->n_vectors = VLIB_FRAME_SIZE;
1913  current_worker_index = ~0;
1914  handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1915  hf = 0;
1916  }
1917  }
1918  else
1919  {
1920  do_handoff = 0;
1921  /* if this is 1st frame */
1922  if (!f)
1923  {
1924  f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1925  to_next = vlib_frame_vector_args (f);
1926  }
1927 
1928  to_next[0] = bi0;
1929  to_next += 1;
1930  f->n_vectors++;
1931  }
1932 
1934  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1935  {
1937  vlib_add_trace (vm, node, b0, sizeof (*t));
1938  t->next_worker_index = next_worker_index;
1939  t->do_handoff = do_handoff;
1940  }
1941  }
1942 
1943  if (f)
1944  vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1945 
1946  if (hf)
1947  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1948 
1949  /* Ship frames to the worker nodes */
1950  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1951  {
1952  if (handoff_queue_elt_by_worker_index[i])
1953  {
1954  hf = handoff_queue_elt_by_worker_index[i];
1955  /*
1956  * It works better to let the handoff node
1957  * rate-adapt, always ship the handoff queue element.
1958  */
1959  if (1 || hf->n_vectors == hf->last_n_vectors)
1960  {
1962  handoff_queue_elt_by_worker_index[i] = 0;
1963  }
1964  else
1965  hf->last_n_vectors = hf->n_vectors;
1966  }
1967  congested_handoff_queue_by_worker_index[i] =
1968  (vlib_frame_queue_t *) (~0);
1969  }
1970  hf = 0;
1971  current_worker_index = ~0;
1972  return frame->n_vectors;
1973 }
1974 
1976  .function = snat_out2in_worker_handoff_fn,
1977  .name = "snat-out2in-worker-handoff",
1978  .vector_size = sizeof (u32),
1980  .type = VLIB_NODE_TYPE_INTERNAL,
1981 
1982  .n_next_nodes = 1,
1983 
1984  .next_nodes = {
1985  [0] = "error-drop",
1986  },
1987 };
1988 
1990 
1991 static uword
1993  vlib_node_runtime_t * node,
1994  vlib_frame_t * frame)
1995 {
1996  u32 n_left_from, * from, * to_next;
1997  snat_out2in_next_t next_index;
1998  u32 pkts_processed = 0;
1999  snat_main_t * sm = &snat_main;
2000 
2001  from = vlib_frame_vector_args (frame);
2002  n_left_from = frame->n_vectors;
2003  next_index = node->cached_next_index;
2004 
2005  while (n_left_from > 0)
2006  {
2007  u32 n_left_to_next;
2008 
2009  vlib_get_next_frame (vm, node, next_index,
2010  to_next, n_left_to_next);
2011 
2012  while (n_left_from > 0 && n_left_to_next > 0)
2013  {
2014  u32 bi0;
2015  vlib_buffer_t * b0;
2016  u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2017  u32 sw_if_index0;
2018  ip4_header_t * ip0;
2019  ip_csum_t sum0;
2020  u32 new_addr0, old_addr0;
2021  u16 new_port0, old_port0;
2022  udp_header_t * udp0;
2023  tcp_header_t * tcp0;
2024  icmp46_header_t * icmp0;
2025  snat_session_key_t key0, sm0;
2026  u32 proto0;
2027  u32 rx_fib_index0;
2028 
2029  /* speculatively enqueue b0 to the current next frame */
2030  bi0 = from[0];
2031  to_next[0] = bi0;
2032  from += 1;
2033  to_next += 1;
2034  n_left_from -= 1;
2035  n_left_to_next -= 1;
2036 
2037  b0 = vlib_get_buffer (vm, bi0);
2038 
2039  ip0 = vlib_buffer_get_current (b0);
2040  udp0 = ip4_next_header (ip0);
2041  tcp0 = (tcp_header_t *) udp0;
2042  icmp0 = (icmp46_header_t *) udp0;
2043 
2044  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2045  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2046 
2047  vnet_feature_next (sw_if_index0, &next0, b0);
2048 
2049  if (PREDICT_FALSE(ip0->ttl == 1))
2050  {
2051  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2052  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2053  ICMP4_time_exceeded_ttl_exceeded_in_transit,
2054  0);
2056  goto trace00;
2057  }
2058 
2059  proto0 = ip_proto_to_snat_proto (ip0->protocol);
2060 
2061  if (PREDICT_FALSE (proto0 == ~0))
2062  goto trace00;
2063 
2064  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2065  {
2066  next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2067  rx_fib_index0, node, next0, ~0, 0, 0);
2068  goto trace00;
2069  }
2070 
2071  key0.addr = ip0->dst_address;
2072  key0.port = udp0->dst_port;
2073  key0.fib_index = rx_fib_index0;
2074 
2075  if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2076  {
2077  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2078  goto trace00;
2079  }
2080 
2081  new_addr0 = sm0.addr.as_u32;
2082  new_port0 = sm0.port;
2083  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2084  old_addr0 = ip0->dst_address.as_u32;
2085  ip0->dst_address.as_u32 = new_addr0;
2086 
2087  sum0 = ip0->checksum;
2088  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2089  ip4_header_t,
2090  dst_address /* changed member */);
2091  ip0->checksum = ip_csum_fold (sum0);
2092 
2093  if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2094  {
2095  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2096  {
2097  old_port0 = tcp0->dst_port;
2098  tcp0->dst_port = new_port0;
2099 
2100  sum0 = tcp0->checksum;
2101  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2102  ip4_header_t,
2103  dst_address /* changed member */);
2104 
2105  sum0 = ip_csum_update (sum0, old_port0, new_port0,
2106  ip4_header_t /* cheat */,
2107  length /* changed member */);
2108  tcp0->checksum = ip_csum_fold(sum0);
2109  }
2110  else
2111  {
2112  old_port0 = udp0->dst_port;
2113  udp0->dst_port = new_port0;
2114  udp0->checksum = 0;
2115  }
2116  }
2117  else
2118  {
2119  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2120  {
2121  sum0 = tcp0->checksum;
2122  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2123  ip4_header_t,
2124  dst_address /* changed member */);
2125 
2126  tcp0->checksum = ip_csum_fold(sum0);
2127  }
2128  }
2129 
2130  trace00:
2131 
2133  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2134  {
2135  snat_out2in_trace_t *t =
2136  vlib_add_trace (vm, node, b0, sizeof (*t));
2137  t->sw_if_index = sw_if_index0;
2138  t->next_index = next0;
2139  }
2140 
2141  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2142 
2143  /* verify speculative enqueue, maybe switch current next frame */
2144  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2145  to_next, n_left_to_next,
2146  bi0, next0);
2147  }
2148 
2149  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2150  }
2151 
2153  SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2154  pkts_processed);
2155  return frame->n_vectors;
2156 }
2157 
2159  .function = snat_out2in_fast_node_fn,
2160  .name = "snat-out2in-fast",
2161  .vector_size = sizeof (u32),
2162  .format_trace = format_snat_out2in_fast_trace,
2163  .type = VLIB_NODE_TYPE_INTERNAL,
2164 
2165  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2166  .error_strings = snat_out2in_error_strings,
2167 
2168  .runtime_data_bytes = sizeof (snat_runtime_t),
2169 
2170  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2171 
2172  /* edit / add dispositions here */
2173  .next_nodes = {
2174  [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2175  [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2176  [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2177  },
2178 };
vlib_node_registration_t snat_out2in_fast_node
(constructor) VLIB_REGISTER_NODE (snat_out2in_fast_node)
Definition: out2in.c:82
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:187
VLIB_NODE_FUNCTION_MULTIARCH(snat_out2in_node, snat_out2in_node_fn)
u32 sessions_per_user_list_head_index
Definition: snat.h:152
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define CLIB_UNUSED(x)
Definition: clib.h:79
u16 ext_host_port
Definition: snat.h:60
static void clib_dlist_init(dlist_elt_t *pool, u32 index)
Definition: dlist.h:36
ip4_address_t src_address
Definition: ip4_packet.h:164
static u8 * format_snat_out2in_fast_trace(u8 *s, va_list *args)
Definition: out2in.c:56
static u32 icmp_out2in_slow_path(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0, f64 now, u32 thread_index, snat_session_t **p_s0)
Definition: out2in.c:581
#define PREDICT_TRUE(x)
Definition: clib.h:98
static int ip4_header_bytes(ip4_header_t *i)
Definition: ip4_packet.h:227
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:192
static void snat_det_reverse(snat_det_map_t *dm, ip4_address_t *out_addr, u16 out_port, ip4_address_t *in_addr)
Definition: snat_det.h:90
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:459
static_always_inline u8 icmp_is_error_message(icmp46_header_t *icmp)
Definition: snat.h:455
static snat_det_map_t * snat_det_map_by_out(snat_main_t *sm, ip4_address_t *out_addr)
Definition: snat_det.h:60
u32 nstaticsessions
Definition: snat.h:154
struct _vlib_node_registration vlib_node_registration_t
uword ip_csum_t
Definition: ip_packet.h:90
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
u32 fib_index
Definition: snat.h:151
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:82
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:419
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:100
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:200
struct _tcp_header tcp_header_t
u32 ip4_fib_table_get_index_for_sw_if_index(u32 sw_if_index)
Definition: ip4_fib.c:221
format_function_t format_ip4_address
Definition: format.h:79
#define static_always_inline
Definition: clib.h:85
static uword ip4_header_checksum_is_valid(ip4_header_t *i)
Definition: ip4_packet.h:260
vlib_node_registration_t snat_det_out2in_node
(constructor) VLIB_REGISTER_NODE (snat_det_out2in_node)
Definition: out2in.c:84
ip_csum_t ip_incremental_checksum(ip_csum_t sum, void *_data, uword n_bytes)
Definition: ip_checksum.c:43
ip4_address_t ext_host_addr
Definition: snat.h:59
ip4_address_t dst_address
Definition: ip4_packet.h:164
#define TCP_FLAG_ACK
Definition: fa_node.h:11
ip4_address_t addr
Definition: snat.h:150
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:164
static_always_inline u8 is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, u32 ip4_addr)
Definition: snat.h:471
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:233
#define foreach_snat_out2in_error
Definition: out2in.c:86
ip4_address_t local_addr
Definition: snat.h:187
static uword snat_out2in_worker_handoff_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: out2in.c:1832
snat_out2in_next_t
Definition: out2in.c:105
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:397
deterministic NAT definitions
snat_det_session_t * sessions
Definition: snat.h:183
u64 key
the key
Definition: bihash_8_8.h:32
static void clib_dlist_addtail(dlist_elt_t *pool, u32 head_index, u32 new_index)
Definition: dlist.h:43
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
snat_main_t snat_main
Definition: jvpp_snat.h:39
#define PREDICT_FALSE(x)
Definition: clib.h:97
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
Definition: threads.h:457
#define TCP_FLAG_FIN
Definition: fa_node.h:7
#define VLIB_FRAME_SIZE
Definition: node.h:329
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
vlib_node_registration_t snat_out2in_node
(constructor) VLIB_REGISTER_NODE (snat_out2in_node)
Definition: out2in.c:81
static_always_inline void vnet_feature_next(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:221
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:216
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:366
int snat_static_mapping_match(snat_main_t *sm, snat_session_key_t match, snat_session_key_t *mapping, u8 by_external, u8 *is_addr_only)
Match SNAT static mapping.
Definition: snat.c:835
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:113
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1131
static u32 icmp_out2in(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0, u32 thread_index, void *d, void *e)
Definition: out2in.c:458
The fine-grained event logger allows lightweight, thread-safe event logging at minimum cost...
u64 value
the value
Definition: bihash_8_8.h:33
static snat_session_t * create_session_for_static_mapping(snat_main_t *sm, vlib_buffer_t *b0, snat_session_key_t in2out, snat_session_key_t out2in, vlib_node_runtime_t *node, u32 thread_index)
Create session for static mapping.
Definition: out2in.c:127
u16 n_vectors
Definition: node.h:345
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:185
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
void icmp4_error_set_vnet_buffer(vlib_buffer_t *b, u8 type, u8 code, u32 data)
Definition: icmp4.c:431
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:85
8 octet key, 8 octet key value pair
Definition: bihash_8_8.h:30
#define ARRAY_LEN(x)
Definition: clib.h:59
ip4_address_t addr
Definition: snat.h:45
u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for packet SNAT translation and create session if needed...
Definition: out2in.c:1702
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:460
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
static uword snat_det_out2in_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: out2in.c:1205
u64 as_u64
Definition: snat.h:75
ip4_address_t addr
Definition: snat.h:72
u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for packet SNAT translation and create session if needed...
Definition: out2in.c:288
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:260
static void clib_dlist_remove(dlist_elt_t *pool, u32 index)
Definition: dlist.h:99
void snat_ipfix_logging_nat44_ses_create(u32 src_ip, u32 nat_src_ip, snat_protocol_t snat_proto, u16 src_port, u16 nat_src_port, u32 vrf_id)
Generate NAT44 session create event.
static snat_det_session_t * snat_det_get_ses_by_out(snat_det_map_t *dm, ip4_address_t *in_addr, u64 out_key)
Definition: snat_det.h:112
u32 value
Definition: dlist.h:32
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
#define vec_elt(v, i)
Get vector value at index i.
static void snat_out2in_unknown_proto(snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip, u32 rx_fib_index)
Definition: out2in.c:615
Definition: defs.h:47
unsigned short u16
Definition: types.h:57
vlib_node_registration_t snat_out2in_worker_handoff_node
(constructor) VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node)
Definition: out2in.c:83
snat_out2in_error_t
Definition: out2in.c:92
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
static char * snat_out2in_error_strings[]
Definition: out2in.c:99
unsigned char u8
Definition: types.h:56
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:269
#define ip_csum_update(sum, old, new, type, field)
Definition: ip_packet.h:139
static_always_inline snat_out2in_error_t icmp_get_key(ip4_header_t *ip0, snat_session_key_t *p_key0)
Definition: out2in.c:230
u32 fib_index
Definition: snat.h:73
static u8 * format_snat_out2in_trace(u8 *s, va_list *args)
Definition: out2in.c:45
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
Definition: threads.h:387
static u32 ip_proto_to_snat_proto(u8 ip_proto)
Definition: snat.h:389
static uword snat_out2in_fast_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: out2in.c:1992
#define vnet_buffer(b)
Definition: buffer.h:304
static void snat_det_ses_close(snat_det_map_t *dm, snat_det_session_t *ses)
Definition: snat_det.h:179
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:144
u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for packet SNAT translation.
Definition: out2in.c:398
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static uword snat_out2in_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: out2in.c:646
u8 data[0]
Packet data.
Definition: buffer.h:152
u16 flags
Copy of main node flags.
Definition: node.h:454
#define SNAT_SESSION_FLAG_STATIC_MAPPING
Definition: snat.h:122
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:196
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:485
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
#define snat_is_session_static(s)
Check if SNAT session is created from static mapping.
Definition: snat.h:375
static u16 ip_csum_fold(ip_csum_t c)
Definition: ip_packet.h:145
static u8 * format_snat_out2in_worker_handoff_trace(u8 *s, va_list *args)
Definition: out2in.c:67
Definition: defs.h:46