FD.io VPP  v18.04-17-g3a0d853
Vector Packet Processing
tcp_output.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vnet/tcp/tcp.h>
17 #include <vnet/lisp-cp/packets.h>
18 #include <math.h>
19 
22 
23 typedef enum _tcp_output_next
24 {
29 
30 #define foreach_tcp4_output_next \
31  _ (DROP, "error-drop") \
32  _ (IP_LOOKUP, "ip4-lookup")
33 
34 #define foreach_tcp6_output_next \
35  _ (DROP, "error-drop") \
36  _ (IP_LOOKUP, "ip6-lookup")
37 
38 static char *tcp_error_strings[] = {
39 #define tcp_error(n,s) s,
40 #include <vnet/tcp/tcp_error.def>
41 #undef tcp_error
42 };
43 
44 typedef struct
45 {
49 
50 u16 dummy_mtu = 1460;
51 
52 u8 *
53 format_tcp_tx_trace (u8 * s, va_list * args)
54 {
55  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
56  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
57  tcp_tx_trace_t *t = va_arg (*args, tcp_tx_trace_t *);
58  u32 indent = format_get_indent (s);
59 
60  s = format (s, "%U\n%U%U",
61  format_tcp_header, &t->tcp_header, 128,
62  format_white_space, indent,
64 
65  return s;
66 }
67 
68 static u8
70 {
71  u8 wnd_scale = 0;
72  while (wnd_scale < TCP_MAX_WND_SCALE && (window >> wnd_scale) > TCP_WND_MAX)
73  wnd_scale++;
74  return wnd_scale;
75 }
76 
77 /**
78  * Update max segment size we're able to process.
79  *
80  * The value is constrained by our interface's MTU and IP options. It is
81  * also what we advertise to our peer.
82  */
83 void
85 {
86  /* TODO find our iface MTU */
87  tc->mss = dummy_mtu - sizeof (tcp_header_t);
88 }
89 
90 /**
91  * TCP's initial window
92  */
95 {
96  /* RFC 6928 recommends the value lower. However at the time our connections
97  * are initialized, fifos may not be allocated. Therefore, advertise the
98  * smallest possible unscaled window size and update once fifos are
99  * assigned to the session.
100  */
101  /*
102  tcp_update_rcv_mss (tc);
103  TCP_IW_N_SEGMENTS * tc->mss;
104  */
105  return TCP_MIN_RX_FIFO_SIZE;
106 }
107 
108 /**
109  * Compute initial window and scale factor. As per RFC1323, window field in
110  * SYN and SYN-ACK segments is never scaled.
111  */
112 u32
114 {
115  u32 max_fifo;
116 
117  /* Initial wnd for SYN. Fifos are not allocated yet.
118  * Use some predefined value. For SYN-ACK we still want the
119  * scale to be computed in the same way */
120  max_fifo = TCP_MAX_RX_FIFO_SIZE;
121 
122  tc->rcv_wscale = tcp_window_compute_scale (max_fifo);
123  tc->rcv_wnd = tcp_initial_wnd_unscaled (tc);
124 
125  return clib_min (tc->rcv_wnd, TCP_WND_MAX);
126 }
127 
128 /**
129  * Compute and return window to advertise, scaled as per RFC1323
130  */
131 u32
133 {
134  if (state < TCP_STATE_ESTABLISHED)
136 
137  tcp_update_rcv_wnd (tc);
138 
139  if (tc->rcv_wnd == 0)
140  {
141  tc->flags |= TCP_CONN_SENT_RCV_WND0;
142  }
143  else
144  {
145  tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
146  }
147 
148  return tc->rcv_wnd >> tc->rcv_wscale;
149 }
150 
151 void
153 {
154  i32 observed_wnd;
155  u32 available_space, max_fifo, wnd;
156 
157  /*
158  * Figure out how much space we have available
159  */
160  available_space = stream_session_max_rx_enqueue (&tc->connection);
161  max_fifo = stream_session_rx_fifo_size (&tc->connection);
162 
163  ASSERT (tc->rcv_opts.mss < max_fifo);
164  if (available_space < tc->rcv_opts.mss && available_space < max_fifo >> 3)
165  available_space = 0;
166 
167  /*
168  * Use the above and what we know about what we've previously advertised
169  * to compute the new window
170  */
171  observed_wnd = (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
172  if (observed_wnd < 0)
173  observed_wnd = 0;
174 
175  /* Bad. Thou shalt not shrink */
176  if (available_space < observed_wnd)
177  {
178  wnd = observed_wnd;
179  TCP_EVT_DBG (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
180  }
181  else
182  {
183  wnd = available_space;
184  }
185 
186  /* Make sure we have a multiple of rcv_wscale */
187  if (wnd && tc->rcv_wscale)
188  {
189  wnd &= ~(1 << tc->rcv_wscale);
190  if (wnd == 0)
191  wnd = 1 << tc->rcv_wscale;
192  }
193 
194  tc->rcv_wnd = clib_min (wnd, TCP_WND_MAX << tc->rcv_wscale);
195 }
196 
197 /**
198  * Write TCP options to segment.
199  */
200 u32
202 {
203  u32 opts_len = 0;
204  u32 buf, seq_len = 4;
205 
206  if (tcp_opts_mss (opts))
207  {
208  *data++ = TCP_OPTION_MSS;
209  *data++ = TCP_OPTION_LEN_MSS;
210  buf = clib_host_to_net_u16 (opts->mss);
211  clib_memcpy (data, &buf, sizeof (opts->mss));
212  data += sizeof (opts->mss);
213  opts_len += TCP_OPTION_LEN_MSS;
214  }
215 
216  if (tcp_opts_wscale (opts))
217  {
218  *data++ = TCP_OPTION_WINDOW_SCALE;
219  *data++ = TCP_OPTION_LEN_WINDOW_SCALE;
220  *data++ = opts->wscale;
221  opts_len += TCP_OPTION_LEN_WINDOW_SCALE;
222  }
223 
224  if (tcp_opts_sack_permitted (opts))
225  {
226  *data++ = TCP_OPTION_SACK_PERMITTED;
228  opts_len += TCP_OPTION_LEN_SACK_PERMITTED;
229  }
230 
231  if (tcp_opts_tstamp (opts))
232  {
233  *data++ = TCP_OPTION_TIMESTAMP;
234  *data++ = TCP_OPTION_LEN_TIMESTAMP;
235  buf = clib_host_to_net_u32 (opts->tsval);
236  clib_memcpy (data, &buf, sizeof (opts->tsval));
237  data += sizeof (opts->tsval);
238  buf = clib_host_to_net_u32 (opts->tsecr);
239  clib_memcpy (data, &buf, sizeof (opts->tsecr));
240  data += sizeof (opts->tsecr);
241  opts_len += TCP_OPTION_LEN_TIMESTAMP;
242  }
243 
244  if (tcp_opts_sack (opts))
245  {
246  int i;
247  u32 n_sack_blocks = clib_min (vec_len (opts->sacks),
249 
250  if (n_sack_blocks != 0)
251  {
252  *data++ = TCP_OPTION_SACK_BLOCK;
253  *data++ = 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
254  for (i = 0; i < n_sack_blocks; i++)
255  {
256  buf = clib_host_to_net_u32 (opts->sacks[i].start);
257  clib_memcpy (data, &buf, seq_len);
258  data += seq_len;
259  buf = clib_host_to_net_u32 (opts->sacks[i].end);
260  clib_memcpy (data, &buf, seq_len);
261  data += seq_len;
262  }
263  opts_len += 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
264  }
265  }
266 
267  /* Terminate TCP options */
268  if (opts_len % 4)
269  {
270  *data++ = TCP_OPTION_EOL;
271  opts_len += TCP_OPTION_LEN_EOL;
272  }
273 
274  /* Pad with zeroes to a u32 boundary */
275  while (opts_len % 4)
276  {
277  *data++ = TCP_OPTION_NOOP;
278  opts_len += TCP_OPTION_LEN_NOOP;
279  }
280  return opts_len;
281 }
282 
283 always_inline int
285 {
286  u8 len = 0;
287 
288  opts->flags |= TCP_OPTS_FLAG_MSS;
289  opts->mss = dummy_mtu; /*XXX discover that */
290  len += TCP_OPTION_LEN_MSS;
291 
292  opts->flags |= TCP_OPTS_FLAG_WSCALE;
293  opts->wscale = wnd_scale;
295 
296  opts->flags |= TCP_OPTS_FLAG_TSTAMP;
297  opts->tsval = tcp_time_now ();
298  opts->tsecr = 0;
300 
301  if (TCP_USE_SACKS)
302  {
303  opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
305  }
306 
307  /* Align to needed boundary */
308  len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
309  return len;
310 }
311 
312 always_inline int
314 {
315  u8 len = 0;
316 
317  opts->flags |= TCP_OPTS_FLAG_MSS;
318  opts->mss = tc->mss;
319  len += TCP_OPTION_LEN_MSS;
320 
321  if (tcp_opts_wscale (&tc->rcv_opts))
322  {
323  opts->flags |= TCP_OPTS_FLAG_WSCALE;
324  opts->wscale = tc->rcv_wscale;
326  }
327 
328  if (tcp_opts_tstamp (&tc->rcv_opts))
329  {
330  opts->flags |= TCP_OPTS_FLAG_TSTAMP;
331  opts->tsval = tcp_time_now ();
332  opts->tsecr = tc->tsval_recent;
334  }
335 
336  if (tcp_opts_sack_permitted (&tc->rcv_opts))
337  {
338  opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
340  }
341 
342  /* Align to needed boundary */
343  len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
344  return len;
345 }
346 
347 always_inline int
349 {
350  u8 len = 0;
351 
352  opts->flags = 0;
353 
354  if (tcp_opts_tstamp (&tc->rcv_opts))
355  {
356  opts->flags |= TCP_OPTS_FLAG_TSTAMP;
357  opts->tsval = tcp_time_now ();
358  opts->tsecr = tc->tsval_recent;
360  }
361  if (tcp_opts_sack_permitted (&tc->rcv_opts))
362  {
363  if (vec_len (tc->snd_sacks))
364  {
365  opts->flags |= TCP_OPTS_FLAG_SACK;
366  opts->sacks = tc->snd_sacks;
367  opts->n_sack_blocks = clib_min (vec_len (tc->snd_sacks),
369  len += 2 + TCP_OPTION_LEN_SACK_BLOCK * opts->n_sack_blocks;
370  }
371  }
372 
373  /* Align to needed boundary */
374  len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
375  return len;
376 }
377 
378 always_inline int
381 {
382  switch (state)
383  {
384  case TCP_STATE_ESTABLISHED:
385  case TCP_STATE_FIN_WAIT_1:
386  return tcp_make_established_options (tc, opts);
387  case TCP_STATE_SYN_RCVD:
388  return tcp_make_synack_options (tc, opts);
389  case TCP_STATE_SYN_SENT:
390  return tcp_make_syn_options (opts, tc->rcv_wscale);
391  default:
392  clib_warning ("State not handled! %d", state);
393  return 0;
394  }
395 }
396 
397 /**
398  * Update snd_mss to reflect the effective segment size that we can send
399  * by taking into account all TCP options, including SACKs
400  */
401 void
403 {
404  /* Compute options to be used for connection. These may be reused when
405  * sending data or to compute the effective mss (snd_mss) */
406  tc->snd_opts_len =
407  tcp_make_options (tc, &tc->snd_opts, TCP_STATE_ESTABLISHED);
408 
409  /* XXX check if MTU has been updated */
410  tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
411  ASSERT (tc->snd_mss > 0);
412 }
413 
414 void
416 {
417  u16 default_min_mss = 536;
418  tcp_update_rcv_mss (tc);
419 
420  /* TODO cache mss and consider PMTU discovery */
421  tc->snd_mss = clib_min (tc->rcv_opts.mss, tc->mss);
422 
423  if (tc->snd_mss < 45)
424  {
425  clib_warning ("snd mss is 0");
426  /* Assume that at least the min default mss works */
427  tc->snd_mss = default_min_mss;
428  tc->rcv_opts.mss = default_min_mss;
429  }
430 
431  /* We should have enough space for 40 bytes of options */
432  ASSERT (tc->snd_mss > 45);
433 
434  /* If we use timestamp option, account for it */
435  if (tcp_opts_tstamp (&tc->rcv_opts))
436  tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
437 }
438 
439 always_inline int
440 tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u32 n_free_buffers)
441 {
443  u32 current_length = vec_len (tm->tx_buffers[thread_index]);
444  u32 n_allocated;
445 
446  vec_validate (tm->tx_buffers[thread_index],
447  current_length + n_free_buffers - 1);
448  n_allocated =
449  vlib_buffer_alloc (vm, &tm->tx_buffers[thread_index][current_length],
450  n_free_buffers);
451  _vec_len (tm->tx_buffers[thread_index]) = current_length + n_allocated;
452  /* buffer shortage, report failure */
453  if (vec_len (tm->tx_buffers[thread_index]) == 0)
454  {
455  clib_warning ("out of buffers");
456  return -1;
457  }
458  return 0;
459 }
460 
461 always_inline int
463 {
464  u32 *my_tx_buffers;
465  u32 thread_index = vlib_get_thread_index ();
466 
467  TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL (thread_index);
468 
469  if (PREDICT_FALSE (vec_len (tm->tx_buffers[thread_index]) == 0))
470  {
471  if (tcp_alloc_tx_buffers (tm, thread_index, VLIB_FRAME_SIZE))
472  return -1;
473  }
474  my_tx_buffers = tm->tx_buffers[thread_index];
475  *bidx = my_tx_buffers[vec_len (my_tx_buffers) - 1];
476  _vec_len (my_tx_buffers) -= 1;
477  return 0;
478 }
479 
480 always_inline void
482 {
483  _vec_len (tm->tx_buffers[vlib_get_thread_index ()]) += 1;
484 }
485 
486 always_inline void *
488 {
489  if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
491  /* Zero all flags but free list index and trace flag */
492  b->flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
493  b->current_data = 0;
494  b->current_length = 0;
496  vnet_buffer (b)->tcp.flags = 0;
497 
498  /* Leave enough space for headers */
500 }
501 
502 always_inline void *
504 {
505  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
506  b->flags &= VLIB_BUFFER_NON_DEFAULT_FREELIST;
507  b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
509  vnet_buffer (b)->tcp.flags = 0;
511  /* Leave enough space for headers */
513 }
514 
515 /**
516  * Prepare ACK
517  */
518 void
520  u8 flags)
521 {
522  tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
523  u8 tcp_opts_len, tcp_hdr_opts_len;
524  tcp_header_t *th;
525  u16 wnd;
526 
527  wnd = tcp_window_to_advertise (tc, state);
528 
529  /* Make and write options */
530  tcp_opts_len = tcp_make_established_options (tc, snd_opts);
531  tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
532 
533  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
534  tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
535 
536  tcp_options_write ((u8 *) (th + 1), snd_opts);
537  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
538 }
539 
540 /**
541  * Convert buffer to ACK
542  */
543 void
545 {
547 
548  tcp_reuse_buffer (vm, b);
549  tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK);
550  TCP_EVT_DBG (TCP_EVT_ACK_SENT, tc);
551  vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_ACK;
552  tc->rcv_las = tc->rcv_nxt;
553 }
554 
555 /**
556  * Convert buffer to FIN-ACK
557  */
558 void
560 {
562  u8 flags = 0;
563 
564  tcp_reuse_buffer (vm, b);
565 
566  flags = TCP_FLAG_FIN | TCP_FLAG_ACK;
567  tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, flags);
568 
569  /* Reset flags, make sure ack is sent */
570  vnet_buffer (b)->tcp.flags &= ~TCP_BUF_FLAG_DUPACK;
571 }
572 
573 /**
574  * Convert buffer to SYN
575  */
576 void
578 {
579  u8 tcp_hdr_opts_len, tcp_opts_len;
580  tcp_header_t *th;
581  u16 initial_wnd;
582  tcp_options_t snd_opts;
583 
584  initial_wnd = tcp_initial_window_to_advertise (tc);
585 
586  /* Make and write options */
587  memset (&snd_opts, 0, sizeof (snd_opts));
588  tcp_opts_len = tcp_make_syn_options (&snd_opts, tc->rcv_wscale);
589  tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
590 
591  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
592  tc->rcv_nxt, tcp_hdr_opts_len, TCP_FLAG_SYN,
593  initial_wnd);
594  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
595  tcp_options_write ((u8 *) (th + 1), &snd_opts);
596 }
597 
598 /**
599  * Convert buffer to SYN-ACK
600  */
601 void
603 {
605  tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
606  u8 tcp_opts_len, tcp_hdr_opts_len;
607  tcp_header_t *th;
608  u16 initial_wnd;
609 
610  memset (snd_opts, 0, sizeof (*snd_opts));
611  tcp_reuse_buffer (vm, b);
612 
613  initial_wnd = tcp_initial_window_to_advertise (tc);
614  tcp_opts_len = tcp_make_synack_options (tc, snd_opts);
615  tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
616 
617  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
618  tc->rcv_nxt, tcp_hdr_opts_len,
619  TCP_FLAG_SYN | TCP_FLAG_ACK, initial_wnd);
620  tcp_options_write ((u8 *) (th + 1), snd_opts);
621 
622  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
623  vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_ACK;
624 
625  /* Init retransmit timer. Use update instead of set because of
626  * retransmissions */
628  TCP_EVT_DBG (TCP_EVT_SYNACK_SENT, tc);
629 }
630 
631 always_inline void
633  u8 is_ip4, u32 fib_index, u8 flush)
634 {
635  tcp_main_t *tm = vnet_get_tcp_main ();
636  u32 thread_index = vlib_get_thread_index ();
637  u32 *to_next, next_index;
638  vlib_frame_t *f;
639 
640  b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
641  b->error = 0;
642 
643  vnet_buffer (b)->sw_if_index[VLIB_TX] = fib_index;
644  vnet_buffer (b)->sw_if_index[VLIB_RX] = 0;
645 
646  /* Send to IP lookup */
647  next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
649 
650  f = tm->ip_lookup_tx_frames[!is_ip4][thread_index];
651  if (!f)
652  {
653  f = vlib_get_frame_to_node (vm, next_index);
654  ASSERT (f);
655  tm->ip_lookup_tx_frames[!is_ip4][thread_index] = f;
656  }
657 
658  to_next = vlib_frame_vector_args (f);
659  to_next[f->n_vectors] = bi;
660  f->n_vectors += 1;
661  if (flush || f->n_vectors == VLIB_FRAME_SIZE)
662  {
663  vlib_put_frame_to_node (vm, next_index, f);
664  tm->ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
665  }
666 }
667 
668 always_inline void
670  u8 is_ip4, u32 fib_index)
671 {
672  tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, fib_index, 1);
673 }
674 
675 always_inline void
677  u8 is_ip4, u32 fib_index)
678 {
679  tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, fib_index, 0);
680 }
681 
682 always_inline void
684  u8 is_ip4, u8 flush)
685 {
686  tcp_main_t *tm = vnet_get_tcp_main ();
687  u32 thread_index = vlib_get_thread_index ();
688  u32 *to_next, next_index;
689  vlib_frame_t *f;
690 
691  b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
692  b->error = 0;
693 
694  /* Decide where to send the packet */
695  next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
697 
698  /* Get frame to v4/6 output node */
699  f = tm->tx_frames[!is_ip4][thread_index];
700  if (!f)
701  {
702  f = vlib_get_frame_to_node (vm, next_index);
703  ASSERT (f);
704  tm->tx_frames[!is_ip4][thread_index] = f;
705  }
706  to_next = vlib_frame_vector_args (f);
707  to_next[f->n_vectors] = bi;
708  f->n_vectors += 1;
709  if (flush || f->n_vectors == VLIB_FRAME_SIZE)
710  {
711  vlib_put_frame_to_node (vm, next_index, f);
712  tm->tx_frames[!is_ip4][thread_index] = 0;
713  }
714 }
715 
716 always_inline void
718 {
719  tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 0);
720 }
721 
722 always_inline void
724  u8 is_ip4)
725 {
726  tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 1);
727 }
728 
729 int
731  tcp_state_t state, u8 thread_index, u8 is_ip4)
732 {
733  ip4_header_t *ih4;
734  ip6_header_t *ih6;
735  tcp_header_t *th0;
736  ip4_address_t src_ip40, dst_ip40;
737  ip6_address_t src_ip60, dst_ip60;
738  u16 src_port, dst_port;
739  u32 tmp;
740  u32 seq, ack;
741  u8 flags;
742 
743  /* Find IP and TCP headers */
744  th0 = tcp_buffer_hdr (b0);
745 
746  /* Save src and dst ip */
747  if (is_ip4)
748  {
749  ih4 = vlib_buffer_get_current (b0);
750  ASSERT ((ih4->ip_version_and_header_length & 0xF0) == 0x40);
751  src_ip40.as_u32 = ih4->src_address.as_u32;
752  dst_ip40.as_u32 = ih4->dst_address.as_u32;
753  }
754  else
755  {
756  ih6 = vlib_buffer_get_current (b0);
757  ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60);
758  clib_memcpy (&src_ip60, &ih6->src_address, sizeof (ip6_address_t));
759  clib_memcpy (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t));
760  }
761 
762  src_port = th0->src_port;
763  dst_port = th0->dst_port;
764 
765  /* Try to determine what/why we're actually resetting */
766  if (state == TCP_STATE_CLOSED)
767  {
768  if (!tcp_syn (th0))
769  return -1;
770 
771  tmp = clib_net_to_host_u32 (th0->seq_number);
772 
773  /* Got a SYN for no listener. */
774  flags = TCP_FLAG_RST | TCP_FLAG_ACK;
775  ack = clib_host_to_net_u32 (tmp + 1);
776  seq = 0;
777  }
778  else
779  {
780  flags = TCP_FLAG_RST;
781  seq = th0->ack_number;
782  ack = 0;
783  }
784 
785  tcp_reuse_buffer (vm, b0);
786  tcp_trajectory_add_start (b0, 4);
787  th0 = vlib_buffer_push_tcp_net_order (b0, dst_port, src_port, seq, ack,
788  sizeof (tcp_header_t), flags, 0);
789 
790  if (is_ip4)
791  {
792  ih4 = vlib_buffer_push_ip4 (vm, b0, &dst_ip40, &src_ip40,
793  IP_PROTOCOL_TCP, 1);
794  th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih4);
795  }
796  else
797  {
798  int bogus = ~0;
799  ih6 = vlib_buffer_push_ip6 (vm, b0, &dst_ip60, &src_ip60,
800  IP_PROTOCOL_TCP);
801  th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih6, &bogus);
802  ASSERT (!bogus);
803  }
804 
805  return 0;
806 }
807 
808 /**
809  * Send reset without reusing existing buffer
810  *
811  * It extracts connection info out of original packet
812  */
813 void
815 {
816  vlib_buffer_t *b;
817  u32 bi, sw_if_index, fib_index;
818  tcp_main_t *tm = vnet_get_tcp_main ();
820  u8 tcp_hdr_len, flags = 0;
821  tcp_header_t *th, *pkt_th;
822  u32 seq, ack;
823  ip4_header_t *ih4, *pkt_ih4;
824  ip6_header_t *ih6, *pkt_ih6;
825  fib_protocol_t fib_proto;
826 
828  return;
829 
830  b = vlib_get_buffer (vm, bi);
831  sw_if_index = vnet_buffer (pkt)->sw_if_index[VLIB_RX];
832  fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
833  fib_index = fib_table_get_index_for_sw_if_index (fib_proto, sw_if_index);
834  tcp_init_buffer (vm, b);
835 
836  /* Make and write options */
837  tcp_hdr_len = sizeof (tcp_header_t);
838 
839  if (is_ip4)
840  {
841  pkt_ih4 = vlib_buffer_get_current (pkt);
842  pkt_th = ip4_next_header (pkt_ih4);
843  }
844  else
845  {
846  pkt_ih6 = vlib_buffer_get_current (pkt);
847  pkt_th = ip6_next_header (pkt_ih6);
848  }
849 
850  if (tcp_ack (pkt_th))
851  {
852  flags = TCP_FLAG_RST;
853  seq = pkt_th->ack_number;
854  ack = (tc && tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
855  }
856  else
857  {
858  flags = TCP_FLAG_RST | TCP_FLAG_ACK;
859  seq = 0;
860  ack = clib_host_to_net_u32 (vnet_buffer (pkt)->tcp.seq_end);
861  }
862 
863  th = vlib_buffer_push_tcp_net_order (b, pkt_th->dst_port, pkt_th->src_port,
864  seq, ack, tcp_hdr_len, flags, 0);
865 
866  /* Swap src and dst ip */
867  if (is_ip4)
868  {
869  ASSERT ((pkt_ih4->ip_version_and_header_length & 0xF0) == 0x40);
870  ih4 = vlib_buffer_push_ip4 (vm, b, &pkt_ih4->dst_address,
871  &pkt_ih4->src_address, IP_PROTOCOL_TCP, 1);
872  th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
873  }
874  else
875  {
876  int bogus = ~0;
877  ASSERT ((pkt_ih6->ip_version_traffic_class_and_flow_label & 0xF0) ==
878  0x60);
879  ih6 = vlib_buffer_push_ip6 (vm, b, &pkt_ih6->dst_address,
880  &pkt_ih6->src_address, IP_PROTOCOL_TCP);
881  th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
882  ASSERT (!bogus);
883  }
884 
885  tcp_enqueue_to_ip_lookup_now (vm, b, bi, is_ip4, fib_index);
886  TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
887 }
888 
889 /**
890  * Build and set reset packet for connection
891  */
892 void
894 {
896  tcp_main_t *tm = vnet_get_tcp_main ();
897  vlib_buffer_t *b;
898  u32 bi;
899  tcp_header_t *th;
900  u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
901  u8 flags;
902 
904  return;
905  b = vlib_get_buffer (vm, bi);
906  tcp_init_buffer (vm, b);
907 
908  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
909  tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
910  advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
911  flags = TCP_FLAG_RST;
912  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
913  tc->rcv_nxt, tcp_hdr_opts_len, flags,
914  advertise_wnd);
915  opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
916  ASSERT (opts_write_len == tc->snd_opts_len);
917  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
918  if (tc->c_is_ip4)
919  {
920  ip4_header_t *ih4;
921  ih4 = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip.ip4,
922  &tc->c_rmt_ip.ip4, IP_PROTOCOL_TCP, 0);
923  th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
924  }
925  else
926  {
927  int bogus = ~0;
928  ip6_header_t *ih6;
929  ih6 = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip.ip6,
930  &tc->c_rmt_ip.ip6, IP_PROTOCOL_TCP);
931  th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
932  ASSERT (!bogus);
933  }
934  tcp_enqueue_to_ip_lookup_now (vm, b, bi, tc->c_is_ip4, tc->c_fib_index);
935  TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
936 }
937 
938 void
940 {
943  if (tc->c_is_ip4)
944  {
945  ip4_header_t *ih;
946  ih = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip4,
947  &tc->c_rmt_ip4, IP_PROTOCOL_TCP, 1);
948  th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih);
949  }
950  else
951  {
952  ip6_header_t *ih;
953  int bogus = ~0;
954 
955  ih = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip6,
956  &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
957  th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih, &bogus);
958  ASSERT (!bogus);
959  }
960 }
961 
962 /**
963  * Send SYN
964  *
965  * Builds a SYN packet for a half-open connection and sends it to ipx_lookup.
966  * The packet is not forwarded through tcpx_output to avoid doing lookups
967  * in the half_open pool.
968  */
969 void
971 {
972  vlib_buffer_t *b;
973  u32 bi;
974  tcp_main_t *tm = vnet_get_tcp_main ();
976 
977  /*
978  * Setup retransmit and establish timers before requesting buffer
979  * such that we can return if we've ran out.
980  */
981  tcp_timer_set (tc, TCP_TIMER_ESTABLISH, TCP_ESTABLISH_TIME);
982  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
983  tc->rto * TCP_TO_TIMER_TICK);
984 
986  return;
987 
988  b = vlib_get_buffer (vm, bi);
989  tcp_init_buffer (vm, b);
990  tcp_make_syn (tc, b);
991 
992  /* Measure RTT with this */
993  tc->rtt_ts = tcp_time_now ();
994  tc->rtt_seq = tc->snd_nxt;
995  tc->rto_boff = 0;
996 
997  tcp_push_ip_hdr (tm, tc, b);
998  tcp_enqueue_to_ip_lookup (vm, b, bi, tc->c_is_ip4, tc->c_fib_index);
999  TCP_EVT_DBG (TCP_EVT_SYN_SENT, tc);
1000 }
1001 
1002 /**
1003  * Flush tx frame populated by retransmits and timer pops
1004  */
1005 void
1006 tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
1007 {
1008  if (tcp_main.tx_frames[!is_ip4][thread_index])
1009  {
1010  u32 next_index;
1011  next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
1012  vlib_put_frame_to_node (vm, next_index,
1013  tcp_main.tx_frames[!is_ip4][thread_index]);
1014  tcp_main.tx_frames[!is_ip4][thread_index] = 0;
1015  }
1016 }
1017 
1018 /**
1019  * Flush ip lookup tx frames populated by timer pops
1020  */
1021 always_inline void
1023 {
1024  if (tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index])
1025  {
1026  u32 next_index;
1027  next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
1028  vlib_put_frame_to_node (vm, next_index,
1029  tcp_main.ip_lookup_tx_frames[!is_ip4]
1030  [thread_index]);
1031  tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
1032  }
1033 }
1034 
1035 /**
1036  * Flush v4 and v6 tcp and ip-lookup tx frames for thread index
1037  */
1038 void
1040 {
1042  tcp_flush_frame_to_output (vm, thread_index, 1);
1043  tcp_flush_frame_to_output (vm, thread_index, 0);
1044  tcp_flush_frame_to_ip_lookup (vm, thread_index, 1);
1045  tcp_flush_frame_to_ip_lookup (vm, thread_index, 0);
1046 }
1047 
1048 /**
1049  * Send FIN
1050  */
1051 void
1053 {
1054  tcp_main_t *tm = vnet_get_tcp_main ();
1056  vlib_buffer_t *b;
1057  u32 bi;
1058  u8 fin_snt = 0;
1059 
1061  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
1062  return;
1063  b = vlib_get_buffer (vm, bi);
1064  fin_snt = tc->flags & TCP_CONN_FINSNT;
1065  if (fin_snt)
1066  tc->snd_nxt = tc->snd_una;
1067  tcp_make_fin (tc, b);
1068  tcp_enqueue_to_output_now (vm, b, bi, tc->c_is_ip4);
1069  if (!fin_snt)
1070  {
1071  tc->flags |= TCP_CONN_FINSNT;
1072  tc->flags &= ~TCP_CONN_FINPNDG;
1073  /* Account for the FIN */
1074  tc->snd_una_max += 1;
1075  tc->snd_nxt = tc->snd_una_max;
1076  }
1077  else
1078  {
1079  tc->snd_nxt = tc->snd_una_max;
1080  }
1081  TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc);
1082 }
1083 
1086 {
1087  switch (next_state)
1088  {
1089  case TCP_STATE_ESTABLISHED:
1090  return TCP_FLAG_ACK;
1091  case TCP_STATE_SYN_RCVD:
1092  return TCP_FLAG_SYN | TCP_FLAG_ACK;
1093  case TCP_STATE_SYN_SENT:
1094  return TCP_FLAG_SYN;
1095  case TCP_STATE_LAST_ACK:
1096  case TCP_STATE_FIN_WAIT_1:
1097  if (tc->snd_nxt + 1 < tc->snd_una_max)
1098  return TCP_FLAG_ACK;
1099  else
1100  return TCP_FLAG_FIN;
1101  default:
1102  clib_warning ("Shouldn't be here!");
1103  }
1104  return 0;
1105 }
1106 
1107 /**
1108  * Push TCP header and update connection variables
1109  */
1110 static void
1112  tcp_state_t next_state, u8 compute_opts)
1113 {
1114  u32 advertise_wnd, data_len;
1115  u8 tcp_hdr_opts_len, opts_write_len, flags;
1116  tcp_header_t *th;
1117 
1120  || (b->flags & VLIB_BUFFER_NEXT_PRESENT));
1121  vnet_buffer (b)->tcp.flags = 0;
1122 
1123  if (compute_opts)
1124  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1125 
1126  tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
1127  advertise_wnd = tcp_window_to_advertise (tc, next_state);
1128  flags = tcp_make_state_flags (tc, next_state);
1129 
1130  /* Push header and options */
1131  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
1132  tc->rcv_nxt, tcp_hdr_opts_len, flags,
1133  advertise_wnd);
1134  opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
1135 
1136  ASSERT (opts_write_len == tc->snd_opts_len);
1137  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
1138 
1139  /*
1140  * Update connection variables
1141  */
1142 
1143  tc->snd_nxt += data_len;
1144  tc->rcv_las = tc->rcv_nxt;
1145 
1146  /* TODO this is updated in output as well ... */
1147  if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1148  {
1149  tc->snd_una_max = tc->snd_nxt;
1150  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
1151  }
1152 
1153  TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
1154 }
1155 
1156 void
1158 {
1159  tcp_main_t *tm = vnet_get_tcp_main ();
1161 
1162  vlib_buffer_t *b;
1163  u32 bi;
1164 
1165  /* Get buffer */
1166  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
1167  return;
1168  b = vlib_get_buffer (vm, bi);
1169 
1170  /* Fill in the ACK */
1171  tcp_make_ack (tc, b);
1172  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1173 }
1174 
1175 /**
1176  * Delayed ack timer handler
1177  *
1178  * Sends delayed ACK when timer expires
1179  */
1180 void
1182 {
1183  u32 thread_index = vlib_get_thread_index ();
1184  tcp_connection_t *tc;
1185 
1186  tc = tcp_connection_get (index, thread_index);
1187  tc->timers[TCP_TIMER_DELACK] = TCP_TIMER_HANDLE_INVALID;
1188  tcp_send_ack (tc);
1189 }
1190 
1191 /**
1192  * Build a retransmit segment
1193  *
1194  * @return the number of bytes in the segment or 0 if there's nothing to
1195  * retransmit
1196  */
1197 u32
1199  u32 max_deq_bytes, vlib_buffer_t ** b)
1200 {
1201  tcp_main_t *tm = vnet_get_tcp_main ();
1203  int n_bytes = 0;
1204  u32 start, bi, available_bytes, seg_size;
1205  u8 *data;
1206 
1207  ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1208  ASSERT (max_deq_bytes != 0);
1209 
1210  /*
1211  * Make sure we can retransmit something
1212  */
1213  available_bytes = stream_session_tx_fifo_max_dequeue (&tc->connection);
1214  ASSERT (available_bytes >= offset);
1215  available_bytes -= offset;
1216  if (!available_bytes)
1217  return 0;
1218  max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes);
1219  max_deq_bytes = clib_min (available_bytes, max_deq_bytes);
1220 
1221  /* Start is beyond snd_congestion */
1222  start = tc->snd_una + offset;
1223  if (seq_geq (start, tc->snd_congestion))
1224  goto done;
1225 
1226  /* Don't overshoot snd_congestion */
1227  if (seq_gt (start + max_deq_bytes, tc->snd_congestion))
1228  {
1229  max_deq_bytes = tc->snd_congestion - start;
1230  if (max_deq_bytes == 0)
1231  goto done;
1232  }
1233 
1234  seg_size = max_deq_bytes + MAX_HDRS_LEN;
1235 
1236  /*
1237  * Prepare options
1238  */
1239  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1240 
1241  /*
1242  * Allocate and fill in buffer(s)
1243  */
1244 
1245  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
1246  return 0;
1247  *b = vlib_get_buffer (vm, bi);
1248  data = tcp_init_buffer (vm, *b);
1249 
1250  /* Easy case, buffer size greater than mss */
1251  if (PREDICT_TRUE (seg_size <= tm->bytes_per_buffer))
1252  {
1253  n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
1254  max_deq_bytes);
1255  ASSERT (n_bytes == max_deq_bytes);
1256  b[0]->current_length = n_bytes;
1257  tcp_push_hdr_i (tc, *b, tc->state, 0);
1258  }
1259  /* Split mss into multiple buffers */
1260  else
1261  {
1262  u32 chain_bi = ~0, n_bufs_per_seg;
1263  u32 thread_index = vlib_get_thread_index ();
1264  u16 n_peeked, len_to_deq, available_bufs;
1265  vlib_buffer_t *chain_b, *prev_b;
1266  int i;
1267 
1268  n_bufs_per_seg = ceil ((double) seg_size / tm->bytes_per_buffer);
1269 
1270  /* Make sure we have enough buffers */
1271  available_bufs = vec_len (tm->tx_buffers[thread_index]);
1272  if (n_bufs_per_seg > available_bufs)
1273  {
1274  if (tcp_alloc_tx_buffers (tm, thread_index,
1275  VLIB_FRAME_SIZE - available_bufs))
1276  {
1277  tcp_return_buffer (tm);
1278  *b = 0;
1279  return 0;
1280  }
1281  }
1282 
1283  n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
1284  tm->bytes_per_buffer -
1285  MAX_HDRS_LEN);
1286  b[0]->current_length = n_bytes;
1287  b[0]->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1289  max_deq_bytes -= n_bytes;
1290 
1291  chain_b = *b;
1292  for (i = 1; i < n_bufs_per_seg; i++)
1293  {
1294  prev_b = chain_b;
1295  len_to_deq = clib_min (max_deq_bytes, tm->bytes_per_buffer);
1296  tcp_get_free_buffer_index (tm, &chain_bi);
1297  ASSERT (chain_bi != (u32) ~ 0);
1298  chain_b = vlib_get_buffer (vm, chain_bi);
1299  chain_b->current_data = 0;
1300  data = vlib_buffer_get_current (chain_b);
1301  n_peeked = stream_session_peek_bytes (&tc->connection, data,
1302  offset + n_bytes, len_to_deq);
1303  ASSERT (n_peeked == len_to_deq);
1304  n_bytes += n_peeked;
1305  chain_b->current_length = n_peeked;
1306  chain_b->flags &= VLIB_BUFFER_NON_DEFAULT_FREELIST;
1307  chain_b->next_buffer = 0;
1308 
1309  /* update previous buffer */
1310  prev_b->next_buffer = chain_bi;
1311  prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1312 
1313  max_deq_bytes -= n_peeked;
1314  b[0]->total_length_not_including_first_buffer += n_peeked;
1315  }
1316 
1317  tcp_push_hdr_i (tc, *b, tc->state, 0);
1318  }
1319 
1320  ASSERT (n_bytes > 0);
1321  ASSERT (((*b)->current_data + (*b)->current_length) <=
1322  tm->bytes_per_buffer);
1323 
1324  if (tcp_in_fastrecovery (tc))
1325  tc->snd_rxt_bytes += n_bytes;
1326 
1327 done:
1328  TCP_EVT_DBG (TCP_EVT_CC_RTX, tc, offset, n_bytes);
1329  return n_bytes;
1330 }
1331 
1332 /**
1333  * Reset congestion control, switch cwnd to loss window and try again.
1334  */
1335 static void
1337 {
1338  tc->prev_ssthresh = tc->ssthresh;
1339  tc->prev_cwnd = tc->cwnd;
1340 
1341  /* Cleanly recover cc (also clears up fast retransmit) */
1342  if (tcp_in_fastrecovery (tc))
1344 
1345  /* Start again from the beginning */
1346  tc->ssthresh = clib_max (tcp_flight_size (tc) / 2, 2 * tc->snd_mss);
1347  tc->cwnd = tcp_loss_wnd (tc);
1348  tc->snd_congestion = tc->snd_una_max;
1349  tc->rtt_ts = 0;
1350  tcp_recovery_on (tc);
1351 }
1352 
1353 static void
1355 {
1356  tcp_main_t *tm = vnet_get_tcp_main ();
1358  u32 thread_index = vlib_get_thread_index ();
1359  tcp_connection_t *tc;
1360  vlib_buffer_t *b = 0;
1361  u32 bi, n_bytes;
1362 
1363  if (is_syn)
1364  {
1365  tc = tcp_half_open_connection_get (index);
1366  /* Note: the connection may have transitioned to ESTABLISHED... */
1367  if (PREDICT_FALSE (tc == 0))
1368  return;
1369  tc->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID;
1370  }
1371  else
1372  {
1373  tc = tcp_connection_get (index, thread_index);
1374  /* Note: the connection may have been closed and pool_put */
1375  if (PREDICT_FALSE (tc == 0))
1376  return;
1377  tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID;
1378  }
1379 
1380  if (tc->state >= TCP_STATE_ESTABLISHED)
1381  {
1382  /* Lost FIN, retransmit and return */
1383  if (tcp_is_lost_fin (tc))
1384  {
1385  tcp_send_fin (tc);
1386  tc->rto_boff += 1;
1387  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1388  return;
1389  }
1390 
1391  /* Shouldn't be here */
1392  if (tc->snd_una == tc->snd_una_max)
1393  {
1394  tcp_recovery_off (tc);
1395  return;
1396  }
1397 
1398  /* We're not in recovery so make sure rto_boff is 0 */
1399  if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
1400  {
1401  tc->rto_boff = 0;
1402  tcp_update_rto (tc);
1403  }
1404 
1405  /* Increment RTO backoff (also equal to number of retries) and go back
1406  * to first un-acked byte */
1407  tc->rto_boff += 1;
1408 
1409  /* First retransmit timeout */
1410  if (tc->rto_boff == 1)
1411  tcp_rtx_timeout_cc (tc);
1412 
1413  tc->snd_nxt = tc->snd_una;
1414  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1415 
1416  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1);
1417 
1418  /* Send one segment. Note that n_bytes may be zero due to buffer shortfall */
1419  n_bytes = tcp_prepare_retransmit_segment (tc, 0, tc->snd_mss, &b);
1420 
1421  /* TODO be less aggressive about this */
1422  scoreboard_clear (&tc->sack_sb);
1423 
1424  if (n_bytes == 0)
1425  {
1427  return;
1428  }
1429 
1430  bi = vlib_get_buffer_index (vm, b);
1431 
1432  /* For first retransmit, record timestamp (Eifel detection RFC3522) */
1433  if (tc->rto_boff == 1)
1434  tc->snd_rxt_ts = tcp_time_now ();
1435 
1436  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1438  }
1439  /* Retransmit for SYN */
1440  else if (tc->state == TCP_STATE_SYN_SENT)
1441  {
1442  /* Half-open connection actually moved to established but we were
1443  * waiting for syn retransmit to pop to call cleanup from the right
1444  * thread. */
1445  if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1446  {
1448  {
1449  clib_warning ("could not remove half-open connection");
1450  ASSERT (0);
1451  }
1452  return;
1453  }
1454 
1455  /* Try without increasing RTO a number of times. If this fails,
1456  * start growing RTO exponentially */
1457  tc->rto_boff += 1;
1458  if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1459  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1460 
1461  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
1462  tc->rto * TCP_TO_TIMER_TICK);
1463 
1464  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
1465  return;
1466 
1467  b = vlib_get_buffer (vm, bi);
1468  tcp_init_buffer (vm, b);
1469  tcp_make_syn (tc, b);
1470 
1471  tc->rtt_ts = 0;
1472  TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 0);
1473 
1474  /* This goes straight to ipx_lookup. Retransmit timer set already */
1475  tcp_push_ip_hdr (tm, tc, b);
1476  tcp_enqueue_to_ip_lookup (vm, b, bi, tc->c_is_ip4, tc->c_fib_index);
1477  }
1478  /* Retransmit SYN-ACK */
1479  else if (tc->state == TCP_STATE_SYN_RCVD)
1480  {
1481  tc->rto_boff += 1;
1482  if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1483  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1484  tc->rtt_ts = 0;
1485 
1486  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
1487  {
1489  return;
1490  }
1491 
1492  b = vlib_get_buffer (vm, bi);
1493  tcp_make_synack (tc, b);
1494  TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 1);
1495 
1496  /* Retransmit timer already updated, just enqueue to output */
1497  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1498  }
1499  else
1500  {
1501  ASSERT (tc->state == TCP_STATE_CLOSED);
1502  return;
1503  }
1504 }
1505 
1506 void
1508 {
1509  tcp_timer_retransmit_handler_i (index, 0);
1510 }
1511 
1512 void
1514 {
1515  tcp_timer_retransmit_handler_i (index, 1);
1516 }
1517 
1518 /**
1519  * Got 0 snd_wnd from peer, try to do something about it.
1520  *
1521  */
1522 void
1524 {
1525  tcp_main_t *tm = vnet_get_tcp_main ();
1527  u32 thread_index = vlib_get_thread_index ();
1528  tcp_connection_t *tc;
1529  vlib_buffer_t *b;
1530  u32 bi, max_snd_bytes, available_bytes, offset;
1531  int n_bytes = 0;
1532  u8 *data;
1533 
1534  tc = tcp_connection_get_if_valid (index, thread_index);
1535 
1536  if (!tc)
1537  return;
1538 
1539  /* Make sure timer handle is set to invalid */
1540  tc->timers[TCP_TIMER_PERSIST] = TCP_TIMER_HANDLE_INVALID;
1541 
1542  /* Problem already solved or worse */
1543  if (tc->state == TCP_STATE_CLOSED || tc->state > TCP_STATE_ESTABLISHED
1544  || tc->snd_wnd > tc->snd_mss || tcp_in_recovery (tc))
1545  return;
1546 
1547  available_bytes = stream_session_tx_fifo_max_dequeue (&tc->connection);
1548  offset = tc->snd_una_max - tc->snd_una;
1549 
1550  /* Reprogram persist if no new bytes available to send. We may have data
1551  * next time */
1552  if (!available_bytes)
1553  {
1554  tcp_persist_timer_set (tc);
1555  return;
1556  }
1557 
1558  if (available_bytes <= offset)
1559  {
1560  ASSERT (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT));
1561  return;
1562  }
1563 
1564  /* Increment RTO backoff */
1565  tc->rto_boff += 1;
1566  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1567 
1568  /*
1569  * Try to force the first unsent segment (or buffer)
1570  */
1571  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
1572  return;
1573  b = vlib_get_buffer (vm, bi);
1574  data = tcp_init_buffer (vm, b);
1575 
1576  tcp_validate_txf_size (tc, offset);
1577  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1578  max_snd_bytes = clib_min (tc->snd_mss, tm->bytes_per_buffer - MAX_HDRS_LEN);
1579  n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
1580  max_snd_bytes);
1581  b->current_length = n_bytes;
1582  ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)
1583  || tc->snd_nxt == tc->snd_una_max
1584  || tc->rto_boff > 1));
1585 
1586  tcp_push_hdr_i (tc, b, tc->state, 0);
1587  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1588 
1589  /* Just sent new data, enable retransmit */
1591 }
1592 
1593 /**
1594  * Retransmit first unacked segment
1595  */
1596 void
1598 {
1600  vlib_buffer_t *b;
1601  u32 bi, old_snd_nxt, n_bytes;
1602 
1603  old_snd_nxt = tc->snd_nxt;
1604  tc->snd_nxt = tc->snd_una;
1605 
1606  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1607  n_bytes = tcp_prepare_retransmit_segment (tc, 0, tc->snd_mss, &b);
1608  if (!n_bytes)
1609  return;
1610  bi = vlib_get_buffer_index (vm, b);
1611  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1612 
1613  tc->snd_nxt = old_snd_nxt;
1614 }
1615 
1616 /**
1617  * Do fast retransmit with SACKs
1618  */
1619 void
1621 {
1623  u32 n_written = 0, offset, max_bytes;
1624  vlib_buffer_t *b = 0;
1625  sack_scoreboard_hole_t *hole;
1626  sack_scoreboard_t *sb;
1627  u32 bi, old_snd_nxt;
1628  int snd_space;
1629  u8 snd_limited = 0, can_rescue = 0;
1630 
1631  ASSERT (tcp_in_fastrecovery (tc));
1632  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
1633 
1634  old_snd_nxt = tc->snd_nxt;
1635  sb = &tc->sack_sb;
1636  snd_space = tcp_available_snd_space (tc);
1637 
1638  hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
1639  while (hole && snd_space > 0)
1640  {
1641  hole = scoreboard_next_rxt_hole (sb, hole,
1643  &can_rescue, &snd_limited);
1644  if (!hole)
1645  {
1646  if (!can_rescue || !(seq_lt (sb->rescue_rxt, tc->snd_una)
1647  || seq_gt (sb->rescue_rxt,
1648  tc->snd_congestion)))
1649  break;
1650 
1651  /* If rescue rxt undefined or less than snd_una then one segment of
1652  * up to SMSS octets that MUST include the highest outstanding
1653  * unSACKed sequence number SHOULD be returned, and RescueRxt set to
1654  * RecoveryPoint. HighRxt MUST NOT be updated.
1655  */
1656  max_bytes = clib_min (tc->snd_mss,
1657  tc->snd_congestion - tc->snd_una);
1658  max_bytes = clib_min (max_bytes, snd_space);
1659  offset = tc->snd_congestion - tc->snd_una - max_bytes;
1660  sb->rescue_rxt = tc->snd_congestion;
1661  tc->snd_nxt = tc->snd_una + offset;
1662  n_written = tcp_prepare_retransmit_segment (tc, offset, max_bytes,
1663  &b);
1664  ASSERT (n_written);
1665  bi = vlib_get_buffer_index (vm, b);
1666  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1667  break;
1668  }
1669 
1670  max_bytes = clib_min (hole->end - sb->high_rxt, snd_space);
1671  max_bytes = snd_limited ? clib_min (max_bytes, tc->snd_mss) : max_bytes;
1672  if (max_bytes == 0)
1673  break;
1674  offset = sb->high_rxt - tc->snd_una;
1675  tc->snd_nxt = sb->high_rxt;
1676  n_written = tcp_prepare_retransmit_segment (tc, offset, max_bytes, &b);
1677 
1678  /* Nothing left to retransmit */
1679  if (n_written == 0)
1680  break;
1681 
1682  bi = vlib_get_buffer_index (vm, b);
1683  sb->high_rxt += n_written;
1684  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1685  ASSERT (n_written <= snd_space);
1686  snd_space -= n_written;
1687  }
1688 
1689  /* If window allows, send 1 SMSS of new data */
1690  tc->snd_nxt = old_snd_nxt;
1691 }
1692 
1693 /**
1694  * Fast retransmit without SACK info
1695  */
1696 void
1698 {
1700  u32 n_written = 0, offset = 0, bi, old_snd_nxt;
1701  int snd_space;
1702  vlib_buffer_t *b;
1703 
1704  ASSERT (tcp_in_fastrecovery (tc));
1705  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
1706 
1707  /* Start resending from first un-acked segment */
1708  old_snd_nxt = tc->snd_nxt;
1709  tc->snd_nxt = tc->snd_una;
1710  snd_space = tcp_available_snd_space (tc);
1711 
1712  while (snd_space > 0)
1713  {
1714  offset += n_written;
1715  n_written = tcp_prepare_retransmit_segment (tc, offset, snd_space, &b);
1716 
1717  /* Nothing left to retransmit */
1718  if (n_written == 0)
1719  break;
1720 
1721  bi = vlib_get_buffer_index (vm, b);
1722  tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
1723  snd_space -= n_written;
1724  }
1725 
1726  /* Restore snd_nxt. If window allows, send 1 SMSS of new data */
1727  tc->snd_nxt = old_snd_nxt;
1728 }
1729 
1730 /**
1731  * Do fast retransmit
1732  */
1733 void
1735 {
1736  if (tcp_opts_sack_permitted (&tc->rcv_opts)
1737  && scoreboard_first_hole (&tc->sack_sb))
1739  else
1741 }
1742 
1745 {
1746  stream_session_t *s = session_get (tc->c_s_index, tc->c_thread_index);
1747  return svm_fifo_has_ooo_data (s->server_rx_fifo);
1748 }
1749 
1752  vlib_node_runtime_t * node,
1753  vlib_frame_t * from_frame, int is_ip4)
1754 {
1755  u32 n_left_from, next_index, *from, *to_next;
1756  u32 my_thread_index = vm->thread_index;
1757 
1758  from = vlib_frame_vector_args (from_frame);
1759  n_left_from = from_frame->n_vectors;
1760  next_index = node->cached_next_index;
1761  tcp_set_time_now (my_thread_index);
1762 
1763  while (n_left_from > 0)
1764  {
1765  u32 n_left_to_next;
1766 
1767  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1768 
1769  while (n_left_from > 0 && n_left_to_next > 0)
1770  {
1771  u32 bi0;
1772  vlib_buffer_t *b0;
1773  tcp_connection_t *tc0;
1774  tcp_tx_trace_t *t0;
1775  tcp_header_t *th0 = 0;
1776  u32 error0 = TCP_ERROR_PKTS_SENT, next0 = TCP_OUTPUT_NEXT_IP_LOOKUP;
1777 
1778  if (n_left_from > 1)
1779  {
1780  vlib_buffer_t *pb;
1781  pb = vlib_get_buffer (vm, from[1]);
1782  vlib_prefetch_buffer_header (pb, STORE);
1783  CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
1784  }
1785 
1786  bi0 = from[0];
1787  to_next[0] = bi0;
1788  from += 1;
1789  to_next += 1;
1790  n_left_from -= 1;
1791  n_left_to_next -= 1;
1792 
1793  b0 = vlib_get_buffer (vm, bi0);
1794  tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
1795  my_thread_index);
1796  if (PREDICT_FALSE (tc0 == 0 || tc0->state == TCP_STATE_CLOSED))
1797  {
1798  error0 = TCP_ERROR_INVALID_CONNECTION;
1799  next0 = TCP_OUTPUT_NEXT_DROP;
1800  goto done;
1801  }
1802 
1803  th0 = vlib_buffer_get_current (b0);
1804  TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
1805 
1806  if (is_ip4)
1807  {
1808  vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
1809  IP_PROTOCOL_TCP, 1);
1810  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
1811  vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
1812  th0->checksum = 0;
1813  }
1814  else
1815  {
1816  ip6_header_t *ih0;
1817  ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
1818  &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
1819  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
1820  vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
1821  vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
1822  th0->checksum = 0;
1823  }
1824 
1825  /* Filter out DUPACKs if there are no OOO segments left */
1826  if (PREDICT_FALSE
1827  (vnet_buffer (b0)->tcp.flags & TCP_BUF_FLAG_DUPACK))
1828  {
1829  if (!tcp_session_has_ooo_data (tc0))
1830  {
1831  error0 = TCP_ERROR_FILTERED_DUPACKS;
1832  next0 = TCP_OUTPUT_NEXT_DROP;
1833  goto done;
1834  }
1835  }
1836 
1837  /* Stop DELACK timer and fix flags */
1838  tc0->flags &= ~(TCP_CONN_SNDACK);
1839  tcp_timer_reset (tc0, TCP_TIMER_DELACK);
1840 
1841  /* If not retransmitting
1842  * 1) update snd_una_max (SYN, SYNACK, FIN)
1843  * 2) If we're not tracking an ACK, start tracking */
1844  if (seq_lt (tc0->snd_una_max, tc0->snd_nxt))
1845  {
1846  tc0->snd_una_max = tc0->snd_nxt;
1847  if (tc0->rtt_ts == 0)
1848  {
1849  tc0->rtt_ts = tcp_time_now ();
1850  tc0->rtt_seq = tc0->snd_nxt;
1851  }
1852  }
1853 
1854  /* Set the retransmit timer if not set already and not
1855  * doing a pure ACK */
1856  if (!tcp_timer_is_active (tc0, TCP_TIMER_RETRANSMIT)
1857  && tc0->snd_nxt != tc0->snd_una)
1858  {
1860  tc0->rto_boff = 0;
1861  }
1862 
1863 #if 0
1864  /* Make sure we haven't lost route to our peer */
1865  if (PREDICT_FALSE (tc0->last_fib_check
1866  < tc0->snd_opts.tsval + TCP_FIB_RECHECK_PERIOD))
1867  {
1868  if (PREDICT_TRUE
1869  (tc0->c_rmt_fei == tcp_lookup_rmt_in_fib (tc0)))
1870  {
1871  tc0->last_fib_check = tc0->snd_opts.tsval;
1872  }
1873  else
1874  {
1875  clib_warning ("lost connection to peer");
1876  tcp_connection_reset (tc0);
1877  goto done;
1878  }
1879  }
1880 
1881  /* Use pre-computed dpo to set next node */
1882  next0 = tc0->c_rmt_dpo.dpoi_next_node;
1883  vnet_buffer (b0)->ip.adj_index[VLIB_TX] = tc0->c_rmt_dpo.dpoi_index;
1884 #endif
1885 
1886  vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
1887  vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
1888 
1889  b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
1890  done:
1891  b0->error = node->errors[error0];
1892  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1893  {
1894  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
1895  if (th0)
1896  {
1897  clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
1898  }
1899  else
1900  {
1901  memset (&t0->tcp_header, 0, sizeof (t0->tcp_header));
1902  }
1903  clib_memcpy (&t0->tcp_connection, tc0,
1904  sizeof (t0->tcp_connection));
1905  }
1906 
1907  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1908  n_left_to_next, bi0, next0);
1909  }
1910 
1911  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1912  }
1913 
1914  return from_frame->n_vectors;
1915 }
1916 
1917 static uword
1919  vlib_frame_t * from_frame)
1920 {
1921  return tcp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */ );
1922 }
1923 
1924 static uword
1926  vlib_frame_t * from_frame)
1927 {
1928  return tcp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */ );
1929 }
1930 
1931 /* *INDENT-OFF* */
1933 {
1934  .function = tcp4_output,.name = "tcp4-output",
1935  /* Takes a vector of packets. */
1936  .vector_size = sizeof (u32),
1937  .n_errors = TCP_N_ERROR,
1938  .error_strings = tcp_error_strings,
1939  .n_next_nodes = TCP_OUTPUT_N_NEXT,
1940  .next_nodes = {
1941 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
1943 #undef _
1944  },
1945  .format_buffer = format_tcp_header,
1946  .format_trace = format_tcp_tx_trace,
1947 };
1948 /* *INDENT-ON* */
1949 
1951 
1952 /* *INDENT-OFF* */
1954 {
1955  .function = tcp6_output,
1956  .name = "tcp6-output",
1957  /* Takes a vector of packets. */
1958  .vector_size = sizeof (u32),
1959  .n_errors = TCP_N_ERROR,
1960  .error_strings = tcp_error_strings,
1961  .n_next_nodes = TCP_OUTPUT_N_NEXT,
1962  .next_nodes = {
1963 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
1965 #undef _
1966  },
1967  .format_buffer = format_tcp_header,
1968  .format_trace = format_tcp_tx_trace,
1969 };
1970 /* *INDENT-ON* */
1971 
1973 
1974 u32
1976 {
1977  tcp_connection_t *tc;
1978 
1979  tc = (tcp_connection_t *) tconn;
1980  tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0);
1981  ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
1982 
1983  if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
1984  {
1985  tc->rtt_ts = tcp_time_now ();
1986  tc->rtt_seq = tc->snd_nxt;
1987  }
1988  tcp_trajectory_add_start (b, 3);
1989  return 0;
1990 }
1991 
1992 typedef enum _tcp_reset_next
1993 {
1998 
1999 #define foreach_tcp4_reset_next \
2000  _(DROP, "error-drop") \
2001  _(IP_LOOKUP, "ip4-lookup")
2002 
2003 #define foreach_tcp6_reset_next \
2004  _(DROP, "error-drop") \
2005  _(IP_LOOKUP, "ip6-lookup")
2006 
2007 static uword
2009  vlib_frame_t * from_frame, u8 is_ip4)
2010 {
2011  u32 n_left_from, next_index, *from, *to_next;
2012  u32 my_thread_index = vm->thread_index;
2013 
2014  from = vlib_frame_vector_args (from_frame);
2015  n_left_from = from_frame->n_vectors;
2016 
2017  next_index = node->cached_next_index;
2018 
2019  while (n_left_from > 0)
2020  {
2021  u32 n_left_to_next;
2022 
2023  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2024 
2025  while (n_left_from > 0 && n_left_to_next > 0)
2026  {
2027  u32 bi0;
2028  vlib_buffer_t *b0;
2029  tcp_tx_trace_t *t0;
2030  tcp_header_t *th0;
2031  u32 error0 = TCP_ERROR_RST_SENT, next0 = TCP_RESET_NEXT_IP_LOOKUP;
2032 
2033  bi0 = from[0];
2034  to_next[0] = bi0;
2035  from += 1;
2036  to_next += 1;
2037  n_left_from -= 1;
2038  n_left_to_next -= 1;
2039 
2040  b0 = vlib_get_buffer (vm, bi0);
2041 
2042  if (tcp_make_reset_in_place (vm, b0, vnet_buffer (b0)->tcp.flags,
2043  my_thread_index, is_ip4))
2044  {
2045  error0 = TCP_ERROR_LOOKUP_DROPS;
2046  next0 = TCP_RESET_NEXT_DROP;
2047  goto done;
2048  }
2049 
2050  /* Prepare to send to IP lookup */
2051  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
2052  next0 = TCP_RESET_NEXT_IP_LOOKUP;
2053 
2054  done:
2055  b0->error = node->errors[error0];
2056  b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
2057  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2058  {
2059  th0 = vlib_buffer_get_current (b0);
2060  if (is_ip4)
2061  th0 = ip4_next_header ((ip4_header_t *) th0);
2062  else
2063  th0 = ip6_next_header ((ip6_header_t *) th0);
2064  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
2065  clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
2066  }
2067 
2068  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2069  n_left_to_next, bi0, next0);
2070  }
2071  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2072  }
2073  return from_frame->n_vectors;
2074 }
2075 
2076 static uword
2078  vlib_frame_t * from_frame)
2079 {
2080  return tcp46_send_reset_inline (vm, node, from_frame, 1);
2081 }
2082 
2083 static uword
2085  vlib_frame_t * from_frame)
2086 {
2087  return tcp46_send_reset_inline (vm, node, from_frame, 0);
2088 }
2089 
2090 /* *INDENT-OFF* */
2092  .function = tcp4_send_reset,
2093  .name = "tcp4-reset",
2094  .vector_size = sizeof (u32),
2095  .n_errors = TCP_N_ERROR,
2096  .error_strings = tcp_error_strings,
2097  .n_next_nodes = TCP_RESET_N_NEXT,
2098  .next_nodes = {
2099 #define _(s,n) [TCP_RESET_NEXT_##s] = n,
2101 #undef _
2102  },
2103  .format_trace = format_tcp_tx_trace,
2104 };
2105 /* *INDENT-ON* */
2106 
2108 
2109 /* *INDENT-OFF* */
2111  .function = tcp6_send_reset,
2112  .name = "tcp6-reset",
2113  .vector_size = sizeof (u32),
2114  .n_errors = TCP_N_ERROR,
2115  .error_strings = tcp_error_strings,
2116  .n_next_nodes = TCP_RESET_N_NEXT,
2117  .next_nodes = {
2118 #define _(s,n) [TCP_RESET_NEXT_##s] = n,
2120 #undef _
2121  },
2122  .format_trace = format_tcp_tx_trace,
2123 };
2124 /* *INDENT-ON* */
2125 
2127 
2128 /*
2129  * fd.io coding-style-patch-verification: ON
2130  *
2131  * Local Variables:
2132  * eval: (c-set-style "gnu")
2133  * End:
2134  */
void tcp_make_fin(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to FIN-ACK.
Definition: tcp_output.c:559
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:434
#define tcp_in_cong_recovery(tc)
Definition: tcp.h:330
#define TCP_FIB_RECHECK_PERIOD
Recheck every 1s.
Definition: tcp.h:31
static void tcp_enqueue_to_ip_lookup_i(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index, u8 flush)
Definition: tcp_output.c:632
#define TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL(thread_index)
Definition: tcp_debug.h:784
End of options.
Definition: tcp_packet.h:104
#define clib_min(x, y)
Definition: clib.h:340
#define TCP_OPTION_LEN_EOL
Definition: tcp_packet.h:163
#define CLIB_UNUSED(x)
Definition: clib.h:79
#define tcp_in_recovery(tc)
Definition: tcp.h:324
static void tcp_retransmit_timer_set(tcp_connection_t *tc)
Definition: tcp.h:723
#define TCP_OPTION_LEN_SACK_PERMITTED
Definition: tcp_packet.h:167
static void tcp_flush_frame_to_ip_lookup(vlib_main_t *vm, u8 thread_index, u8 is_ip4)
Flush ip lookup tx frames populated by timer pops.
Definition: tcp_output.c:1022
#define seq_leq(_s1, _s2)
Definition: tcp.h:531
void tcp_timer_retransmit_handler(u32 index)
Definition: tcp_output.c:1507
ip4_address_t src_address
Definition: ip4_packet.h:164
struct _transport_connection transport_connection_t
#define TCP_TO_TIMER_TICK
Definition: tcp.h:96
Selective Ack permitted.
Definition: tcp_packet.h:108
#define TCP_FLAG_SYN
Definition: fa_node.h:10
#define TCP_MIN_RX_FIFO_SIZE
Definition: tcp.h:36
void tcp_send_reset_w_pkt(tcp_connection_t *tc, vlib_buffer_t *pkt, u8 is_ip4)
Send reset without reusing existing buffer.
Definition: tcp_output.c:814
static u8 svm_fifo_has_ooo_data(svm_fifo_t *f)
Definition: svm_fifo.h:112
#define tcp_opts_tstamp(_to)
Definition: tcp_packet.h:157
static uword tcp4_send_reset(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_output.c:2077
void tcp_make_synack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN-ACK.
Definition: tcp_output.c:602
#define PREDICT_TRUE(x)
Definition: clib.h:106
static void tcp_enqueue_to_output(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4)
Definition: tcp_output.c:717
static tcp_connection_t * tcp_connection_get_if_valid(u32 conn_index, u32 thread_index)
Definition: tcp.h:458
static int tcp_make_syn_options(tcp_options_t *opts, u8 wnd_scale)
Definition: tcp_output.c:284
struct _sack_scoreboard sack_scoreboard_t
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
Definition: fib_table.c:940
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
Definition: tcp.h:499
void tcp_update_rcv_mss(tcp_connection_t *tc)
Update max segment size we&#39;re able to process.
Definition: tcp_output.c:84
struct _tcp_main tcp_main_t
u32 thread_index
Definition: main.h:176
void tcp_flush_frame_to_output(vlib_main_t *vm, u8 thread_index, u8 is_ip4)
Flush tx frame populated by retransmits and timer pops.
Definition: tcp_output.c:1006
#define TCP_OPTS_ALIGN
Definition: tcp_packet.h:174
static u32 tcp_initial_wnd_unscaled(tcp_connection_t *tc)
TCP&#39;s initial window.
Definition: tcp_output.c:94
#define tcp_recovery_off(tc)
Definition: tcp.h:322
enum _tcp_output_next tcp_output_next_t
int i
static u32 tcp_session_has_ooo_data(tcp_connection_t *tc)
Definition: tcp_output.c:1744
static u32 format_get_indent(u8 *s)
Definition: format.h:72
struct _tcp_connection tcp_connection_t
static int tcp_alloc_tx_buffers(tcp_main_t *tm, u8 thread_index, u32 n_free_buffers)
Definition: tcp_output.c:440
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define tcp_opts_sack(_to)
Definition: tcp_packet.h:159
void tcp_make_ack_i(tcp_connection_t *tc, vlib_buffer_t *b, tcp_state_t state, u8 flags)
Prepare ACK.
Definition: tcp_output.c:519
static void scoreboard_clear(sack_scoreboard_t *sb)
Definition: tcp.h:845
static uword tcp46_send_reset_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, u8 is_ip4)
Definition: tcp_output.c:2008
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:415
No operation.
Definition: tcp_packet.h:105
int tcp_make_reset_in_place(vlib_main_t *vm, vlib_buffer_t *b0, tcp_state_t state, u8 thread_index, u8 is_ip4)
Definition: tcp_output.c:730
u8 n_sack_blocks
Number of SACKs blocks.
Definition: tcp_packet.h:152
struct _tcp_header tcp_header_t
static u32 tcp_available_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
Definition: tcp.h:619
static u32 stream_session_max_rx_enqueue(transport_connection_t *tc)
Definition: session.h:415
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
Definition: tcp.c:129
ip6_address_t src_address
Definition: ip6_packet.h:342
static void tcp_enqueue_to_output_now(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4)
Definition: tcp_output.c:723
struct _sack_scoreboard_hole sack_scoreboard_hole_t
u8 wscale
Window scale advertised.
Definition: tcp_packet.h:148
enum fib_protocol_t_ fib_protocol_t
Protocol Type.
#define TCP_OPTS_MAX_SACK_BLOCKS
Definition: tcp_packet.h:175
#define TCP_MAX_RX_FIFO_SIZE
Definition: tcp.h:35
u16 dummy_mtu
Definition: tcp_output.c:50
vlib_node_registration_t ip4_lookup_node
(constructor) VLIB_REGISTER_NODE (ip4_lookup_node)
Definition: ip4_forward.c:103
static void tcp_timer_retransmit_handler_i(u32 index, u8 is_syn)
Definition: tcp_output.c:1354
#define foreach_tcp4_reset_next
Definition: tcp_output.c:1999
Limit MSS.
Definition: tcp_packet.h:106
static void * tcp_init_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Definition: tcp_output.c:503
void tcp_make_syn(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN.
Definition: tcp_output.c:577
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:104
#define seq_gt(_s1, _s2)
Definition: tcp.h:532
void tcp_push_ip_hdr(tcp_main_t *tm, tcp_connection_t *tc, vlib_buffer_t *b)
Definition: tcp_output.c:939
#define always_inline
Definition: clib.h:92
#define TCP_OPTION_LEN_SACK_BLOCK
Definition: tcp_packet.h:169
ip4_address_t dst_address
Definition: ip4_packet.h:164
#define TCP_FLAG_ACK
Definition: fa_node.h:13
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
tcp_main_t tcp_main
Definition: tcp.c:29
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
Definition: tcp.h:428
int i32
Definition: types.h:81
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:191
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:182
u8 * format_tcp_tx_trace(u8 *s, va_list *args)
Definition: tcp_output.c:53
enum _tcp_state tcp_state_t
void tcp_fast_retransmit_no_sack(tcp_connection_t *tc)
Fast retransmit without SACK info.
Definition: tcp_output.c:1697
#define TCP_RTO_MAX
Definition: tcp.h:107
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:233
static u32 tcp_time_now(void)
Definition: tcp.h:656
sack_block_t * sacks
SACK blocks.
Definition: tcp_packet.h:151
struct _stream_session_t stream_session_t
#define TCP_ESTABLISH_TIME
Definition: tcp.h:99
sack_scoreboard_hole_t * scoreboard_next_rxt_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *start, u8 have_sent_1_smss, u8 *can_rescue, u8 *snd_limited)
Figure out the next hole to retransmit.
Definition: tcp_input.c:642
#define tcp_validate_txf_size(_tc, _a)
Definition: tcp.h:786
#define VLIB_FRAME_SIZE
Definition: node.h:328
#define TCP_EVT_DBG(_evt, _args...)
Definition: tcp_debug.h:235
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:74
static void tcp_timer_set(tcp_connection_t *tc, u8 timer_id, u32 interval)
Definition: tcp.h:689
#define TCP_OPTION_LEN_WINDOW_SCALE
Definition: tcp_packet.h:166
vlib_node_registration_t tcp6_reset_node
(constructor) VLIB_REGISTER_NODE (tcp6_reset_node)
Definition: tcp_output.c:2110
#define TCP_RTO_SYN_RETRIES
Definition: tcp.h:110
#define tcp_trajectory_add_start(b, start)
Definition: tcp.h:441
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
Definition: tcp_output.c:893
static int tcp_make_synack_options(tcp_connection_t *tc, tcp_options_t *opts)
Definition: tcp_output.c:313
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
static void tcp_push_hdr_i(tcp_connection_t *tc, vlib_buffer_t *b, tcp_state_t next_state, u8 compute_opts)
Push TCP header and update connection variables.
Definition: tcp_output.c:1111
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:321
#define MAX_HDRS_LEN
Definition: session.h:31
#define tcp_in_fastrecovery(tc)
Definition: tcp.h:323
static void * vlib_buffer_push_tcp_net_order(vlib_buffer_t *b, u16 sp, u16 dp, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
Definition: tcp.h:922
static int tcp_get_free_buffer_index(tcp_main_t *tm, u32 *bidx)
Definition: tcp_output.c:462
#define tcp_opts_mss(_to)
Definition: tcp_packet.h:156
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:191
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:209
#define TCP_TIMER_HANDLE_INVALID
Definition: tcp.h:93
#define foreach_tcp6_output_next
Definition: tcp_output.c:34
static u32 tcp_flight_size(const tcp_connection_t *tc)
Our estimate of the number of bytes in flight (pipe size)
Definition: tcp.h:556
#define PREDICT_FALSE(x)
Definition: clib.h:105
void tcp_fast_retransmit(tcp_connection_t *tc)
Do fast retransmit.
Definition: tcp_output.c:1734
#define TCP_FLAG_FIN
Definition: fa_node.h:9
static u8 tcp_window_compute_scale(u32 window)
Definition: tcp_output.c:69
static void tcp_rtx_timeout_cc(tcp_connection_t *tc)
Reset congestion control, switch cwnd to loss window and try again.
Definition: tcp_output.c:1336
void tcp_timer_retransmit_syn_handler(u32 index)
Definition: tcp_output.c:1513
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
#define TCP_OPTION_LEN_TIMESTAMP
Definition: tcp_packet.h:168
#define foreach_tcp4_output_next
Definition: tcp_output.c:30
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:130
#define TCP_WND_MAX
Definition: tcp_packet.h:172
Selective Ack block.
Definition: tcp_packet.h:109
void tcp_make_ack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to ACK.
Definition: tcp_output.c:544
#define TCP_FLAG_RST
Definition: fa_node.h:11
static stream_session_t * session_get(u32 si, u32 thread_index)
Definition: session.h:241
#define TCP_MAX_WND_SCALE
Definition: tcp_packet.h:173
static void tcp_timer_reset(tcp_connection_t *tc, u8 timer_id)
Definition: tcp.h:699
static sack_scoreboard_hole_t * scoreboard_first_hole(sack_scoreboard_t *sb)
Definition: tcp.h:829
#define tcp_fastrecovery_sent_1_smss(tc)
Definition: tcp.h:326
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
static void * vlib_buffer_push_tcp(vlib_buffer_t *b, u16 sp_net, u16 dp_net, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
Definition: tcp.h:957
tcp_header_t tcp_header
Definition: tcp_output.c:46
u16 n_vectors
Definition: node.h:344
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:74
vlib_main_t * vm
Definition: buffer.c:294
static void tcp_enqueue_to_ip_lookup(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
Definition: tcp_output.c:676
u32 stream_session_tx_fifo_max_dequeue(transport_connection_t *tc)
Definition: session.c:410
#define clib_warning(format, args...)
Definition: error.h:59
static u8 tcp_make_state_flags(tcp_connection_t *tc, tcp_state_t next_state)
Definition: tcp_output.c:1085
#define clib_memcpy(a, b, c)
Definition: string.h:75
format_function_t format_tcp_header
Definition: format.h:102
#define TCP_USE_SACKS
Disable only for testing.
Definition: tcp.h:39
#define tcp_recovery_on(tc)
Definition: tcp.h:321
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
fib_node_index_t tcp_lookup_rmt_in_fib(tcp_connection_t *tc)
u16 mss
Option flags, see above.
Definition: tcp_packet.h:147
static void * ip6_next_header(ip6_header_t *i)
Definition: ip6_packet.h:369
static void tcp_return_buffer(tcp_main_t *tm)
Definition: tcp_output.c:481
static void tcp_timer_update(tcp_connection_t *tc, u8 timer_id, u32 interval)
Definition: tcp.h:711
u16 ip6_tcp_udp_icmp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip6_header_t *ip0, int *bogus_lengthp)
Definition: ip6_forward.c:865
vlib_node_registration_t ip6_lookup_node
(constructor) VLIB_REGISTER_NODE (ip6_lookup_node)
Definition: ip6_forward.c:467
static int tcp_make_established_options(tcp_connection_t *tc, tcp_options_t *opts)
Definition: tcp_output.c:348
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:456
#define ASSERT(truth)
#define tcp_syn(_th)
Definition: tcp_packet.h:80
unsigned int u32
Definition: types.h:88
static uword tcp6_output(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_output.c:1925
u16 ip4_tcp_udp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip4_header_t *ip0)
Definition: ip4_forward.c:1041
u32 tcp_push_header(transport_connection_t *tconn, vlib_buffer_t *b)
Definition: tcp_output.c:1975
#define seq_geq(_s1, _s2)
Definition: tcp.h:533
u32 tcp_window_to_advertise(tcp_connection_t *tc, tcp_state_t state)
Compute and return window to advertise, scaled as per RFC1323.
Definition: tcp_output.c:132
vhost_vring_state_t state
Definition: vhost-user.h:82
void tcp_retransmit_first_unacked(tcp_connection_t *tc)
Retransmit first unacked segment.
Definition: tcp_output.c:1597
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
static void tcp_enqueue_to_ip_lookup_now(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
Definition: tcp_output.c:669
void tcp_init_mss(tcp_connection_t *tc)
Definition: tcp_output.c:415
static uword tcp6_send_reset(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_output.c:2084
static uword tcp46_output_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip4)
Definition: tcp_output.c:1751
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
Definition: tcp_output.c:1052
#define clib_max(x, y)
Definition: clib.h:333
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
VLIB_NODE_FUNCTION_MULTIARCH(tcp4_output_node, tcp4_output)
void tcp_send_ack(tcp_connection_t *tc)
Definition: tcp_output.c:1157
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:159
#define seq_lt(_s1, _s2)
Definition: tcp.h:530
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
u32 ip_version_traffic_class_and_flow_label
Definition: ip6_packet.h:329
#define tcp_opts_wscale(_to)
Definition: tcp_packet.h:158
static sack_scoreboard_hole_t * scoreboard_get_hole(sack_scoreboard_t *sb, u32 index)
Definition: tcp.h:805
Definition: defs.h:47
unsigned short u16
Definition: types.h:57
void tcp_connection_reset(tcp_connection_t *tc)
Notify session that connection has been reset.
Definition: tcp.c:221
u32 tsval
Timestamp value.
Definition: tcp_packet.h:149
u32 tsecr
Echoed/reflected time stamp.
Definition: tcp_packet.h:150
static void * vlib_buffer_push_ip6(vlib_main_t *vm, vlib_buffer_t *b, ip6_address_t *src, ip6_address_t *dst, int proto)
Push IPv6 header to buffer.
Definition: ip6.h:592
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
static u32 tcp_set_time_now(u32 thread_index)
Definition: tcp.h:662
static u8 tcp_is_lost_fin(tcp_connection_t *tc)
Definition: tcp.h:631
#define foreach_tcp6_reset_next
Definition: tcp_output.c:2003
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:553
static void tcp_retransmit_timer_update(tcp_connection_t *tc)
Definition: tcp.h:767
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
void tcp_timer_delack_handler(u32 index)
Delayed ack timer handler.
Definition: tcp_output.c:1181
#define TCP_OPTION_LEN_MSS
Definition: tcp_packet.h:165
void tcp_update_snd_mss(tcp_connection_t *tc)
Update snd_mss to reflect the effective segment size that we can send by taking into account all TCP ...
Definition: tcp_output.c:402
struct clib_bihash_value offset
template key/value backing page structure
static void tcp_retransmit_timer_force_update(tcp_connection_t *tc)
Definition: tcp.h:737
u8 * format_tcp_connection(u8 *s, va_list *args)
Definition: tcp.c:741
u32 tcp_initial_window_to_advertise(tcp_connection_t *tc)
Compute initial window and scale factor.
Definition: tcp_output.c:113
static u32 stream_session_rx_fifo_size(transport_connection_t *tc)
Definition: session.h:422
void tcp_flush_frames_to_output(u8 thread_index)
Flush v4 and v6 tcp and ip-lookup tx frames for thread index.
Definition: tcp_output.c:1039
#define vnet_buffer(b)
Definition: buffer.h:372
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
Definition: tcp.h:449
void tcp_update_rcv_wnd(tcp_connection_t *tc)
Definition: tcp_output.c:152
void tcp_update_rto(tcp_connection_t *tc)
Definition: tcp_input.c:419
u8 data[0]
Packet data.
Definition: buffer.h:179
void tcp_fast_retransmit_sack(tcp_connection_t *tc)
Do fast retransmit with SACKs.
Definition: tcp_output.c:1620
#define TCP_OPTION_LEN_NOOP
Definition: tcp_packet.h:164
void tcp_send_syn(tcp_connection_t *tc)
Send SYN.
Definition: tcp_output.c:970
vlib_node_registration_t tcp6_output_node
(constructor) VLIB_REGISTER_NODE (tcp6_output_node)
Definition: tcp_output.c:21
Window scale.
Definition: tcp_packet.h:107
enum _tcp_reset_next tcp_reset_next_t
#define tcp_opts_sack_permitted(_to)
Definition: tcp_packet.h:160
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:432
tcp_connection_t tcp_connection
Definition: tcp_output.c:47
static u32 tcp_loss_wnd(const tcp_connection_t *tc)
Definition: tcp.h:592
static void * tcp_reuse_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Definition: tcp_output.c:487
u8 ip_version_and_header_length
Definition: ip4_packet.h:132
Timestamps.
Definition: tcp_packet.h:110
u32 tcp_prepare_retransmit_segment(tcp_connection_t *tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t **b)
Build a retransmit segment.
Definition: tcp_output.c:1198
vlib_node_registration_t tcp4_reset_node
(constructor) VLIB_REGISTER_NODE (tcp4_reset_node)
Definition: tcp_output.c:2091
vlib_node_registration_t tcp4_output_node
(constructor) VLIB_REGISTER_NODE (tcp4_output_node)
Definition: tcp_output.c:20
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:347
static void tcp_enqueue_to_output_i(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u8 flush)
Definition: tcp_output.c:683
static void tcp_persist_timer_set(tcp_connection_t *tc)
Definition: tcp.h:744
static tcp_main_t * vnet_get_tcp_main()
Definition: tcp.h:422
int stream_session_peek_bytes(transport_connection_t *tc, u8 *buffer, u32 offset, u32 max_bytes)
Definition: session.c:419
static char * tcp_error_strings[]
Definition: tcp_output.c:38
static uword tcp4_output(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_output.c:1918
static void * vlib_buffer_push_ip4(vlib_main_t *vm, vlib_buffer_t *b, ip4_address_t *src, ip4_address_t *dst, int proto, u8 csum_offload)
Push IPv4 header to buffer.
Definition: ip4.h:345
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
void tcp_cc_fastrecovery_exit(tcp_connection_t *tc)
Definition: tcp_input.c:950
void tcp_timer_persist_handler(u32 index)
Got 0 snd_wnd from peer, try to do something about it.
Definition: tcp_output.c:1523
#define tcp_ack(_th)
Definition: tcp_packet.h:83
static u8 tcp_timer_is_active(tcp_connection_t *tc, tcp_timers_e timer)
Definition: tcp.h:781
Definition: defs.h:46
ip6_address_t dst_address
Definition: ip6_packet.h:342
u32 tcp_options_write(u8 *data, tcp_options_t *opts)
Write TCP options to segment.
Definition: tcp_output.c:201
static int tcp_make_options(tcp_connection_t *tc, tcp_options_t *opts, tcp_state_t state)
Definition: tcp_output.c:379