FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
tcp.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 /**
17  * @file
18  * @brief TCP host stack utilities
19  */
20 
21 #include <vnet/tcp/tcp.h>
22 #include <vnet/tcp/tcp_inlines.h>
23 #include <vnet/session/session.h>
24 #include <vnet/fib/fib.h>
25 #include <vnet/dpo/load_balance.h>
26 #include <math.h>
27 
29 
30 typedef struct
31 {
34  ip46_address_t ip;
38 
39 static void
41 {
42  u32 ai;
43  if (args->is_add)
44  {
45  adj_nbr_add_or_lock (args->nh_proto, args->link_type, &args->ip,
46  args->sw_if_index);
47  }
48  else
49  {
51  args->sw_if_index);
52  if (ai != ADJ_INDEX_INVALID)
53  adj_unlock (ai);
54  }
55 }
56 
57 static void
59 {
60  tcp_add_del_adj_args_t args = {
62  .link_type = VNET_LINK_IP6,
63  .ip = tc->c_rmt_ip,
64  .sw_if_index = tc->sw_if_index,
65  .is_add = is_add
66  };
68  sizeof (args));
69 }
70 
71 static void
73 {
74  tc->cc_algo->init (tc);
75 }
76 
77 static void
79 {
80  if (tc->cc_algo->cleanup)
81  tc->cc_algo->cleanup (tc);
82 }
83 
84 void
86  const tcp_cc_algorithm_t * vft)
87 {
89  vec_validate (tm->cc_algos, type);
90 
91  tm->cc_algos[type] = *vft;
92  hash_set_mem (tm->cc_algo_by_name, vft->name, type);
93 }
94 
97 {
99  return &tm->cc_algos[type];
100 }
101 
104 {
105  tcp_main_t *tm = vnet_get_tcp_main ();
106  tcp_cc_algo_register (++tm->cc_last_type, vft);
107  return tm->cc_last_type;
108 }
109 
110 static u32
112 {
113  tcp_main_t *tm = &tcp_main;
115  void *iface_ip;
116 
117  pool_get (tm->listener_pool, listener);
118  clib_memset (listener, 0, sizeof (*listener));
119 
120  listener->c_c_index = listener - tm->listener_pool;
121  listener->c_lcl_port = lcl->port;
122 
123  /* If we are provided a sw_if_index, bind using one of its ips */
124  if (ip_is_zero (&lcl->ip, 1) && lcl->sw_if_index != ENDPOINT_INVALID_INDEX)
125  {
126  if ((iface_ip = ip_interface_get_first_ip (lcl->sw_if_index,
127  lcl->is_ip4)))
128  ip_set (&lcl->ip, iface_ip, lcl->is_ip4);
129  }
130  ip_copy (&listener->c_lcl_ip, &lcl->ip, lcl->is_ip4);
131  listener->c_is_ip4 = lcl->is_ip4;
132  listener->c_proto = TRANSPORT_PROTO_TCP;
133  listener->c_s_index = session_index;
134  listener->c_fib_index = lcl->fib_index;
135  listener->state = TCP_STATE_LISTEN;
136  listener->cc_algo = tcp_cc_algo_get (tcp_cfg.cc_algo);
137 
138  tcp_connection_timers_init (listener);
139 
140  TCP_EVT (TCP_EVT_BIND, listener);
141 
142  return listener->c_c_index;
143 }
144 
145 static u32
147 {
148  return tcp_connection_bind (session_index, tep);
149 }
150 
151 static void
152 tcp_connection_unbind (u32 listener_index)
153 {
154  tcp_main_t *tm = vnet_get_tcp_main ();
155  tcp_connection_t *tc;
156 
157  tc = pool_elt_at_index (tm->listener_pool, listener_index);
158 
159  TCP_EVT (TCP_EVT_UNBIND, tc);
160 
161  /* Poison the entry */
162  if (CLIB_DEBUG > 0)
163  clib_memset (tc, 0xFA, sizeof (*tc));
164 
165  pool_put_index (tm->listener_pool, listener_index);
166 }
167 
168 static u32
169 tcp_session_unbind (u32 listener_index)
170 {
171  tcp_connection_unbind (listener_index);
172  return 0;
173 }
174 
175 static transport_connection_t *
177 {
178  tcp_main_t *tm = vnet_get_tcp_main ();
179  tcp_connection_t *tc;
180  tc = pool_elt_at_index (tm->listener_pool, listener_index);
181  return &tc->connection;
182 }
183 
184 /**
185  * Cleanup half-open connection
186  *
187  */
188 static void
190 {
191  tcp_main_t *tm = vnet_get_tcp_main ();
192  clib_spinlock_lock_if_init (&tm->half_open_lock);
193  if (CLIB_DEBUG)
194  clib_memset (tc, 0xFA, sizeof (*tc));
195  pool_put (tm->half_open_connections, tc);
196  clib_spinlock_unlock_if_init (&tm->half_open_lock);
197 }
198 
199 /**
200  * Try to cleanup half-open connection
201  *
202  * If called from a thread that doesn't own tc, the call won't have any
203  * effect.
204  *
205  * @param tc - connection to be cleaned up
206  * @return non-zero if cleanup failed.
207  */
208 int
210 {
211  tcp_worker_ctx_t *wrk;
212 
213  /* Make sure this is the owning thread */
214  if (tc->c_thread_index != vlib_get_thread_index ())
215  return 1;
216 
217  session_half_open_delete_notify (TRANSPORT_PROTO_TCP, tc->c_s_ho_handle);
218  wrk = tcp_get_worker (tc->c_thread_index);
219  tcp_timer_reset (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN);
221  return 0;
222 }
223 
224 static tcp_connection_t *
226 {
227  tcp_main_t *tm = vnet_get_tcp_main ();
228  tcp_connection_t *tc = 0;
229  ASSERT (vlib_get_thread_index () == 0);
230  pool_get (tm->half_open_connections, tc);
231  clib_memset (tc, 0, sizeof (*tc));
232  tc->c_c_index = tc - tm->half_open_connections;
233  return tc;
234 }
235 
236 /**
237  * Cleans up connection state.
238  *
239  * No notifications.
240  */
241 void
243 {
244  TCP_EVT (TCP_EVT_DELETE, tc);
245 
246  /* Cleanup local endpoint if this was an active connect */
247  if (!(tc->cfg_flags & TCP_CFG_F_NO_ENDPOINT))
248  transport_endpoint_cleanup (TRANSPORT_PROTO_TCP, &tc->c_lcl_ip,
249  tc->c_lcl_port);
250 
251  /* Check if connection is not yet fully established */
252  if (tc->state == TCP_STATE_SYN_SENT)
253  {
254  /* Try to remove the half-open connection. If this is not the owning
255  * thread, tc won't be removed. Retransmit or establish timers will
256  * eventually expire and call again cleanup on the right thread. */
258  tc->flags |= TCP_CONN_HALF_OPEN_DONE;
259  }
260  else
261  {
262  /* Make sure all timers are cleared */
264 
265  if (!tc->c_is_ip4 && ip6_address_is_link_local_unicast (&tc->c_rmt_ip6))
266  tcp_add_del_adjacency (tc, 0);
267 
268  tcp_cc_cleanup (tc);
269  vec_free (tc->snd_sacks);
270  vec_free (tc->snd_sacks_fl);
271  vec_free (tc->rcv_opts.sacks);
272  pool_free (tc->sack_sb.holes);
273 
274  if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
275  tcp_bt_cleanup (tc);
276 
277  tcp_connection_free (tc);
278  }
279 }
280 
281 /**
282  * Connection removal.
283  *
284  * This should be called only once connection enters CLOSED state. Note
285  * that it notifies the session of the removal event, so if the goal is to
286  * just remove the connection, call tcp_connection_cleanup instead.
287  */
288 void
290 {
291  session_transport_delete_notify (&tc->connection);
293 }
294 
296 tcp_connection_alloc (u8 thread_index)
297 {
298  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
299  tcp_connection_t *tc;
300 
301  pool_get (wrk->connections, tc);
302  clib_memset (tc, 0, sizeof (*tc));
303  tc->c_c_index = tc - wrk->connections;
304  tc->c_thread_index = thread_index;
305  return tc;
306 }
307 
310 {
311  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
312  tcp_connection_t *tc;
313 
314  pool_get (wrk->connections, tc);
315  clib_memcpy_fast (tc, base, sizeof (*tc));
316  tc->c_c_index = tc - wrk->connections;
317  tc->c_thread_index = thread_index;
318  return tc;
319 }
320 
321 void
323 {
324  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
325  if (CLIB_DEBUG)
326  {
327  clib_memset (tc, 0xFA, sizeof (*tc));
328  pool_put (wrk->connections, tc);
329  return;
330  }
331  pool_put (wrk->connections, tc);
332 }
333 
334 void
336 {
337  tcp_cleanup_req_t *req;
338  clib_time_type_t now;
339 
340  now = transport_time_now (tc->c_thread_index);
341  clib_fifo_add2 (wrk->pending_cleanups, req);
342  req->connection_index = tc->c_c_index;
343  req->free_time = now + tcp_cfg.cleanup_time;
344 }
345 
346 /**
347  * Begin connection closing procedure.
348  *
349  * If at the end the connection is not in CLOSED state, it is not removed.
350  * Instead, we rely on on TCP to advance through state machine to either
351  * 1) LAST_ACK (passive close) whereby when the last ACK is received
352  * tcp_connection_del is called. This notifies session of the delete and
353  * calls cleanup.
354  * 2) TIME_WAIT (active close) whereby after 2MSL the 2MSL timer triggers
355  * and cleanup is called.
356  *
357  * N.B. Half-close connections are not supported
358  */
359 void
361 {
362  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
363 
364  TCP_EVT (TCP_EVT_CLOSE, tc);
365 
366  /* Send/Program FIN if needed and switch state */
367  switch (tc->state)
368  {
369  case TCP_STATE_SYN_SENT:
370  /* Try to cleanup. If not on the right thread, mark as half-open done.
371  * Connection will be cleaned up when establish timer pops */
373  break;
374  case TCP_STATE_SYN_RCVD:
376  tcp_send_fin (tc);
377  tcp_connection_set_state (tc, TCP_STATE_FIN_WAIT_1);
378  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
379  tcp_cfg.finwait1_time);
380  break;
381  case TCP_STATE_ESTABLISHED:
382  /* If closing with unread data, reset the connection */
383  if (transport_max_rx_dequeue (&tc->connection))
384  {
385  tcp_send_reset (tc);
387  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
388  session_transport_closed_notify (&tc->connection);
389  tcp_program_cleanup (tcp_get_worker (tc->c_thread_index), tc);
390  tcp_worker_stats_inc (wrk, rst_unread, 1);
391  break;
392  }
393  if (!transport_max_tx_dequeue (&tc->connection))
394  tcp_send_fin (tc);
395  else
396  tc->flags |= TCP_CONN_FINPNDG;
397  tcp_connection_set_state (tc, TCP_STATE_FIN_WAIT_1);
398  /* Set a timer in case the peer stops responding. Otherwise the
399  * connection will be stuck here forever. */
400  ASSERT (tc->timers[TCP_TIMER_WAITCLOSE] == TCP_TIMER_HANDLE_INVALID);
401  tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
402  tcp_cfg.finwait1_time);
403  break;
404  case TCP_STATE_CLOSE_WAIT:
405  if (!transport_max_tx_dequeue (&tc->connection))
406  {
407  tcp_send_fin (tc);
409  tcp_connection_set_state (tc, TCP_STATE_LAST_ACK);
410  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
411  tcp_cfg.lastack_time);
412  }
413  else
414  tc->flags |= TCP_CONN_FINPNDG;
415  break;
416  case TCP_STATE_FIN_WAIT_1:
417  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
418  tcp_cfg.finwait1_time);
419  break;
420  case TCP_STATE_CLOSED:
421  /* Cleanup should've been programmed already */
422  break;
423  default:
424  TCP_DBG ("state: %u", tc->state);
425  }
426 }
427 
428 static void
429 tcp_session_close (u32 conn_index, u32 thread_index)
430 {
431  tcp_connection_t *tc;
432  tc = tcp_connection_get (conn_index, thread_index);
434 }
435 
436 static void
437 tcp_session_cleanup (u32 conn_index, u32 thread_index)
438 {
439  tcp_connection_t *tc;
440  tc = tcp_connection_get (conn_index, thread_index);
441  if (!tc)
442  return;
443  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
445 }
446 
447 static void
449 {
450  tcp_worker_ctx_t *wrk;
451  tcp_connection_t *tc;
452 
453  tc = tcp_half_open_connection_get (conn_index);
454  wrk = tcp_get_worker (tc->c_thread_index);
455  tcp_timer_reset (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN);
457 }
458 
459 static void
460 tcp_session_reset (u32 conn_index, u32 thread_index)
461 {
462  tcp_connection_t *tc;
463  tc = tcp_connection_get (conn_index, thread_index);
464  tcp_send_reset (tc);
467  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
468  session_transport_closed_notify (&tc->connection);
469  tcp_program_cleanup (tcp_get_worker (thread_index), tc);
470 }
471 
472 /**
473  * Initialize all connection timers as invalid
474  */
475 void
477 {
478  int i;
479 
480  /* Set all to invalid */
481  for (i = 0; i < TCP_N_TIMERS; i++)
482  {
483  tc->timers[i] = TCP_TIMER_HANDLE_INVALID;
484  }
485 
486  tc->rto = TCP_RTO_INIT;
487 }
488 
489 /**
490  * Stop all connection timers
491  */
492 void
494 {
495  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
496  int i;
497 
498  for (i = 0; i < TCP_N_TIMERS; i++)
499  tcp_timer_reset (&wrk->timer_wheel, tc, i);
500 }
501 
502 #if 0
503 typedef struct ip4_tcp_hdr
504 {
506  tcp_header_t tcp;
507 } ip4_tcp_hdr_t;
508 
509 typedef struct ip6_tcp_hdr
510 {
512  tcp_header_t tcp;
513 } ip6_tcp_hdr_t;
514 
515 static void
516 tcp_connection_select_lb_bucket (tcp_connection_t * tc, const dpo_id_t * dpo,
517  dpo_id_t * result)
518 {
519  const dpo_id_t *choice;
520  load_balance_t *lb;
521  int hash;
522 
523  lb = load_balance_get (dpo->dpoi_index);
524  if (tc->c_is_ip4)
525  {
526  ip4_tcp_hdr_t hdr;
527  clib_memset (&hdr, 0, sizeof (hdr));
528  hdr.ip.protocol = IP_PROTOCOL_TCP;
529  hdr.ip.address_pair.src.as_u32 = tc->c_lcl_ip.ip4.as_u32;
530  hdr.ip.address_pair.dst.as_u32 = tc->c_rmt_ip.ip4.as_u32;
531  hdr.tcp.src_port = tc->c_lcl_port;
532  hdr.tcp.dst_port = tc->c_rmt_port;
533  hash = ip4_compute_flow_hash (&hdr.ip, lb->lb_hash_config);
534  }
535  else
536  {
537  ip6_tcp_hdr_t hdr;
538  clib_memset (&hdr, 0, sizeof (hdr));
539  hdr.ip.protocol = IP_PROTOCOL_TCP;
540  clib_memcpy_fast (&hdr.ip.src_address, &tc->c_lcl_ip.ip6,
541  sizeof (ip6_address_t));
542  clib_memcpy_fast (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6,
543  sizeof (ip6_address_t));
544  hdr.tcp.src_port = tc->c_lcl_port;
545  hdr.tcp.dst_port = tc->c_rmt_port;
546  hash = ip6_compute_flow_hash (&hdr.ip, lb->lb_hash_config);
547  }
548  choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1);
549  dpo_copy (result, choice);
550 }
551 
554 {
556  u32 fib_index;
557 
558  clib_memcpy_fast (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr));
559  prefix.fp_proto = tc->c_is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
560  prefix.fp_len = tc->c_is_ip4 ? 32 : 128;
561  fib_index = fib_table_find (prefix.fp_proto, tc->c_fib_index);
562  return fib_table_lookup (fib_index, &prefix);
563 }
564 
565 static int
566 tcp_connection_stack_on_fib_entry (tcp_connection_t * tc)
567 {
568  dpo_id_t choice = DPO_INVALID;
569  u32 output_node_index;
570  fib_entry_t *fe;
571 
572  fe = fib_entry_get (tc->c_rmt_fei);
573  if (fe->fe_lb.dpoi_type != DPO_LOAD_BALANCE)
574  return -1;
575 
576  tcp_connection_select_lb_bucket (tc, &fe->fe_lb, &choice);
577 
578  output_node_index =
579  tc->c_is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
580  dpo_stack_from_node (output_node_index, &tc->c_rmt_dpo, &choice);
581  return 0;
582 }
583 
584 /** Stack tcp connection on peer's fib entry.
585  *
586  * This ultimately populates the dpo the connection will use to send packets.
587  */
588 static void
589 tcp_connection_fib_attach (tcp_connection_t * tc)
590 {
591  tc->c_rmt_fei = tcp_lookup_rmt_in_fib (tc);
592 
593  ASSERT (tc->c_rmt_fei != FIB_NODE_INDEX_INVALID);
594 
595  tcp_connection_stack_on_fib_entry (tc);
596 }
597 #endif /* 0 */
598 
599 /**
600  * Generate random iss as per rfc6528
601  */
602 static u32
604 {
605  tcp_main_t *tm = &tcp_main;
606  u64 tmp;
607 
608  if (tc->c_is_ip4)
609  tmp = (u64) tc->c_lcl_ip.ip4.as_u32 << 32 | (u64) tc->c_rmt_ip.ip4.as_u32;
610  else
611  tmp = tc->c_lcl_ip.ip6.as_u64[0] ^ tc->c_lcl_ip.ip6.as_u64[1]
612  ^ tc->c_rmt_ip.ip6.as_u64[0] ^ tc->c_rmt_ip.ip6.as_u64[1];
613 
614  tmp ^= tm->iss_seed.first | ((u64) tc->c_lcl_port << 16 | tc->c_rmt_port);
615  tmp ^= tm->iss_seed.second;
616  tmp = clib_xxhash (tmp) + clib_cpu_time_now ();
617  return ((tmp >> 32) ^ (tmp & 0xffffffff));
618 }
619 
620 /**
621  * Initialize max segment size we're able to process.
622  *
623  * The value is constrained by the output interface's MTU and by the size
624  * of the IP and TCP headers (see RFC6691). It is also what we advertise
625  * to our peer.
626  */
627 static void
629 {
630  u8 ip_hdr_len;
631 
632  /* Already provided at connection init time */
633  if (tc->mss)
634  return;
635 
636  ip_hdr_len = tc->c_is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
637  tc->mss = tcp_cfg.default_mtu - sizeof (tcp_header_t) - ip_hdr_len;
638 }
639 
640 static void
642 {
643  u16 default_min_mss = 536;
644 
645  tcp_init_rcv_mss (tc);
646 
647  /* TODO consider PMTU discovery */
648  tc->snd_mss = clib_min (tc->rcv_opts.mss, tc->mss);
649 
650  if (tc->snd_mss < 45)
651  {
652  /* Assume that at least the min default mss works */
653  tc->snd_mss = default_min_mss;
654  tc->rcv_opts.mss = default_min_mss;
655  }
656 
657  /* We should have enough space for 40 bytes of options */
658  ASSERT (tc->snd_mss > 45);
659 
660  /* If we use timestamp option, account for it */
661  if (tcp_opts_tstamp (&tc->rcv_opts))
662  tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
663 }
664 
665 /**
666  * Initialize connection send variables.
667  */
668 void
670 {
671  /*
672  * We use the time to randomize iss and for setting up the initial
673  * timestamp. Make sure it's updated otherwise syn and ack in the
674  * handshake may make it look as if time has flown in the opposite
675  * direction for us.
676  */
678 
679  tcp_init_rcv_mss (tc);
680  tc->iss = tcp_generate_random_iss (tc);
681  tc->snd_una = tc->iss;
682  tc->snd_nxt = tc->iss + 1;
683  tc->snd_una_max = tc->snd_nxt;
684  tc->srtt = 0.1 * THZ; /* 100 ms */
685 
686  if (!tcp_cfg.csum_offload)
687  tc->cfg_flags |= TCP_CFG_F_NO_CSUM_OFFLOAD;
688 }
689 
690 void
692 {
693  u32 byte_rate;
694  byte_rate = tc->cwnd / (tc->srtt * TCP_TICK);
695  transport_connection_tx_pacer_init (&tc->connection, byte_rate, tc->cwnd);
696  tc->mrtt_us = (u32) ~ 0;
697 }
698 
699 /** Initialize tcp connection variables
700  *
701  * Should be called after having received a msg from the peer, i.e., a SYN or
702  * a SYNACK, such that connection options have already been exchanged. */
703 void
705 {
707  tcp_init_mss (tc);
708  scoreboard_init (&tc->sack_sb);
709  if (tc->state == TCP_STATE_SYN_RCVD)
710  tcp_init_snd_vars (tc);
711 
712  tcp_cc_init (tc);
713 
714  if (!tc->c_is_ip4 && ip6_address_is_link_local_unicast (&tc->c_rmt_ip6))
715  tcp_add_del_adjacency (tc, 1);
716 
717  /* tcp_connection_fib_attach (tc); */
718 
719  if (transport_connection_is_tx_paced (&tc->connection)
720  || tcp_cfg.enable_tx_pacing)
721  tcp_enable_pacing (tc);
722 
723  if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
724  tcp_bt_init (tc);
725 
726  if (!tcp_cfg.allow_tso)
727  tc->cfg_flags |= TCP_CFG_F_NO_TSO;
728 
729  tc->start_ts = tcp_time_now_us (tc->c_thread_index);
730 }
731 
732 static int
733 tcp_alloc_custom_local_endpoint (tcp_main_t * tm, ip46_address_t * lcl_addr,
734  u16 * lcl_port, u8 is_ip4)
735 {
736  int index, port;
737  if (is_ip4)
738  {
739  index = tm->last_v4_addr_rotor++;
740  if (tm->last_v4_addr_rotor >= vec_len (tcp_cfg.ip4_src_addrs))
741  tm->last_v4_addr_rotor = 0;
742  lcl_addr->ip4.as_u32 = tcp_cfg.ip4_src_addrs[index].as_u32;
743  }
744  else
745  {
746  index = tm->last_v6_addr_rotor++;
747  if (tm->last_v6_addr_rotor >= vec_len (tcp_cfg.ip6_src_addrs))
748  tm->last_v6_addr_rotor = 0;
749  clib_memcpy_fast (&lcl_addr->ip6, &tcp_cfg.ip6_src_addrs[index],
750  sizeof (ip6_address_t));
751  }
752  port = transport_alloc_local_port (TRANSPORT_PROTO_TCP, lcl_addr);
753  if (port < 1)
754  return SESSION_E_NOPORT;
755  *lcl_port = port;
756  return 0;
757 }
758 
759 static int
761 {
762  tcp_main_t *tm = vnet_get_tcp_main ();
763  tcp_connection_t *tc;
764  ip46_address_t lcl_addr;
765  u16 lcl_port;
766  int rv;
767 
768  /*
769  * Allocate local endpoint
770  */
771  if ((rmt->is_ip4 && vec_len (tcp_cfg.ip4_src_addrs))
772  || (!rmt->is_ip4 && vec_len (tcp_cfg.ip6_src_addrs)))
773  rv = tcp_alloc_custom_local_endpoint (tm, &lcl_addr, &lcl_port,
774  rmt->is_ip4);
775  else
776  rv = transport_alloc_local_endpoint (TRANSPORT_PROTO_TCP,
777  rmt, &lcl_addr, &lcl_port);
778 
779  if (rv)
780  {
781  if (rv != SESSION_E_PORTINUSE)
782  return rv;
783 
784  if (session_lookup_connection (rmt->fib_index, &lcl_addr, &rmt->ip,
785  lcl_port, rmt->port, TRANSPORT_PROTO_UDP,
786  rmt->is_ip4))
787  return SESSION_E_PORTINUSE;
788 
789  /* 5-tuple is available so increase lcl endpoint refcount and proceed
790  * with connection allocation */
791  transport_share_local_endpoint (TRANSPORT_PROTO_UDP, &lcl_addr,
792  lcl_port);
793  }
794 
795  /*
796  * Create connection and send SYN
797  */
798  clib_spinlock_lock_if_init (&tm->half_open_lock);
800  ip_copy (&tc->c_rmt_ip, &rmt->ip, rmt->is_ip4);
801  ip_copy (&tc->c_lcl_ip, &lcl_addr, rmt->is_ip4);
802  tc->c_rmt_port = rmt->port;
803  tc->c_lcl_port = clib_host_to_net_u16 (lcl_port);
804  tc->c_is_ip4 = rmt->is_ip4;
805  tc->c_proto = TRANSPORT_PROTO_TCP;
806  tc->c_fib_index = rmt->fib_index;
807  tc->cc_algo = tcp_cc_algo_get (tcp_cfg.cc_algo);
808  /* The other connection vars will be initialized after SYN ACK */
810  tc->mss = rmt->mss;
811 
812  TCP_EVT (TCP_EVT_OPEN, tc);
813  tc->state = TCP_STATE_SYN_SENT;
814  tcp_init_snd_vars (tc);
815  tcp_send_syn (tc);
816  clib_spinlock_unlock_if_init (&tm->half_open_lock);
817 
818  return tc->c_c_index;
819 }
820 
821 static u8 *
822 format_tcp_session (u8 * s, va_list * args)
823 {
824  u32 tci = va_arg (*args, u32);
825  u32 thread_index = va_arg (*args, u32);
826  u32 verbose = va_arg (*args, u32);
827  tcp_connection_t *tc;
828 
829  tc = tcp_connection_get (tci, thread_index);
830  if (tc)
831  s = format (s, "%U", format_tcp_connection, tc, verbose);
832  else
833  s = format (s, "empty\n");
834  return s;
835 }
836 
837 static u8 *
838 format_tcp_listener_session (u8 * s, va_list * args)
839 {
840  u32 tci = va_arg (*args, u32);
841  u32 __clib_unused thread_index = va_arg (*args, u32);
842  u32 verbose = va_arg (*args, u32);
844  s = format (s, "%-50U", format_tcp_connection_id, tc);
845  if (verbose)
846  s = format (s, "%-15U", format_tcp_state, tc->state);
847  return s;
848 }
849 
850 static u8 *
851 format_tcp_half_open_session (u8 * s, va_list * args)
852 {
853  u32 tci = va_arg (*args, u32);
854  u32 __clib_unused thread_index = va_arg (*args, u32);
856  return format (s, "%U", format_tcp_connection_id, tc);
857 }
858 
859 static transport_connection_t *
860 tcp_session_get_transport (u32 conn_index, u32 thread_index)
861 {
862  tcp_connection_t *tc = tcp_connection_get (conn_index, thread_index);
863  if (PREDICT_FALSE (!tc))
864  return 0;
865  return &tc->connection;
866 }
867 
868 static transport_connection_t *
870 {
872  return &tc->connection;
873 }
874 
875 static u16
877 {
878  u16 goal_size = tc->snd_mss;
879 
880  goal_size = TCP_MAX_GSO_SZ - tc->snd_mss % TCP_MAX_GSO_SZ;
881  goal_size = clib_min (goal_size, tc->snd_wnd / 2);
882 
883  return goal_size > tc->snd_mss ? goal_size : tc->snd_mss;
884 }
885 
888 {
889  if (PREDICT_FALSE (tc->snd_wnd < tc->snd_mss))
890  {
891  return tc->snd_wnd <= snd_space ? tc->snd_wnd : 0;
892  }
893 
894  /* If not snd_wnd constrained and we can't write at least a segment,
895  * don't try at all */
896  if (PREDICT_FALSE (snd_space < tc->snd_mss))
897  return snd_space < tc->cwnd ? 0 : snd_space;
898 
899  /* round down to mss multiple */
900  return snd_space - (snd_space % tc->snd_mss);
901 }
902 
903 /**
904  * Compute tx window session is allowed to fill.
905  *
906  * Takes into account available send space, snd_mss and the congestion
907  * state of the connection. If possible, the value returned is a multiple
908  * of snd_mss.
909  *
910  * @param tc tcp connection
911  * @return number of bytes session is allowed to write
912  */
913 static inline u32
915 {
916  int snd_space;
917 
919  || tc->state == TCP_STATE_CLOSED))
920  return 0;
921 
922  snd_space = tcp_available_output_snd_space (tc);
923 
924  /* If we got dupacks or sacked bytes but we're not yet in recovery, try
925  * to force the peer to send enough dupacks to start retransmitting as
926  * per Limited Transmit (RFC3042)
927  */
928  if (PREDICT_FALSE (tc->rcv_dupacks || tc->sack_sb.sacked_bytes))
929  {
930  int snt_limited, n_pkts;
931 
932  n_pkts = tcp_opts_sack_permitted (&tc->rcv_opts)
933  ? tc->sack_sb.reorder - 1 : 2;
934 
935  if ((seq_lt (tc->limited_transmit, tc->snd_nxt - n_pkts * tc->snd_mss)
936  || seq_gt (tc->limited_transmit, tc->snd_nxt)))
937  tc->limited_transmit = tc->snd_nxt;
938 
939  ASSERT (seq_leq (tc->limited_transmit, tc->snd_nxt));
940 
941  snt_limited = tc->snd_nxt - tc->limited_transmit;
942  snd_space = clib_max (n_pkts * tc->snd_mss - snt_limited, 0);
943  }
944  return tcp_round_snd_space (tc, snd_space);
945 }
946 
947 u32
949 {
950  return tcp_snd_space_inline (tc);
951 }
952 
953 static int
956 {
957  tcp_connection_t *tc = (tcp_connection_t *) trans_conn;
958 
959  /* Ensure snd_mss does accurately reflect the amount of data we can push
960  * in a segment. This also makes sure that options are updated according to
961  * the current state of the connection. */
963 
964  if (PREDICT_FALSE (tc->cfg_flags & TCP_CFG_F_TSO))
966  else
967  sp->snd_mss = tc->snd_mss;
968 
970  tc->snd_wnd - (tc->snd_nxt - tc->snd_una));
971 
972  ASSERT (seq_geq (tc->snd_nxt, tc->snd_una));
973  /* This still works if fast retransmit is on */
974  sp->tx_offset = tc->snd_nxt - tc->snd_una;
975 
976  sp->flags = sp->snd_space ? 0 : TRANSPORT_SND_F_DESCHED;
977 
978  return 0;
979 }
980 
981 static void
983 {
984  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
985 
986  switch (tc->state)
987  {
988  case TCP_STATE_CLOSE_WAIT:
990  /* App never returned with a close */
991  if (!(tc->flags & TCP_CONN_FINPNDG))
992  {
993  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
994  session_transport_closed_notify (&tc->connection);
995  tcp_program_cleanup (wrk, tc);
996  tcp_worker_stats_inc (wrk, to_closewait, 1);
997  break;
998  }
999 
1000  /* Send FIN either way and switch to LAST_ACK. */
1001  tcp_cong_recovery_off (tc);
1002  /* Make sure we don't try to send unsent data */
1003  tc->snd_nxt = tc->snd_una;
1004  tcp_send_fin (tc);
1005  tcp_connection_set_state (tc, TCP_STATE_LAST_ACK);
1006  session_transport_closed_notify (&tc->connection);
1007 
1008  /* Make sure we don't wait in LAST ACK forever */
1009  tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
1010  tcp_cfg.lastack_time);
1011  tcp_worker_stats_inc (wrk, to_closewait2, 1);
1012 
1013  /* Don't delete the connection yet */
1014  break;
1015  case TCP_STATE_FIN_WAIT_1:
1017  if (tc->flags & TCP_CONN_FINPNDG)
1018  {
1019  /* If FIN pending, we haven't sent everything, but we did try.
1020  * Notify session layer that transport is closed. */
1021  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1022  tcp_send_reset (tc);
1023  tcp_program_cleanup (wrk, tc);
1024  }
1025  else
1026  {
1027  /* We've sent the fin but no progress. Close the connection and
1028  * to make sure everything is flushed, setup a cleanup timer */
1029  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1030  tcp_program_cleanup (wrk, tc);
1031  }
1032  session_transport_closed_notify (&tc->connection);
1033  tcp_worker_stats_inc (wrk, to_finwait1, 1);
1034  break;
1035  case TCP_STATE_LAST_ACK:
1037  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1038  session_transport_closed_notify (&tc->connection);
1039  tcp_program_cleanup (wrk, tc);
1040  tcp_worker_stats_inc (wrk, to_lastack, 1);
1041  break;
1042  case TCP_STATE_CLOSING:
1044  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1045  session_transport_closed_notify (&tc->connection);
1046  tcp_program_cleanup (wrk, tc);
1047  tcp_worker_stats_inc (wrk, to_closing, 1);
1048  break;
1049  case TCP_STATE_FIN_WAIT_2:
1050  tcp_send_reset (tc);
1052  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1053  session_transport_closed_notify (&tc->connection);
1054  tcp_program_cleanup (wrk, tc);
1055  tcp_worker_stats_inc (wrk, to_finwait2, 1);
1056  break;
1057  case TCP_STATE_TIME_WAIT:
1058  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1059  tcp_program_cleanup (wrk, tc);
1060  break;
1061  default:
1062  clib_warning ("waitclose in state: %U", format_tcp_state, tc->state);
1063  break;
1064  }
1065 }
1066 
1067 /* *INDENT-OFF* */
1069 {
1074 };
1075 /* *INDENT-ON* */
1076 
1077 static void
1079 {
1080  u32 n_timers, connection_index, timer_id, thread_index, timer_handle;
1081  tcp_connection_t *tc;
1082  int i;
1083 
1084  if (!(n_timers = clib_fifo_elts (wrk->pending_timers)))
1085  return;
1086 
1087  thread_index = wrk->vm->thread_index;
1088  for (i = 0; i < clib_min (n_timers, wrk->max_timers_per_loop); i++)
1089  {
1090  clib_fifo_sub1 (wrk->pending_timers, timer_handle);
1091  connection_index = timer_handle & 0x0FFFFFFF;
1092  timer_id = timer_handle >> 28;
1093 
1094  if (PREDICT_TRUE (timer_id != TCP_TIMER_RETRANSMIT_SYN))
1095  tc = tcp_connection_get (connection_index, thread_index);
1096  else
1097  tc = tcp_half_open_connection_get (connection_index);
1098 
1099  if (PREDICT_FALSE (!tc))
1100  continue;
1101 
1102  /* Skip if the timer is not pending. Probably it was reset while
1103  * wating for dispatch */
1104  if (PREDICT_FALSE (!(tc->pending_timers & (1 << timer_id))))
1105  continue;
1106 
1107  tc->pending_timers &= ~(1 << timer_id);
1108 
1109  /* Skip timer if it was rearmed while pending dispatch */
1110  if (PREDICT_FALSE (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID))
1111  continue;
1112 
1113  (*timer_expiration_handlers[timer_id]) (tc);
1114  }
1115 
1116  if (thread_index == 0 && clib_fifo_elts (wrk->pending_timers))
1118 }
1119 
1120 static void
1122 {
1123  u32 thread_index = wrk->vm->thread_index;
1124  tcp_cleanup_req_t *req;
1125  tcp_connection_t *tc;
1126 
1127  while (clib_fifo_elts (wrk->pending_cleanups))
1128  {
1129  req = clib_fifo_head (wrk->pending_cleanups);
1130  if (req->free_time > now)
1131  break;
1132  clib_fifo_sub2 (wrk->pending_cleanups, req);
1133  tc = tcp_connection_get (req->connection_index, thread_index);
1134  if (PREDICT_FALSE (!tc))
1135  continue;
1136  session_transport_delete_notify (&tc->connection);
1138  }
1139 }
1140 
1141 static void
1142 tcp_update_time (f64 now, u8 thread_index)
1143 {
1144  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1145 
1146  tcp_set_time_now (wrk);
1147  tcp_handle_cleanups (wrk, now);
1148  tw_timer_expire_timers_16t_2w_512sl (&wrk->timer_wheel, now);
1150 }
1151 
1152 static void
1154 {
1155  tcp_connection_t *tc = (tcp_connection_t *) tconn;
1156  if (tc->flags & TCP_CONN_PSH_PENDING)
1157  return;
1158  tc->flags |= TCP_CONN_PSH_PENDING;
1159  tc->psh_seq = tc->snd_una + transport_max_tx_dequeue (tconn) - 1;
1160 }
1161 
1162 /* *INDENT-OFF* */
1164  .enable = vnet_tcp_enable_disable,
1165  .start_listen = tcp_session_bind,
1166  .stop_listen = tcp_session_unbind,
1167  .push_header = tcp_session_push_header,
1168  .get_connection = tcp_session_get_transport,
1169  .get_listener = tcp_session_get_listener,
1170  .get_half_open = tcp_half_open_session_get_transport,
1171  .connect = tcp_session_open,
1172  .close = tcp_session_close,
1173  .cleanup = tcp_session_cleanup,
1174  .cleanup_ho = tcp_session_cleanup_ho,
1175  .reset = tcp_session_reset,
1176  .send_params = tcp_session_send_params,
1177  .update_time = tcp_update_time,
1178  .flush_data = tcp_session_flush_data,
1179  .custom_tx = tcp_session_custom_tx,
1180  .format_connection = format_tcp_session,
1181  .format_listener = format_tcp_listener_session,
1182  .format_half_open = format_tcp_half_open_session,
1183  .transport_options = {
1184  .name = "tcp",
1185  .short_name = "T",
1186  .tx_type = TRANSPORT_TX_PEEK,
1187  .service_type = TRANSPORT_SERVICE_VC,
1188  },
1189 };
1190 /* *INDENT-ON* */
1191 
1192 void
1194 {
1195  if (!transport_connection_is_tx_paced (&tc->connection))
1196  return;
1197 
1198  f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
1199 
1200  transport_connection_tx_pacer_update (&tc->connection,
1202  srtt * CLIB_US_TIME_FREQ);
1203 }
1204 
1205 void
1207  u32 start_bucket)
1208 {
1209  f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
1210  transport_connection_tx_pacer_reset (&tc->connection,
1212  start_bucket,
1213  srtt * CLIB_US_TIME_FREQ);
1214 }
1215 
1216 void
1218 {
1219  if (tcp_in_cong_recovery (tc) || tcp_snd_space_inline (tc))
1220  transport_connection_reschedule (&tc->connection);
1221 }
1222 
1223 static void
1225 {
1226  u32 thread_index = vlib_get_thread_index (), n_left, max_per_loop;
1227  u32 connection_index, timer_id, n_expired, max_loops;
1228  tcp_worker_ctx_t *wrk;
1229  tcp_connection_t *tc;
1230  int i;
1231 
1232  wrk = tcp_get_worker (thread_index);
1233  n_expired = vec_len (expired_timers);
1234  tcp_worker_stats_inc (wrk, timer_expirations, n_expired);
1235  n_left = clib_fifo_elts (wrk->pending_timers);
1236 
1237  /*
1238  * Invalidate all timer handles before dispatching. This avoids dangling
1239  * index references to timer wheel pool entries that have been freed.
1240  */
1241  for (i = 0; i < n_expired; i++)
1242  {
1243  connection_index = expired_timers[i] & 0x0FFFFFFF;
1244  timer_id = expired_timers[i] >> 28;
1245 
1246  if (timer_id != TCP_TIMER_RETRANSMIT_SYN)
1247  tc = tcp_connection_get (connection_index, thread_index);
1248  else
1249  tc = tcp_half_open_connection_get (connection_index);
1250 
1251  TCP_EVT (TCP_EVT_TIMER_POP, connection_index, timer_id);
1252 
1253  tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID;
1254  tc->pending_timers |= (1 << timer_id);
1255  }
1256 
1257  clib_fifo_add (wrk->pending_timers, expired_timers, n_expired);
1258 
1259  max_loops = clib_max (1, 0.5 * TCP_TIMER_TICK * wrk->vm->loops_per_second);
1260  max_per_loop = clib_max ((n_left + n_expired) / max_loops, 10);
1261  max_per_loop = clib_min (max_per_loop, VLIB_FRAME_SIZE);
1262  wrk->max_timers_per_loop = clib_max (n_left ? wrk->max_timers_per_loop : 0,
1263  max_per_loop);
1264 
1265  if (thread_index == 0)
1267 }
1268 
1269 static void
1271 {
1273  tw_timer_wheel_16t_2w_512sl_t *tw;
1274  /* *INDENT-OFF* */
1275  foreach_vlib_main (({
1276  tw = &tm->wrk_ctx[ii].timer_wheel;
1277  tw_timer_wheel_init_16t_2w_512sl (tw, tcp_expired_timers_dispatch,
1278  TCP_TIMER_TICK, ~0);
1279  tw->last_run_time = vlib_time_now (vm);
1280  }));
1281  /* *INDENT-ON* */
1282 }
1283 
1284 static void
1286 {
1287  u32 default_seed = random_default_seed ();
1288  u64 time_now = clib_cpu_time_now ();
1289 
1290  tm->iss_seed.first = (u64) random_u32 (&default_seed) << 32;
1291  tm->iss_seed.second = random_u64 (&time_now);
1292 }
1293 
1294 static clib_error_t *
1296 {
1298  u32 num_threads, n_workers, prealloc_conn_per_wrk;
1299  tcp_connection_t *tc __attribute__ ((unused));
1300  tcp_main_t *tm = vnet_get_tcp_main ();
1301  tcp_worker_ctx_t *wrk;
1302  clib_error_t *error = 0;
1303  int thread;
1304 
1305  if ((error = vlib_call_init_function (vm, ip_main_init)))
1306  return error;
1307  if ((error = vlib_call_init_function (vm, ip4_lookup_init)))
1308  return error;
1309  if ((error = vlib_call_init_function (vm, ip6_lookup_init)))
1310  return error;
1311 
1312  /*
1313  * Registrations
1314  */
1315 
1316  ip4_register_protocol (IP_PROTOCOL_TCP, tcp4_input_node.index);
1317  ip6_register_protocol (IP_PROTOCOL_TCP, tcp6_input_node.index);
1318 
1319  /*
1320  * Initialize data structures
1321  */
1322 
1323  num_threads = 1 /* main thread */ + vtm->n_threads;
1324  vec_validate (tm->wrk_ctx, num_threads - 1);
1325  n_workers = num_threads == 1 ? 1 : vtm->n_threads;
1326  prealloc_conn_per_wrk = tcp_cfg.preallocated_connections / n_workers;
1327 
1328  wrk = &tm->wrk_ctx[0];
1330  tcp4_output_node.index);
1332  tcp6_output_node.index);
1333 
1334  for (thread = 0; thread < num_threads; thread++)
1335  {
1336  wrk = &tm->wrk_ctx[thread];
1337 
1338  vec_validate (wrk->pending_deq_acked, 255);
1339  vec_validate (wrk->pending_disconnects, 255);
1340  vec_validate (wrk->pending_resets, 255);
1344  wrk->vm = vlib_mains[thread];
1345  wrk->max_timers_per_loop = 10;
1346 
1347  if (thread > 0)
1348  {
1349  wrk->tco_next_node[0] = tm->wrk_ctx[0].tco_next_node[0];
1350  wrk->tco_next_node[1] = tm->wrk_ctx[0].tco_next_node[1];
1351  }
1352 
1353  /*
1354  * Preallocate connections. Assume that thread 0 won't
1355  * use preallocated threads when running multi-core
1356  */
1357  if ((thread > 0 || num_threads == 1) && prealloc_conn_per_wrk)
1358  pool_init_fixed (wrk->connections, prealloc_conn_per_wrk);
1359  }
1360 
1361  /*
1362  * Use a preallocated half-open connection pool?
1363  */
1364  if (tcp_cfg.preallocated_half_open_connections)
1365  pool_init_fixed (tm->half_open_connections,
1366  tcp_cfg.preallocated_half_open_connections);
1367 
1368  if (num_threads > 1)
1369  {
1370  clib_spinlock_init (&tm->half_open_lock);
1371  }
1372 
1375 
1376  tm->bytes_per_buffer = vlib_buffer_get_default_data_size (vm);
1377  tm->cc_last_type = TCP_CC_LAST;
1378 
1379  tm->ipl_next_node[0] = vlib_node_get_next (vm, session_queue_node.index,
1380  ip4_lookup_node.index);
1381  tm->ipl_next_node[1] = vlib_node_get_next (vm, session_queue_node.index,
1382  ip6_lookup_node.index);
1383  return error;
1384 }
1385 
1386 clib_error_t *
1388 {
1389  if (is_en)
1390  {
1391  if (tcp_main.is_enabled)
1392  return 0;
1393 
1394  return tcp_main_enable (vm);
1395  }
1396  else
1397  {
1398  tcp_main.is_enabled = 0;
1399  }
1400 
1401  return 0;
1402 }
1403 
1404 void
1405 tcp_punt_unknown (vlib_main_t * vm, u8 is_ip4, u8 is_add)
1406 {
1407  tcp_main_t *tm = &tcp_main;
1408  if (is_ip4)
1409  tm->punt_unknown4 = is_add;
1410  else
1411  tm->punt_unknown6 = is_add;
1412 }
1413 
1414 /**
1415  * Initialize default values for tcp parameters
1416  */
1417 static void
1419 {
1420  /* Initial wnd for SYN. Fifos are not allocated at that point so use some
1421  * predefined value. For SYN-ACK we still want the scale to be computed in
1422  * the same way */
1423  tcp_cfg.max_rx_fifo = 32 << 20;
1424  tcp_cfg.min_rx_fifo = 4 << 10;
1425 
1426  tcp_cfg.default_mtu = 1500;
1427  tcp_cfg.initial_cwnd_multiplier = 0;
1428  tcp_cfg.enable_tx_pacing = 1;
1429  tcp_cfg.allow_tso = 0;
1430  tcp_cfg.csum_offload = 1;
1431  tcp_cfg.cc_algo = TCP_CC_CUBIC;
1432  tcp_cfg.rwnd_min_update_ack = 1;
1433 
1434  /* Time constants defined as timer tick (100ms) multiples */
1435  tcp_cfg.delack_time = 1; /* 0.1s */
1436  tcp_cfg.closewait_time = 20; /* 2s */
1437  tcp_cfg.timewait_time = 100; /* 10s */
1438  tcp_cfg.finwait1_time = 600; /* 60s */
1439  tcp_cfg.lastack_time = 300; /* 30s */
1440  tcp_cfg.finwait2_time = 300; /* 30s */
1441  tcp_cfg.closing_time = 300; /* 30s */
1442  tcp_cfg.cleanup_time = 0.1; /* 100ms */
1443 }
1444 
1445 static clib_error_t *
1447 {
1448  tcp_main_t *tm = vnet_get_tcp_main ();
1449  ip_main_t *im = &ip_main;
1450  ip_protocol_info_t *pi;
1451 
1452  /* Session layer, and by implication tcp, are disabled by default */
1453  tm->is_enabled = 0;
1454 
1455  /* Register with IP for header parsing */
1456  pi = ip_get_protocol_info (im, IP_PROTOCOL_TCP);
1457  if (pi == 0)
1458  return clib_error_return (0, "TCP protocol info AWOL");
1461 
1462  /* Register as transport with session layer */
1463  transport_register_protocol (TRANSPORT_PROTO_TCP, &tcp_proto,
1465  transport_register_protocol (TRANSPORT_PROTO_TCP, &tcp_proto,
1467 
1469 
1470  tm->cc_algo_by_name = hash_create_string (0, sizeof (uword));
1471 
1472  return 0;
1473 }
1474 
1476 
1477 /*
1478  * fd.io coding-style-patch-verification: ON
1479  *
1480  * Local Variables:
1481  * eval: (c-set-style "gnu")
1482  * End:
1483  */
static void tcp_session_close(u32 conn_index, u32 thread_index)
Definition: tcp.c:429
u32 * pending_timers
Definition: tcp.h:104
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:509
void dpo_stack_from_node(u32 child_node_index, dpo_id_t *dpo, const dpo_id_t *parent)
Stack one DPO object on another, and thus establish a child parent relationship.
Definition: dpo.c:531
fib_protocol_t fp_proto
protocol type
Definition: fib_types.h:212
#define ENDPOINT_INVALID_INDEX
static void tcp_handle_cleanups(tcp_worker_ctx_t *wrk, clib_time_type_t now)
Definition: tcp.c:1121
static void tcp_add_del_adjacency(tcp_connection_t *tc, u8 is_add)
Definition: tcp.c:58
#define TCP_TIMER_HANDLE_INVALID
Definition: tcp_types.h:81
An entry in a FIB table.
Definition: fib_entry.h:305
#define clib_min(x, y)
Definition: clib.h:327
#define clib_fifo_head(v)
Definition: fifo.h:254
static void tcp_half_open_connection_free(tcp_connection_t *tc)
Cleanup half-open connection.
Definition: tcp.c:189
static u8 * format_tcp_listener_session(u8 *s, va_list *args)
Definition: tcp.c:838
f64 clib_time_type_t
Definition: time.h:203
tcp_cleanup_req_t * pending_cleanups
Definition: tcp.h:115
u32 * pending_disconnects
vector of pending disconnect notifications
Definition: tcp.h:86
void ip_copy(ip46_address_t *dst, ip46_address_t *src, u8 is_ip4)
Definition: ip.c:81
static clib_time_type_t transport_time_now(u32 thread_index)
Definition: session.h:538
static uword random_default_seed(void)
Default random seed (unix/linux user-mode)
Definition: random.h:91
void ip6_register_protocol(u32 protocol, u32 node_index)
Definition: ip6_forward.c:1664
#define clib_fifo_sub2(f, p)
Definition: fifo.h:232
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
Definition: tcp_inlines.h:30
static uword clib_fifo_elts(void *v)
Definition: fifo.h:66
static clib_error_t * ip4_lookup_init(vlib_main_t *vm)
Definition: ip4_forward.c:1148
void session_queue_run_on_main_thread(vlib_main_t *vm)
Definition: session.c:1650
vlib_node_registration_t tcp4_output_node
(constructor) VLIB_REGISTER_NODE (tcp4_output_node)
Definition: tcp_output.c:2303
#define THZ
TCP tick frequency.
Definition: tcp_types.h:26
void transport_share_local_endpoint(u8 proto, ip46_address_t *lcl_ip, u16 port)
Definition: transport.c:456
#define tcp_opts_tstamp(_to)
Definition: tcp_packet.h:156
#define PREDICT_TRUE(x)
Definition: clib.h:121
static transport_connection_t * tcp_half_open_session_get_transport(u32 conn_index)
Definition: tcp.c:869
unsigned long u64
Definition: types.h:89
void * ip_interface_get_first_ip(u32 sw_if_index, u8 is_ip4)
Definition: ip_interface.c:174
void ip_set(ip46_address_t *dst, void *src, u8 is_ip4)
Definition: ip.c:93
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static u32 ip4_compute_flow_hash(const ip4_header_t *ip, flow_hash_config_t flow_hash_config)
Definition: ip4.h:310
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
void session_transport_delete_notify(transport_connection_t *tc)
Notification from transport that connection is being deleted.
Definition: session.c:970
static int tcp_alloc_custom_local_endpoint(tcp_main_t *tm, ip46_address_t *lcl_addr, u16 *lcl_port, u8 is_ip4)
Definition: tcp.c:733
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:333
static tcp_connection_t * tcp_half_open_connection_new(void)
Definition: tcp.c:225
flow_hash_config_t lb_hash_config
the hash config to use when selecting a bucket.
Definition: load_balance.h:161
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:127
struct _tcp_main tcp_main_t
u32 thread_index
Definition: main.h:249
void tcp_connection_timers_reset(tcp_connection_t *tc)
Stop all connection timers.
Definition: tcp.c:493
struct _tcp_connection tcp_connection_t
f64 loops_per_second
Definition: main.h:290
vlib_node_registration_t tcp6_output_node
(constructor) VLIB_REGISTER_NODE (tcp6_output_node)
Definition: tcp_output.c:2323
void dpo_copy(dpo_id_t *dst, const dpo_id_t *src)
atomic copy a data-plane object.
Definition: dpo.c:262
static u64 clib_cpu_time_now(void)
Definition: time.h:81
static u32 tcp_set_time_now(tcp_worker_ctx_t *wrk)
Definition: tcp_inlines.h:219
timer_expiration_handler tcp_timer_retransmit_handler
static u64 clib_xxhash(u64 key)
Definition: xxhash.h:58
u32 tcp_session_push_header(transport_connection_t *tconn, vlib_buffer_t *b)
Definition: tcp_output.c:986
u32 * pending_resets
vector of pending reset notifications
Definition: tcp.h:89
void tcp_update_burst_snd_vars(tcp_connection_t *tc)
Update burst send vars.
Definition: tcp_output.c:298
#define hash_set_mem(h, key, value)
Definition: hash.h:275
vlib_main_t * vm
Definition: in2out_ed.c:1582
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
#define TCP_TICK
TCP tick period (s)
Definition: tcp_types.h:25
static u64 random_u64(u64 *seed)
64-bit random number generator Again, constants courtesy of Donald Knuth.
Definition: random.h:126
Definition: ip.h:107
vl_api_prefix_t prefix
Definition: ip.api:144
static void tcp_initialize_iss_seed(tcp_main_t *tm)
Definition: tcp.c:1285
void ip4_register_protocol(u32 protocol, u32 node_index)
Definition: ip4_forward.c:1933
static void tcp_initialize_timer_wheels(tcp_main_t *tm)
Definition: tcp.c:1270
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:252
struct _tcp_header tcp_header_t
fib_protocol_t nh_proto
Definition: tcp.c:32
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
Definition: tcp.c:209
vlib_node_registration_t session_queue_node
(constructor) VLIB_REGISTER_NODE (session_queue_node)
vlib_main_t ** vlib_mains
Definition: buffer.c:332
#define tcp_in_cong_recovery(tc)
Definition: tcp_types.h:429
u32 * pending_deq_acked
vector of pending ack dequeues
Definition: tcp.h:83
unsigned char u8
Definition: types.h:56
enum fib_protocol_t_ fib_protocol_t
Protocol Type.
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
vlib_node_registration_t ip4_lookup_node
(constructor) VLIB_REGISTER_NODE (ip4_lookup_node)
Definition: ip4_forward.c:104
tcp_cc_algorithm_t * tcp_cc_algo_get(tcp_cc_algorithm_type_e type)
Definition: tcp.c:96
tcp_connection_t * connections
worker&#39;s pool of connections
Definition: tcp.h:80
struct _tcp_cc_algorithm tcp_cc_algorithm_t
Definition: tcp_types.h:256
unformat_function_t * unformat_pg_edit
Definition: ip.h:88
format_function_t format_tcp_connection
Definition: tcp.h:351
void tcp_init_snd_vars(tcp_connection_t *tc)
Initialize connection send variables.
Definition: tcp.c:669
#define tcp_cfg
Definition: tcp.h:271
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
u16 lb_n_buckets_minus_1
number of buckets in the load-balance - 1.
Definition: load_balance.h:121
void tcp_connection_timers_init(tcp_connection_t *tc)
Initialize all connection timers as invalid.
Definition: tcp.c:476
#define seq_leq(_s1, _s2)
Definition: tcp_packet.h:178
tcp_main_t tcp_main
Definition: tcp.c:28
static void tcp_connection_unbind(u32 listener_index)
Definition: tcp.c:152
Aggregate type for a prefix.
Definition: fib_types.h:203
#define clib_error_return(e, args...)
Definition: error.h:99
void() timer_expiration_handler(tcp_connection_t *tc)
Definition: tcp.h:29
void adj_unlock(adj_index_t adj_index)
Release a reference counting lock on the adjacency.
Definition: adj.c:348
timer_expiration_handler tcp_timer_retransmit_syn_handler
pthread_t thread[MAX_CONNS]
Definition: main.c:142
int transport_alloc_local_endpoint(u8 proto, transport_endpoint_cfg_t *rmt_cfg, ip46_address_t *lcl_addr, u16 *lcl_port)
Definition: transport.c:565
void session_half_open_delete_notify(transport_proto_t tp, session_handle_t ho_handle)
Definition: session.c:292
u32 tcp_snd_space(tcp_connection_t *tc)
Definition: tcp.c:948
unsigned int u32
Definition: types.h:88
static void tcp_timer_reset(tcp_timer_wheel_t *tw, tcp_connection_t *tc, u8 timer_id)
Definition: tcp_timer.h:31
u32 fib_table_find(fib_protocol_t proto, u32 table_id)
Get the index of the FIB for a Table-ID.
Definition: fib_table.c:1097
u16 fp_len
The mask length.
Definition: fib_types.h:207
#define vlib_call_init_function(vm, x)
Definition: init.h:270
#define VLIB_FRAME_SIZE
Definition: node.h:377
fib_node_index_t fib_table_lookup(u32 fib_index, const fib_prefix_t *prefix)
Perfom a longest prefix match in the non-forwarding table.
Definition: fib_table.c:68
static u32 tcp_connection_bind(u32 session_index, transport_endpoint_t *lcl)
Definition: tcp.c:111
#define tcp_in_fastrecovery(tc)
Definition: tcp_types.h:419
#define hash_create_string(elts, value_bytes)
Definition: hash.h:690
static void tcp_timer_set(tcp_timer_wheel_t *tw, tcp_connection_t *tc, u8 timer_id, u32 interval)
Definition: tcp_timer.h:21
static void tcp_cc_init(tcp_connection_t *tc)
Definition: tcp.c:72
struct _transport_proto_vft transport_proto_vft_t
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
vl_api_fib_path_type_t type
Definition: fib_types.api:123
static u32 tcp_session_unbind(u32 listener_index)
Definition: tcp.c:169
static timer_expiration_handler * timer_expiration_handlers[TCP_N_TIMERS]
Definition: tcp.c:1068
void tcp_bt_init(tcp_connection_t *tc)
Byte tracker initialize.
Definition: tcp_bt.c:663
The identity of a DPO is a combination of its type and its instance number/index of objects of that t...
Definition: dpo.h:170
static void tcp_dispatch_pending_timers(tcp_worker_ctx_t *wrk)
Definition: tcp.c:1078
static clib_error_t * tcp_main_enable(vlib_main_t *vm)
Definition: tcp.c:1295
vlib_main_t * vm
convenience pointer to this thread&#39;s vlib main
Definition: tcp.h:92
#define ADJ_INDEX_INVALID
Invalid ADJ index - used when no adj is known likewise blazoned capitals INVALID speak volumes where ...
Definition: adj_types.h:36
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
timer_expiration_handler tcp_timer_persist_handler
#define CLIB_US_TIME_FREQ
Definition: time.h:207
static transport_connection_t * tcp_session_get_transport(u32 conn_index, u32 thread_index)
Definition: tcp.c:860
static ip_protocol_info_t * ip_get_protocol_info(ip_main_t *im, u32 protocol)
Definition: ip.h:134
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
Definition: tcp_inlines.h:67
#define clib_fifo_sub1(f, e)
Definition: fifo.h:224
ip46_address_t fp_addr
The address type is not deriveable from the fp_addr member.
Definition: fib_types.h:226
void tcp_connection_tx_pacer_reset(tcp_connection_t *tc, u32 window, u32 start_bucket)
Definition: tcp.c:1206
dpo_type_t dpoi_type
the type
Definition: dpo.h:174
format_function_t * format_header
Definition: ip.h:79
tcp_connection_t * tcp_connection_alloc_w_base(u8 thread_index, tcp_connection_t *base)
Definition: tcp.c:309
static const dpo_id_t * load_balance_get_bucket_i(const load_balance_t *lb, u32 bucket)
Definition: load_balance.h:229
format_function_t format_tcp_connection_id
Definition: tcp.h:352
static void tcp_init_rcv_mss(tcp_connection_t *tc)
Initialize max segment size we&#39;re able to process.
Definition: tcp.c:628
void tcp_send_syn(tcp_connection_t *tc)
Send SYN.
Definition: tcp_output.c:799
unsigned short u16
Definition: types.h:57
#define TCP_TIMER_TICK
Timer tick in seconds.
Definition: tcp_types.h:83
tcp_connection_t * tcp_connection_alloc(u8 thread_index)
Definition: tcp.c:296
void tcp_bt_cleanup(tcp_connection_t *tc)
Byte tracker cleanup.
Definition: tcp_bt.c:652
load-balancing over a choice of [un]equal cost paths
Definition: dpo.h:102
static transport_connection_t * tcp_session_get_listener(u32 listener_index)
Definition: tcp.c:176
static u32 transport_max_rx_dequeue(transport_connection_t *tc)
Definition: session.h:510
void tcp_connection_tx_pacer_update(tcp_connection_t *tc)
Definition: tcp.c:1193
static u32 ip6_compute_flow_hash(const ip6_header_t *ip, flow_hash_config_t flow_hash_config)
Definition: ip6.h:382
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:302
static void tcp_expired_timers_dispatch(u32 *expired_timers)
Definition: tcp.c:1224
void transport_connection_tx_pacer_init(transport_connection_t *tc, u64 rate_bytes_per_sec, u32 initial_bucket)
Initialize tx pacer for connection.
Definition: transport.c:712
static void tcp_session_cleanup(u32 conn_index, u32 thread_index)
Definition: tcp.c:437
The FIB DPO provieds;.
Definition: load_balance.h:106
tcp_timer_wheel_t timer_wheel
worker timer wheel
Definition: tcp.h:118
#define PREDICT_FALSE(x)
Definition: clib.h:120
#define always_inline
Definition: ipsec.h:28
clib_error_t * vnet_tcp_enable_disable(vlib_main_t *vm, u8 is_en)
Definition: tcp.c:1387
static void tcp_session_cleanup_ho(u32 conn_index)
Definition: tcp.c:448
static u8 transport_connection_is_tx_paced(transport_connection_t *tc)
Check if transport connection is paced.
Definition: transport.h:317
static u64 tcp_cc_get_pacing_rate(tcp_connection_t *tc)
Definition: tcp_cc.h:68
void transport_connection_tx_pacer_reset(transport_connection_t *tc, u64 rate_bytes_per_sec, u32 start_bucket, clib_us_time_t rtt)
Definition: transport.c:695
#define foreach_vlib_main(body)
Definition: threads.h:242
static u32 tcp_available_output_snd_space(const tcp_connection_t *tc)
Definition: tcp_inlines.h:156
#define TCP_OPTION_LEN_TIMESTAMP
Definition: tcp_packet.h:167
static int tcp_session_send_params(transport_connection_t *trans_conn, transport_send_params_t *sp)
Definition: tcp.c:954
#define TCP_DBG(_fmt, _args...)
Definition: tcp_debug.h:146
void tcp_program_cleanup(tcp_worker_ctx_t *wrk, tcp_connection_t *tc)
Definition: tcp.c:335
void tcp_connection_free(tcp_connection_t *tc)
Definition: tcp.c:322
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:96
#define pool_free(p)
Free a pool.
Definition: pool.h:427
void transport_connection_reschedule(transport_connection_t *tc)
Definition: transport.c:756
clib_error_t * ip_main_init(vlib_main_t *vm)
Definition: ip_init.c:45
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:219
ip_main_t ip_main
Definition: ip_init.c:42
static void tcp_add_del_adj_cb(tcp_add_del_adj_args_t *args)
Definition: tcp.c:40
void scoreboard_init(sack_scoreboard_t *sb)
Definition: tcp_sack.c:268
void tcp_connection_close(tcp_connection_t *tc)
Begin connection closing procedure.
Definition: tcp.c:360
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
Definition: tcp_output.c:740
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
virtual circuit service
static void tcp_session_flush_data(transport_connection_t *tconn)
Definition: tcp.c:1153
format_function_t format_tcp_state
Definition: tcp.h:347
#define clib_warning(format, args...)
Definition: error.h:59
format_function_t format_tcp_header
Definition: format.h:100
struct _transport_connection transport_connection_t
u32 fib_node_index_t
A typedef of a node index.
Definition: fib_types.h:30
vlib_node_registration_t tcp6_input_node
(constructor) VLIB_REGISTER_NODE (tcp6_input_node)
Definition: tcp_input.c:3021
fib_node_index_t tcp_lookup_rmt_in_fib(tcp_connection_t *tc)
fib_entry_t * fib_entry_get(fib_node_index_t index)
Definition: fib_entry.c:51
#define pool_init_fixed(pool, max_elts)
initialize a fixed-size, preallocated pool
Definition: pool.h:86
void transport_endpoint_cleanup(u8 proto, ip46_address_t *lcl_ip, u16 port)
Definition: transport.c:421
void transport_register_protocol(transport_proto_t transport_proto, const transport_proto_vft_t *vft, fib_protocol_t fib_proto, u32 output_node)
Register transport virtual function table.
Definition: transport.c:246
transport_snd_flags_t flags
Definition: transport.h:61
static u32 transport_max_tx_dequeue(transport_connection_t *tc)
Definition: session.h:503
#define seq_geq(_s1, _s2)
Definition: tcp_packet.h:180
vlib_node_registration_t ip6_lookup_node
(constructor) VLIB_REGISTER_NODE (ip6_lookup_node)
Definition: ip6_forward.c:741
#define pool_put_index(p, i)
Free pool element with given index.
Definition: pool.h:331
#define ASSERT(truth)
static int tcp_session_open(transport_endpoint_cfg_t *rmt)
Definition: tcp.c:760
tcp_cc_algorithm_type_e tcp_cc_algo_new_type(const tcp_cc_algorithm_t *vft)
Register new cc algo type.
Definition: tcp.c:103
unformat_function_t unformat_pg_tcp_header
Definition: format.h:102
static void tcp_configuration_init(void)
Initialize default values for tcp parameters.
Definition: tcp.c:1418
enum vnet_link_t_ vnet_link_t
Link Type: A description of the protocol of packets on the link.
static load_balance_t * load_balance_get(index_t lbi)
Definition: load_balance.h:220
static void tcp_init_mss(tcp_connection_t *tc)
Definition: tcp.c:641
uword vlib_node_get_next(vlib_main_t *vm, uword node_index, uword next_node_index)
Definition: node.c:151
static void tcp_cong_recovery_off(tcp_connection_t *tc)
Definition: tcp_types.h:433
u32 max_timers_per_loop
Definition: tcp.h:98
static u32 tcp_generate_random_iss(tcp_connection_t *tc)
Generate random iss as per rfc6528.
Definition: tcp.c:603
clib_time_type_t free_time
Definition: tcp.h:71
reliable transport protos
#define seq_gt(_s1, _s2)
Definition: tcp_packet.h:179
static uword ip6_address_is_link_local_unicast(const ip6_address_t *a)
Definition: ip6_packet.h:253
void tcp_cc_algo_register(tcp_cc_algorithm_type_e type, const tcp_cc_algorithm_t *vft)
Register exiting cc algo type.
Definition: tcp.c:85
#define clib_max(x, y)
Definition: clib.h:320
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define TCP_MAX_GSO_SZ
Definition: tcp_types.h:36
static clib_error_t * tcp_init(vlib_main_t *vm)
Definition: tcp.c:1446
void transport_connection_tx_pacer_update(transport_connection_t *tc, u64 bytes_per_sec, clib_us_time_t rtt)
Update tx pacer pacing rate.
Definition: transport.c:722
dpo_id_t fe_lb
The load-balance used for forwarding.
Definition: fib_entry.h:331
u8 ip_is_zero(ip46_address_t *ip46_address, u8 is_ip4)
Definition: ip.c:20
static const transport_proto_vft_t tcp_proto
Definition: tcp.c:1163
void tcp_punt_unknown(vlib_main_t *vm, u8 is_ip4, u8 is_add)
Definition: tcp.c:1405
vlib_node_registration_t tcp4_input_node
(constructor) VLIB_REGISTER_NODE (tcp4_input_node)
Definition: tcp_input.c:3001
static void tcp_timer_update(tcp_timer_wheel_t *tw, tcp_connection_t *tc, u8 timer_id, u32 interval)
Definition: tcp_timer.h:43
static u32 tcp_session_bind(u32 session_index, transport_endpoint_t *tep)
Definition: tcp.c:146
index_t dpoi_index
the index of objects of that type
Definition: dpo.h:186
vl_api_address_t ip
Definition: l2.api:501
ip46_address_t ip
Definition: tcp.c:34
#define FIB_NODE_INDEX_INVALID
Definition: fib_types.h:31
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
Definition: tcp_output.c:863
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void vlib_rpc_call_main_thread(void *callback, u8 *args, u32 arg_size)
Definition: threads.c:1938
static void tcp_cc_cleanup(tcp_connection_t *tc)
Definition: tcp.c:78
#define TCP_RTO_INIT
Definition: tcp_types.h:92
static tcp_connection_t * tcp_listener_get(u32 tli)
Definition: tcp_inlines.h:58
static u32 tcp_round_snd_space(tcp_connection_t *tc, u32 snd_space)
Definition: tcp.c:887
static tcp_worker_ctx_t * tcp_get_worker(u32 thread_index)
Definition: tcp.h:282
void session_transport_closed_notify(transport_connection_t *tc)
Notification from transport that it is closed.
Definition: session.c:1036
u64 uword
Definition: types.h:112
void tcp_connection_init_vars(tcp_connection_t *tc)
Initialize tcp connection variables.
Definition: tcp.c:704
#define DPO_INVALID
An initialiser for DPOs declared on the stack.
Definition: dpo.h:197
u32 index
Definition: flow_types.api:221
static u8 * format_tcp_session(u8 *s, va_list *args)
Definition: tcp.c:822
u16 port
Definition: lb_types.api:72
transport_connection_t * session_lookup_connection(u32 fib_index, ip46_address_t *lcl, ip46_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u8 is_ip4)
static f64 tcp_time_now_us(u32 thread_index)
Definition: tcp_inlines.h:213
static void tcp_connection_set_state(tcp_connection_t *tc, tcp_state_t state)
Definition: tcp_inlines.h:51
enum _tcp_cc_algorithm_type tcp_cc_algorithm_type_e
static void tcp_timer_waitclose_handler(tcp_connection_t *tc)
Definition: tcp.c:982
#define clib_fifo_add2(f, p)
Definition: fifo.h:200
vnet_link_t link_type
Definition: tcp.c:33
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
#define clib_fifo_add(f, e, n)
Definition: fifo.h:208
static u16 tcp_session_cal_goal_size(tcp_connection_t *tc)
Definition: tcp.c:876
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
void tcp_connection_cleanup(tcp_connection_t *tc)
Cleans up connection state.
Definition: tcp.c:242
void tcp_connection_del(tcp_connection_t *tc)
Connection removal.
Definition: tcp.c:289
adj_index_t adj_nbr_add_or_lock(fib_protocol_t nh_proto, vnet_link_t link_type, const ip46_address_t *nh_addr, u32 sw_if_index)
Neighbour Adjacency sub-type.
Definition: adj_nbr.c:236
void tcp_reschedule(tcp_connection_t *tc)
Definition: tcp.c:1217
static u8 * format_tcp_half_open_session(u8 *s, va_list *args)
Definition: tcp.c:851
#define tcp_opts_sack_permitted(_to)
Definition: tcp_packet.h:159
static u32 tcp_snd_space_inline(tcp_connection_t *tc)
Compute tx window session is allowed to fill.
Definition: tcp.c:914
int transport_alloc_local_port(u8 proto, ip46_address_t *ip)
Allocate local port and add if successful add entry to local endpoint table to mark the pair as used...
Definition: transport.c:475
static clib_error_t * ip6_lookup_init(vlib_main_t *vm)
Definition: ip6_forward.c:2785
static void tcp_update_time(f64 now, u8 thread_index)
Definition: tcp.c:1142
void tcp_enable_pacing(tcp_connection_t *tc)
Definition: tcp.c:691
static tcp_main_t * vnet_get_tcp_main()
Definition: tcp.h:276
u32 tco_next_node[2]
Session layer edge indices to tcp output.
Definition: tcp.h:101
static void tcp_session_reset(u32 conn_index, u32 thread_index)
Definition: tcp.c:460
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:104
#define seq_lt(_s1, _s2)
Definition: tcp_packet.h:177
int tcp_session_custom_tx(void *conn, transport_send_params_t *sp)
Definition: tcp_output.c:2013
#define tcp_worker_stats_inc(_wrk, _stat, _val)
Definition: tcp.h:125
adj_index_t adj_nbr_find(fib_protocol_t nh_proto, vnet_link_t link_type, const ip46_address_t *nh_addr, u32 sw_if_index)
Lookup neighbor adjancency.
Definition: adj_nbr.c:109
#define TCP_EVT(_evt, _args...)
Definition: tcp_debug.h:145
u32 connection_index
Definition: tcp.h:72