25 #define tcp_error(n,s) s, 31 #define foreach_tcp_state_next \ 32 _ (DROP4, "ip4-drop") \ 33 _ (DROP6, "ip6-drop") \ 34 _ (TCP4_OUTPUT, "tcp4-output") \ 35 _ (TCP6_OUTPUT, "tcp6-output") 37 typedef enum _tcp_established_next
39 #define _(s,n) TCP_ESTABLISHED_NEXT_##s, 45 typedef enum _tcp_rcv_process_next
47 #define _(s,n) TCP_RCV_PROCESS_NEXT_##s, 53 typedef enum _tcp_syn_sent_next
55 #define _(s,n) TCP_SYN_SENT_NEXT_##s, 61 typedef enum _tcp_listen_next
63 #define _(s,n) TCP_LISTEN_NEXT_##s, 70 typedef enum _tcp_state_next
72 #define _(s,n) TCP_NEXT_##s, 78 #define tcp_next_output(is_ip4) (is_ip4 ? TCP_NEXT_TCP4_OUTPUT \ 79 : TCP_NEXT_TCP6_OUTPUT) 81 #define tcp_next_drop(is_ip4) (is_ip4 ? TCP_NEXT_DROP4 \ 114 return (
seq_geq (end_seq, tc->rcv_las)
115 &&
seq_leq (seq, tc->rcv_nxt + tc->rcv_wnd));
148 &&
seq_leq (tc->rcv_las, seq_end))
151 tc->tsval_recent = tc->rcv_opts.tsval;
159 switch (tc->rst_state)
161 case TCP_STATE_SYN_RCVD:
166 case TCP_STATE_SYN_SENT:
170 case TCP_STATE_ESTABLISHED:
174 case TCP_STATE_CLOSE_WAIT:
175 case TCP_STATE_FIN_WAIT_1:
176 case TCP_STATE_FIN_WAIT_2:
177 case TCP_STATE_CLOSING:
178 case TCP_STATE_LAST_ACK:
181 case TCP_STATE_CLOSED:
182 case TCP_STATE_TIME_WAIT:
185 TCP_DBG (
"reset state: %u", tc->state);
194 tc->rst_state = tc->state;
209 TCP_EVT (TCP_EVT_RST_RCVD, tc);
212 case TCP_STATE_SYN_RCVD:
216 case TCP_STATE_SYN_SENT:
220 case TCP_STATE_ESTABLISHED:
227 case TCP_STATE_CLOSE_WAIT:
228 case TCP_STATE_FIN_WAIT_1:
229 case TCP_STATE_FIN_WAIT_2:
230 case TCP_STATE_CLOSING:
231 case TCP_STATE_LAST_ACK:
240 case TCP_STATE_CLOSED:
241 case TCP_STATE_TIME_WAIT:
244 TCP_DBG (
"reset state: %u", tc->state);
265 *error0 = TCP_ERROR_CONNECTION_CLOSED;
271 *error0 = TCP_ERROR_SEGMENT_INVALID;
277 *error0 = TCP_ERROR_OPTIONS;
283 *error0 = TCP_ERROR_PAWS;
292 tc0->tsval_recent = tc0->rcv_opts.tsval;
313 &&
vnet_buffer (b0)->tcp.seq_number == tc0->rcv_nxt - 1)
316 if (tc0->state == TCP_STATE_SYN_RCVD)
319 TCP_EVT (TCP_EVT_SYN_RCVD, tc0, 0);
320 *error0 = TCP_ERROR_SYNS_RCVD;
325 TCP_EVT (TCP_EVT_SYNACK_RCVD, tc0);
326 *error0 = TCP_ERROR_SYN_ACKS_RCVD;
333 if (tc0->rcv_wnd < tc0->snd_mss
334 && tc0->rcv_nxt ==
vnet_buffer (b0)->tcp.seq_number)
342 tc0->rcv_las - tc0->rcv_wnd)
344 tc0->rcv_nxt + tc0->rcv_wnd))
347 *error0 = TCP_ERROR_RCV_WND;
351 if ((tc0->flags & TCP_CONN_ZERO_RWND_SENT)
353 tc0->rcv_las + tc0->rcv_opts.mss))
354 *error0 = TCP_ERROR_ZERO_RWND;
375 *error0 = TCP_ERROR_RST_RCVD;
386 *error0 = TCP_ERROR_SPURIOUS_SYN;
412 *error = TCP_ERROR_ACK_INVALID;
417 tc->bytes_acked =
vnet_buffer (b)->tcp.ack_number - tc->snd_una;
419 *error = TCP_ERROR_ACK_OK;
439 err = mrtt - tc->srtt;
443 tc->srtt =
clib_max ((
int) tc->srtt + (err >> 3), 1);
444 diff = (
clib_abs (err) - (int) tc->rttvar) >> 2;
445 tc->rttvar =
clib_max ((
int) tc->rttvar + diff, 1);
451 tc->rttvar = mrtt >> 1;
476 if ((tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
485 if (tc->rtt_ts &&
seq_geq (ack, tc->rtt_seq))
488 tc->mrtt_us = tc->mrtt_us + (sample - tc->mrtt_us) * 0.125;
499 mrtt =
clib_max (now - tc->rcv_opts.tsecr, 1);
529 tc->mrtt_us =
clib_max (tc->mrtt_us, 0.0001);
538 if (tc->rto_boff && mrtt > 1 *
THZ)
555 u32 *pending_deq_acked;
563 for (i = 0; i <
vec_len (pending_deq_acked); i++)
566 tc->flags &= ~TCP_CONN_DEQ_PENDING;
577 if (
seq_leq (tc->psh_seq, tc->snd_una))
578 tc->flags &= ~TCP_CONN_PSH_PENDING;
599 if (!(tc->flags & TCP_CONN_DEQ_PENDING))
602 tc->flags |= TCP_CONN_DEQ_PENDING;
604 tc->burst_acked += tc->bytes_acked;
618 if (
seq_lt (tc->snd_wl1, seq)
619 || (tc->snd_wl1 == seq &&
seq_leq (tc->snd_wl2, ack)))
621 tc->snd_wnd = snd_wnd;
666 tc->snd_congestion = tc->snd_nxt;
667 tc->cwnd_acc_bytes = 0;
668 tc->snd_rxt_bytes = 0;
669 tc->rxt_delivered = 0;
670 tc->prr_delivered = 0;
671 tc->prr_start = tc->snd_una;
672 tc->prev_ssthresh = tc->ssthresh;
673 tc->prev_cwnd = tc->cwnd;
682 tc->cwnd += 3 * tc->snd_mss;
684 tc->fr_occurences += 1;
685 TCP_EVT (TCP_EVT_CC_EVT, tc, 4);
691 tc->cwnd = tc->prev_cwnd;
692 tc->ssthresh = tc->prev_ssthresh;
694 ASSERT (tc->rto_boff == 0);
695 TCP_EVT (TCP_EVT_CC_EVT, tc, 5);
716 return (tc->sack_sb.lost_bytes
718 < tc->sack_sb.sacked_bytes));
736 if (
seq_leq (tc->snd_una, tc->snd_congestion)
737 && ((!(tc->cwnd > tc->snd_mss
738 && tc->bytes_acked <= 4 * tc->snd_mss))
739 || (tc->rcv_opts.tsecr != tc->tsecr_last_ack)))
768 if (tc->sack_sb.sacked_bytes)
770 tc->snd_congestion = tc->snd_nxt;
775 tc->rxt_delivered = 0;
776 tc->snd_rxt_bytes = 0;
778 tc->prr_delivered = 0;
780 tc->flags &= ~TCP_CONN_RXT_PENDING;
783 if (hole && hole->start == tc->snd_una && hole->end == tc->snd_nxt)
792 TCP_EVT (TCP_EVT_CC_EVT, tc, 3);
794 ASSERT (tc->rto_boff == 0);
817 (
seq_leq (tc->snd_congestion, tc->snd_una - tc->bytes_acked)
818 &&
seq_gt (tc->snd_congestion, tc->snd_una)))
819 tc->snd_congestion = tc->snd_una - 1;
843 TCP_EVT (TCP_EVT_DUPACK_RCVD, tc, 1);
870 if (!tc->bytes_acked && tc->sack_sb.rxt_sacked)
873 tc->rxt_delivered += tc->sack_sb.rxt_sacked;
874 tc->prr_delivered += tc->bytes_acked + tc->sack_sb.last_sacked_bytes
875 - tc->sack_sb.last_bytes_delivered;
881 tc->rcv_dupacks += 1;
882 TCP_EVT (TCP_EVT_DUPACK_RCVD, tc, 1);
884 tc->rxt_delivered =
clib_min (tc->rxt_delivered + tc->bytes_acked,
887 tc->prr_delivered +=
clib_min (tc->snd_mss,
888 tc->snd_nxt - tc->snd_una);
890 tc->prr_delivered += tc->bytes_acked -
clib_min (tc->bytes_acked,
902 if (
seq_geq (tc->snd_una, tc->snd_congestion))
907 tc->tsecr_last_ack = tc->rcv_opts.tsecr;
922 if (!tc->bytes_acked)
949 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
962 return ((
vnet_buffer (b)->tcp.ack_number == prev_snd_una)
963 &&
seq_gt (tc->snd_nxt, tc->snd_una)
965 && (prev_snd_wnd == tc->snd_wnd));
973 u32 prev_snd_wnd,
u32 prev_snd_una,
u8 * is_dack)
977 *is_dack = tc->sack_sb.last_sacked_bytes
990 u32 prev_snd_wnd, prev_snd_una;
1008 tc->errors.above_ack_wnd += 1;
1009 *error = TCP_ERROR_ACK_FUTURE;
1017 tc->errors.below_ack_wnd += 1;
1018 *error = TCP_ERROR_ACK_OLD;
1039 prev_snd_wnd = tc->snd_wnd;
1040 prev_snd_una = tc->snd_una;
1043 clib_net_to_host_u16 (th->window) << tc->snd_wscale);
1044 tc->bytes_acked =
vnet_buffer (b)->tcp.ack_number - tc->snd_una;
1048 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
1051 if (tc->bytes_acked)
1057 TCP_EVT (TCP_EVT_ACK_RCVD, tc);
1066 tc->dupacks_in += is_dack;
1069 *error = TCP_ERROR_ACK_OK;
1072 *error = TCP_ERROR_ACK_DUP;
1082 *error = TCP_ERROR_ACK_OK;
1099 u32 thread_index, *pending_disconnects, *pending_resets;
1107 for (i = 0; i <
vec_len (pending_disconnects); i++)
1120 for (i = 0; i <
vec_len (pending_resets); i++)
1140 tc->flags |= TCP_CONN_FINRCVD;
1148 TCP_EVT (TCP_EVT_FIN_RCVD, tc);
1149 *error = TCP_ERROR_FIN_RCVD;
1157 int written, error = TCP_ERROR_ENQUEUED;
1163 tc->bytes_in += written;
1165 TCP_EVT (TCP_EVT_INPUT, tc, 0, data_len, written);
1170 tc->rcv_nxt += written;
1173 else if (written > data_len)
1175 tc->rcv_nxt += written;
1176 TCP_EVT (TCP_EVT_CC_INPUT, tc, data_len, written);
1178 else if (written > 0)
1181 tc->rcv_nxt += written;
1182 error = TCP_ERROR_PARTIALLY_ENQUEUED;
1187 if (tc->rcv_wnd < tc->snd_mss)
1188 return TCP_ERROR_ZERO_RWND;
1190 return TCP_ERROR_FIFO_FULL;
1223 TCP_EVT (TCP_EVT_INPUT, tc, 1, data_len, 0);
1224 return TCP_ERROR_FIFO_FULL;
1227 TCP_EVT (TCP_EVT_INPUT, tc, 1, data_len, data_len);
1228 tc->bytes_in += data_len;
1236 s0 =
session_get (tc->c_s_index, tc->c_thread_index);
1244 start = tc->rcv_nxt +
offset;
1248 TCP_EVT (TCP_EVT_CC_SACKS, tc);
1252 return TCP_ERROR_ENQUEUED_OOO;
1267 || (tc->flags & TCP_CONN_SNDACK) != 0
1284 if (!(b->
flags & VLIB_BUFFER_NEXT_PRESENT))
1291 n_bytes_to_drop -= discard;
1293 while (n_bytes_to_drop);
1294 if (n_bytes_to_drop > first)
1312 u32 error, n_bytes_to_drop, n_data_bytes;
1317 tc->data_segs_in += 1;
1331 tc->errors.below_data_wnd++;
1332 error = TCP_ERROR_SEGMENT_OLD;
1338 n_bytes_to_drop = tc->rcv_nxt -
vnet_buffer (b)->tcp.seq_number;
1339 n_data_bytes -= n_bytes_to_drop;
1343 error = TCP_ERROR_SEGMENT_OLD;
1354 tc->rcv_las + tc->rcv_wnd);
1406 s =
format (s,
"%d -> %d (%U)",
1407 clib_net_to_host_u16 (t->
tcp_header.dst_port),
1450 if (b0->
flags & VLIB_BUFFER_IS_TRACED)
1474 #define tcp_maybe_inc_counter(node_id, err, count) \ 1476 if (next0 != tcp_next_drop (is_ip4)) \ 1477 tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \ 1478 tcp6_##node_id##_node.index, is_ip4, err, \ 1481 #define tcp_inc_counter(node_id, err, count) \ 1482 tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \ 1483 tcp6_##node_id##_node.index, is_ip4, \ 1485 #define tcp_maybe_inc_err_counter(cnts, err) \ 1487 cnts[err] += (next0 != tcp_next_drop (is_ip4)); \ 1489 #define tcp_inc_err_counter(cnts, err, val) \ 1493 #define tcp_store_err_counters(node_id, cnts) \ 1496 for (i = 0; i < TCP_N_ERROR; i++) \ 1498 tcp_inc_counter(node_id, i, cnts[i]); \ 1508 u32 n_left_from, *from, *first_buffer;
1517 while (n_left_from > 0)
1519 u32 bi0, error0 = TCP_ERROR_ACK_OK;
1524 if (n_left_from > 1)
1542 error0 = TCP_ERROR_INVALID_CONNECTION;
1577 err_counters[TCP_ERROR_MSG_QUEUE_FULL] = errors;
1603 .name =
"tcp4-established",
1605 .vector_size =
sizeof (
u32),
1611 #define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n, 1622 .name =
"tcp6-established",
1624 .vector_size =
sizeof (
u32),
1630 #define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n, 1650 if (tc->c_lcl_port == 0 && tc->state == TCP_STATE_LISTEN)
1653 u8 is_ip_valid = 0, val_l, val_r;
1655 if (tc->connection.is_ip4)
1660 &tc->connection.lcl_ip.ip4);
1661 val_l = val_l ||
ip_is_zero (&tc->connection.lcl_ip, 1);
1663 &tc->connection.rmt_ip.ip4);
1664 val_r = val_r || tc->state == TCP_STATE_LISTEN;
1665 is_ip_valid = val_l && val_r;
1672 &tc->connection.lcl_ip.ip6);
1673 val_l = val_l ||
ip_is_zero (&tc->connection.lcl_ip, 0);
1675 &tc->connection.rmt_ip.ip6);
1676 val_r = val_r || tc->state == TCP_STATE_LISTEN;
1677 is_ip_valid = val_l && val_r;
1680 u8 is_valid = (tc->c_lcl_port == hdr->dst_port
1681 && (tc->state == TCP_STATE_LISTEN
1682 || tc->c_rmt_port == hdr->src_port) && is_ip_valid);
1688 tc->c_proto, tc->c_is_ip4);
1692 if (tmp->lcl_port == hdr->dst_port
1693 && tmp->rmt_port == hdr->src_port)
1695 TCP_DBG (
"half-open is valid!");
1724 TRANSPORT_PROTO_TCP,
1725 thread_index, &is_filtered);
1739 TRANSPORT_PROTO_TCP,
1740 thread_index, &is_filtered);
1758 tcp->dst_port, TRANSPORT_PROTO_TCP, 1);
1766 tcp->dst_port, TRANSPORT_PROTO_TCP, 1);
1771 (TRANSPORT_PROTO_TCP,
1784 u32 sw_if_idx, lb_idx;
1793 ip6_address_t *dst_addr = &(tc->c_rmt_ip.ip6);
1808 tc->cfg_flags |= TCP_CFG_F_TSO;
1815 u32 n_left_from, *from, *first_buffer, errors = 0;
1822 while (n_left_from > 0)
1824 u32 bi0, ack0, seq0, error0 = TCP_ERROR_NONE;
1839 error0 = TCP_ERROR_INVALID_CONNECTION;
1849 my_thread_index, is_ip4));
1850 error0 = TCP_ERROR_SPURIOUS_SYN_ACK;
1861 || tcp0->src_port != tc0->c_rmt_port))
1863 error0 = TCP_ERROR_INVALID_CONNECTION;
1870 error0 = TCP_ERROR_SEGMENT_INVALID;
1891 if (
seq_leq (ack0, tc0->iss) ||
seq_gt (ack0, tc0->snd_nxt))
1895 error0 = TCP_ERROR_RCV_WND;
1900 if (
seq_gt (tc0->snd_una, ack0))
1902 error0 = TCP_ERROR_ACK_INVALID;
1917 error0 = TCP_ERROR_RST_RCVD;
1932 error0 = TCP_ERROR_SEGMENT_INVALID;
1939 error0 = TCP_ERROR_OPTIONS;
1947 new_tc0->irs = seq0;
1953 new_tc0->tsval_recent = new_tc0->rcv_opts.tsval;
1958 new_tc0->snd_wscale = new_tc0->rcv_opts.wscale;
1960 new_tc0->rcv_wscale = 0;
1962 new_tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
1963 << new_tc0->snd_wscale;
1964 new_tc0->snd_wl1 = seq0;
1965 new_tc0->snd_wl2 = ack0;
1975 new_tc0->snd_una = ack0;
1976 new_tc0->state = TCP_STATE_ESTABLISHED;
1979 new_tc0->rcv_las = new_tc0->rcv_nxt;
1988 error0 = TCP_ERROR_CREATE_SESSION_FAIL;
1992 new_tc0->tx_fifo_size =
1996 TCP_EVT (TCP_EVT_SYNACK_RCVD, new_tc0);
1997 error0 = TCP_ERROR_SYN_ACKS_RCVD;
2002 new_tc0->state = TCP_STATE_SYN_RCVD;
2010 TCP_EVT (TCP_EVT_RST_SENT, tc0);
2011 error0 = TCP_ERROR_CREATE_SESSION_FAIL;
2015 new_tc0->tx_fifo_size =
2017 new_tc0->rtt_ts = 0;
2020 error0 = TCP_ERROR_SYNS_RCVD;
2024 if (!(new_tc0->cfg_flags & TCP_CFG_F_NO_TSO))
2032 if (error0 == TCP_ERROR_ACK_OK)
2033 error0 = TCP_ERROR_SYN_ACKS_RCVD;
2047 tc0->flags |= TCP_CONN_HALF_OPEN_DONE;
2087 .name =
"tcp4-syn-sent",
2089 .vector_size =
sizeof (
u32),
2095 #define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n, 2106 .name =
"tcp6-syn-sent",
2108 .vector_size =
sizeof (
u32),
2114 #define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n, 2132 u32 n_left_from, *from, max_dequeue;
2137 while (n_left_from > 0)
2139 u32 bi0, error0 = TCP_ERROR_NONE;
2154 error0 = TCP_ERROR_INVALID_CONNECTION;
2168 if (tmp->state != tc0->state)
2170 if (tc0->state != TCP_STATE_CLOSED)
2182 error0 = TCP_ERROR_CONNECTION_CLOSED;
2197 case TCP_STATE_SYN_RCVD:
2200 if (tc0->rcv_nxt !=
vnet_buffer (b0)->tcp.seq_number || is_fin0)
2203 error0 = TCP_ERROR_SEGMENT_INVALID;
2216 error0 = TCP_ERROR_SEGMENT_INVALID;
2225 tc0->state = TCP_STATE_ESTABLISHED;
2226 TCP_EVT (TCP_EVT_STATE_CHANGE, tc0);
2228 if (!(tc0->cfg_flags & TCP_CFG_F_NO_TSO))
2233 tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
2234 << tc0->rcv_opts.wscale;
2242 error0 = TCP_ERROR_MSG_QUEUE_FULL;
2248 error0 = TCP_ERROR_ACK_OK;
2250 case TCP_STATE_ESTABLISHED:
2257 case TCP_STATE_FIN_WAIT_1:
2265 if (tc0->flags & TCP_CONN_FINPNDG)
2269 if (max_dequeue <= tc0->burst_acked)
2272 else if ((tc0->flags & TCP_CONN_FINRCVD) && tc0->bytes_acked)
2277 else if (tc0->snd_una == tc0->snd_nxt)
2285 if (tc0->flags & TCP_CONN_FINRCVD)
2300 if (tc0->burst_acked > 1)
2302 tc0->burst_acked - 1);
2303 tc0->burst_acked = 0;
2306 case TCP_STATE_FIN_WAIT_2:
2312 tc0->burst_acked = 0;
2314 case TCP_STATE_CLOSE_WAIT:
2319 if (!(tc0->flags & TCP_CONN_FINPNDG))
2324 if (max_dequeue > tc0->burst_acked)
2333 case TCP_STATE_CLOSING:
2340 if (tc0->snd_una != tc0->snd_nxt)
2351 case TCP_STATE_LAST_ACK:
2360 if (is_fin0 && tc0->snd_una != tc0->snd_nxt)
2378 case TCP_STATE_TIME_WAIT:
2404 case TCP_STATE_ESTABLISHED:
2405 case TCP_STATE_FIN_WAIT_1:
2406 case TCP_STATE_FIN_WAIT_2:
2410 case TCP_STATE_CLOSE_WAIT:
2411 case TCP_STATE_CLOSING:
2412 case TCP_STATE_LAST_ACK:
2413 case TCP_STATE_TIME_WAIT:
2423 TCP_EVT (TCP_EVT_FIN_RCVD, tc0);
2427 case TCP_STATE_ESTABLISHED:
2436 case TCP_STATE_SYN_RCVD:
2447 case TCP_STATE_CLOSE_WAIT:
2448 case TCP_STATE_CLOSING:
2449 case TCP_STATE_LAST_ACK:
2452 case TCP_STATE_FIN_WAIT_1:
2455 if (tc0->flags & TCP_CONN_FINPNDG)
2460 tc0->flags |= TCP_CONN_FINRCVD;
2473 case TCP_STATE_FIN_WAIT_2:
2483 case TCP_STATE_TIME_WAIT:
2491 error0 = TCP_ERROR_FIN_RCVD;
2530 .name =
"tcp4-rcv-process",
2532 .vector_size =
sizeof (
u32),
2538 #define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n, 2549 .name =
"tcp6-rcv-process",
2551 .vector_size =
sizeof (
u32),
2557 #define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n, 2572 u32 n_left_from, *from, n_syns = 0, *first_buffer;
2578 while (n_left_from > 0)
2580 u32 bi, error = TCP_ERROR_NONE;
2596 if (tc->state != TCP_STATE_TIME_WAIT)
2598 error = TCP_ERROR_CREATE_EXISTS;
2611 error = TCP_ERROR_CREATE_EXISTS;
2637 error = TCP_ERROR_OPTIONS;
2644 child->state = TCP_STATE_SYN_RCVD;
2645 child->c_fib_index = lc->c_fib_index;
2646 child->cc_algo = lc->cc_algo;
2651 lc->c_thread_index, 0 ))
2654 error = TCP_ERROR_CREATE_SESSION_FAIL;
2662 TCP_EVT (TCP_EVT_SYN_RCVD, child, 1);
2676 n_syns += (error == TCP_ERROR_NONE);
2700 .name =
"tcp4-listen",
2702 .vector_size =
sizeof (
u32),
2708 #define _(s,n) [TCP_LISTEN_NEXT_##s] = n, 2719 .name =
"tcp6-listen",
2721 .vector_size =
sizeof (
u32),
2727 #define _(s,n) [TCP_LISTEN_NEXT_##s] = n, 2735 typedef enum _tcp_input_next
2747 #define foreach_tcp4_input_next \ 2748 _ (DROP, "ip4-drop") \ 2749 _ (LISTEN, "tcp4-listen") \ 2750 _ (RCV_PROCESS, "tcp4-rcv-process") \ 2751 _ (SYN_SENT, "tcp4-syn-sent") \ 2752 _ (ESTABLISHED, "tcp4-established") \ 2753 _ (RESET, "tcp4-reset") \ 2754 _ (PUNT, "ip4-punt") 2756 #define foreach_tcp6_input_next \ 2757 _ (DROP, "ip6-drop") \ 2758 _ (LISTEN, "tcp6-listen") \ 2759 _ (RCV_PROCESS, "tcp6-rcv-process") \ 2760 _ (SYN_SENT, "tcp6-syn-sent") \ 2761 _ (ESTABLISHED, "tcp6-established") \ 2762 _ (RESET, "tcp6-reset") \ 2763 _ (PUNT, "ip6-punt") 2765 #define filter_flags (TCP_FLAG_SYN|TCP_FLAG_ACK|TCP_FLAG_RST|TCP_FLAG_FIN) 2776 for (i = 0; i < n_bufs; i++)
2778 if (bs[i]->
flags & VLIB_BUFFER_IS_TRACED)
2792 if (*error == TCP_ERROR_FILTERED || *error == TCP_ERROR_WRONG_THREAD)
2796 else if ((is_ip4 && tm->punt_unknown4) || (!is_ip4 && tm->punt_unknown6))
2799 *error = TCP_ERROR_PUNT;
2804 *error = TCP_ERROR_NO_LISTENER;
2819 *next = tm->dispatch_table[tc->state][
flags].next;
2820 error = tm->dispatch_table[tc->state][
flags].error;
2826 if (error == TCP_ERROR_DISPATCH)
2827 clib_warning (
"tcp conn %u disp error state %U flags %U",
2851 while (n_left_from >= 4)
2853 u32 error0 = TCP_ERROR_NO_LISTENER, error1 = TCP_ERROR_NO_LISTENER;
2876 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
2877 vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
2887 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
2899 vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
2913 while (n_left_from > 0)
2916 u32 error0 = TCP_ERROR_NO_LISTENER;
2918 if (n_left_from > 1)
2930 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
2970 .name =
"tcp4-input-nolookup",
2972 .vector_size =
sizeof (
u32),
2978 #define _(s,n) [TCP_INPUT_NEXT_##s] = n, 2990 .name =
"tcp6-input-nolookup",
2992 .vector_size =
sizeof (
u32),
2998 #define _(s,n) [TCP_INPUT_NEXT_##s] = n, 3024 .name =
"tcp4-input",
3026 .vector_size =
sizeof (
u32),
3032 #define _(s,n) [TCP_INPUT_NEXT_##s] = n, 3044 .name =
"tcp6-input",
3046 .vector_size =
sizeof (
u32),
3052 #define _(s,n) [TCP_INPUT_NEXT_##s] = n, 3061 #ifndef CLIB_MARCH_VARIANT 3066 for (i = 0; i <
ARRAY_LEN (tm->dispatch_table); i++)
3067 for (j = 0; j <
ARRAY_LEN (tm->dispatch_table[i]); j++)
3070 tm->dispatch_table[
i][j].error = TCP_ERROR_DISPATCH;
3073 #define _(t,f,n,e) \ 3075 tm->dispatch_table[TCP_STATE_##t][f].next = (n); \ 3076 tm->dispatch_table[TCP_STATE_##t][f].error = (e); \ 3085 TCP_ERROR_ACK_INVALID);
3087 TCP_ERROR_SEGMENT_INVALID);
3089 TCP_ERROR_SEGMENT_INVALID);
3091 TCP_ERROR_INVALID_CONNECTION);
3094 TCP_ERROR_SEGMENT_INVALID);
3096 TCP_ERROR_SEGMENT_INVALID);
3098 TCP_ERROR_SEGMENT_INVALID);
3100 TCP_ERROR_SEGMENT_INVALID);
3102 TCP_ERROR_SEGMENT_INVALID);
3104 TCP_ERROR_SEGMENT_INVALID);
3284 TCP_ERROR_CONNECTION_CLOSED);
3288 TCP_ERROR_CONNECTION_CLOSED);
u16 lb_n_buckets
number of buckets in the load-balance.
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
u32 connection_index
Index of the transport connection associated to the session.
void tcp_program_retransmit(tcp_connection_t *tc)
#define TCP_TIMER_HANDLE_INVALID
u32 * pending_disconnects
vector of pending disconnect notifications
static u32 ip6_fib_table_fwding_lookup(u32 fib_index, const ip6_address_t *dst)
static void tcp_persist_timer_set(tcp_timer_wheel_t *tw, tcp_connection_t *tc)
static u32 tcp_time_now(void)
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
transport_connection_t * session_lookup_connection_wt6(u32 fib_index, ip6_address_t *lcl, ip6_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *result)
Lookup connection with ip6 and transport layer information.
vnet_main_t * vnet_get_main(void)
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
#define THZ
TCP tick frequency.
#define tcp_opts_tstamp(_to)
#define clib_memcpy_fast(a, b, c)
void session_transport_delete_notify(transport_connection_t *tc)
Notification from transport that connection is being deleted.
svm_fifo_t * rx_fifo
Pointers to rx/tx buffers.
#define tcp_fastrecovery_first_off(tc)
struct _tcp_main tcp_main_t
void tcp_connection_timers_reset(tcp_connection_t *tc)
Stop all connection timers.
u16 current_length
Nbytes between current data and the end of this buffer.
int session_main_flush_enqueue_events(u8 transport_proto, u32 thread_index)
Flushes queue of sessions that are to be notified of new data enqueued events.
struct _tcp_connection tcp_connection_t
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static u32 tcp_set_time_now(tcp_worker_ctx_t *wrk)
void session_transport_reset_notify(transport_connection_t *tc)
Notify application that connection has been reset.
u32 dpo_get_urpf(const dpo_id_t *dpo)
Get a uRPF interface for the DPO.
u32 * pending_resets
vector of pending reset notifications
#define tcp_disconnect_pending_on(tc)
static void tcp_cc_congestion(tcp_connection_t *tc)
static u32 tcp_time_now_w_thread(u32 thread_index)
#define timestamp_lt(_t1, _t2)
static session_t * session_get(u32 si, u32 thread_index)
#define TCP_TICK
TCP tick period (s)
#define tcp_disconnect_pending_off(tc)
tcp_connection_t tcp_connection
#define VLIB_NODE_FN(node)
int session_enqueue_stream_connection(transport_connection_t *tc, vlib_buffer_t *b, u32 offset, u8 queue_event, u8 is_in_order)
u64 session_lookup_half_open_handle(transport_connection_t *tc)
vlib_error_t * errors
Vector of errors for this node.
format_function_t format_tcp_flags
static u8 tcp_is_descheduled(tcp_connection_t *tc)
struct _tcp_header tcp_header_t
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
#define tcp_in_cong_recovery(tc)
u32 * pending_deq_acked
vector of pending ack dequeues
void session_transport_closing_notify(transport_connection_t *tc)
Notification from transport that connection is being closed.
#define timestamp_leq(_t1, _t2)
void tcp_init_snd_vars(tcp_connection_t *tc)
Initialize connection send variables.
static void tcp_persist_timer_reset(tcp_timer_wheel_t *tw, tcp_connection_t *tc)
#define VLIB_INIT_FUNCTION(x)
static int tcp_options_parse(tcp_header_t *th, tcp_options_t *to, u8 is_syn)
Parse TCP header options.
void tcp_bt_sample_delivery_rate(tcp_connection_t *tc, tcp_rate_sample_t *rs)
Generate a delivery rate sample from recently acked bytes.
#define seq_leq(_s1, _s2)
transport_connection_t * session_lookup_connection_wt4(u32 fib_index, ip4_address_t *lcl, ip4_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *result)
Lookup connection with ip4 and transport layer information.
vnet_hw_interface_flags_t flags
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
#define tcp_fastrecovery_off(tc)
static u32 ooo_segment_length(svm_fifo_t *f, ooo_segment_t *s)
static void * ip4_next_header(ip4_header_t *i)
static sack_scoreboard_hole_t * scoreboard_first_hole(sack_scoreboard_t *sb)
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
#define vlib_call_init_function(vm, x)
#define tcp_validate_txf_size(_tc, _a)
#define tcp_fastrecovery_on(tc)
static void tcp_retransmit_timer_update(tcp_timer_wheel_t *tw, tcp_connection_t *tc)
static void tcp_timer_set(tcp_timer_wheel_t *tw, tcp_connection_t *tc, u8 timer_id, u32 interval)
static void tcp_cc_recovered(tcp_connection_t *tc)
static void svm_fifo_newest_ooo_segment_reset(svm_fifo_t *f)
static heap_elt_t * first(heap_header_t *h)
vlib_error_t error
Error code for buffers to be enqueued to error handler.
static void tcp_retransmit_timer_reset(tcp_timer_wheel_t *tw, tcp_connection_t *tc)
The identity of a DPO is a combination of its type and its instance number/index of objects of that t...
void tcp_update_sack_list(tcp_connection_t *tc, u32 start, u32 end)
Build SACK list as per RFC2018.
vlib_main_t * vm
convenience pointer to this thread's vlib main
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
void tcp_send_ack(tcp_connection_t *tc)
void tcp_connection_tx_pacer_reset(tcp_connection_t *tc, u32 window, u32 start_bucket)
tcp_connection_t * tcp_connection_alloc_w_base(u8 thread_index, tcp_connection_t *base)
static const dpo_id_t * load_balance_get_bucket_i(const load_balance_t *lb, u32 bucket)
format_function_t format_tcp_connection_id
#define TCP_DUPACK_THRESHOLD
tcp_connection_t * tcp_connection_alloc(u8 thread_index)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
void tcp_connection_tx_pacer_update(tcp_connection_t *tc)
#define TCP_PAWS_IDLE
24 days
tcp_timer_wheel_t timer_wheel
worker timer wheel
int ip6_address_compare(ip6_address_t *a1, ip6_address_t *a2)
static u8 tcp_is_lost_fin(tcp_connection_t *tc)
static ooo_segment_t * svm_fifo_newest_ooo_segment(svm_fifo_t *f)
static void tcp_cc_rcv_ack(tcp_connection_t *tc, tcp_rate_sample_t *rs)
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
#define TCP_DBG(_fmt, _args...)
#define tcp_recovery_off(tc)
void tcp_program_cleanup(tcp_worker_ctx_t *wrk, tcp_connection_t *tc)
void tcp_connection_free(tcp_connection_t *tc)
#define VLIB_REGISTER_NODE(x,...)
#define CLIB_PREFETCH(addr, size, type)
int ip4_address_compare(ip4_address_t *a1, ip4_address_t *a2)
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
#define tcp_disconnect_pending(tc)
void tcp_program_dupack(tcp_connection_t *tc)
sll srl srl sll sra u16x4 i
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
format_function_t format_tcp_state
static void tcp_update_rto(tcp_connection_t *tc)
#define clib_warning(format, args...)
#define tcp_in_recovery(tc)
Don't register connection in lookup.
struct _transport_connection transport_connection_t
f64 rtt_time
RTT for sample.
static void tcp_cc_undo_recovery(tcp_connection_t *tc)
static u32 tcp_available_cc_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
vlib_main_t vlib_node_runtime_t * node
static void * ip6_next_header(ip6_header_t *i)
static u32 transport_max_tx_dequeue(transport_connection_t *tc)
void tcp_send_synack(tcp_connection_t *tc)
#define seq_geq(_s1, _s2)
static load_balance_t * load_balance_get(index_t lbi)
static void tcp_cong_recovery_off(tcp_connection_t *tc)
static index_t ip4_fib_forwarding_lookup(u32 fib_index, const ip4_address_t *addr)
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
#define tcp_fastrecovery_first_on(tc)
int session_stream_accept_notify(transport_connection_t *tc)
struct _sack_scoreboard_hole sack_scoreboard_hole_t
static vlib_main_t * vlib_get_main(void)
static clib_error_t * tcp_init(vlib_main_t *vm)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
u8 ip_is_zero(ip46_address_t *ip46_address, u8 is_ip4)
u8 tcp_scoreboard_is_sane_post_recovery(tcp_connection_t *tc)
Test that scoreboard is sane after recovery.
#define tcp_opts_wscale(_to)
void tcp_send_reset_w_pkt(tcp_connection_t *tc, vlib_buffer_t *pkt, u32 thread_index, u8 is_ip4)
Send reset without reusing existing buffer.
static void tcp_timer_update(tcp_timer_wheel_t *tw, tcp_connection_t *tc, u8 timer_id, u32 interval)
void scoreboard_clear(sack_scoreboard_t *sb)
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define TCP_ALWAYS_ACK
On/off delayed acks.
u32 next_buffer
Next buffer for this linked-list of buffers.
static tcp_connection_t * tcp_listener_get(u32 tli)
static tcp_worker_ctx_t * tcp_get_worker(u32 thread_index)
void session_transport_closed_notify(transport_connection_t *tc)
Notification from transport that it is closed.
VLIB buffer representation.
int session_stream_connect_notify(transport_connection_t *tc, session_error_t err)
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
void tcp_connection_init_vars(tcp_connection_t *tc)
Initialize tcp connection variables.
static void tcp_init_w_buffer(tcp_connection_t *tc, vlib_buffer_t *b, u8 is_ip4)
Initialize connection by gleaning network and rcv params from buffer.
session_t * session_lookup_listener6(u32 fib_index, ip6_address_t *lcl, u16 lcl_port, u8 proto, u8 use_wildcard)
static f64 tcp_time_now_us(u32 thread_index)
void scoreboard_init_rxt(sack_scoreboard_t *sb, u32 snd_una)
static void tcp_connection_set_state(tcp_connection_t *tc, tcp_state_t state)
static u32 ooo_segment_offset_prod(svm_fifo_t *f, ooo_segment_t *s)
struct clib_bihash_value offset
template key/value backing page structure
static u32 vlib_num_workers()
void tcp_connection_cleanup(tcp_connection_t *tc)
Cleans up connection state.
void tcp_connection_del(tcp_connection_t *tc)
Connection removal.
f64 end
end of the time range
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
void tcp_reschedule(tcp_connection_t *tc)
u16 flags
Copy of main node flags.
u32 session_tx_fifo_dequeue_drop(transport_connection_t *tc, u32 max_bytes)
static u8 tcp_timer_is_active(tcp_connection_t *tc, tcp_timers_e timer)
void tcp_program_ack(tcp_connection_t *tc)
#define tcp_opts_sack_permitted(_to)
static u32 tcp_tstamp(tcp_connection_t *tc)
Generate timestamp for tcp connection.
static void tcp_cc_rcv_cong_ack(tcp_connection_t *tc, tcp_cc_ack_t ack_type, tcp_rate_sample_t *rs)
int session_stream_accept(transport_connection_t *tc, u32 listener_index, u32 thread_index, u8 notify)
Accept a stream session.
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
#define VLIB_NODE_FLAG_TRACE
tcp_bts_flags_t flags
Rate sample flags from bt sample.
#define CLIB_CACHE_LINE_BYTES
static transport_connection_t * transport_get_listener(transport_proto_t tp, u32 conn_index)
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
static tcp_connection_t * tcp_get_connection_from_transport(transport_connection_t *tconn)
static tcp_main_t * vnet_get_tcp_main()
session_t * session_lookup_listener4(u32 fib_index, ip4_address_t *lcl, u16 lcl_port, u8 proto, u8 use_wildcard)
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static tcp_connection_t * tcp_input_lookup_buffer(vlib_buffer_t *b, u8 thread_index, u32 *error, u8 is_ip4, u8 is_nolookup)
static u32 transport_tx_fifo_size(transport_connection_t *tc)
transport_connection_t * session_lookup_half_open_connection(u64 handle, u8 proto, u8 is_ip4)
void tcp_rcv_sacks(tcp_connection_t *tc, u32 ack)
#define TCP_EVT(_evt, _args...)