23 #define tcp_error(n,s) s, 29 #define foreach_tcp_state_next \ 30 _ (DROP4, "ip4-drop") \ 31 _ (DROP6, "ip6-drop") \ 32 _ (TCP4_OUTPUT, "tcp4-output") \ 33 _ (TCP6_OUTPUT, "tcp6-output") 35 typedef enum _tcp_established_next
37 #define _(s,n) TCP_ESTABLISHED_NEXT_##s, 43 typedef enum _tcp_rcv_process_next
45 #define _(s,n) TCP_RCV_PROCESS_NEXT_##s, 51 typedef enum _tcp_syn_sent_next
53 #define _(s,n) TCP_SYN_SENT_NEXT_##s, 59 typedef enum _tcp_listen_next
61 #define _(s,n) TCP_LISTEN_NEXT_##s, 68 typedef enum _tcp_state_next
70 #define _(s,n) TCP_NEXT_##s, 76 #define tcp_next_output(is_ip4) (is_ip4 ? TCP_NEXT_TCP4_OUTPUT \ 77 : TCP_NEXT_TCP6_OUTPUT) 79 #define tcp_next_drop(is_ip4) (is_ip4 ? TCP_NEXT_DROP4 \ 115 return (
seq_geq (end_seq, tc->rcv_las)
116 &&
seq_leq (seq, tc->rcv_nxt + tc->rcv_wnd));
130 u8 opt_len, opts_len, kind;
135 data = (
const u8 *) (th + 1);
138 to->
flags &= (TCP_OPTS_FLAG_SACK_PERMITTED | TCP_OPTS_FLAG_WSCALE
139 | TCP_OPTS_FLAG_SACK);
141 for (; opts_len > 0; opts_len -= opt_len, data += opt_len)
161 if (opt_len < 2 || opt_len > opts_len)
171 to->
flags |= TCP_OPTS_FLAG_MSS;
172 to->
mss = clib_net_to_host_u16 (*(
u16 *) (data + 2));
178 to->
flags |= TCP_OPTS_FLAG_WSCALE;
191 to->
flags |= TCP_OPTS_FLAG_TSTAMP;
192 to->
tsval = clib_net_to_host_u32 (*(
u32 *) (data + 2));
193 to->
tsecr = clib_net_to_host_u32 (*(
u32 *) (data + 6));
198 to->
flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
202 if ((to->
flags & TCP_OPTS_FLAG_SACK_PERMITTED) == 0 ||
tcp_syn (th))
209 to->
flags |= TCP_OPTS_FLAG_SACK;
214 b.start = clib_net_to_host_u32 (*(
u32 *) (data + 2 + 8 * j));
215 b.end = clib_net_to_host_u32 (*(
u32 *) (data + 6 + 8 * j));
257 &&
seq_leq (tc->rcv_las, seq_end))
260 tc->tsval_recent = tc->rcv_opts.tsval;
277 u32 * next0,
u32 * error0)
283 *error0 = TCP_ERROR_CONNECTION_CLOSED;
289 *error0 = TCP_ERROR_SEGMENT_INVALID;
296 *error0 = TCP_ERROR_OPTIONS;
302 *error0 = TCP_ERROR_PAWS;
314 tc0->tsval_recent = 0;
315 clib_warning (
"paws failed - really old segment. REALLY?");
334 *error0 = TCP_ERROR_RCV_WND;
337 if (!(tc0->rcv_wnd == 0
338 && tc0->rcv_nxt ==
vnet_buffer (b0)->tcp.seq_number))
355 *error0 = TCP_ERROR_RST_RCVD;
365 if (tc0->state == TCP_STATE_SYN_RCVD)
415 err = mrtt - tc->srtt;
419 tc->srtt =
clib_max ((
int) tc->srtt + (err >> 3), 1);
420 diff = (
clib_abs (err) - (int) tc->rttvar) >> 2;
421 tc->rttvar =
clib_max ((
int) tc->rttvar + diff, 1);
427 tc->rttvar = mrtt >> 1;
463 if (tc->rtt_ts &&
seq_geq (ack, tc->rtt_seq))
502 tc->bytes_acked + tc->sack_sb.snd_una_adv);
521 return ((
vnet_buffer (b)->tcp.ack_number == prev_snd_una)
522 &&
seq_gt (tc->snd_una_max, tc->snd_una)
524 && (prev_snd_wnd == tc->snd_wnd));
532 u32 prev_snd_wnd,
u32 prev_snd_una,
u8 * is_dack)
536 *is_dack = tc->sack_sb.last_sacked_bytes
546 return hole - sb->holes;
552 return hole->end - hole->start;
603 next->prev = hole->prev;
607 sb->tail = hole->prev;
613 prev->next = hole->next;
617 sb->head = hole->next;
625 memset (hole, 0xfe,
sizeof (*hole));
638 memset (hole, 0,
sizeof (*hole));
647 hole->prev = prev_index;
648 hole->next = prev->next;
651 next->prev = hole_index;
653 sb->tail = hole_index;
655 prev->next = hole_index;
659 sb->head = hole_index;
671 u32 bytes = 0, blks = 0;
674 sb->sacked_bytes = 0;
679 if (
seq_gt (sb->high_sacked, left->end))
681 bytes = sb->high_sacked - left->end;
685 while ((right = left)
691 bytes += right->start - left->end;
704 bytes += right->start - left->end;
706 while ((right = left));
709 sb->sacked_bytes = bytes;
721 u8 * can_rescue,
u8 * snd_limited)
726 while (hole &&
seq_leq (hole->end, sb->high_rxt) && hole->is_lost)
737 if (hole->is_lost &&
seq_lt (hole->start, sb->high_sacked))
744 if (!have_sent_1_smss)
750 else if (
seq_lt (hole->start, sb->high_sacked))
766 if (hole &&
seq_lt (sb->high_rxt, hole->start))
767 sb->high_rxt = hole->start;
779 seq =
seq_gt (seq, hole->start) ? seq : hole->start;
780 sb->cur_rxt_hole = sb->head;
803 sb->sacked_bytes = 0;
804 sb->last_sacked_bytes = 0;
805 sb->last_bytes_delivered = 0;
824 return (!hole || (
seq_geq (hole->start, tc->snd_una)
825 &&
seq_lt (hole->end, tc->snd_una_max)));
834 u32 blk_index = 0, old_sacked_bytes, hole_index;
837 sb->last_sacked_bytes = 0;
838 sb->last_bytes_delivered = 0;
845 old_sacked_bytes = sb->sacked_bytes;
848 blk = tc->rcv_opts.sacks;
849 while (blk <
vec_end (tc->rcv_opts.sacks))
851 if (
seq_lt (blk->start, blk->end)
852 &&
seq_gt (blk->start, tc->snd_una)
853 &&
seq_gt (blk->start, ack) &&
seq_leq (blk->end, tc->snd_una_max))
858 vec_del1 (tc->rcv_opts.sacks, blk - tc->rcv_opts.sacks);
862 if (
seq_gt (ack, tc->snd_una))
864 tmp.start = tc->snd_una;
869 if (
vec_len (tc->rcv_opts.sacks) == 0)
875 for (i = 0; i <
vec_len (tc->rcv_opts.sacks); i++)
876 for (j = i + 1; j <
vec_len (tc->rcv_opts.sacks); j++)
877 if (
seq_lt (tc->rcv_opts.sacks[j].start, tc->rcv_opts.sacks[i].start))
879 tmp = tc->rcv_opts.sacks[
i];
880 tc->rcv_opts.sacks[
i] = tc->rcv_opts.sacks[j];
881 tc->rcv_opts.sacks[j] = tmp;
888 tc->snd_una, tc->snd_una_max);
890 tmp = tc->rcv_opts.sacks[
vec_len (tc->rcv_opts.sacks) - 1];
891 sb->high_sacked = tmp.end;
897 tmp = tc->rcv_opts.sacks[
vec_len (tc->rcv_opts.sacks) - 1];
899 if (
seq_gt (tc->snd_una_max, last_hole->end))
901 if (
seq_geq (last_hole->start, sb->high_sacked))
903 last_hole->end = tc->snd_una_max;
906 else if (
seq_lt (sb->high_sacked, tc->snd_una_max))
914 if (
seq_gt (tmp.end, sb->high_sacked))
915 sb->high_sacked = tmp.end;
920 while (hole && blk_index < vec_len (tc->rcv_opts.sacks))
922 blk = &tc->rcv_opts.sacks[blk_index];
923 if (
seq_leq (blk->start, hole->start))
926 if (
seq_geq (blk->end, hole->end))
935 if (
seq_lt (ack, next_hole->start))
936 sb->snd_una_adv = next_hole->start - ack;
937 sb->last_bytes_delivered +=
938 next_hole->start - hole->end;
943 sb->snd_una_adv = sb->high_sacked - ack;
944 sb->last_bytes_delivered += sb->high_sacked - hole->end;
954 if (
seq_gt (blk->end, hole->start))
956 hole->start = blk->end;
964 if (
seq_lt (blk->end, hole->end))
972 hole->end = blk->start;
976 else if (
seq_lt (blk->start, hole->end))
978 hole->end = blk->start;
987 if (hole->start == ack + sb->snd_una_adv
988 && hole->end == tc->snd_una_max)
993 sb->last_sacked_bytes = sb->sacked_bytes
994 - (old_sacked_bytes - sb->last_bytes_delivered);
996 ASSERT (sb->sacked_bytes == 0
997 || sb->sacked_bytes < tc->snd_una_max -
seq_max (tc->snd_una, ack));
998 ASSERT (sb->last_sacked_bytes + sb->lost_bytes <= tc->snd_una_max
1001 || sb->holes[sb->head].start == ack + sb->snd_una_adv);
1016 if (
seq_lt (tc->snd_wl1, seq)
1017 || (tc->snd_wl1 == seq &&
seq_leq (tc->snd_wl2, ack)))
1019 tc->snd_wnd = snd_wnd;
1024 if (tc->snd_wnd < tc->snd_mss)
1053 tc->snd_congestion = tc->snd_una_max;
1054 tc->cwnd_acc_bytes = 0;
1055 tc->cc_algo->congestion (tc);
1065 tc->snd_nxt = tc->snd_una_max;
1073 tc->cc_algo->recovered (tc);
1074 tc->snd_rxt_bytes = 0;
1075 tc->rcv_dupacks = 0;
1076 tc->snd_nxt = tc->snd_una_max;
1085 tc->cwnd = tc->prev_cwnd;
1086 tc->ssthresh = tc->prev_ssthresh;
1087 tc->snd_nxt = tc->snd_una_max;
1088 tc->rcv_dupacks = 0;
1091 ASSERT (tc->rto_boff == 0);
1120 ASSERT (tc->rto_boff == 0);
1132 tc->cc_algo->rcv_ack (tc);
1133 tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1136 tc->rcv_dupacks = 0;
1143 (
seq_leq (tc->snd_congestion, tc->snd_una - tc->bytes_acked)
1144 &&
seq_gt (tc->snd_congestion, tc->snd_una)))
1145 tc->snd_congestion = tc->snd_una - 1;
1171 if (tc->bytes_acked)
1183 ASSERT (tc->snd_una != tc->snd_una_max
1184 || tc->sack_sb.last_sacked_bytes);
1209 if (
seq_leq (tc->snd_una, tc->snd_congestion)
1210 && ((!(tc->cwnd > tc->snd_mss
1211 && tc->bytes_acked <= 4 * tc->snd_mss))
1212 || (tc->rcv_opts.tsecr != tc->tsecr_last_ack)))
1214 tc->rcv_dupacks = 0;
1227 tc->cwnd = tc->ssthresh + tc->rcv_dupacks * tc->snd_mss;
1228 ASSERT (tc->cwnd >= tc->snd_mss);
1234 tc->snd_una + tc->snd_mss);
1243 else if (!tc->bytes_acked
1253 if (!tc->bytes_acked)
1263 if (
seq_lt (tc->snd_una, tc->snd_congestion))
1268 if (
seq_geq (tc->snd_una, tc->snd_congestion))
1273 tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1277 tc->snd_nxt = tc->snd_una_max;
1280 tc->cc_algo->rcv_ack (tc);
1281 tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1291 tc->rcv_dupacks = 0;
1296 tc->cc_algo->rcv_ack (tc);
1297 tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1303 ASSERT (tc->bytes_acked + tc->sack_sb.snd_una_adv
1304 >= tc->sack_sb.last_bytes_delivered
1305 || (tc->flags & TCP_CONN_FINSNT));
1307 if (
seq_lt (tc->snd_una, tc->sack_sb.high_rxt))
1311 rxt_delivered = tc->bytes_acked + tc->sack_sb.snd_una_adv
1312 - tc->sack_sb.last_bytes_delivered;
1313 ASSERT (tc->snd_rxt_bytes >= rxt_delivered);
1314 tc->snd_rxt_bytes -= rxt_delivered;
1320 tc->snd_rxt_bytes = 0;
1338 u32 prev_snd_wnd, prev_snd_una;
1352 if (
seq_gt (tc->snd_nxt, tc->snd_una_max))
1353 tc->snd_una_max = tc->snd_nxt;
1364 *error = TCP_ERROR_ACK_INVALID;
1374 *error = TCP_ERROR_ACK_FUTURE;
1380 *error = TCP_ERROR_ACK_OLD;
1396 prev_snd_wnd = tc->snd_wnd;
1397 prev_snd_una = tc->snd_una;
1400 clib_net_to_host_u16 (th->window) << tc->snd_wscale);
1401 tc->bytes_acked =
vnet_buffer (b)->tcp.ack_number - tc->snd_una;
1402 tc->snd_una =
vnet_buffer (b)->tcp.ack_number + tc->sack_sb.snd_una_adv;
1405 if (tc->bytes_acked)
1419 *error = TCP_ERROR_ACK_DUP;
1429 *error = TCP_ERROR_ACK_OK;
1437 for (i = 1; i <
vec_len (sacks); i++)
1439 if (sacks[i - 1].end == sacks[i].start)
1464 if (
seq_lt (tc->rcv_nxt, start))
1467 block->start = start;
1472 for (i = 0; i <
vec_len (tc->snd_sacks); i++)
1475 if (
seq_leq (tc->snd_sacks[i].start, tc->rcv_nxt))
1479 if (block && (
seq_geq (tc->snd_sacks[i].end, new_list[0].start)
1480 &&
seq_leq (tc->snd_sacks[i].start, new_list[0].end)))
1482 if (
seq_lt (tc->snd_sacks[i].start, new_list[0].start))
1483 new_list[0].start = tc->snd_sacks[i].start;
1484 if (
seq_lt (new_list[0].end, tc->snd_sacks[i].end))
1485 new_list[0].end = tc->snd_sacks[i].end;
1492 vec_add1 (new_list, tc->snd_sacks[i]);
1504 tc->snd_sacks = new_list;
1514 for (
i = 0;
i <
vec_len (tc->snd_sacks);
i++)
1515 bytes += tc->snd_sacks[
i].end - tc->snd_sacks[
i].start;
1524 int written, error = TCP_ERROR_ENQUEUED;
1531 TCP_EVT_DBG (TCP_EVT_INPUT, tc, 0, data_len, written);
1536 tc->rcv_nxt += written;
1539 else if (written > data_len)
1541 tc->rcv_nxt += written;
1544 tc->flags |= TCP_CONN_SNDACK;
1545 TCP_EVT_DBG (TCP_EVT_CC_INPUT, tc, data_len, written);
1547 else if (written > 0)
1550 tc->rcv_nxt += written;
1554 tc->flags |= TCP_CONN_SNDACK;
1556 error = TCP_ERROR_PARTIALLY_ENQUEUED;
1560 tc->flags |= TCP_CONN_SNDACK;
1561 return TCP_ERROR_FIFO_FULL;
1595 return TCP_ERROR_FIFO_FULL;
1598 TCP_EVT_DBG (TCP_EVT_INPUT, tc, 1, data_len, data_len);
1606 s0 =
session_get (tc->c_s_index, tc->c_thread_index);
1614 start = tc->rcv_nxt +
offset;
1622 return TCP_ERROR_ENQUEUED_OOO;
1635 || (tc->flags & TCP_CONN_SENT_RCV_WND0) != 0
1637 || (tc->flags & TCP_CONN_SNDACK) != 0
1654 if (!(b->
flags & VLIB_BUFFER_NEXT_PRESENT))
1661 n_bytes_to_drop -= discard;
1663 while (n_bytes_to_drop);
1664 if (n_bytes_to_drop > first)
1681 u32 error, n_bytes_to_drop, n_data_bytes;
1699 error = TCP_ERROR_SEGMENT_OLD;
1706 n_bytes_to_drop = tc->rcv_nxt -
vnet_buffer (b)->tcp.seq_number;
1707 n_data_bytes -= n_bytes_to_drop;
1711 error = TCP_ERROR_SEGMENT_OLD;
1761 s =
format (s,
"%U\n%U%U",
1776 s =
format (s,
"%d -> %d (%U)",
1777 clib_net_to_host_u16 (t->
tcp_header.dst_port),
1819 if (b0->
flags & VLIB_BUFFER_IS_TRACED)
1843 #define tcp_maybe_inc_counter(node_id, err, count) \ 1845 if (next0 != tcp_next_drop (is_ip4)) \ 1846 tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \ 1847 tcp6_##node_id##_node.index, is_ip4, err, \ 1850 #define tcp_inc_counter(node_id, err, count) \ 1851 tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \ 1852 tcp6_##node_id##_node.index, is_ip4, \ 1854 #define tcp_maybe_inc_err_counter(cnts, err) \ 1856 cnts[err] += (next0 != tcp_next_drop (is_ip4)); \ 1858 #define tcp_inc_err_counter(cnts, err, val) \ 1862 #define tcp_store_err_counters(node_id, cnts) \ 1865 for (i = 0; i < TCP_N_ERROR; i++) \ 1867 tcp_inc_counter(node_id, i, cnts[i]); \ 1876 u32 n_left_from, next_index, *from, *to_next;
1887 while (n_left_from > 0)
1892 while (n_left_from > 0 && n_left_to_next > 0)
1900 if (n_left_from > 1)
1913 n_left_to_next -= 1;
1921 error0 = TCP_ERROR_INVALID_CONNECTION;
1971 tc0->state = TCP_STATE_CLOSE_WAIT;
1981 n_left_to_next, bi0, next0);
1989 err_counters[TCP_ERROR_EVENT_FIFO_FULL] = errors;
2014 .name =
"tcp4-established",
2016 .vector_size =
sizeof (
u32),
2022 #define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n, 2036 .name =
"tcp6-established",
2038 .vector_size =
sizeof (
u32),
2044 #define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n, 2068 if (tc->c_lcl_port == 0 && tc->state == TCP_STATE_LISTEN)
2071 u8 is_valid = (tc->c_lcl_port == hdr->dst_port
2072 && (tc->state == TCP_STATE_LISTEN
2073 || tc->c_rmt_port == hdr->src_port));
2079 tc->c_proto, tc->c_is_ip4);
2083 if (tmp->lcl_port == hdr->dst_port
2084 && tmp->rmt_port == hdr->src_port)
2086 TCP_DBG (
"half-open is valid!");
2115 thread_index, &is_filtered);
2130 thread_index, &is_filtered);
2142 u32 n_left_from, next_index, *from, *to_next;
2150 while (n_left_from > 0)
2156 while (n_left_from > 0 && n_left_to_next > 0)
2158 u32 bi0, ack0, seq0;
2171 n_left_to_next -= 1;
2176 tcp.connection_index);
2179 error0 = TCP_ERROR_INVALID_CONNECTION;
2189 my_thread_index, is_ip4));
2200 || tcp0->src_port != tc0->c_rmt_port))
2225 if (
seq_leq (ack0, tc0->iss) ||
seq_gt (ack0, tc0->snd_nxt))
2234 if (
seq_gt (tc0->snd_una, ack0))
2278 pool_get (tm->connections[my_thread_index], new_tc0);
2280 new_tc0->c_c_index = new_tc0 - tm->connections[my_thread_index];
2281 new_tc0->c_thread_index = my_thread_index;
2283 new_tc0->irs = seq0;
2285 new_tc0->timers[TCP_TIMER_RETRANSMIT_SYN] =
2292 tc0->flags |= TCP_CONN_HALF_OPEN_DONE;
2296 new_tc0->tsval_recent = new_tc0->rcv_opts.tsval;
2301 new_tc0->snd_wscale = new_tc0->rcv_opts.wscale;
2303 new_tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
2304 << new_tc0->snd_wscale;
2305 new_tc0->snd_wl1 = seq0;
2306 new_tc0->snd_wl2 = ack0;
2316 new_tc0->snd_una = ack0;
2317 new_tc0->state = TCP_STATE_ESTABLISHED;
2320 new_tc0->rcv_las = new_tc0->rcv_nxt;
2333 new_tc0->flags |= TCP_CONN_SNDACK;
2342 new_tc0->state = TCP_STATE_SYN_RCVD;
2366 if (error0 == TCP_ERROR_ACK_OK)
2367 error0 = TCP_ERROR_SYN_ACKS_RCVD;
2380 ((b0->
flags & VLIB_BUFFER_IS_TRACED) && tcp0 != 0))
2389 n_left_to_next, bi0, next0);
2419 .name =
"tcp4-syn-sent",
2421 .vector_size =
sizeof (
u32),
2427 #define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n, 2441 .name =
"tcp6-syn-sent",
2443 .vector_size =
sizeof (
u32),
2449 #define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n, 2470 u32 n_left_from, next_index, *from, *to_next, n_fins = 0;
2477 while (n_left_from > 0)
2483 while (n_left_from > 0 && n_left_to_next > 0)
2497 n_left_to_next -= 1;
2504 error0 = TCP_ERROR_INVALID_CONNECTION;
2519 my_thread_index, is_ip4);
2520 if (tmp->state != tc0->state)
2532 error0 = TCP_ERROR_CONNECTION_CLOSED;
2551 case TCP_STATE_SYN_RCVD:
2560 TCP_DBG (
"connection not accepted");
2562 error0 = TCP_ERROR_ACK_INVALID;
2570 tc0->state = TCP_STATE_ESTABLISHED;
2575 tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
2576 << tc0->rcv_opts.wscale;
2584 error0 = TCP_ERROR_ACK_OK;
2586 case TCP_STATE_ESTABLISHED:
2596 case TCP_STATE_FIN_WAIT_1:
2607 if (tc0->flags & TCP_CONN_FINPNDG)
2614 else if (tc0->snd_una == tc0->snd_una_max)
2616 tc0->state = TCP_STATE_FIN_WAIT_2;
2626 case TCP_STATE_FIN_WAIT_2:
2636 case TCP_STATE_CLOSE_WAIT:
2643 if (tc0->flags & TCP_CONN_FINPNDG)
2650 tc0->state = TCP_STATE_LAST_ACK;
2656 case TCP_STATE_CLOSING:
2666 tc0->state = TCP_STATE_TIME_WAIT;
2672 case TCP_STATE_LAST_ACK:
2679 error0 = TCP_ERROR_ACK_INVALID;
2682 error0 = TCP_ERROR_ACK_OK;
2685 if (is_fin0 && tc0->snd_una != tc0->snd_una_max)
2691 tc0->state = TCP_STATE_CLOSED;
2703 case TCP_STATE_TIME_WAIT:
2729 case TCP_STATE_ESTABLISHED:
2730 case TCP_STATE_FIN_WAIT_1:
2731 case TCP_STATE_FIN_WAIT_2:
2740 case TCP_STATE_CLOSE_WAIT:
2741 case TCP_STATE_CLOSING:
2742 case TCP_STATE_LAST_ACK:
2743 case TCP_STATE_TIME_WAIT:
2755 case TCP_STATE_ESTABLISHED:
2756 case TCP_STATE_SYN_RCVD:
2761 tc0->snd_una_max = tc0->snd_nxt;
2765 tc0->state = TCP_STATE_CLOSE_WAIT;
2768 case TCP_STATE_CLOSE_WAIT:
2769 case TCP_STATE_CLOSING:
2770 case TCP_STATE_LAST_ACK:
2773 case TCP_STATE_FIN_WAIT_1:
2774 tc0->state = TCP_STATE_CLOSING;
2781 case TCP_STATE_FIN_WAIT_2:
2783 tc0->state = TCP_STATE_TIME_WAIT;
2790 case TCP_STATE_TIME_WAIT:
2811 n_left_to_next, bi0, next0);
2842 .name =
"tcp4-rcv-process",
2844 .vector_size =
sizeof (
u32),
2850 #define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n, 2864 .name =
"tcp6-rcv-process",
2866 .vector_size =
sizeof (
u32),
2872 #define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n, 2892 u32 n_left_from, next_index, *from, *to_next, n_syns = 0;
2900 while (n_left_from > 0)
2906 while (n_left_from > 0 && n_left_to_next > 0)
2923 n_left_to_next -= 1;
2956 my_thread_index, is_ip4);
2959 error0 = TCP_ERROR_CREATE_EXISTS;
2965 child0->c_lcl_port = th0->dst_port;
2966 child0->c_rmt_port = th0->src_port;
2967 child0->c_is_ip4 = is_ip4;
2968 child0->state = TCP_STATE_SYN_RCVD;
2969 child0->c_fib_index = lc0->c_fib_index;
2991 child0->rcv_nxt =
vnet_buffer (b0)->tcp.seq_number + 1;
2992 child0->rcv_las = child0->rcv_nxt;
2999 child0->tsval_recent = child0->rcv_opts.tsval;
3004 child0->snd_wscale = child0->rcv_opts.wscale;
3006 child0->snd_wnd = clib_net_to_host_u16 (th0->window)
3007 << child0->snd_wscale;
3008 child0->snd_wl1 =
vnet_buffer (b0)->tcp.seq_number;
3009 child0->snd_wl2 =
vnet_buffer (b0)->tcp.ack_number;
3019 error0 = TCP_ERROR_CREATE_SESSION_FAIL;
3037 n_syns += (error0 == TCP_ERROR_NONE);
3041 n_left_to_next, bi0, next0);
3069 .name =
"tcp4-listen",
3071 .vector_size =
sizeof (
u32),
3077 #define _(s,n) [TCP_LISTEN_NEXT_##s] = n, 3091 .name =
"tcp6-listen",
3093 .vector_size =
sizeof (
u32),
3099 #define _(s,n) [TCP_LISTEN_NEXT_##s] = n, 3112 typedef enum _tcp_input_next
3124 #define foreach_tcp4_input_next \ 3125 _ (DROP, "ip4-drop") \ 3126 _ (LISTEN, "tcp4-listen") \ 3127 _ (RCV_PROCESS, "tcp4-rcv-process") \ 3128 _ (SYN_SENT, "tcp4-syn-sent") \ 3129 _ (ESTABLISHED, "tcp4-established") \ 3130 _ (RESET, "tcp4-reset") \ 3131 _ (PUNT, "ip4-punt") 3133 #define foreach_tcp6_input_next \ 3134 _ (DROP, "ip6-drop") \ 3135 _ (LISTEN, "tcp6-listen") \ 3136 _ (RCV_PROCESS, "tcp6-rcv-process") \ 3137 _ (SYN_SENT, "tcp6-syn-sent") \ 3138 _ (ESTABLISHED, "tcp6-established") \ 3139 _ (RESET, "tcp6-reset") \ 3140 _ (PUNT, "ip6-punt") 3142 #define filter_flags (TCP_FLAG_SYN|TCP_FLAG_ACK|TCP_FLAG_RST|TCP_FLAG_FIN) 3153 for (i = 0; i < n_bufs; i++)
3155 if (bs[i]->
flags & VLIB_BUFFER_IS_TRACED)
3169 if (*error == TCP_ERROR_FILTERED)
3173 else if ((is_ip4 && tm->punt_unknown4) || (!is_ip4 && tm->punt_unknown6))
3176 *error = TCP_ERROR_PUNT;
3181 *error = TCP_ERROR_NO_LISTENER;
3190 int n_advance_bytes, n_data_bytes;
3201 n_data_bytes = clib_net_to_host_u16 (ip4->
length) - n_advance_bytes;
3206 *error = TCP_ERROR_LENGTH;
3213 thread_index, &is_filtered);
3223 n_advance_bytes +=
sizeof (ip6[0]);
3227 *error = TCP_ERROR_LENGTH;
3234 thread_index, &is_filtered);
3237 vnet_buffer (b)->tcp.seq_number = clib_net_to_host_u32 (tcp->seq_number);
3238 vnet_buffer (b)->tcp.ack_number = clib_net_to_host_u32 (tcp->ack_number);
3239 vnet_buffer (b)->tcp.data_offset = n_advance_bytes;
3243 *error = is_filtered ? TCP_ERROR_FILTERED : *error;
3257 *next = tm->dispatch_table[tc->state][
flags].next;
3258 *error = tm->dispatch_table[tc->state][
flags].error;
3267 if (*error == TCP_ERROR_DISPATCH)
3291 while (n_left_from >= 4)
3293 u32 error0 = TCP_ERROR_NO_LISTENER, error1 = TCP_ERROR_NO_LISTENER;
3314 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3315 vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
3325 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3334 vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
3345 while (n_left_from > 0)
3348 u32 error0 = TCP_ERROR_NO_LISTENER;
3350 if (n_left_from > 1)
3361 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3397 .name =
"tcp4-input",
3399 .vector_size =
sizeof (
u32),
3405 #define _(s,n) [TCP_INPUT_NEXT_##s] = n, 3420 .name =
"tcp6-input",
3422 .vector_size =
sizeof (
u32),
3428 #define _(s,n) [TCP_INPUT_NEXT_##s] = n, 3443 for (i = 0; i <
ARRAY_LEN (tm->dispatch_table); i++)
3444 for (j = 0; j <
ARRAY_LEN (tm->dispatch_table[i]); j++)
3447 tm->dispatch_table[
i][j].error = TCP_ERROR_DISPATCH;
3450 #define _(t,f,n,e) \ 3452 tm->dispatch_table[TCP_STATE_##t][f].next = (n); \ 3453 tm->dispatch_table[TCP_STATE_##t][f].error = (e); \ 3519 TCP_ERROR_CONNECTION_CLOSED);
#define tcp_in_cong_recovery(tc)
#define tcp_fastrecovery_1_smss_off(tc)
#define tcp_in_recovery(tc)
static void tcp_retransmit_timer_set(tcp_connection_t *tc)
#define TCP_OPTION_LEN_SACK_PERMITTED
#define seq_leq(_s1, _s2)
void tcp_make_fin(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to FIN-ACK.
struct _sack_block sack_block_t
#define timestamp_leq(_t1, _t2)
struct _transport_connection transport_connection_t
#define TCP_TIMEWAIT_TIME
#define tcp_opts_tstamp(_to)
void tcp_fast_retransmit(tcp_connection_t *tc)
Do fast retransmit.
tcp_connection_t * tcp_connection_new(u8 thread_index)
struct _sack_scoreboard sack_scoreboard_t
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
struct _tcp_main tcp_main_t
void tcp_connection_timers_reset(tcp_connection_t *tc)
Stop all connection timers.
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
#define tcp_recovery_off(tc)
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
struct _tcp_connection tcp_connection_t
static u32 tcp_available_cc_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
#define tcp_opts_sack(_to)
void tcp_fast_retransmit_sack(tcp_connection_t *tc)
Do fast retransmit with SACKs.
tcp_connection_t tcp_connection
static tcp_connection_t * tcp_get_connection_from_transport(transport_connection_t *tconn)
int session_enqueue_stream_connection(transport_connection_t *tc, vlib_buffer_t *b, u32 offset, u8 queue_event, u8 is_in_order)
u64 session_lookup_half_open_handle(transport_connection_t *tc)
void tcp_send_reset_w_pkt(tcp_connection_t *tc, vlib_buffer_t *pkt, u8 is_ip4)
Send reset without reusing existing buffer.
vlib_error_t * errors
Vector of errors for this node.
format_function_t format_tcp_flags
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
u8 n_sack_blocks
Number of SACKs blocks.
struct _tcp_header tcp_header_t
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
struct _sack_scoreboard_hole sack_scoreboard_hole_t
u8 wscale
Window scale advertised.
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define tcp_fastrecovery_on(tc)
void tcp_flush_frame_to_output(vlib_main_t *vm, u8 thread_index, u8 is_ip4)
Flush tx frame populated by retransmits and timer pops.
memset(h->entries, 0, sizeof(h->entries[0])*entries)
void tcp_init_snd_vars(tcp_connection_t *tc)
Initialize connection send variables.
#define VLIB_INIT_FUNCTION(x)
#define TCP_CLOSEWAIT_TIME
void stream_session_accept_notify(transport_connection_t *tc)
#define TCP_OPTION_LEN_SACK_BLOCK
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
enum _tcp_state tcp_state_t
#define TCP_ALWAYS_ACK
On/off delayed acks.
vhost_vring_state_t state
static u32 ooo_segment_length(svm_fifo_t *f, ooo_segment_t *s)
static void * ip4_next_header(ip4_header_t *i)
static u32 tcp_time_now(void)
sack_block_t * sacks
SACK blocks.
#define vec_end(v)
End (last data address) of vector.
struct _stream_session_t stream_session_t
#define vlib_call_init_function(vm, x)
#define TCP_MAX_SACK_BLOCKS
Max number of SACK blocks stored.
#define tcp_validate_txf_size(_tc, _a)
#define TCP_EVT_DBG(_evt, _args...)
#define timestamp_lt(_t1, _t2)
static void tcp_timer_set(tcp_connection_t *tc, u8 timer_id, u32 interval)
#define TCP_OPTION_LEN_WINDOW_SCALE
static void svm_fifo_newest_ooo_segment_reset(svm_fifo_t *f)
static heap_elt_t * first(heap_header_t *h)
u32 stream_session_dequeue_drop(transport_connection_t *tc, u32 max_bytes)
#define TCP_INVALID_SACK_HOLE_INDEX
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
u16 current_length
Nbytes between current data and the end of this buffer.
#define tcp_in_fastrecovery(tc)
void tcp_retransmit_first_unacked(tcp_connection_t *tc)
Retransmit first unacked segment.
static u32 ooo_segment_offset(svm_fifo_t *f, ooo_segment_t *s)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
#define pool_put(P, E)
Free an object E in pool P.
#define TCP_TIMER_HANDLE_INVALID
void tcp_fast_retransmit_no_sack(tcp_connection_t *tc)
Fast retransmit without SACK info.
#define vec_del1(v, i)
Delete the element at index I.
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
int stream_session_accept(transport_connection_t *tc, u32 listener_index, u8 notify)
Accept a stream session.
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
#define TCP_OPTION_LEN_TIMESTAMP
static ooo_segment_t * svm_fifo_newest_ooo_segment(svm_fifo_t *f)
vlib_error_t error
Error code for buffers to be enqueued to error handler.
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
static stream_session_t * session_get(u32 si, u32 thread_index)
#define TCP_DBG(_fmt, _args...)
#define TCP_MAX_WND_SCALE
static void tcp_timer_reset(tcp_connection_t *tc, u8 timer_id)
#define VLIB_REGISTER_NODE(x,...)
#define CLIB_PREFETCH(addr, size, type)
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
#define vec_free(V)
Free vector's memory (no header).
#define TCP_DUPACK_THRESHOLD
format_function_t format_tcp_state
#define clib_warning(format, args...)
#define clib_memcpy(a, b, c)
void tcp_make_synack(tcp_connection_t *ts, vlib_buffer_t *b)
Convert buffer to SYN-ACK.
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
u16 mss
Option flags, see above.
static void * ip6_next_header(ip6_header_t *i)
void tcp_make_ack(tcp_connection_t *ts, vlib_buffer_t *b)
Convert buffer to ACK.
void stream_session_disconnect_notify(transport_connection_t *tc)
Notification from transport that connection is being closed.
transport_connection_t * session_lookup_connection_wt4(u32 fib_index, ip4_address_t *lcl, ip4_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *is_filtered)
Lookup connection with ip4 and transport layer information.
static void tcp_timer_update(tcp_connection_t *tc, u8 timer_id, u32 interval)
#define TCP_PAWS_IDLE
24 days
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
u32 session_tx_fifo_max_dequeue(transport_connection_t *tc)
#define seq_geq(_s1, _s2)
u32 next_buffer
Next buffer for this linked-list of buffers.
static void transport_add_tx_event(transport_connection_t *tc)
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
transport_connection_t * session_lookup_connection_wt6(u32 fib_index, ip6_address_t *lcl, ip6_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *is_filtered)
Lookup connection with ip6 and transport layer information.
static vlib_main_t * vlib_get_main(void)
static clib_error_t * tcp_init(vlib_main_t *vm)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
struct _vlib_node_registration vlib_node_registration_t
#define tcp_opts_wscale(_to)
void tcp_connection_reset(tcp_connection_t *tc)
Notify session that connection has been reset.
u32 tsval
Timestamp value.
u32 tsecr
Echoed/reflected time stamp.
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static u32 tcp_set_time_now(u32 thread_index)
static u8 tcp_is_lost_fin(tcp_connection_t *tc)
static void tcp_retransmit_timer_update(tcp_connection_t *tc)
#define seq_max(_s1, _s2)
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
void tcp_connection_init_vars(tcp_connection_t *tc)
Initialize tcp connection variables.
#define TCP_OPTION_LEN_MSS
struct clib_bihash_value offset
template key/value backing page structure
static void tcp_retransmit_timer_force_update(tcp_connection_t *tc)
#define tcp_scoreboard_trace_add(_tc, _ack)
u8 * format_tcp_connection(u8 *s, va_list *args)
int session_manager_flush_enqueue_events(u8 transport_proto, u32 thread_index)
Flushes queue of sessions that are to be notified of new data enqueued events.
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
static int tcp_header_bytes(tcp_header_t *t)
int session_stream_connect_notify(transport_connection_t *tc, u8 is_fail)
void tcp_connection_cleanup(tcp_connection_t *tc)
Cleans up connection state.
u16 flags
Copy of main node flags.
#define tcp_opts_sack_permitted(_to)
static int ip4_header_bytes(const ip4_header_t *i)
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
#define VLIB_NODE_FLAG_TRACE
#define CLIB_CACHE_LINE_BYTES
#define TCP_SYN_RCVD_TIME
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static void tcp_persist_timer_set(tcp_connection_t *tc)
static tcp_main_t * vnet_get_tcp_main()
#define tcp_fastrecovery_off(tc)
static void tcp_retransmit_timer_reset(tcp_connection_t *tc)
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static u8 tcp_timer_is_active(tcp_connection_t *tc, tcp_timers_e timer)
transport_connection_t * session_lookup_half_open_connection(u64 handle, u8 proto, u8 is_ip4)
static tcp_connection_t * tcp_listener_get(u32 tli)
static void tcp_persist_timer_reset(tcp_connection_t *tc)
static uword pool_elts(void *v)
Number of active elements in a pool.