19 typedef enum _tcp_output_next
28 #define foreach_tcp4_output_next \ 29 _ (DROP, "error-drop") \ 30 _ (IP_LOOKUP, "ip4-lookup") \ 31 _ (IP_REWRITE, "ip4-rewrite") \ 34 #define foreach_tcp6_output_next \ 35 _ (DROP, "error-drop") \ 36 _ (IP_LOOKUP, "ip6-lookup") \ 37 _ (IP_REWRITE, "ip6-rewrite") \ 38 _ (IP_ARP, "ip6-discover-neighbor") 41 #define tcp_error(n,s) s, 68 #ifndef CLIB_MARCH_VARIANT 104 if (tc->state != TCP_STATE_SYN_RCVD ||
tcp_opts_wscale (&tc->rcv_opts))
115 u32 available_space, wnd;
134 observed_wnd = (
i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
140 TCP_EVT (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
144 wnd = available_space;
148 if (wnd && tc->rcv_wscale)
150 wnd &= ~((1 << tc->rcv_wscale) - 1);
152 wnd = 1 << tc->rcv_wscale;
164 if (state < TCP_STATE_ESTABLISHED)
168 return tc->rcv_wnd >> tc->rcv_wscale;
178 u32 buf, seq_len = 4;
184 buf = clib_host_to_net_u16 (opts->
mss);
186 data +=
sizeof (opts->
mss);
209 buf = clib_host_to_net_u32 (opts->
tsval);
211 data +=
sizeof (opts->
tsval);
212 buf = clib_host_to_net_u32 (opts->
tsecr);
214 data +=
sizeof (opts->
tsecr);
228 buf = clib_host_to_net_u32 (opts->
sacks[i].start);
231 buf = clib_host_to_net_u32 (opts->
sacks[i].end);
260 opts->
flags |= TCP_OPTS_FLAG_MSS;
264 opts->
flags |= TCP_OPTS_FLAG_WSCALE;
265 opts->
wscale = tc->rcv_wscale;
268 opts->
flags |= TCP_OPTS_FLAG_TSTAMP;
275 opts->
flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
289 opts->
flags |= TCP_OPTS_FLAG_MSS;
295 opts->
flags |= TCP_OPTS_FLAG_WSCALE;
296 opts->
wscale = tc->rcv_wscale;
302 opts->
flags |= TCP_OPTS_FLAG_TSTAMP;
304 opts->
tsecr = tc->tsval_recent;
310 opts->
flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
328 opts->
flags |= TCP_OPTS_FLAG_TSTAMP;
330 opts->
tsecr = tc->tsval_recent;
337 opts->
flags |= TCP_OPTS_FLAG_SACK;
338 if (tc->snd_sack_pos >=
vec_len (tc->snd_sacks))
339 tc->snd_sack_pos = 0;
340 opts->
sacks = &tc->snd_sacks[tc->snd_sack_pos];
360 case TCP_STATE_ESTABLISHED:
361 case TCP_STATE_CLOSE_WAIT:
362 case TCP_STATE_FIN_WAIT_1:
363 case TCP_STATE_LAST_ACK:
364 case TCP_STATE_CLOSING:
365 case TCP_STATE_FIN_WAIT_2:
366 case TCP_STATE_TIME_WAIT:
367 case TCP_STATE_CLOSED:
369 case TCP_STATE_SYN_RCVD:
371 case TCP_STATE_SYN_SENT:
397 TCP_STATE_ESTABLISHED);
400 tc->snd_mss =
clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
408 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
411 if (tc->snd_una == tc->snd_nxt)
423 if (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
426 b->
flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
436 #ifndef CLIB_MARCH_VARIANT 440 ASSERT ((b->
flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
441 b->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
454 ip46_address_t *
src, ip46_address_t *
dst)
457 u16 payload_length_host_byte_order;
462 clib_host_to_net_u16 (IP_PROTOCOL_TCP);
465 for (i = 0; i <
ARRAY_LEN (src->ip6.as_uword); i++)
474 payload_length_host_byte_order,
NULL, 0,
480 ip46_address_t *
src, ip46_address_t *
dst)
483 u32 payload_length_host_byte_order;
487 clib_host_to_net_u32 (payload_length_host_byte_order +
488 (IP_PROTOCOL_TCP << 16));
494 payload_length_host_byte_order,
NULL, 0,
502 if (
PREDICT_FALSE (tc->cfg_flags & TCP_CFG_F_NO_CSUM_OFFLOAD))
509 (vm, b, &tc->c_lcl_ip, &tc->c_rmt_ip);
512 (vm, b, &tc->c_lcl_ip, &tc->c_rmt_ip);
516 b->
flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
529 u8 tcp_opts_len, tcp_hdr_opts_len;
537 tcp_hdr_opts_len = tcp_opts_len +
sizeof (
tcp_header_t);
540 tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
546 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
561 TCP_EVT (TCP_EVT_ACK_SENT, tc);
562 tc->rcv_las = tc->rcv_nxt;
580 u8 tcp_hdr_opts_len, tcp_opts_len;
590 tcp_hdr_opts_len = tcp_opts_len +
sizeof (
tcp_header_t);
595 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
607 u8 tcp_opts_len, tcp_hdr_opts_len;
614 tcp_hdr_opts_len = tcp_opts_len +
sizeof (
tcp_header_t);
617 tc->rcv_nxt, tcp_hdr_opts_len,
621 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
630 u32 *to_next, next_index;
633 b->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
683 b->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
725 src_port = th0->src_port;
726 dst_port = th0->dst_port;
729 if (state == TCP_STATE_CLOSED)
734 tmp = clib_net_to_host_u32 (th0->seq_number);
738 ack = clib_host_to_net_u32 (tmp + 1);
744 seq = th0->ack_number;
771 #ifndef CLIB_MARCH_VARIANT 818 seq = pkt_th->ack_number;
819 ack = (tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
825 ack = clib_host_to_net_u32 (
vnet_buffer (pkt)->tcp.seq_end);
829 seq, ack, tcp_hdr_len, flags, 0);
848 tc->ipv6_flow_label);
854 TCP_EVT (TCP_EVT_RST_SENT, tc);
856 TCP_ERROR_RST_SENT, 1);
870 u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
879 tcp_hdr_opts_len = tc->snd_opts_len +
sizeof (
tcp_header_t);
883 tc->rcv_nxt, tcp_hdr_opts_len, flags,
887 ASSERT (opts_write_len == tc->snd_opts_len);
888 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
890 TCP_EVT (TCP_EVT_RST_SENT, tc);
892 TCP_ERROR_RST_SENT, 1);
907 IP_PROTOCOL_TCP, tc->ipv6_flow_label);
945 tc->rtt_seq = tc->snd_nxt;
950 TCP_EVT (TCP_EVT_SYN_SENT, tc);
974 TCP_EVT (TCP_EVT_SYNACK_SENT, tc);
1015 fin_snt = tc->
flags & TCP_CONN_FINSNT;
1027 tc->flags |= TCP_CONN_FINSNT;
1032 if ((tc->flags & TCP_CONN_SNDACK) && !tc->pending_dupacks)
1033 tc->flags &= ~TCP_CONN_SNDACK;
1040 TCP_EVT (TCP_EVT_FIN_SENT, tc);
1045 tc->flags |= TCP_CONN_FINSNT;
1046 tc->flags &= ~TCP_CONN_FINPNDG;
1047 tc->snd_una_max =
seq_max (tc->snd_una_max, tc->snd_nxt);
1057 u8 compute_opts,
u8 maybe_burst,
u8 update_snd_nxt)
1060 u32 advertise_wnd, data_len;
1069 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
1074 tcp_hdr_opts_len = tc->snd_opts_len +
sizeof (
tcp_header_t);
1077 advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
1083 if (
seq_geq (tc->psh_seq, snd_nxt)
1084 &&
seq_lt (tc->psh_seq, snd_nxt + data_len))
1085 flags |= TCP_FLAG_PSH;
1088 tc->rcv_nxt, tcp_hdr_opts_len, flags,
1094 tm->wrk_ctx[tc->c_thread_index].cached_opts,
1100 ASSERT (len == tc->snd_opts_len);
1108 tc->snd_nxt += data_len;
1109 tc->rcv_las = tc->rcv_nxt;
1111 tc->bytes_out += data_len;
1112 tc->data_segs_out += 1;
1133 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
1139 tc->snd_una_max =
seq_max (tc->snd_nxt, tc->snd_una_max);
1145 tc->rtt_seq = tc->snd_nxt;
1178 if (!(tc->flags & TCP_CONN_SNDACK))
1181 tc->flags |= TCP_CONN_SNDACK;
1188 if (!(tc->flags & TCP_CONN_SNDACK))
1191 tc->flags |= TCP_CONN_SNDACK;
1193 if (tc->pending_dupacks < 255)
1194 tc->pending_dupacks += 1;
1200 if (!(tc->flags & TCP_CONN_RXT_PENDING))
1203 tc->flags |= TCP_CONN_RXT_PENDING;
1234 if (tc->rcv_wnd >=
tcp_cfg.rwnd_min_update_ack * tc->snd_mss)
1284 ASSERT (n_bytes == max_deq_bytes);
1292 u32 chain_bi = ~0, n_bufs_per_seg, n_bufs;
1293 u16 n_peeked, len_to_deq;
1298 n_bufs_per_seg = ceil ((
double) seg_size / bytes_per_buffer);
1313 TRANSPORT_MAX_HDRS_LEN);
1315 b[0]->
flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1317 max_deq_bytes -= n_bytes;
1320 for (i = 1; i < n_bufs_per_seg; i++)
1323 len_to_deq =
clib_min (max_deq_bytes, bytes_per_buffer);
1331 ASSERT (n_peeked == len_to_deq);
1332 n_bytes += n_peeked;
1338 prev_b->
flags |= VLIB_BUFFER_NEXT_PRESENT;
1340 max_deq_bytes -= n_peeked;
1355 ASSERT (((*b)->current_data + (*b)->current_length) <= bytes_per_buffer);
1371 u32 start, available_bytes;
1374 ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1375 ASSERT (max_deq_bytes != 0);
1381 ASSERT (available_bytes >= offset);
1382 available_bytes -=
offset;
1383 if (!available_bytes)
1386 max_deq_bytes =
clib_min (tc->snd_mss, max_deq_bytes);
1387 max_deq_bytes =
clib_min (available_bytes, max_deq_bytes);
1389 start = tc->snd_una +
offset;
1396 tc->snd_rxt_bytes += n_bytes;
1398 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
1401 tc->bytes_retrans += n_bytes;
1402 tc->segs_retrans += 1;
1403 TCP_EVT (TCP_EVT_CC_RTX, tc, offset, n_bytes);
1415 if (!sb->is_reneging && (!hole || hole->start == tc->snd_una))
1427 TCP_EVT (TCP_EVT_CC_EVT, tc, 6);
1429 tc->prev_ssthresh = tc->ssthresh;
1430 tc->prev_cwnd = tc->cwnd;
1441 tc->cwnd_acc_bytes = 0;
1442 tc->tr_occurences += 1;
1458 if (
PREDICT_FALSE (tc == 0 || tc->state == TCP_STATE_SYN_SENT))
1462 if (tc->state == TCP_STATE_CLOSED)
1465 if (tc->state >= TCP_STATE_ESTABLISHED)
1467 TCP_EVT (TCP_EVT_CC_EVT, tc, 2);
1470 if (tc->flags & TCP_CONN_FINSNT)
1480 if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_nxt)
1481 || (tc->rto_boff > 0 &&
seq_geq (tc->snd_una, tc->snd_congestion)
1515 tc->snd_congestion = tc->snd_nxt;
1519 n_bytes =
clib_min (tc->snd_mss, tc->snd_nxt - tc->snd_una);
1534 if (tc->rto_boff == 1)
1547 else if (tc->state == TCP_STATE_SYN_RCVD)
1549 TCP_EVT (TCP_EVT_CC_EVT, tc, 2);
1577 TCP_EVT (TCP_EVT_SYN_RXT, tc, 1);
1584 ASSERT (tc->state == TCP_STATE_CLOSED);
1604 if (
PREDICT_FALSE (tc == 0 || tc->state != TCP_STATE_SYN_SENT))
1610 if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1613 TCP_DBG (
"could not remove half-open connection");
1617 TCP_EVT (TCP_EVT_CC_EVT, tc, 2);
1644 TCP_EVT (TCP_EVT_SYN_RXT, tc, 0);
1662 u32 bi, max_snd_bytes, available_bytes,
offset;
1675 if (tc->state == TCP_STATE_CLOSED || tc->snd_wnd > tc->snd_mss
1676 || (tc->flags & TCP_CONN_FINSNT))
1680 offset = tc->snd_nxt - tc->snd_una;
1684 if (!available_bytes)
1690 if (available_bytes <= offset)
1710 max_snd_bytes =
clib_min (tc->snd_mss,
1716 || tc->snd_nxt == tc->snd_una_max
1717 || tc->rto_boff > 1));
1719 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
1727 tc->snd_una_max =
seq_max (tc->snd_nxt, tc->snd_una_max);
1745 TCP_EVT (TCP_EVT_CC_EVT, tc, 1);
1761 u32 offset, n_segs = 0, n_written, bi, available_wnd;
1765 offset = tc->snd_nxt - tc->snd_una;
1766 available_wnd = tc->snd_wnd -
offset;
1767 burst_size =
clib_min (burst_size, available_wnd / tc->snd_mss);
1769 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
1772 while (n_segs < burst_size)
1780 offset += n_written;
1783 if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
1786 tc->snd_nxt += n_written;
1787 tc->snd_una_max =
seq_max (tc->snd_nxt, tc->snd_una_max);
1804 prr_out = tc->snd_rxt_bytes + (tc->snd_nxt - tc->snd_congestion);
1806 if (pipe > tc->ssthresh)
1808 space = ((int) tc->prr_delivered * ((
f64) tc->ssthresh / tc->prev_cwnd))
1814 limit =
clib_max ((
int) (tc->prr_delivered - prr_out), 0) + tc->snd_mss;
1815 space =
clib_min (tc->ssthresh - pipe, limit);
1817 space =
clib_max (space, prr_out ? 0 : tc->snd_mss);
1825 u32 tx_adv_sack = sb->high_sacked - tc->snd_congestion;
1826 f64 rr = (
f64) tc->ssthresh / tc->prev_cwnd;
1831 return (tx_adv_sack > (tc->snd_una - tc->prr_start) * rr);
1838 - (tc->snd_nxt - tc->snd_una));
1841 #define scoreboard_rescue_rxt_valid(_sb, _tc) \ 1842 (seq_geq (_sb->rescue_rxt, _tc->snd_una) \ 1843 && seq_leq (_sb->rescue_rxt, _tc->snd_congestion)) 1852 u32 n_written = 0,
offset, max_bytes, n_segs = 0;
1853 u8 snd_limited = 0, can_rescue = 0;
1854 u32 bi, max_deq, burst_bytes;
1864 burst_size =
clib_min (burst_size, burst_bytes / tc->snd_mss);
1876 if (snd_space < tc->snd_mss)
1883 &&
seq_gt (sb->high_sacked, tc->snd_congestion)
1884 && tc->rxt_head != tc->snd_una
1887 max_bytes =
clib_min (tc->snd_mss, tc->snd_congestion - tc->snd_una);
1898 tc->rxt_head = tc->snd_una;
1899 tc->rxt_delivered += n_written;
1900 tc->prr_delivered += n_written;
1901 ASSERT (tc->rxt_delivered <= tc->snd_rxt_bytes);
1906 TCP_EVT (TCP_EVT_CC_EVT, tc, 0);
1910 max_deq -= tc->snd_nxt - tc->snd_una;
1912 while (snd_space > 0 && n_segs < burst_size)
1919 if (max_deq > tc->snd_mss)
1926 av_wnd = (int) tc->snd_wnd - (tc->snd_nxt - tc->snd_una);
1927 av_wnd =
clib_max (av_wnd - tc->snd_mss, 0);
1928 snd_space =
clib_min (snd_space, av_wnd);
1929 snd_space =
clib_min (max_deq, snd_space);
1930 burst_size =
clib_min (burst_size - n_segs,
1931 snd_space / tc->snd_mss);
1934 if (max_deq > n_segs_new * tc->snd_mss)
1937 n_segs += n_segs_new;
1951 max_bytes =
clib_min (tc->snd_mss, hole->end - hole->start);
1952 max_bytes =
clib_min (max_bytes, snd_space);
1953 offset = hole->end - tc->snd_una - max_bytes;
1959 sb->rescue_rxt = tc->snd_congestion;
1966 max_bytes =
clib_min (hole->end - sb->high_rxt, snd_space);
1967 max_bytes = snd_limited ?
clib_min (max_bytes, tc->snd_mss) : max_bytes;
1971 offset = sb->high_rxt - tc->snd_una;
1974 ASSERT (n_written <= snd_space);
1983 sb->high_rxt += n_written;
1986 snd_space -= n_written;
2006 u32 n_written = 0,
offset = 0, bi, max_deq, n_segs_now, max_bytes;
2007 u32 burst_bytes, sent_bytes;
2009 int snd_space, n_segs = 0;
2014 TCP_EVT (TCP_EVT_CC_EVT, tc, 0);
2017 burst_size =
clib_min (burst_size, burst_bytes / tc->snd_mss);
2025 cc_limited = snd_space < burst_bytes;
2032 while (snd_space > 0 && n_segs < burst_size)
2035 tc->snd_congestion - tc->snd_una -
offset);
2047 snd_space -= n_written;
2052 if (n_segs == burst_size)
2058 if (snd_space < tc->snd_mss || tc->snd_mss == 0)
2062 max_deq -= tc->snd_nxt - tc->snd_una;
2065 snd_space =
clib_min (max_deq, snd_space);
2066 burst_size =
clib_min (burst_size - n_segs, snd_space / tc->snd_mss);
2068 if (n_segs_now && max_deq > n_segs_now * tc->snd_mss)
2070 n_segs += n_segs_now;
2076 sent_bytes =
clib_min (n_segs * tc->snd_mss, burst_bytes);
2077 sent_bytes = cc_limited ? burst_bytes : sent_bytes;
2088 if (!tc->pending_dupacks)
2091 || tc->state != TCP_STATE_ESTABLISHED)
2104 tc->pending_dupacks = 0;
2109 tc->snd_sack_pos = 0;
2115 n_acks =
clib_min (n_acks, tc->pending_dupacks);
2117 for (j = 0; j <
clib_min (n_acks, max_burst_size); j++)
2120 if (n_acks < max_burst_size)
2122 tc->pending_dupacks = 0;
2123 tc->snd_sack_pos = 0;
2124 tc->dupacks_out += n_acks;
2129 TCP_DBG (
"constrained by burst size");
2130 tc->pending_dupacks = n_acks - max_burst_size;
2131 tc->dupacks_out += max_burst_size;
2133 return max_burst_size;
2164 tc->flags &= ~TCP_CONN_RXT_PENDING;
2166 max_burst_size -= n_segs;
2169 if (!(tc->flags & TCP_CONN_SNDACK))
2172 tc->flags &= ~TCP_CONN_SNDACK;
2175 if (n_segs && !tc->pending_dupacks)
2178 if (!max_burst_size)
2181 return max_burst_size;
2192 u16 * next0,
u32 * error0)
2205 *error0 = TCP_ERROR_LINK_LOCAL_RW;
2217 *error0 = TCP_ERROR_LINK_LOCAL_RW;
2224 u32 * to_next,
u32 n_bufs)
2232 for (i = 0; i < n_bufs; i++)
2235 if (!(b->
flags & VLIB_BUFFER_IS_TRACED))
2259 IP_PROTOCOL_TCP, tc0->ipv6_flow_label);
2277 ASSERT ((b->
flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID) != 0);
2278 ASSERT ((b->
flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID) != 0);
2279 b->
flags |= VNET_BUFFER_F_GSO;
2292 if (tc0->next_node_index)
2294 *next0 = tc0->next_node_index;
2295 vnet_buffer (b0)->tcp.next_node_opaque = tc0->next_node_opaque;
2347 while (n_left_from >= 4)
2386 b[0]->
error = error_node->
errors[TCP_ERROR_INVALID_CONNECTION];
2398 b[1]->
error = error_node->
errors[TCP_ERROR_INVALID_CONNECTION];
2407 while (n_left_from > 0)
2411 if (n_left_from > 1)
2428 b[0]->
error = error_node->
errors[TCP_ERROR_INVALID_CONNECTION];
2458 .name =
"tcp4-output",
2460 .vector_size =
sizeof (
u32),
2466 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n, 2478 .name =
"tcp6-output",
2480 .vector_size =
sizeof (
u32),
2486 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n, 2495 typedef enum _tcp_reset_next
2502 #define foreach_tcp4_reset_next \ 2503 _(DROP, "error-drop") \ 2504 _(IP_LOOKUP, "ip4-lookup") 2506 #define foreach_tcp6_reset_next \ 2507 _(DROP, "error-drop") \ 2508 _(IP_LOOKUP, "ip6-lookup") 2514 u32 n_left_from, next_index, *from, *to_next;
2522 while (n_left_from > 0)
2528 while (n_left_from > 0 && n_left_to_next > 0)
2541 n_left_to_next -= 1;
2546 my_thread_index, is_ip4))
2548 error0 = TCP_ERROR_LOOKUP_DROPS;
2559 b0->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
2573 n_left_to_next, bi0, next0);
2594 .name =
"tcp4-reset",
2595 .vector_size =
sizeof (
u32),
2600 #define _(s,n) [TCP_RESET_NEXT_##s] = n, 2610 .name =
"tcp6-reset",
2611 .vector_size =
sizeof (
u32),
2616 #define _(s,n) [TCP_RESET_NEXT_##s] = n, void tcp_make_fin(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to FIN-ACK.
#define tcp_in_cong_recovery(tc)
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static void tcp_check_if_gso(tcp_connection_t *tc, vlib_buffer_t *b)
static void tcp_check_sack_reneging(tcp_connection_t *tc)
void session_flush_frames_main_thread(vlib_main_t *vm)
static u32 tcp_options_write(u8 *data, tcp_options_t *opts)
Write TCP options to segment.
static int tcp_send_acks(tcp_connection_t *tc, u32 max_burst_size)
#define TCP_OPTION_LEN_EOL
u16 ip4_tcp_compute_checksum_custom(vlib_main_t *vm, vlib_buffer_t *p0, ip46_address_t *src, ip46_address_t *dst)
#define tcp_in_recovery(tc)
static f64 tcp_time_now_us(u32 thread_index)
static void tcp_retransmit_timer_set(tcp_connection_t *tc)
static u32 transport_rx_fifo_size(transport_connection_t *tc)
#define TCP_OPTION_LEN_SACK_PERMITTED
#define seq_leq(_s1, _s2)
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
#define tcp_node_index(node_id, is_ip4)
static void session_add_pending_tx_buffer(session_type_t st, u32 thread_index, u32 bi)
int session_tx_fifo_peek_bytes(transport_connection_t *tc, u8 *buffer, u32 offset, u32 max_bytes)
#define TCP_TO_TIMER_TICK
Factor for converting ticks to timer ticks.
#define tcp_opts_tstamp(_to)
void tcp_make_synack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN-ACK.
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
static void tcp_flush_frame_to_ip_lookup(tcp_worker_ctx_t *wrk, u8 is_ip4)
Flush ip lookup tx frames populated by timer pops.
static tcp_connection_t * tcp_connection_get_if_valid(u32 conn_index, u32 thread_index)
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
struct _sack_scoreboard sack_scoreboard_t
static int tcp_do_retransmit(tcp_connection_t *tc, u32 max_burst_size)
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
sack_scoreboard_hole_t * scoreboard_last_hole(sack_scoreboard_t *sb)
#define tcp_zero_rwnd_sent_off(tc)
void session_add_self_custom_tx_evt(transport_connection_t *tc, u8 has_prio)
struct _tcp_main tcp_main_t
void tcp_connection_timers_reset(tcp_connection_t *tc)
Stop all connection timers.
This packet is to be rewritten and forwarded to the next processing node.
u16 current_length
Nbytes between current data and the end of this buffer.
static int tcp_transmit_unsent(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 burst_size)
static u32 tcp_initial_wnd_unscaled(tcp_connection_t *tc)
TCP's initial window.
enum _tcp_output_next tcp_output_next_t
void tcp_timer_delack_handler(u32 index, u32 thread_index)
Delayed ack timer handler.
static ip_csum_t ip_csum_with_carry(ip_csum_t sum, ip_csum_t x)
struct _tcp_connection tcp_connection_t
static u32 tcp_available_cc_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
#define tcp_opts_sack(_to)
#define VLIB_NODE_FN(node)
static void tcp_push_ip_hdr(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, vlib_buffer_t *b)
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
static uword tcp46_send_reset_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, u8 is_ip4)
vlib_error_t * errors
Vector of errors for this node.
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
u8 n_sack_blocks
Number of SACKs blocks.
struct _tcp_header tcp_header_t
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
#define scoreboard_rescue_rxt_valid(_sb, _tc)
struct _sack_scoreboard_hole sack_scoreboard_hole_t
u8 wscale
Option flags, see above.
enum fib_protocol_t_ fib_protocol_t
Protocol Type.
#define TCP_OPTS_MAX_SACK_BLOCKS
vlib_node_registration_t ip4_lookup_node
(constructor) VLIB_REGISTER_NODE (ip4_lookup_node)
#define tcp_csum_offload(tc)
#define foreach_tcp4_reset_next
static u32 tcp_prepare_retransmit_segment(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t **b)
Build a retransmit segment.
#define tcp_zero_rwnd_sent_on(tc)
static u16 ip_calculate_l4_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip_csum_t sum0, u32 payload_length, u8 *iph, u32 ip_header_size, u8 *l4h)
void session_transport_closing_notify(transport_connection_t *tc)
Notification from transport that connection is being closed.
static uword tcp46_output_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_ip4)
static void * tcp_init_buffer(vlib_main_t *vm, vlib_buffer_t *b)
static ip_adjacency_t * adj_get(adj_index_t adj_index)
Get a pointer to an adjacency object from its index.
void tcp_make_syn(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN.
static int tcp_prepare_segment(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t **b)
Allocate a new buffer and build a new tcp segment.
static void tcp_connection_set_state(tcp_connection_t *tc, tcp_state_t state)
vl_api_interface_index_t sw_if_index
u8 * format_tcp_connection_id(u8 *s, va_list *args)
sack_scoreboard_hole_t * scoreboard_get_hole(sack_scoreboard_t *sb, u32 index)
#define TCP_OPTION_LEN_SACK_BLOCK
static void tcp_cc_loss(tcp_connection_t *tc)
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
enum _tcp_state tcp_state_t
#define TCP_ALWAYS_ACK
On/off delayed acks.
static void * ip4_next_header(ip4_header_t *i)
static u32 tcp_time_now(void)
sack_block_t * sacks
SACK blocks.
static void tcp46_output_trace_frame(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *to_next, u32 n_bufs)
#define TCP_ESTABLISH_TIME
sack_scoreboard_hole_t * scoreboard_next_rxt_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *start, u8 have_sent_1_smss, u8 *can_rescue, u8 *snd_limited)
Figure out the next hole to retransmit.
#define tcp_validate_txf_size(_tc, _a)
static void tcp_enqueue_to_ip_lookup_now(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
static void tcp_push_hdr_i(tcp_connection_t *tc, vlib_buffer_t *b, u32 snd_nxt, u8 compute_opts, u8 maybe_burst, u8 update_snd_nxt)
Push TCP header and update connection variables.
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
u32 tcp_session_push_header(transport_connection_t *tconn, vlib_buffer_t *b)
#define TCP_OPTION_LEN_WINDOW_SCALE
vlib_node_registration_t tcp6_reset_node
(constructor) VLIB_REGISTER_NODE (tcp6_reset_node)
#define TCP_RTO_SYN_RETRIES
#define tcp_zero_rwnd_sent(tc)
vlib_error_t error
Error code for buffers to be enqueued to error handler.
#define tcp_trajectory_add_start(b, start)
#define TRANSPORT_MAX_HDRS_LEN
static session_type_t session_type_from_proto_and_ip(transport_proto_t proto, u8 is_ip4)
vlib_main_t * vm
convenience pointer to this thread's vlib main
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
void tcp_send_synack(tcp_connection_t *tc)
#define ADJ_INDEX_INVALID
Invalid ADJ index - used when no adj is known likewise blazoned capitals INVALID speak volumes where ...
static int tcp_make_synack_options(tcp_connection_t *tc, tcp_options_t *opts)
static int tcp_make_syn_options(tcp_connection_t *tc, tcp_options_t *opts)
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
static int tcp_retransmit_sack(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 burst_size)
Do retransmit with SACKs.
#define tcp_in_fastrecovery(tc)
void tcp_connection_tx_pacer_reset(tcp_connection_t *tc, u32 window, u32 start_bucket)
static void * vlib_buffer_push_tcp_net_order(vlib_buffer_t *b, u16 sp, u16 dp, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
#define tcp_opts_mss(_to)
void tcp_flush_frames_to_output(tcp_worker_ctx_t *wrk)
Flush v4 and v6 tcp and ip-lookup tx frames for thread index.
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
static void tcp_output_handle_link_local(tcp_connection_t *tc0, vlib_buffer_t *b0, u16 *next0, u32 *error0)
#define foreach_tcp6_output_next
static u32 tcp_flight_size(const tcp_connection_t *tc)
Our estimate of the number of bytes in flight (pipe size)
void tcp_timer_persist_handler(u32 index, u32 thread_index)
Got 0 snd_wnd from peer, try to do something about it.
void tcp_program_dupack(tcp_connection_t *tc)
static int tcp_make_reset_in_place(vlib_main_t *vm, vlib_buffer_t *b0, tcp_state_t state, u8 thread_index, u8 is_ip4)
int tcp_fastrecovery_prr_snd_space(tcp_connection_t *tc)
Estimate send space using proportional rate reduction (RFC6937)
static u8 tcp_window_compute_scale(u32 window)
int tcp_session_custom_tx(void *conn, u32 max_burst_size)
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
#define TCP_OPTION_LEN_TIMESTAMP
#define foreach_tcp4_output_next
#define TCP_RXT_MAX_BURST
static void tcp_enqueue_to_ip_lookup(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
#define TCP_DBG(_fmt, _args...)
#define TCP_MAX_WND_SCALE
static void tcp_timer_reset(tcp_connection_t *tc, u8 timer_id)
static void tcp_output_handle_packet(tcp_connection_t *tc0, vlib_buffer_t *b0, vlib_node_runtime_t *error_node, u16 *next0, u8 is_ip4)
void scoreboard_init_rxt(sack_scoreboard_t *sb, u32 snd_una)
This packet matches an "incomplete adjacency" and packets need to be passed to ARP to find rewrite st...
#define VLIB_REGISTER_NODE(x,...)
static void * vlib_buffer_push_tcp(vlib_buffer_t *b, u16 sp_net, u16 dp_net, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
void scoreboard_clear_reneging(sack_scoreboard_t *sb, u32 start, u32 end)
#define CLIB_PREFETCH(addr, size, type)
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
void tcp_send_window_update_ack(tcp_connection_t *tc)
Send window update ack.
void tcp_program_retransmit(tcp_connection_t *tc)
static u32 tcp_tstamp(tcp_connection_t *tc)
Generate timestamp for tcp connection.
void tcp_send_reset_w_pkt(tcp_connection_t *tc, vlib_buffer_t *pkt, u32 thread_index, u8 is_ip4)
Send reset without reusing existing buffer.
format_function_t format_tcp_state
#define clib_warning(format, args...)
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
void tcp_bt_track_tx(tcp_connection_t *tc, u32 len)
Track a tcp tx burst.
struct _transport_connection transport_connection_t
#define TCP_USE_SACKS
Disable only for testing.
#define tcp_recovery_on(tc)
static u32 tcp_window_to_advertise(tcp_connection_t *tc, tcp_state_t state)
Compute and return window to advertise, scaled as per RFC1323.
#define tcp_fastrecovery_first(tc)
u32 adj_index_t
An index for adjacencies.
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
u16 mss
Maximum segment size advertised.
vlib_main_t vlib_node_runtime_t * node
void tcp_timer_retransmit_syn_handler(u32 tc_index, u32 thread_index)
SYN retransmit timer handler.
static void * ip6_next_header(ip6_header_t *i)
static int tcp_retransmit_no_sack(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 burst_size)
Fast retransmit without SACK info.
static void tcp_make_ack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to ACK.
static u32 transport_max_tx_dequeue(transport_connection_t *tc)
static void tcp_timer_update(tcp_connection_t *tc, u8 timer_id, u32 interval)
u16 ip6_tcp_udp_icmp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip6_header_t *ip0, int *bogus_lengthp)
vlib_node_registration_t ip6_lookup_node
(constructor) VLIB_REGISTER_NODE (ip6_lookup_node)
static int tcp_make_established_options(tcp_connection_t *tc, tcp_options_t *opts)
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
static void tcp_cc_init_rxt_timeout(tcp_connection_t *tc)
Reset congestion control, switch cwnd to loss window and try again.
static void tcp_output_push_ip(vlib_main_t *vm, vlib_buffer_t *b0, tcp_connection_t *tc0, u8 is_ip4)
static u8 * format_tcp_tx_trace(u8 *s, va_list *args)
u16 ip4_tcp_udp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip4_header_t *ip0)
void transport_connection_tx_pacer_reset_bucket(transport_connection_t *tc, u32 bucket)
Reset tx pacer bucket.
void tcp_update_burst_snd_vars(tcp_connection_t *tc)
Update burst send vars.
#define seq_geq(_s1, _s2)
#define TRANSPORT_PACER_MIN_BURST
static uword ip6_address_is_link_local_unicast(const ip6_address_t *a)
#define clib_mem_unaligned(pointer, type)
#define tcp_fastrecovery_first_off(tc)
static void tcp_update_rcv_wnd(tcp_connection_t *tc)
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
void tcp_send_ack(tcp_connection_t *tc)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
void transport_connection_tx_pacer_update_bytes(transport_connection_t *tc, u32 bytes)
int tcp_retransmit_first_unacked(tcp_worker_ctx_t *wrk, tcp_connection_t *tc)
Retransmit first unacked segment.
template key/value backing page structure
#define tcp_opts_wscale(_to)
void tcp_timer_retransmit_handler(u32 tc_index, u32 thread_index)
void tcp_bt_check_app_limited(tcp_connection_t *tc)
Check if sample to be generated is app limited.
u32 tsval
Timestamp value.
u32 tsecr
Echoed/reflected time stamp.
static void * vlib_buffer_push_ip6(vlib_main_t *vm, vlib_buffer_t *b, ip6_address_t *src, ip6_address_t *dst, int proto)
Push IPv6 header to buffer.
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static u8 tcp_max_tx_deq(tcp_connection_t *tc)
ip_lookup_next_t lookup_next_index
Next hop after ip4-lookup.
u32 next_buffer
Next buffer for this linked-list of buffers.
#define foreach_tcp6_reset_next
sack_scoreboard_hole_t * scoreboard_first_hole(sack_scoreboard_t *sb)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
static tcp_worker_ctx_t * tcp_get_worker(u32 thread_index)
void session_transport_closed_notify(transport_connection_t *tc)
Notification from transport that it is closed.
static void tcp_retransmit_timer_update(tcp_connection_t *tc)
VLIB buffer representation.
#define seq_max(_s1, _s2)
static void tcp_enqueue_to_ip_lookup_i(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index, u8 flush)
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
static void tcp_make_ack_i(tcp_connection_t *tc, vlib_buffer_t *b, tcp_state_t state, u8 flags)
Prepare ACK.
#define TCP_OPTION_LEN_MSS
u16 ip6_tcp_compute_checksum_custom(vlib_main_t *vm, vlib_buffer_t *p0, ip46_address_t *src, ip46_address_t *dst)
struct clib_bihash_value offset
template key/value backing page structure
static void tcp_retransmit_timer_force_update(tcp_connection_t *tc)
u32 tcp_initial_window_to_advertise(tcp_connection_t *tc)
Compute initial window and scale factor.
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
static void tcp_cc_event(tcp_connection_t *tc, tcp_cc_event_t evt)
void tcp_update_rto(tcp_connection_t *tc)
int session_stream_connect_notify(transport_connection_t *tc, u8 is_fail)
vl_api_dhcp_client_state_t state
static u32 vlib_num_workers()
void tcp_connection_cleanup(tcp_connection_t *tc)
Cleans up connection state.
static u32 tcp_buffer_len(vlib_buffer_t *b)
static u8 tcp_retransmit_should_retry_head(tcp_connection_t *tc, sack_scoreboard_t *sb)
#define TCP_OPTION_LEN_NOOP
void tcp_send_syn(tcp_connection_t *tc)
Send SYN.
vlib_node_registration_t tcp6_output_node
(constructor) VLIB_REGISTER_NODE (tcp6_output_node)
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
u16 flags
Copy of main node flags.
static u16 tcp_compute_checksum(tcp_connection_t *tc, vlib_buffer_t *b)
enum _tcp_reset_next tcp_reset_next_t
static u32 transport_max_rx_enqueue(transport_connection_t *tc)
#define tcp_opts_sack_permitted(_to)
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
tcp_connection_t tcp_connection
void tcp_program_ack(tcp_connection_t *tc)
static void * vlib_buffer_push_ip6_custom(vlib_main_t *vm, vlib_buffer_t *b, ip6_address_t *src, ip6_address_t *dst, int proto, u32 flow_label)
Push IPv6 header to buffer.
vlib_frame_t * ip_lookup_tx_frames[2]
tx frames for ip 4/6 lookup nodes
static void * tcp_reuse_buffer(vlib_main_t *vm, vlib_buffer_t *b)
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
vlib_node_registration_t tcp4_reset_node
(constructor) VLIB_REGISTER_NODE (tcp4_reset_node)
#define VLIB_NODE_FLAG_TRACE
vlib_node_registration_t tcp4_output_node
(constructor) VLIB_REGISTER_NODE (tcp4_output_node)
#define CLIB_CACHE_LINE_BYTES
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
static void tcp_enqueue_to_output(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4)
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
static void tcp_persist_timer_set(tcp_connection_t *tc)
static tcp_main_t * vnet_get_tcp_main()
static char * tcp_error_strings[]
static void * vlib_buffer_push_ip4(vlib_main_t *vm, vlib_buffer_t *b, ip4_address_t *src, ip4_address_t *dst, int proto, u8 csum_offload)
Push IPv4 header to buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static u32 tcp_set_time_now(tcp_worker_ctx_t *wrk)
void tcp_bt_track_rxt(tcp_connection_t *tc, u32 start, u32 end)
Track a tcp retransmission.
u32 transport_connection_tx_pacer_burst(transport_connection_t *tc)
Get tx pacer max burst.
static u8 tcp_timer_is_active(tcp_connection_t *tc, tcp_timers_e timer)
static void tcp_cc_congestion(tcp_connection_t *tc)
u32 * tx_buffers
tx buffer free list
adj_index_t adj_nbr_find(fib_protocol_t nh_proto, vnet_link_t link_type, const ip46_address_t *nh_addr, u32 sw_if_index)
Lookup neighbor adjancency.
#define TCP_EVT(_evt, _args...)
static int tcp_make_options(tcp_connection_t *tc, tcp_options_t *opts, tcp_state_t state)
static uword pool_elts(void *v)
Number of active elements in a pool.