23 typedef enum _tcp_output_next
30 #define foreach_tcp4_output_next \ 31 _ (DROP, "error-drop") \ 32 _ (IP_LOOKUP, "ip4-lookup") 34 #define foreach_tcp6_output_next \ 35 _ (DROP, "error-drop") \ 36 _ (IP_LOOKUP, "ip6-lookup") 39 #define tcp_error(n,s) s, 134 if (state < TCP_STATE_ESTABLISHED)
139 if (tc->rcv_wnd == 0)
141 tc->flags |= TCP_CONN_SENT_RCV_WND0;
145 tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
148 return tc->rcv_wnd >> tc->rcv_wscale;
155 u32 available_space, max_fifo, wnd;
163 ASSERT (tc->rcv_opts.mss < max_fifo);
164 if (available_space < tc->rcv_opts.mss && available_space < max_fifo >> 3)
171 observed_wnd = (
i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
172 if (observed_wnd < 0)
176 if (available_space < observed_wnd)
179 TCP_EVT_DBG (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
183 wnd = available_space;
187 if (wnd && tc->rcv_wscale)
189 wnd &= ~(1 << tc->rcv_wscale);
191 wnd = 1 << tc->rcv_wscale;
204 u32 buf, seq_len = 4;
210 buf = clib_host_to_net_u16 (opts->
mss);
212 data +=
sizeof (opts->
mss);
235 buf = clib_host_to_net_u32 (opts->
tsval);
237 data +=
sizeof (opts->
tsval);
238 buf = clib_host_to_net_u32 (opts->
tsecr);
240 data +=
sizeof (opts->
tsecr);
250 if (n_sack_blocks != 0)
254 for (i = 0; i < n_sack_blocks; i++)
256 buf = clib_host_to_net_u32 (opts->
sacks[i].start);
259 buf = clib_host_to_net_u32 (opts->
sacks[i].end);
288 opts->
flags |= TCP_OPTS_FLAG_MSS;
292 opts->
flags |= TCP_OPTS_FLAG_WSCALE;
296 opts->
flags |= TCP_OPTS_FLAG_TSTAMP;
303 opts->
flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
317 opts->
flags |= TCP_OPTS_FLAG_MSS;
323 opts->
flags |= TCP_OPTS_FLAG_WSCALE;
324 opts->
wscale = tc->rcv_wscale;
330 opts->
flags |= TCP_OPTS_FLAG_TSTAMP;
332 opts->
tsecr = tc->tsval_recent;
338 opts->
flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
356 opts->
flags |= TCP_OPTS_FLAG_TSTAMP;
358 opts->
tsecr = tc->tsval_recent;
365 opts->
flags |= TCP_OPTS_FLAG_SACK;
366 opts->
sacks = tc->snd_sacks;
384 case TCP_STATE_ESTABLISHED:
385 case TCP_STATE_FIN_WAIT_1:
387 case TCP_STATE_SYN_RCVD:
389 case TCP_STATE_SYN_SENT:
410 tc->snd_mss =
clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
417 u16 default_min_mss = 536;
421 tc->snd_mss =
clib_min (tc->rcv_opts.mss, tc->mss);
423 if (tc->snd_mss < 45)
427 tc->snd_mss = default_min_mss;
428 tc->rcv_opts.mss = default_min_mss;
432 ASSERT (tc->snd_mss > 45);
443 u32 current_length =
vec_len (tm->tx_buffers[thread_index]);
447 current_length + n_free_buffers - 1);
451 _vec_len (tm->tx_buffers[thread_index]) = current_length + n_allocated;
453 if (
vec_len (tm->tx_buffers[thread_index]) == 0)
474 my_tx_buffers = tm->tx_buffers[thread_index];
475 *bidx = my_tx_buffers[
vec_len (my_tx_buffers) - 1];
476 _vec_len (my_tx_buffers) -= 1;
489 if (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
492 b->
flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
505 ASSERT ((b->
flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
506 b->
flags &= VLIB_BUFFER_NON_DEFAULT_FREELIST;
507 b->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
523 u8 tcp_opts_len, tcp_hdr_opts_len;
531 tcp_hdr_opts_len = tcp_opts_len +
sizeof (
tcp_header_t);
534 tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
537 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
552 tc->rcv_las = tc->rcv_nxt;
570 vnet_buffer (b)->tcp.flags &= ~TCP_BUF_FLAG_DUPACK;
579 u8 tcp_hdr_opts_len, tcp_opts_len;
587 memset (&snd_opts, 0,
sizeof (snd_opts));
589 tcp_hdr_opts_len = tcp_opts_len +
sizeof (
tcp_header_t);
594 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
606 u8 tcp_opts_len, tcp_hdr_opts_len;
610 memset (snd_opts, 0,
sizeof (*snd_opts));
615 tcp_hdr_opts_len = tcp_opts_len +
sizeof (
tcp_header_t);
618 tc->rcv_nxt, tcp_hdr_opts_len,
622 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
633 u8 is_ip4,
u32 fib_index,
u8 flush)
637 u32 *to_next, next_index;
640 b->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
650 f = tm->ip_lookup_tx_frames[!is_ip4][thread_index];
655 tm->ip_lookup_tx_frames[!is_ip4][thread_index] = f;
664 tm->ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
670 u8 is_ip4,
u32 fib_index)
677 u8 is_ip4,
u32 fib_index)
688 u32 *to_next, next_index;
691 b->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
699 f = tm->tx_frames[!is_ip4][thread_index];
704 tm->tx_frames[!is_ip4][thread_index] = f;
712 tm->tx_frames[!is_ip4][thread_index] = 0;
738 u16 src_port, dst_port;
762 src_port = th0->src_port;
763 dst_port = th0->dst_port;
766 if (state == TCP_STATE_CLOSED)
771 tmp = clib_net_to_host_u32 (th0->seq_number);
775 ack = clib_host_to_net_u32 (tmp + 1);
781 seq = th0->ack_number;
817 u32 bi, sw_if_index, fib_index;
853 seq = pkt_th->ack_number;
854 ack = (tc && tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
860 ack = clib_host_to_net_u32 (
vnet_buffer (pkt)->tcp.seq_end);
864 seq, ack, tcp_hdr_len, flags, 0);
900 u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
909 tcp_hdr_opts_len = tc->snd_opts_len +
sizeof (
tcp_header_t);
913 tc->rcv_nxt, tcp_hdr_opts_len, flags,
916 ASSERT (opts_write_len == tc->snd_opts_len);
917 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
922 &tc->c_rmt_ip.ip4, IP_PROTOCOL_TCP, 0);
930 &tc->c_rmt_ip.ip6, IP_PROTOCOL_TCP);
947 &tc->c_rmt_ip4, IP_PROTOCOL_TCP, 1);
956 &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
994 tc->rtt_seq = tc->snd_nxt;
1008 if (
tcp_main.tx_frames[!is_ip4][thread_index])
1013 tcp_main.tx_frames[!is_ip4][thread_index]);
1014 tcp_main.tx_frames[!is_ip4][thread_index] = 0;
1024 if (
tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index])
1029 tcp_main.ip_lookup_tx_frames[!is_ip4]
1031 tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
1064 fin_snt = tc->flags & TCP_CONN_FINSNT;
1066 tc->snd_nxt = tc->snd_una;
1071 tc->flags |= TCP_CONN_FINSNT;
1072 tc->flags &= ~TCP_CONN_FINPNDG;
1074 tc->snd_una_max += 1;
1075 tc->snd_nxt = tc->snd_una_max;
1079 tc->snd_nxt = tc->snd_una_max;
1089 case TCP_STATE_ESTABLISHED:
1091 case TCP_STATE_SYN_RCVD:
1093 case TCP_STATE_SYN_SENT:
1095 case TCP_STATE_LAST_ACK:
1096 case TCP_STATE_FIN_WAIT_1:
1097 if (tc->snd_nxt + 1 < tc->snd_una_max)
1114 u32 advertise_wnd, data_len;
1115 u8 tcp_hdr_opts_len, opts_write_len,
flags;
1120 || (b->
flags & VLIB_BUFFER_NEXT_PRESENT));
1126 tcp_hdr_opts_len = tc->snd_opts_len +
sizeof (
tcp_header_t);
1132 tc->rcv_nxt, tcp_hdr_opts_len, flags,
1136 ASSERT (opts_write_len == tc->snd_opts_len);
1137 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
1143 tc->snd_nxt += data_len;
1144 tc->rcv_las = tc->rcv_nxt;
1147 if (
seq_gt (tc->snd_nxt, tc->snd_una_max))
1149 tc->snd_una_max = tc->snd_nxt;
1204 u32 start, bi, available_bytes, seg_size;
1207 ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1208 ASSERT (max_deq_bytes != 0);
1214 ASSERT (available_bytes >= offset);
1215 available_bytes -=
offset;
1216 if (!available_bytes)
1218 max_deq_bytes =
clib_min (tc->snd_mss, max_deq_bytes);
1219 max_deq_bytes =
clib_min (available_bytes, max_deq_bytes);
1222 start = tc->snd_una +
offset;
1223 if (
seq_geq (start, tc->snd_congestion))
1227 if (
seq_gt (start + max_deq_bytes, tc->snd_congestion))
1229 max_deq_bytes = tc->snd_congestion - start;
1230 if (max_deq_bytes == 0)
1255 ASSERT (n_bytes == max_deq_bytes);
1262 u32 chain_bi = ~0, n_bufs_per_seg;
1264 u16 n_peeked, len_to_deq, available_bufs;
1268 n_bufs_per_seg = ceil ((
double) seg_size / tm->bytes_per_buffer);
1271 available_bufs =
vec_len (tm->tx_buffers[thread_index]);
1272 if (n_bufs_per_seg > available_bufs)
1284 tm->bytes_per_buffer -
1287 b[0]->
flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1289 max_deq_bytes -= n_bytes;
1292 for (i = 1; i < n_bufs_per_seg; i++)
1295 len_to_deq =
clib_min (max_deq_bytes, tm->bytes_per_buffer);
1302 offset + n_bytes, len_to_deq);
1303 ASSERT (n_peeked == len_to_deq);
1304 n_bytes += n_peeked;
1306 chain_b->
flags &= VLIB_BUFFER_NON_DEFAULT_FREELIST;
1311 prev_b->
flags |= VLIB_BUFFER_NEXT_PRESENT;
1313 max_deq_bytes -= n_peeked;
1321 ASSERT (((*b)->current_data + (*b)->current_length) <=
1322 tm->bytes_per_buffer);
1325 tc->snd_rxt_bytes += n_bytes;
1328 TCP_EVT_DBG (TCP_EVT_CC_RTX, tc, offset, n_bytes);
1338 tc->prev_ssthresh = tc->ssthresh;
1339 tc->prev_cwnd = tc->cwnd;
1348 tc->snd_congestion = tc->snd_una_max;
1380 if (tc->state >= TCP_STATE_ESTABLISHED)
1392 if (tc->snd_una == tc->snd_una_max)
1410 if (tc->rto_boff == 1)
1413 tc->snd_nxt = tc->snd_una;
1433 if (tc->rto_boff == 1)
1440 else if (tc->state == TCP_STATE_SYN_SENT)
1445 if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1449 clib_warning (
"could not remove half-open connection");
1479 else if (tc->state == TCP_STATE_SYN_RCVD)
1501 ASSERT (tc->state == TCP_STATE_CLOSED);
1530 u32 bi, max_snd_bytes, available_bytes,
offset;
1543 if (tc->state == TCP_STATE_CLOSED || tc->state > TCP_STATE_ESTABLISHED
1548 offset = tc->snd_una_max - tc->snd_una;
1552 if (!available_bytes)
1558 if (available_bytes <= offset)
1583 || tc->snd_nxt == tc->snd_una_max
1584 || tc->rto_boff > 1));
1601 u32 bi, old_snd_nxt, n_bytes;
1603 old_snd_nxt = tc->snd_nxt;
1604 tc->snd_nxt = tc->snd_una;
1613 tc->snd_nxt = old_snd_nxt;
1627 u32 bi, old_snd_nxt;
1629 u8 snd_limited = 0, can_rescue = 0;
1634 old_snd_nxt = tc->snd_nxt;
1639 while (hole && snd_space > 0)
1643 &can_rescue, &snd_limited);
1646 if (!can_rescue || !(
seq_lt (sb->rescue_rxt, tc->snd_una)
1647 ||
seq_gt (sb->rescue_rxt,
1648 tc->snd_congestion)))
1657 tc->snd_congestion - tc->snd_una);
1658 max_bytes =
clib_min (max_bytes, snd_space);
1659 offset = tc->snd_congestion - tc->snd_una - max_bytes;
1660 sb->rescue_rxt = tc->snd_congestion;
1661 tc->snd_nxt = tc->snd_una +
offset;
1670 max_bytes =
clib_min (hole->end - sb->high_rxt, snd_space);
1671 max_bytes = snd_limited ?
clib_min (max_bytes, tc->snd_mss) : max_bytes;
1674 offset = sb->high_rxt - tc->snd_una;
1675 tc->snd_nxt = sb->high_rxt;
1683 sb->high_rxt += n_written;
1685 ASSERT (n_written <= snd_space);
1686 snd_space -= n_written;
1690 tc->snd_nxt = old_snd_nxt;
1700 u32 n_written = 0,
offset = 0, bi, old_snd_nxt;
1708 old_snd_nxt = tc->snd_nxt;
1709 tc->snd_nxt = tc->snd_una;
1712 while (snd_space > 0)
1723 snd_space -= n_written;
1727 tc->snd_nxt = old_snd_nxt;
1755 u32 n_left_from, next_index, *from, *to_next;
1763 while (n_left_from > 0)
1769 while (n_left_from > 0 && n_left_to_next > 0)
1778 if (n_left_from > 1)
1791 n_left_to_next -= 1;
1796 if (
PREDICT_FALSE (tc0 == 0 || tc0->state == TCP_STATE_CLOSED))
1798 error0 = TCP_ERROR_INVALID_CONNECTION;
1809 IP_PROTOCOL_TCP, 1);
1810 b0->
flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
1818 &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
1819 b0->
flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
1827 (
vnet_buffer (b0)->tcp.flags & TCP_BUF_FLAG_DUPACK))
1831 error0 = TCP_ERROR_FILTERED_DUPACKS;
1838 tc0->flags &= ~(TCP_CONN_SNDACK);
1844 if (
seq_lt (tc0->snd_una_max, tc0->snd_nxt))
1846 tc0->snd_una_max = tc0->snd_nxt;
1847 if (tc0->rtt_ts == 0)
1850 tc0->rtt_seq = tc0->snd_nxt;
1857 && tc0->snd_nxt != tc0->snd_una)
1871 tc0->last_fib_check = tc0->snd_opts.tsval;
1882 next0 = tc0->c_rmt_dpo.dpoi_next_node;
1889 b0->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
1908 n_left_to_next, bi0, next0);
1936 .vector_size =
sizeof (
u32),
1941 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n, 1956 .name =
"tcp6-output",
1958 .vector_size =
sizeof (
u32),
1963 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n, 1981 ASSERT (
seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
1986 tc->rtt_seq = tc->snd_nxt;
1992 typedef enum _tcp_reset_next
1999 #define foreach_tcp4_reset_next \ 2000 _(DROP, "error-drop") \ 2001 _(IP_LOOKUP, "ip4-lookup") 2003 #define foreach_tcp6_reset_next \ 2004 _(DROP, "error-drop") \ 2005 _(IP_LOOKUP, "ip6-lookup") 2011 u32 n_left_from, next_index, *from, *to_next;
2019 while (n_left_from > 0)
2025 while (n_left_from > 0 && n_left_to_next > 0)
2038 n_left_to_next -= 1;
2043 my_thread_index, is_ip4))
2045 error0 = TCP_ERROR_LOOKUP_DROPS;
2056 b0->
flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
2069 n_left_to_next, bi0, next0);
2093 .name =
"tcp4-reset",
2094 .vector_size =
sizeof (
u32),
2099 #define _(s,n) [TCP_RESET_NEXT_##s] = n, 2112 .name =
"tcp6-reset",
2113 .vector_size =
sizeof (
u32),
2118 #define _(s,n) [TCP_RESET_NEXT_##s] = n, void tcp_make_fin(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to FIN-ACK.
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
#define tcp_in_cong_recovery(tc)
#define TCP_FIB_RECHECK_PERIOD
Recheck every 1s.
static void tcp_enqueue_to_ip_lookup_i(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index, u8 flush)
#define TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL(thread_index)
#define TCP_OPTION_LEN_EOL
#define tcp_in_recovery(tc)
static void tcp_retransmit_timer_set(tcp_connection_t *tc)
#define TCP_OPTION_LEN_SACK_PERMITTED
static void tcp_flush_frame_to_ip_lookup(vlib_main_t *vm, u8 thread_index, u8 is_ip4)
Flush ip lookup tx frames populated by timer pops.
#define seq_leq(_s1, _s2)
void tcp_timer_retransmit_handler(u32 index)
struct _transport_connection transport_connection_t
#define TCP_TO_TIMER_TICK
#define TCP_MIN_RX_FIFO_SIZE
void tcp_send_reset_w_pkt(tcp_connection_t *tc, vlib_buffer_t *pkt, u8 is_ip4)
Send reset without reusing existing buffer.
static u8 svm_fifo_has_ooo_data(svm_fifo_t *f)
#define tcp_opts_tstamp(_to)
static uword tcp4_send_reset(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
void tcp_make_synack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN-ACK.
static void tcp_enqueue_to_output(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4)
static tcp_connection_t * tcp_connection_get_if_valid(u32 conn_index, u32 thread_index)
static int tcp_make_syn_options(tcp_options_t *opts, u8 wnd_scale)
struct _sack_scoreboard sack_scoreboard_t
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
void tcp_update_rcv_mss(tcp_connection_t *tc)
Update max segment size we're able to process.
struct _tcp_main tcp_main_t
void tcp_flush_frame_to_output(vlib_main_t *vm, u8 thread_index, u8 is_ip4)
Flush tx frame populated by retransmits and timer pops.
static u32 tcp_initial_wnd_unscaled(tcp_connection_t *tc)
TCP's initial window.
#define tcp_recovery_off(tc)
enum _tcp_output_next tcp_output_next_t
static u32 tcp_session_has_ooo_data(tcp_connection_t *tc)
struct _tcp_connection tcp_connection_t
static int tcp_alloc_tx_buffers(tcp_main_t *tm, u8 thread_index, u32 n_free_buffers)
#define tcp_opts_sack(_to)
void tcp_make_ack_i(tcp_connection_t *tc, vlib_buffer_t *b, tcp_state_t state, u8 flags)
Prepare ACK.
static void scoreboard_clear(sack_scoreboard_t *sb)
static uword tcp46_send_reset_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, u8 is_ip4)
vlib_error_t * errors
Vector of errors for this node.
int tcp_make_reset_in_place(vlib_main_t *vm, vlib_buffer_t *b0, tcp_state_t state, u8 thread_index, u8 is_ip4)
u8 n_sack_blocks
Number of SACKs blocks.
struct _tcp_header tcp_header_t
static u32 tcp_available_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
static u32 stream_session_max_rx_enqueue(transport_connection_t *tc)
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
static void tcp_enqueue_to_output_now(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4)
struct _sack_scoreboard_hole sack_scoreboard_hole_t
u8 wscale
Window scale advertised.
enum fib_protocol_t_ fib_protocol_t
Protocol Type.
#define TCP_OPTS_MAX_SACK_BLOCKS
#define TCP_MAX_RX_FIFO_SIZE
vlib_node_registration_t ip4_lookup_node
(constructor) VLIB_REGISTER_NODE (ip4_lookup_node)
static void tcp_timer_retransmit_handler_i(u32 index, u8 is_syn)
#define foreach_tcp4_reset_next
static void * tcp_init_buffer(vlib_main_t *vm, vlib_buffer_t *b)
void tcp_make_syn(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN.
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
void tcp_push_ip_hdr(tcp_main_t *tm, tcp_connection_t *tc, vlib_buffer_t *b)
#define TCP_OPTION_LEN_SACK_BLOCK
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
u8 * format_tcp_tx_trace(u8 *s, va_list *args)
enum _tcp_state tcp_state_t
void tcp_fast_retransmit_no_sack(tcp_connection_t *tc)
Fast retransmit without SACK info.
static void * ip4_next_header(ip4_header_t *i)
static u32 tcp_time_now(void)
sack_block_t * sacks
SACK blocks.
struct _stream_session_t stream_session_t
#define TCP_ESTABLISH_TIME
sack_scoreboard_hole_t * scoreboard_next_rxt_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *start, u8 have_sent_1_smss, u8 *can_rescue, u8 *snd_limited)
Figure out the next hole to retransmit.
#define tcp_validate_txf_size(_tc, _a)
#define TCP_EVT_DBG(_evt, _args...)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
static void tcp_timer_set(tcp_connection_t *tc, u8 timer_id, u32 interval)
#define TCP_OPTION_LEN_WINDOW_SCALE
vlib_node_registration_t tcp6_reset_node
(constructor) VLIB_REGISTER_NODE (tcp6_reset_node)
#define TCP_RTO_SYN_RETRIES
#define tcp_trajectory_add_start(b, start)
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
static int tcp_make_synack_options(tcp_connection_t *tc, tcp_options_t *opts)
u16 current_length
Nbytes between current data and the end of this buffer.
static void tcp_push_hdr_i(tcp_connection_t *tc, vlib_buffer_t *b, tcp_state_t next_state, u8 compute_opts)
Push TCP header and update connection variables.
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
#define tcp_in_fastrecovery(tc)
static void * vlib_buffer_push_tcp_net_order(vlib_buffer_t *b, u16 sp, u16 dp, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
static int tcp_get_free_buffer_index(tcp_main_t *tm, u32 *bidx)
#define tcp_opts_mss(_to)
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
#define TCP_TIMER_HANDLE_INVALID
#define foreach_tcp6_output_next
static u32 tcp_flight_size(const tcp_connection_t *tc)
Our estimate of the number of bytes in flight (pipe size)
void tcp_fast_retransmit(tcp_connection_t *tc)
Do fast retransmit.
static u8 tcp_window_compute_scale(u32 window)
static void tcp_rtx_timeout_cc(tcp_connection_t *tc)
Reset congestion control, switch cwnd to loss window and try again.
void tcp_timer_retransmit_syn_handler(u32 index)
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
#define TCP_OPTION_LEN_TIMESTAMP
#define foreach_tcp4_output_next
vlib_error_t error
Error code for buffers to be enqueued to error handler.
void tcp_make_ack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to ACK.
static stream_session_t * session_get(u32 si, u32 thread_index)
#define TCP_MAX_WND_SCALE
static void tcp_timer_reset(tcp_connection_t *tc, u8 timer_id)
static sack_scoreboard_hole_t * scoreboard_first_hole(sack_scoreboard_t *sb)
#define tcp_fastrecovery_sent_1_smss(tc)
#define VLIB_REGISTER_NODE(x,...)
static void * vlib_buffer_push_tcp(vlib_buffer_t *b, u16 sp_net, u16 dp_net, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
static_always_inline uword vlib_get_thread_index(void)
#define CLIB_PREFETCH(addr, size, type)
static void tcp_enqueue_to_ip_lookup(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
u32 stream_session_tx_fifo_max_dequeue(transport_connection_t *tc)
#define clib_warning(format, args...)
static u8 tcp_make_state_flags(tcp_connection_t *tc, tcp_state_t next_state)
#define clib_memcpy(a, b, c)
#define TCP_USE_SACKS
Disable only for testing.
#define tcp_recovery_on(tc)
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
fib_node_index_t tcp_lookup_rmt_in_fib(tcp_connection_t *tc)
u16 mss
Option flags, see above.
static void * ip6_next_header(ip6_header_t *i)
static void tcp_return_buffer(tcp_main_t *tm)
static void tcp_timer_update(tcp_connection_t *tc, u8 timer_id, u32 interval)
u16 ip6_tcp_udp_icmp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip6_header_t *ip0, int *bogus_lengthp)
vlib_node_registration_t ip6_lookup_node
(constructor) VLIB_REGISTER_NODE (ip6_lookup_node)
static int tcp_make_established_options(tcp_connection_t *tc, tcp_options_t *opts)
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
static uword tcp6_output(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
u16 ip4_tcp_udp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip4_header_t *ip0)
u32 tcp_push_header(transport_connection_t *tconn, vlib_buffer_t *b)
#define seq_geq(_s1, _s2)
u32 tcp_window_to_advertise(tcp_connection_t *tc, tcp_state_t state)
Compute and return window to advertise, scaled as per RFC1323.
vhost_vring_state_t state
void tcp_retransmit_first_unacked(tcp_connection_t *tc)
Retransmit first unacked segment.
u32 next_buffer
Next buffer for this linked-list of buffers.
static void tcp_enqueue_to_ip_lookup_now(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
void tcp_init_mss(tcp_connection_t *tc)
static uword tcp6_send_reset(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
static uword tcp46_output_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip4)
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
static vlib_main_t * vlib_get_main(void)
VLIB_NODE_FUNCTION_MULTIARCH(tcp4_output_node, tcp4_output)
void tcp_send_ack(tcp_connection_t *tc)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
#define tcp_opts_wscale(_to)
static sack_scoreboard_hole_t * scoreboard_get_hole(sack_scoreboard_t *sb, u32 index)
void tcp_connection_reset(tcp_connection_t *tc)
Notify session that connection has been reset.
u32 tsval
Timestamp value.
u32 tsecr
Echoed/reflected time stamp.
static void * vlib_buffer_push_ip6(vlib_main_t *vm, vlib_buffer_t *b, ip6_address_t *src, ip6_address_t *dst, int proto)
Push IPv6 header to buffer.
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static u32 tcp_set_time_now(u32 thread_index)
static u8 tcp_is_lost_fin(tcp_connection_t *tc)
#define foreach_tcp6_reset_next
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
static void tcp_retransmit_timer_update(tcp_connection_t *tc)
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
void tcp_timer_delack_handler(u32 index)
Delayed ack timer handler.
#define TCP_OPTION_LEN_MSS
void tcp_update_snd_mss(tcp_connection_t *tc)
Update snd_mss to reflect the effective segment size that we can send by taking into account all TCP ...
struct clib_bihash_value offset
template key/value backing page structure
static void tcp_retransmit_timer_force_update(tcp_connection_t *tc)
u8 * format_tcp_connection(u8 *s, va_list *args)
u32 tcp_initial_window_to_advertise(tcp_connection_t *tc)
Compute initial window and scale factor.
static u32 stream_session_rx_fifo_size(transport_connection_t *tc)
void tcp_flush_frames_to_output(u8 thread_index)
Flush v4 and v6 tcp and ip-lookup tx frames for thread index.
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
void tcp_update_rcv_wnd(tcp_connection_t *tc)
void tcp_update_rto(tcp_connection_t *tc)
void tcp_fast_retransmit_sack(tcp_connection_t *tc)
Do fast retransmit with SACKs.
#define TCP_OPTION_LEN_NOOP
void tcp_send_syn(tcp_connection_t *tc)
Send SYN.
vlib_node_registration_t tcp6_output_node
(constructor) VLIB_REGISTER_NODE (tcp6_output_node)
enum _tcp_reset_next tcp_reset_next_t
#define tcp_opts_sack_permitted(_to)
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
tcp_connection_t tcp_connection
static u32 tcp_loss_wnd(const tcp_connection_t *tc)
static void * tcp_reuse_buffer(vlib_main_t *vm, vlib_buffer_t *b)
u32 tcp_prepare_retransmit_segment(tcp_connection_t *tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t **b)
Build a retransmit segment.
vlib_node_registration_t tcp4_reset_node
(constructor) VLIB_REGISTER_NODE (tcp4_reset_node)
vlib_node_registration_t tcp4_output_node
(constructor) VLIB_REGISTER_NODE (tcp4_output_node)
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
static void tcp_enqueue_to_output_i(vlib_main_t *vm, vlib_buffer_t *b, u32 bi, u8 is_ip4, u8 flush)
static void tcp_persist_timer_set(tcp_connection_t *tc)
static tcp_main_t * vnet_get_tcp_main()
int stream_session_peek_bytes(transport_connection_t *tc, u8 *buffer, u32 offset, u32 max_bytes)
static char * tcp_error_strings[]
static uword tcp4_output(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
static void * vlib_buffer_push_ip4(vlib_main_t *vm, vlib_buffer_t *b, ip4_address_t *src, ip4_address_t *dst, int proto, u8 csum_offload)
Push IPv4 header to buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
void tcp_cc_fastrecovery_exit(tcp_connection_t *tc)
void tcp_timer_persist_handler(u32 index)
Got 0 snd_wnd from peer, try to do something about it.
static u8 tcp_timer_is_active(tcp_connection_t *tc, tcp_timers_e timer)
u32 tcp_options_write(u8 *data, tcp_options_t *opts)
Write TCP options to segment.
static int tcp_make_options(tcp_connection_t *tc, tcp_options_t *opts, tcp_state_t state)