FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
tcp_inlines.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef SRC_VNET_TCP_TCP_INLINES_H_
17 #define SRC_VNET_TCP_TCP_INLINES_H_
18 
19 #include <vnet/tcp/tcp.h>
20 
23 {
24  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
25  return (tcp_header_t *) (b->data + b->current_data
26  + vnet_buffer (b)->tcp.hdr_offset);
27 }
28 
30 tcp_connection_get (u32 conn_index, u32 thread_index)
31 {
32  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
33  if (PREDICT_FALSE (pool_is_free_index (wrk->connections, conn_index)))
34  return 0;
35  return pool_elt_at_index (wrk->connections, conn_index);
36 }
37 
39 tcp_connection_get_if_valid (u32 conn_index, u32 thread_index)
40 {
41  tcp_worker_ctx_t *wrk;
42  if (thread_index >= vec_len (tcp_main.wrk_ctx))
43  return 0;
44  wrk = tcp_get_worker (thread_index);
45  if (pool_is_free_index (wrk->connections, conn_index))
46  return 0;
47  return pool_elt_at_index (wrk->connections, conn_index);
48 }
49 
50 always_inline void
52 {
53  tc->state = state;
54  TCP_EVT (TCP_EVT_STATE_CHANGE, tc);
55 }
56 
59 {
60  tcp_connection_t *tc = 0;
61  if (!pool_is_free_index (tcp_main.listener_pool, tli))
62  tc = pool_elt_at_index (tcp_main.listener_pool, tli);
63  return tc;
64 }
65 
68 {
69  tcp_connection_t *tc = 0;
70  clib_spinlock_lock_if_init (&tcp_main.half_open_lock);
71  if (!pool_is_free_index (tcp_main.half_open_connections, conn_index))
72  tc = pool_elt_at_index (tcp_main.half_open_connections, conn_index);
73  clib_spinlock_unlock_if_init (&tcp_main.half_open_lock);
74  return tc;
75 }
76 
77 /**
78  * Our estimate of the number of bytes that have left the network
79  */
82 {
83  if (tcp_opts_sack_permitted (&tc->rcv_opts))
84  return tc->sack_sb.sacked_bytes + tc->sack_sb.lost_bytes;
85  else
86  return clib_min (tc->rcv_dupacks * tc->snd_mss,
87  tc->snd_nxt - tc->snd_una);
88 }
89 
90 /**
91  * Our estimate of the number of bytes in flight (pipe size)
92  */
95 {
96  int flight_size;
97 
98  flight_size = (int) (tc->snd_nxt - tc->snd_una) - tcp_bytes_out (tc)
99  + tc->snd_rxt_bytes - tc->rxt_delivered;
100 
101  ASSERT (flight_size >= 0);
102 
103  return flight_size;
104 }
105 
106 /**
107  * Initial cwnd as per RFC5681
108  */
111 {
112  if (tcp_cfg.initial_cwnd_multiplier > 0)
113  return tcp_cfg.initial_cwnd_multiplier * tc->snd_mss;
114 
115  if (tc->snd_mss > 2190)
116  return 2 * tc->snd_mss;
117  else if (tc->snd_mss > 1095)
118  return 3 * tc->snd_mss;
119  else
120  return 4 * tc->snd_mss;
121 }
122 
123 /*
124  * Accumulate acked bytes for cwnd increase
125  *
126  * Once threshold bytes are accumulated, snd_mss bytes are added
127  * to the cwnd.
128  */
129 always_inline void
131 {
132  tc->cwnd_acc_bytes += bytes;
133  if (tc->cwnd_acc_bytes >= thresh)
134  {
135  u32 inc = tc->cwnd_acc_bytes / thresh;
136  tc->cwnd_acc_bytes -= inc * thresh;
137  tc->cwnd += inc * tc->snd_mss;
138  tc->cwnd = clib_min (tc->cwnd, tc->tx_fifo_size);
139  }
140 }
141 
144 {
145  /* Whatever we have in flight + the packet we're about to send */
146  return tcp_flight_size (tc) + tc->snd_mss;
147 }
148 
151 {
152  return clib_min (tc->cwnd, tc->snd_wnd);
153 }
154 
157 {
158  u32 available_wnd = tcp_available_snd_wnd (tc);
159  int flight_size = (int) (tc->snd_nxt - tc->snd_una);
160 
161  if (available_wnd <= flight_size)
162  return 0;
163 
164  return available_wnd - flight_size;
165 }
166 
167 /**
168  * Estimate of how many bytes we can still push into the network
169  */
172 {
173  u32 available_wnd = tcp_available_snd_wnd (tc);
174  u32 flight_size = tcp_flight_size (tc);
175 
176  if (available_wnd <= flight_size)
177  return 0;
178 
179  return available_wnd - flight_size;
180 }
181 
184 {
185  if ((tc->flags & TCP_CONN_FINSNT) && (tc->snd_una_max - tc->snd_una == 1))
186  return 1;
187  return 0;
188 }
189 
192 {
193  return tcp_main.wrk_ctx[vlib_get_thread_index ()].time_now;
194 }
195 
198 {
199  return tcp_main.wrk_ctx[thread_index].time_now;
200 }
201 
202 /**
203  * Generate timestamp for tcp connection
204  */
207 {
208  return (tcp_main.wrk_ctx[tc->c_thread_index].time_now -
209  tc->timestamp_delta);
210 }
211 
213 tcp_time_now_us (u32 thread_index)
214 {
215  return transport_time_now (thread_index);
216 }
217 
220 {
221  return wrk->time_now = (u64) (vlib_time_now (wrk->vm) * TCP_TSTP_HZ);
222 }
223 
225 tcp_input_lookup_buffer (vlib_buffer_t * b, u8 thread_index, u32 * error,
226  u8 is_ip4, u8 is_nolookup)
227 {
228  u32 fib_index = vnet_buffer (b)->ip.fib_index;
229  int n_advance_bytes, n_data_bytes;
231  tcp_header_t *tcp;
232  u8 result = 0;
233 
234  if (is_ip4)
235  {
237  int ip_hdr_bytes = ip4_header_bytes (ip4);
238  if (PREDICT_FALSE (b->current_length < ip_hdr_bytes + sizeof (*tcp)))
239  {
240  *error = TCP_ERROR_LENGTH;
241  return 0;
242  }
243  tcp = ip4_next_header (ip4);
244  vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip4;
245  n_advance_bytes = (ip_hdr_bytes + tcp_header_bytes (tcp));
246  n_data_bytes = clib_net_to_host_u16 (ip4->length) - n_advance_bytes;
247 
248  /* Length check. Checksum computed by ipx_local no need to compute again */
249  if (PREDICT_FALSE (n_data_bytes < 0))
250  {
251  *error = TCP_ERROR_LENGTH;
252  return 0;
253  }
254 
255  if (!is_nolookup)
256  tc = session_lookup_connection_wt4 (fib_index, &ip4->dst_address,
257  &ip4->src_address, tcp->dst_port,
258  tcp->src_port,
259  TRANSPORT_PROTO_TCP, thread_index,
260  &result);
261  }
262  else
263  {
265  if (PREDICT_FALSE (b->current_length < sizeof (*ip6) + sizeof (*tcp)))
266  {
267  *error = TCP_ERROR_LENGTH;
268  return 0;
269  }
270  tcp = ip6_next_header (ip6);
271  vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip6;
272  n_advance_bytes = tcp_header_bytes (tcp);
273  n_data_bytes = clib_net_to_host_u16 (ip6->payload_length)
274  - n_advance_bytes;
275  n_advance_bytes += sizeof (ip6[0]);
276 
277  if (PREDICT_FALSE (n_data_bytes < 0))
278  {
279  *error = TCP_ERROR_LENGTH;
280  return 0;
281  }
282 
283  if (!is_nolookup)
284  {
285  if (PREDICT_FALSE
287  {
288  ip6_main_t *im = &ip6_main;
289  fib_index = vec_elt (im->fib_index_by_sw_if_index,
291  }
292 
293  tc = session_lookup_connection_wt6 (fib_index, &ip6->dst_address,
294  &ip6->src_address,
295  tcp->dst_port, tcp->src_port,
296  TRANSPORT_PROTO_TCP,
297  thread_index, &result);
298  }
299  }
300 
301  if (is_nolookup)
302  tc =
304  tcp.connection_index,
305  thread_index);
306 
307  vnet_buffer (b)->tcp.seq_number = clib_net_to_host_u32 (tcp->seq_number);
308  vnet_buffer (b)->tcp.ack_number = clib_net_to_host_u32 (tcp->ack_number);
309  vnet_buffer (b)->tcp.data_offset = n_advance_bytes;
310  vnet_buffer (b)->tcp.data_len = n_data_bytes;
311  vnet_buffer (b)->tcp.seq_end = vnet_buffer (b)->tcp.seq_number
312  + n_data_bytes;
313  vnet_buffer (b)->tcp.flags = 0;
314 
315  *error = result ? TCP_ERROR_NONE + result : *error;
316 
318 }
319 
320 /**
321  * Initialize connection by gleaning network and rcv params from buffer
322  *
323  * @param tc connection to initialize
324  * @param b buffer whose current data is pointing at ip
325  * @param is_ip4 flag set to 1 if using ip4
326  */
327 always_inline void
329 {
330  tcp_header_t *th = tcp_buffer_hdr (b);
331 
332  tc->c_lcl_port = th->dst_port;
333  tc->c_rmt_port = th->src_port;
334  tc->c_is_ip4 = is_ip4;
335 
336  if (is_ip4)
337  {
339  tc->c_lcl_ip4.as_u32 = ip4->dst_address.as_u32;
340  tc->c_rmt_ip4.as_u32 = ip4->src_address.as_u32;
341  }
342  else
343  {
345  clib_memcpy_fast (&tc->c_lcl_ip6, &ip6->dst_address,
346  sizeof (ip6_address_t));
347  clib_memcpy_fast (&tc->c_rmt_ip6, &ip6->src_address,
348  sizeof (ip6_address_t));
349  }
350 
351  tc->irs = vnet_buffer (b)->tcp.seq_number;
352  tc->rcv_nxt = vnet_buffer (b)->tcp.seq_number + 1;
353  tc->rcv_las = tc->rcv_nxt;
354  tc->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
355  tc->snd_wl1 = vnet_buffer (b)->tcp.seq_number;
356  tc->snd_wl2 = vnet_buffer (b)->tcp.ack_number;
357 
358  /* RFC1323: TSval timestamps sent on {SYN} and {SYN,ACK}
359  * segments are used to initialize PAWS. */
360  if (tcp_opts_tstamp (&tc->rcv_opts))
361  {
362  tc->tsval_recent = tc->rcv_opts.tsval;
363  tc->tsval_recent_age = tcp_time_now ();
364  }
365 
366  if (tcp_opts_wscale (&tc->rcv_opts))
367  tc->snd_wscale = tc->rcv_opts.wscale;
368 
369  tc->snd_wnd = clib_net_to_host_u16 (th->window) << tc->snd_wscale;
370 }
371 
372 always_inline void
374 {
375  tc->rto = clib_min (tc->srtt + (tc->rttvar << 2), TCP_RTO_MAX);
376  tc->rto = clib_max (tc->rto, TCP_RTO_MIN);
377 }
378 
381 {
382  return (transport_connection_is_descheduled (&tc->connection) ? 1 : 0);
383 }
384 
385 /**
386  * Push TCP header to buffer
387  *
388  * @param vm - vlib_main
389  * @param b - buffer to write the header to
390  * @param sp_net - source port net order
391  * @param dp_net - destination port net order
392  * @param seq - sequence number net order
393  * @param ack - ack number net order
394  * @param tcp_hdr_opts_len - header and options length in bytes
395  * @param flags - header flags
396  * @param wnd - window size
397  *
398  * @return - pointer to start of TCP header
399  */
400 always_inline void *
402  u32 ack, u8 tcp_hdr_opts_len, u8 flags,
403  u16 wnd)
404 {
405  tcp_header_t *th;
406 
407  th = vlib_buffer_push_uninit (b, tcp_hdr_opts_len);
408 
409  th->src_port = sp;
410  th->dst_port = dp;
411  th->seq_number = seq;
412  th->ack_number = ack;
413  th->data_offset_and_reserved = (tcp_hdr_opts_len >> 2) << 4;
414  th->flags = flags;
415  th->window = wnd;
416  th->checksum = 0;
417  th->urgent_pointer = 0;
418  vnet_buffer (b)->l4_hdr_offset = (u8 *) th - b->data;
419  b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
420  return th;
421 }
422 
423 /**
424  * Push TCP header to buffer
425  *
426  * @param b - buffer to write the header to
427  * @param sp_net - source port net order
428  * @param dp_net - destination port net order
429  * @param seq - sequence number host order
430  * @param ack - ack number host order
431  * @param tcp_hdr_opts_len - header and options length in bytes
432  * @param flags - header flags
433  * @param wnd - window size
434  *
435  * @return - pointer to start of TCP header
436  */
437 always_inline void *
438 vlib_buffer_push_tcp (vlib_buffer_t * b, u16 sp_net, u16 dp_net, u32 seq,
439  u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
440 {
441  return vlib_buffer_push_tcp_net_order (b, sp_net, dp_net,
442  clib_host_to_net_u32 (seq),
443  clib_host_to_net_u32 (ack),
444  tcp_hdr_opts_len, flags,
445  clib_host_to_net_u16 (wnd));
446 }
447 
448 #endif /* SRC_VNET_TCP_TCP_INLINES_H_ */
449 
450 /*
451  * fd.io coding-style-patch-verification: ON
452  *
453  * Local Variables:
454  * eval: (c-set-style "gnu")
455  * End:
456  */
static void tcp_cwnd_accumulate(tcp_connection_t *tc, u32 thresh, u32 bytes)
Definition: tcp_inlines.h:130
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
static u32 tcp_loss_wnd(const tcp_connection_t *tc)
Definition: tcp_inlines.h:143
#define clib_min(x, y)
Definition: clib.h:327
static clib_time_type_t transport_time_now(u32 thread_index)
Definition: session.h:538
static u32 tcp_time_now(void)
Definition: tcp_inlines.h:191
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:103
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
Definition: tcp_inlines.h:30
ip4_address_t src_address
Definition: ip4_packet.h:125
transport_connection_t * session_lookup_connection_wt6(u32 fib_index, ip6_address_t *lcl, ip6_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *result)
Lookup connection with ip6 and transport layer information.
#define tcp_opts_tstamp(_to)
Definition: tcp_packet.h:156
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:333
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:127
static void * vlib_buffer_push_tcp_net_order(vlib_buffer_t *b, u16 sp, u16 dp, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
Definition: tcp_inlines.h:401
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
struct _tcp_connection tcp_connection_t
static u32 tcp_set_time_now(tcp_worker_ctx_t *wrk)
Definition: tcp_inlines.h:219
static u32 tcp_time_now_w_thread(u32 thread_index)
Definition: tcp_inlines.h:197
static u8 tcp_is_descheduled(tcp_connection_t *tc)
Definition: tcp_inlines.h:380
struct _tcp_header tcp_header_t
ip6_address_t src_address
Definition: ip6_packet.h:310
unsigned char u8
Definition: types.h:56
double f64
Definition: types.h:142
tcp_connection_t * connections
worker&#39;s pool of connections
Definition: tcp.h:80
#define tcp_cfg
Definition: tcp.h:271
vl_api_ip6_address_t ip6
Definition: one.api:424
ip4_address_t dst_address
Definition: ip4_packet.h:125
transport_connection_t * session_lookup_connection_wt4(u32 fib_index, ip4_address_t *lcl, ip4_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *result)
Lookup connection with ip4 and transport layer information.
tcp_main_t tcp_main
Definition: tcp.c:28
#define TCP_RTO_MAX
Definition: tcp_types.h:88
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:196
unsigned int u32
Definition: types.h:88
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
Definition: tcp_inlines.h:22
vlib_main_t * vm
convenience pointer to this thread&#39;s vlib main
Definition: tcp.h:92
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
Definition: tcp_inlines.h:67
unsigned short u16
Definition: types.h:57
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
static u32 tcp_initial_cwnd(const tcp_connection_t *tc)
Initial cwnd as per RFC5681.
Definition: tcp_inlines.h:110
#define PREDICT_FALSE(x)
Definition: clib.h:120
#define always_inline
Definition: ipsec.h:28
vl_api_ip4_address_t ip4
Definition: one.api:376
ip6_main_t ip6_main
Definition: ip6_forward.c:2781
static u8 tcp_is_lost_fin(tcp_connection_t *tc)
Definition: tcp_inlines.h:183
static u32 tcp_available_output_snd_space(const tcp_connection_t *tc)
Definition: tcp_inlines.h:156
static u32 tcp_flight_size(const tcp_connection_t *tc)
Our estimate of the number of bytes in flight (pipe size)
Definition: tcp_inlines.h:94
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:219
static void tcp_update_rto(tcp_connection_t *tc)
Definition: tcp_inlines.h:373
u8 data[]
Packet data.
Definition: buffer.h:181
#define TCP_RTO_MIN
Definition: tcp_types.h:89
struct _transport_connection transport_connection_t
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:299
#define TCP_TSTP_HZ
Timestamp freq.
Definition: tcp_types.h:29
static u32 tcp_available_cc_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
Definition: tcp_inlines.h:171
static void * ip6_next_header(ip6_header_t *i)
Definition: ip6_packet.h:371
static void * vlib_buffer_push_tcp(vlib_buffer_t *b, u16 sp_net, u16 dp_net, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
Definition: tcp_inlines.h:438
#define ASSERT(truth)
static tcp_connection_t * tcp_connection_get_if_valid(u32 conn_index, u32 thread_index)
Definition: tcp_inlines.h:39
static uword ip6_address_is_link_local_unicast(const ip6_address_t *a)
Definition: ip6_packet.h:253
#define clib_max(x, y)
Definition: clib.h:320
#define vec_elt(v, i)
Get vector value at index i.
#define tcp_opts_wscale(_to)
Definition: tcp_packet.h:157
static void * vlib_buffer_push_uninit(vlib_buffer_t *b, u8 size)
Prepend uninitialized data to buffer.
Definition: buffer.h:335
u16 payload_length
Definition: ip6_packet.h:301
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static tcp_connection_t * tcp_listener_get(u32 tli)
Definition: tcp_inlines.h:58
static tcp_worker_ctx_t * tcp_get_worker(u32 thread_index)
Definition: tcp.h:282
VLIB buffer representation.
Definition: buffer.h:102
u32 time_now
Time measured in TCP_TSTAMP_TICK used for time stamps.
Definition: tcp.h:95
static void tcp_init_w_buffer(tcp_connection_t *tc, vlib_buffer_t *b, u8 is_ip4)
Initialize connection by gleaning network and rcv params from buffer.
Definition: tcp_inlines.h:328
static f64 tcp_time_now_us(u32 thread_index)
Definition: tcp_inlines.h:213
static void tcp_connection_set_state(tcp_connection_t *tc, tcp_state_t state)
Definition: tcp_inlines.h:51
#define vnet_buffer(b)
Definition: buffer.h:417
static u32 tcp_bytes_out(const tcp_connection_t *tc)
Our estimate of the number of bytes that have left the network.
Definition: tcp_inlines.h:81
static u32 tcp_available_snd_wnd(const tcp_connection_t *tc)
Definition: tcp_inlines.h:150
static int tcp_header_bytes(tcp_header_t *t)
Definition: tcp_packet.h:93
vl_api_dhcp_client_state_t state
Definition: dhcp.api:201
#define tcp_opts_sack_permitted(_to)
Definition: tcp_packet.h:159
static u32 tcp_tstamp(tcp_connection_t *tc)
Generate timestamp for tcp connection.
Definition: tcp_inlines.h:206
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:190
static tcp_connection_t * tcp_get_connection_from_transport(transport_connection_t *tconn)
Definition: tcp_types.h:446
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:104
u32 * fib_index_by_sw_if_index
Definition: ip6.h:196
static tcp_connection_t * tcp_input_lookup_buffer(vlib_buffer_t *b, u8 thread_index, u32 *error, u8 is_ip4, u8 is_nolookup)
Definition: tcp_inlines.h:225
enum _tcp_state tcp_state_t
vl_api_interface_index_t sw_if_index
Definition: wireguard.api:33
Definition: defs.h:46
ip6_address_t dst_address
Definition: ip6_packet.h:310
#define TCP_EVT(_evt, _args...)
Definition: tcp_debug.h:145
static u8 transport_connection_is_descheduled(transport_connection_t *tc)
Definition: transport.h:202