FD.io VPP  v17.07.01-10-g3be13f0
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <math.h>
17 #include <vlib/vlib.h>
18 #include <vnet/vnet.h>
19 #include <vnet/tcp/tcp.h>
20 #include <vppinfra/elog.h>
24 
26 
27 typedef struct
28 {
32 
33 /* packet trace format function */
34 static u8 *
35 format_session_queue_trace (u8 * s, va_list * args)
36 {
37  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
38  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
39  session_queue_trace_t *t = va_arg (*args, session_queue_trace_t *);
40 
41  s = format (s, "SESSION_QUEUE: session index %d, server thread index %d",
43  return s;
44 }
45 
47 
48 #define foreach_session_queue_error \
49 _(TX, "Packets transmitted") \
50 _(TIMER, "Timer events") \
51 _(NO_BUFFER, "Out of buffers")
52 
53 typedef enum
54 {
55 #define _(sym,str) SESSION_QUEUE_ERROR_##sym,
57 #undef _
60 
61 static char *session_queue_error_strings[] = {
62 #define _(sym,string) string,
64 #undef _
65 };
66 
72 };
73 
74 always_inline void
76  u8 thread_index, svm_fifo_t * fifo,
77  vlib_buffer_t * b0, u32 bi0, u8 n_bufs_per_seg,
78  u32 * left_to_snd0, u16 * n_bufs, u32 * rx_offset,
79  u16 deq_per_buf, u8 peek_data)
80 {
81  vlib_buffer_t *chain_b0, *prev_b0;
82  u32 chain_bi0;
83  u16 len_to_deq0, n_bytes_read;
84  u8 *data0, j;
85 
86  chain_bi0 = bi0;
87  chain_b0 = b0;
88  for (j = 1; j < n_bufs_per_seg; j++)
89  {
90  prev_b0 = chain_b0;
91  len_to_deq0 = clib_min (*left_to_snd0, deq_per_buf);
92 
93  *n_bufs -= 1;
94  chain_bi0 = smm->tx_buffers[thread_index][*n_bufs];
95  _vec_len (smm->tx_buffers[thread_index]) = *n_bufs;
96 
97  chain_b0 = vlib_get_buffer (vm, chain_bi0);
98  chain_b0->current_data = 0;
99  data0 = vlib_buffer_get_current (chain_b0);
100  if (peek_data)
101  {
102  n_bytes_read = svm_fifo_peek (fifo, *rx_offset, len_to_deq0, data0);
103  *rx_offset += n_bytes_read;
104  }
105  else
106  {
107  n_bytes_read = svm_fifo_dequeue_nowait (fifo, len_to_deq0, data0);
108  }
109  ASSERT (n_bytes_read == len_to_deq0);
110  chain_b0->current_length = n_bytes_read;
112 
113  /* update previous buffer */
114  prev_b0->next_buffer = chain_bi0;
115  prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
116 
117  /* update current buffer */
118  chain_b0->next_buffer = 0;
119 
120  *left_to_snd0 -= n_bytes_read;
121  if (*left_to_snd0 == 0)
122  break;
123  }
124 }
125 
126 always_inline int
129  session_fifo_event_t * e0,
130  stream_session_t * s0, u32 thread_index,
131  int *n_tx_packets, u8 peek_data)
132 {
133  u32 n_trace = vlib_get_trace_count (vm, node);
134  u32 left_to_snd0, max_len_to_snd0, len_to_deq0, snd_space0;
135  u32 n_bufs_per_evt, n_frames_per_evt;
137  transport_proto_vft_t *transport_vft;
138  u32 next_index, next0, *to_next, n_left_to_next, bi0;
139  vlib_buffer_t *b0;
140  u32 rx_offset = 0, max_dequeue0, n_bytes_per_seg;
141  u16 snd_mss0, n_bufs_per_seg, n_bufs;
142  u8 *data0;
143  int i, n_bytes_read;
144  u32 n_bytes_per_buf, deq_per_buf;
145  u32 buffers_allocated, buffers_allocated_this_call;
146 
147  next_index = next0 = session_type_to_next[s0->session_type];
148 
149  transport_vft = session_get_transport_vft (s0->session_type);
150  tc0 = transport_vft->get_connection (s0->connection_index, thread_index);
151 
152  /* Make sure we have space to send and there's something to dequeue */
153  snd_mss0 = transport_vft->send_mss (tc0);
154  snd_space0 = transport_vft->send_space (tc0);
155 
156  /* Can't make any progress */
157  if (snd_space0 == 0 || snd_mss0 == 0)
158  {
159  vec_add1 (smm->pending_event_vector[thread_index], *e0);
160  return 0;
161  }
162 
163  if (peek_data)
164  {
165  /* Offset in rx fifo from where to peek data */
166  rx_offset = transport_vft->tx_fifo_offset (tc0);
167  }
168 
169  /* Check how much we can pull. If buffering, subtract the offset */
170  max_dequeue0 = svm_fifo_max_dequeue (s0->server_tx_fifo) - rx_offset;
171 
172  /* Nothing to read return */
173  if (max_dequeue0 == 0)
174  {
175  svm_fifo_unset_event (s0->server_tx_fifo);
176  return 0;
177  }
178 
179  /* Ensure we're not writing more than transport window allows */
180  if (max_dequeue0 < snd_space0)
181  {
182  /* Constrained by tx queue. Try to send only fully formed segments */
183  max_len_to_snd0 = (max_dequeue0 > snd_mss0) ?
184  max_dequeue0 - max_dequeue0 % snd_mss0 : max_dequeue0;
185  /* TODO Nagle ? */
186  }
187  else
188  {
189  max_len_to_snd0 = snd_space0;
190  }
191 
192  n_bytes_per_buf = vlib_buffer_free_list_buffer_size
194  n_bytes_per_seg = MAX_HDRS_LEN + snd_mss0;
195  n_bufs_per_seg = ceil ((double) n_bytes_per_seg / n_bytes_per_buf);
196  n_bufs_per_evt = (ceil ((double) max_len_to_snd0 / n_bytes_per_seg))
197  * n_bufs_per_seg;
198  n_frames_per_evt = ceil ((double) n_bufs_per_evt / VLIB_FRAME_SIZE);
199 
200  deq_per_buf = clib_min (snd_mss0, n_bytes_per_buf);
201 
202  n_bufs = vec_len (smm->tx_buffers[thread_index]);
203  left_to_snd0 = max_len_to_snd0;
204  for (i = 0; i < n_frames_per_evt; i++)
205  {
206  /* Make sure we have at least one full frame of buffers ready */
207  if (PREDICT_FALSE (n_bufs < VLIB_FRAME_SIZE))
208  {
209  vec_validate (smm->tx_buffers[thread_index],
210  n_bufs + 2 * VLIB_FRAME_SIZE - 1);
211 
212  buffers_allocated = 0;
213  do
214  {
215  buffers_allocated_this_call =
217  (vm,
218  &smm->tx_buffers[thread_index][n_bufs + buffers_allocated],
219  2 * VLIB_FRAME_SIZE - buffers_allocated);
220  buffers_allocated += buffers_allocated_this_call;
221  }
222  while (buffers_allocated_this_call > 0
223  && ((buffers_allocated + n_bufs < VLIB_FRAME_SIZE)));
224 
225  n_bufs += buffers_allocated;
226 
227  _vec_len (smm->tx_buffers[thread_index]) = n_bufs;
228 
229  if (PREDICT_FALSE (n_bufs < VLIB_FRAME_SIZE))
230  {
231  vec_add1 (smm->pending_event_vector[thread_index], *e0);
232  return -1;
233  }
234  }
235  /* Allow enqueuing of a new event */
236  svm_fifo_unset_event (s0->server_tx_fifo);
237 
238  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
239  while (left_to_snd0 && n_left_to_next >= n_bufs_per_seg)
240  {
241  /*
242  * Handle first buffer in chain separately
243  */
244 
245  /* Get free buffer */
246  ASSERT (n_bufs >= 1);
247  bi0 = smm->tx_buffers[thread_index][--n_bufs];
248  ASSERT (bi0);
249  _vec_len (smm->tx_buffers[thread_index]) = n_bufs;
250 
251  /* usual speculation, or the enqueue_x1 macro will barf */
252  to_next[0] = bi0;
253  to_next += 1;
254  n_left_to_next -= 1;
255 
256  b0 = vlib_get_buffer (vm, bi0);
257  b0->error = 0;
260  b0->current_data = 0;
262 
263  len_to_deq0 = clib_min (left_to_snd0, deq_per_buf);
264 
266  if (peek_data)
267  {
268  n_bytes_read = svm_fifo_peek (s0->server_tx_fifo, rx_offset,
269  len_to_deq0, data0);
270  /* Keep track of progress locally, transport is also supposed to
271  * increment it independently when pushing the header */
272  rx_offset += n_bytes_read;
273  }
274  else
275  {
276  n_bytes_read = svm_fifo_dequeue_nowait (s0->server_tx_fifo,
277  len_to_deq0, data0);
278  }
279 
280  if (n_bytes_read <= 0)
281  goto dequeue_fail;
282 
283  b0->current_length = n_bytes_read;
284 
285  left_to_snd0 -= n_bytes_read;
286  *n_tx_packets = *n_tx_packets + 1;
287 
288  /*
289  * Fill in the remaining buffers in the chain, if any
290  */
291  if (PREDICT_FALSE (n_bufs_per_seg > 1))
292  session_tx_fifo_chain_tail (smm, vm, thread_index,
293  s0->server_tx_fifo, b0, bi0,
294  n_bufs_per_seg, &left_to_snd0,
295  &n_bufs, &rx_offset, deq_per_buf,
296  peek_data);
297 
298  /* Ask transport to push header after current_length and
299  * total_length_not_including_first_buffer are updated */
300  transport_vft->push_header (tc0, b0);
301 
302  /* *INDENT-OFF* */
303  SESSION_EVT_DBG(SESSION_EVT_DEQ, s0, ({
304  ed->data[0] = e0->event_id;
305  ed->data[1] = max_dequeue0;
306  ed->data[2] = len_to_deq0;
307  ed->data[3] = left_to_snd0;
308  }));
309  /* *INDENT-ON* */
310 
311 
313  if (PREDICT_FALSE (n_trace > 0))
314  {
316  vlib_trace_buffer (vm, node, next_index, b0,
317  1 /* follow_chain */ );
318  vlib_set_trace_count (vm, node, --n_trace);
319  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
320  t0->session_index = s0->session_index;
321  t0->server_thread_index = s0->thread_index;
322  }
323 
324  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
325  to_next, n_left_to_next,
326  bi0, next0);
327  }
328  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
329  }
330 
331  /* If we couldn't dequeue all bytes mark as partially read */
332  if (max_len_to_snd0 < max_dequeue0)
333  {
334  /* If we don't already have new event */
335  if (svm_fifo_set_event (s0->server_tx_fifo))
336  {
337  vec_add1 (smm->pending_event_vector[thread_index], *e0);
338  }
339  }
340  return 0;
341 
342 dequeue_fail:
343  /*
344  * Can't read from fifo. If we don't already have an event, save as partially
345  * read, return buff to free list and return
346  */
347  clib_warning ("dequeue fail");
348 
349  if (svm_fifo_set_event (s0->server_tx_fifo))
350  {
351  vec_add1 (smm->pending_event_vector[thread_index], *e0);
352  }
353  vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1);
354  _vec_len (smm->tx_buffers[thread_index]) += 1;
355 
356  return 0;
357 }
358 
359 int
362  session_fifo_event_t * e0,
363  stream_session_t * s0, u32 thread_index,
364  int *n_tx_pkts)
365 {
366  return session_tx_fifo_read_and_snd_i (vm, node, smm, e0, s0, thread_index,
367  n_tx_pkts, 1);
368 }
369 
370 int
373  session_fifo_event_t * e0,
374  stream_session_t * s0, u32 thread_index,
375  int *n_tx_pkts)
376 {
377  return session_tx_fifo_read_and_snd_i (vm, node, smm, e0, s0, thread_index,
378  n_tx_pkts, 0);
379 }
380 
382 session_event_get_session (session_fifo_event_t * e0, u8 thread_index)
383 {
384  svm_fifo_t *f0;
385  stream_session_t *s0;
386  u32 session_index0;
387 
388  f0 = e0->fifo;
389  session_index0 = f0->master_session_index;
390 
391  /* $$$ add multiple event queues, per vpp worker thread */
392  ASSERT (f0->master_thread_index == thread_index);
393 
394  s0 = stream_session_get_if_valid (session_index0, thread_index);
395 
396  ASSERT (s0 == 0 || s0->thread_index == thread_index);
397 
398  return s0;
399 }
400 
401 void
403 {
406  u32 my_thread_index = vm->thread_index;
407  session_fifo_event_t _e, *e = &_e;
408  stream_session_t *s0;
409  int i, index;
410  i8 *headp;
411 
413  q = smm->vpp_event_queues[my_thread_index];
414 
415  index = q->head;
416 
417  for (i = 0; i < q->cursize; i++)
418  {
419  headp = (i8 *) (&q->data[0] + q->elsize * index);
420  clib_memcpy (e, headp, q->elsize);
421 
422  switch (e->event_type)
423  {
424  case FIFO_EVENT_APP_TX:
425  s0 = session_event_get_session (e, my_thread_index);
426  fformat (stdout, "[%04d] TX session %d\n", i, s0->session_index);
427  break;
428 
430  s0 = stream_session_get_from_handle (e->session_handle);
431  fformat (stdout, "[%04d] disconnect session %d\n", i,
432  s0->session_index);
433  break;
434 
436  s0 = session_event_get_session (e, my_thread_index);
437  fformat (stdout, "[%04d] builtin_rx %d\n", i, s0->session_index);
438  break;
439 
440  case FIFO_EVENT_RPC:
441  fformat (stdout, "[%04d] RPC call %llx with %llx\n",
442  i, (u64) (e->rpc_args.fp), (u64) (e->rpc_args.arg));
443  break;
444 
445  default:
446  fformat (stdout, "[%04d] unhandled event type %d\n",
447  i, e->event_type);
448  break;
449  }
450 
451  index++;
452 
453  if (index == q->maxsize)
454  index = 0;
455  }
456 }
457 
458 static uword
460  vlib_frame_t * frame)
461 {
463  session_fifo_event_t *my_pending_event_vector, *e;
464  session_fifo_event_t *my_fifo_events;
465  u32 n_to_dequeue, n_events;
467  application_t *app;
468  int n_tx_packets = 0;
469  u32 my_thread_index = vm->thread_index;
470  int i, rv;
471  f64 now = vlib_time_now (vm);
472  void (*fp) (void *);
473 
474  SESSION_EVT_DBG (SESSION_EVT_POLL_GAP_TRACK, smm, my_thread_index);
475 
476  /*
477  * Update TCP time
478  */
479  tcp_update_time (now, my_thread_index);
480 
481  /*
482  * Get vpp queue events
483  */
484  q = smm->vpp_event_queues[my_thread_index];
485  if (PREDICT_FALSE (q == 0))
486  return 0;
487 
488  my_fifo_events = smm->free_event_vector[my_thread_index];
489 
490  /* min number of events we can dequeue without blocking */
491  n_to_dequeue = q->cursize;
492  my_pending_event_vector = smm->pending_event_vector[my_thread_index];
493 
494  if (n_to_dequeue == 0 && vec_len (my_pending_event_vector) == 0)
495  return 0;
496 
497  SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 0);
498 
499  /*
500  * If we didn't manage to process previous events try going
501  * over them again without dequeuing new ones.
502  */
503  /* XXX: Block senders to sessions that can't keep up */
504  if (0 && vec_len (my_pending_event_vector) >= 100)
505  {
506  clib_warning ("too many fifo events unsolved");
507  goto skip_dequeue;
508  }
509 
510  /* See you in the next life, don't be late */
511  if (pthread_mutex_trylock (&q->mutex))
512  return 0;
513 
514  for (i = 0; i < n_to_dequeue; i++)
515  {
516  vec_add2 (my_fifo_events, e, 1);
518  }
519 
520  /* The other side of the connection is not polling */
521  if (q->cursize < (q->maxsize / 8))
522  (void) pthread_cond_broadcast (&q->condvar);
523  pthread_mutex_unlock (&q->mutex);
524 
525  vec_append (my_fifo_events, my_pending_event_vector);
526 
527  _vec_len (my_pending_event_vector) = 0;
528  smm->pending_event_vector[my_thread_index] = my_pending_event_vector;
529 
530 skip_dequeue:
531  n_events = vec_len (my_fifo_events);
532  for (i = 0; i < n_events; i++)
533  {
534  stream_session_t *s0; /* $$$ prefetch 1 ahead maybe */
535  session_fifo_event_t *e0;
536 
537  e0 = &my_fifo_events[i];
538 
539  switch (e0->event_type)
540  {
541  case FIFO_EVENT_APP_TX:
542  s0 = session_event_get_session (e0, my_thread_index);
543 
544  if (CLIB_DEBUG && !s0)
545  {
546  clib_warning ("It's dead, Jim!");
547  continue;
548  }
549 
550  if (PREDICT_FALSE (s0->session_state == SESSION_STATE_CLOSED))
551  continue;
552  /* Spray packets in per session type frames, since they go to
553  * different nodes */
554  rv = (smm->session_tx_fns[s0->session_type]) (vm, node, smm, e0, s0,
555  my_thread_index,
556  &n_tx_packets);
557  /* Out of buffers */
558  if (PREDICT_FALSE (rv < 0))
559  {
561  SESSION_QUEUE_ERROR_NO_BUFFER, 1);
562  continue;
563  }
564  break;
566  s0 = stream_session_get_from_handle (e0->session_handle);
568  break;
570  s0 = session_event_get_session (e0, my_thread_index);
571  svm_fifo_unset_event (s0->server_rx_fifo);
572  /* Get session's server */
573  app = application_get (s0->app_index);
574  app->cb_fns.builtin_server_rx_callback (s0);
575  break;
576  case FIFO_EVENT_RPC:
577  fp = e0->rpc_args.fp;
578  (*fp) (e0->rpc_args.arg);
579  break;
580 
581  default:
582  clib_warning ("unhandled event type %d", e0->event_type);
583  }
584  }
585 
586  _vec_len (my_fifo_events) = 0;
587  smm->free_event_vector[my_thread_index] = my_fifo_events;
588 
590  SESSION_QUEUE_ERROR_TX, n_tx_packets);
591 
592  SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 1);
593 
594  return n_tx_packets;
595 }
596 
597 /* *INDENT-OFF* */
599 {
600  .function = session_queue_node_fn,
601  .name = "session-queue",
602  .format_trace = format_session_queue_trace,
603  .type = VLIB_NODE_TYPE_INPUT,
605  .error_strings = session_queue_error_strings,
606  .n_next_nodes = SESSION_QUEUE_N_NEXT,
607  .state = VLIB_NODE_STATE_DISABLED,
608  .next_nodes =
609  {
610  [SESSION_QUEUE_NEXT_DROP] = "error-drop",
611  [SESSION_QUEUE_NEXT_IP4_LOOKUP] = "ip4-lookup",
612  [SESSION_QUEUE_NEXT_IP6_LOOKUP] = "ip6-lookup",
613  [SESSION_QUEUE_NEXT_TCP_IP4_OUTPUT] = "tcp4-output",
614  [SESSION_QUEUE_NEXT_TCP_IP6_OUTPUT] = "tcp6-output",
615  },
616 };
617 /* *INDENT-ON* */
618 
619 /*
620  * fd.io coding-style-patch-verification: ON
621  *
622  * Local Variables:
623  * eval: (c-set-style "gnu")
624  * End:
625  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define clib_min(x, y)
Definition: clib.h:332
#define CLIB_UNUSED(x)
Definition: clib.h:79
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:143
struct _transport_connection transport_connection_t
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:192
static u32 session_type_to_next[]
Definition: node.c:67
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:459
u32 thread_index
Definition: main.h:159
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
struct _vlib_node_registration vlib_node_registration_t
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:561
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
session_queue_error_t
Definition: node.c:53
int session_tx_fifo_dequeue_and_snd(vlib_main_t *vm, vlib_node_runtime_t *node, session_manager_main_t *smm, session_fifo_event_t *e0, stream_session_t *s0, u32 thread_index, int *n_tx_pkts)
Definition: node.c:371
struct _stream_session_t stream_session_t
struct _svm_fifo svm_fifo_t
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:104
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:87
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
stream_session_t * session_event_get_session(session_fifo_event_t *e0, u8 thread_index)
Definition: node.c:382
static char * session_queue_error_strings[]
Definition: node.c:61
#define always_inline
Definition: clib.h:84
static u32 svm_fifo_max_dequeue(svm_fifo_t *f)
Definition: svm_fifo.h:71
static void session_tx_fifo_chain_tail(session_manager_main_t *smm, vlib_main_t *vm, u8 thread_index, svm_fifo_t *fifo, vlib_buffer_t *b0, u32 bi0, u8 n_bufs_per_seg, u32 *left_to_snd0, u16 *n_bufs, u32 *rx_offset, u16 deq_per_buf, u8 peek_data)
Definition: node.c:75
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
Definition: buffer.h:89
char i8
Definition: types.h:45
unsigned long u64
Definition: types.h:89
int session_tx_fifo_peek_and_snd(vlib_main_t *vm, vlib_node_runtime_t *node, session_manager_main_t *smm, session_fifo_event_t *e0, stream_session_t *s0, u32 thread_index, int *n_tx_pkts)
Definition: node.c:360
struct _transport_proto_vft transport_proto_vft_t
static session_manager_main_t * vnet_get_session_manager_main()
Definition: session.h:237
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:300
#define MAX_HDRS_LEN
Definition: session.h:27
vlib_main_t vlib_global_main
Definition: main.c:1653
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:402
#define PREDICT_FALSE(x)
Definition: clib.h:97
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:399
static void svm_fifo_unset_event(svm_fifo_t *f)
Unsets fifo event flag.
Definition: svm_fifo.h:104
int unix_shared_memory_queue_sub_raw(unix_shared_memory_queue_t *q, u8 *elem)
#define VLIB_FRAME_SIZE
Definition: node.h:329
u32 node_index
Node index.
Definition: node.h:441
struct _session_manager_main session_manager_main_t
Definition: session.h:157
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:216
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:366
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:113
word fformat(FILE *f, char *fmt,...)
Definition: format.c:453
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1131
static u8 svm_fifo_set_event(svm_fifo_t *f)
Sets fifo event flag.
Definition: svm_fifo.h:94
The fine-grained event logger allows lightweight, thread-safe event logging at minimum cost...
#define SESSION_EVT_DBG(_evt, _args...)
#define foreach_session_queue_error
Definition: node.c:48
#define VNET_BUFFER_LOCALLY_ORIGINATED
Definition: buffer.h:68
void dump_thread_0_event_queue(void)
Definition: node.c:402
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:69
struct _application application_t
#define ARRAY_LEN(x)
Definition: clib.h:59
static stream_session_t * stream_session_get_if_valid(u64 si, u32 thread_index)
Definition: session.h:287
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
#define vec_append(v1, v2)
Append v2 after v1.
Definition: vec.h:819
void stream_session_disconnect(stream_session_t *s)
Disconnect session and propagate to transport.
Definition: session.c:1012
vlib_node_registration_t session_queue_node
(constructor) VLIB_REGISTER_NODE (session_queue_node)
Definition: node.c:25
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
transport_proto_vft_t * session_get_transport_vft(u8 type)
Definition: session.c:1055
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:103
unsigned short u16
Definition: types.h:57
static void tcp_update_time(f64 now, u32 thread_index)
Definition: tcp.h:546
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:496
application_t * application_get(u32 index)
Definition: application.c:189
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:144
u32 server_thread_index
Definition: node.c:30
static int session_tx_fifo_read_and_snd_i(vlib_main_t *vm, vlib_node_runtime_t *node, session_manager_main_t *smm, session_fifo_event_t *e0, stream_session_t *s0, u32 thread_index, int *n_tx_packets, u8 peek_data)
Definition: node.c:127
static u8 * format_session_queue_trace(u8 *s, va_list *args)
Definition: node.c:35
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:159
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:245
int svm_fifo_peek(svm_fifo_t *f, u32 relative_offset, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:659
int svm_fifo_dequeue_nowait(svm_fifo_t *f, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:593
static uword session_queue_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: node.c:459
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
static stream_session_t * stream_session_get_from_handle(u64 handle)
Definition: session.h:324
struct _unix_shared_memory_queue unix_shared_memory_queue_t