FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
session.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /**
16  * @file
17  * @brief Session and session manager
18  */
19 
20 #include <vnet/session/session.h>
22 #include <vnet/dpo/load_balance.h>
23 #include <vnet/fib/ip4_fib.h>
24 
26 
27 static inline int
28 session_send_evt_to_thread (void *data, void *args, u32 thread_index,
29  session_evt_type_t evt_type)
30 {
31  session_event_t *evt;
32  svm_msg_q_msg_t msg;
33  svm_msg_q_t *mq;
34 
35  mq = session_main_get_vpp_event_queue (thread_index);
36  if (PREDICT_FALSE (svm_msg_q_lock (mq)))
37  return -1;
40  {
41  svm_msg_q_unlock (mq);
42  return -2;
43  }
44  switch (evt_type)
45  {
48  evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
49  evt->rpc_args.fp = data;
50  evt->rpc_args.arg = args;
51  break;
52  case SESSION_IO_EVT_RX:
53  case SESSION_IO_EVT_TX:
57  evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
58  evt->session_index = *(u32 *) data;
59  break;
64  evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
65  evt->session_handle = session_handle ((session_t *) data);
66  break;
67  default:
68  clib_warning ("evt unhandled!");
69  svm_msg_q_unlock (mq);
70  return -1;
71  }
72  evt->event_type = evt_type;
73 
74  svm_msg_q_add_and_unlock (mq, &msg);
75  return 0;
76 }
77 
78 int
80 {
81  return session_send_evt_to_thread (&f->master_session_index, 0,
82  f->master_thread_index, evt_type);
83 }
84 
85 int
87  session_evt_type_t evt_type)
88 {
89  return session_send_evt_to_thread (data, 0, thread_index, evt_type);
90 }
91 
92 int
94 {
95  /* only events supported are disconnect and reset */
96  ASSERT (evt_type == SESSION_CTRL_EVT_CLOSE
97  || evt_type == SESSION_CTRL_EVT_RESET);
98  return session_send_evt_to_thread (s, 0, s->thread_index, evt_type);
99 }
100 
101 void
103  void *rpc_args)
104 {
105  session_send_evt_to_thread (fp, rpc_args, thread_index,
107 }
108 
109 void
110 session_send_rpc_evt_to_thread (u32 thread_index, void *fp, void *rpc_args)
111 {
112  if (thread_index != vlib_get_thread_index ())
113  session_send_rpc_evt_to_thread_force (thread_index, fp, rpc_args);
114  else
115  {
116  void (*fnp) (void *) = fp;
117  fnp (rpc_args);
118  }
119 }
120 
121 void
123 {
124  session_t *s;
125 
126  s = session_get (tc->s_index, tc->thread_index);
128  ASSERT (s->session_state != SESSION_STATE_TRANSPORT_DELETED);
129  if (!(s->flags & SESSION_F_CUSTOM_TX))
130  {
131  s->flags |= SESSION_F_CUSTOM_TX;
132  if (svm_fifo_set_event (s->tx_fifo)
134  {
135  session_worker_t *wrk;
136  session_evt_elt_t *elt;
137  wrk = session_main_get_worker (tc->thread_index);
138  if (has_prio)
139  elt = session_evt_alloc_new (wrk);
140  else
141  elt = session_evt_alloc_old (wrk);
142  elt->evt.session_index = tc->s_index;
143  elt->evt.event_type = SESSION_IO_EVT_TX;
144  tc->flags &= ~TRANSPORT_CONNECTION_F_DESCHED;
145  }
146  }
147 }
148 
149 void
151 {
152  session_worker_t *wrk = session_main_get_worker (tc->thread_index);
153  session_evt_elt_t *elt;
154 
155  ASSERT (tc->thread_index == vlib_get_thread_index ());
156 
157  elt = session_evt_alloc_new (wrk);
158  elt->evt.session_index = tc->s_index;
159  elt->evt.event_type = SESSION_IO_EVT_TX;
160 }
161 
162 static void
164 {
165  u32 thread_index = vlib_get_thread_index ();
166  session_evt_elt_t *elt;
167  session_worker_t *wrk;
168 
169  /* If we are in the handler thread, or being called with the worker barrier
170  * held, just append a new event to pending disconnects vector. */
171  if (vlib_thread_is_main_w_barrier () || thread_index == s->thread_index)
172  {
174  elt = session_evt_alloc_ctrl (wrk);
175  clib_memset (&elt->evt, 0, sizeof (session_event_t));
176  elt->evt.session_handle = session_handle (s);
177  elt->evt.event_type = evt;
178  }
179  else
181 }
182 
183 session_t *
184 session_alloc (u32 thread_index)
185 {
186  session_worker_t *wrk = &session_main.wrk[thread_index];
187  session_t *s;
188  u8 will_expand = 0;
189  pool_get_aligned_will_expand (wrk->sessions, will_expand,
191  /* If we have peekers, let them finish */
192  if (PREDICT_FALSE (will_expand && vlib_num_workers ()))
193  {
197  }
198  else
199  {
201  }
202  clib_memset (s, 0, sizeof (*s));
203  s->session_index = s - wrk->sessions;
204  s->thread_index = thread_index;
206  return s;
207 }
208 
209 void
211 {
212  if (CLIB_DEBUG)
213  {
214  u8 thread_index = s->thread_index;
215  clib_memset (s, 0xFA, sizeof (*s));
216  pool_put (session_main.wrk[thread_index].sessions, s);
217  return;
218  }
219  SESSION_EVT (SESSION_EVT_FREE, s);
220  pool_put (session_main.wrk[s->thread_index].sessions, s);
221 }
222 
223 u8
224 session_is_valid (u32 si, u8 thread_index)
225 {
226  session_t *s;
228 
229  s = pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
230 
231  if (!s)
232  return 1;
233 
234  if (s->thread_index != thread_index || s->session_index != si)
235  return 0;
236 
237  if (s->session_state == SESSION_STATE_TRANSPORT_DELETED
238  || s->session_state <= SESSION_STATE_LISTENING)
239  return 1;
240 
241  tc = session_get_transport (s);
242  if (s->connection_index != tc->c_index
243  || s->thread_index != tc->thread_index || tc->s_index != si)
244  return 0;
245 
246  return 1;
247 }
248 
249 static void
251 {
252  app_worker_t *app_wrk;
253 
254  app_wrk = app_worker_get_if_valid (s->app_wrk_index);
255  if (!app_wrk)
256  return;
257  app_worker_cleanup_notify (app_wrk, s, ntf);
258 }
259 
260 void
262 {
265  session_free (s);
266 }
267 
268 /**
269  * Cleans up session and lookup table.
270  *
271  * Transport connection must still be valid.
272  */
273 static void
275 {
276  int rv;
277 
278  /* Delete from the main lookup table. */
279  if ((rv = session_lookup_del_session (s)))
280  clib_warning ("session %u hash delete rv %d", s->session_index, rv);
281 
283 }
284 
285 void
287 {
289 }
290 
291 void
293  session_handle_t ho_handle)
294 {
295  app_worker_t *app_wrk = app_worker_get (session_handle_data (ho_handle));
296  app_worker_del_half_open (app_wrk, tp, ho_handle);
297 }
298 
299 session_t *
301 {
302  session_t *s;
303  u32 thread_index = tc->thread_index;
304 
305  ASSERT (thread_index == vlib_get_thread_index ()
306  || transport_protocol_is_cl (tc->proto));
307 
308  s = session_alloc (thread_index);
309  s->session_type = session_type_from_proto_and_ip (tc->proto, tc->is_ip4);
310  s->session_state = SESSION_STATE_CLOSED;
311 
312  /* Attach transport to session and vice versa */
313  s->connection_index = tc->c_index;
314  tc->s_index = s->session_index;
315  return s;
316 }
317 
318 /**
319  * Discards bytes from buffer chain
320  *
321  * It discards n_bytes_to_drop starting at first buffer after chain_b
322  */
323 always_inline void
325  vlib_buffer_t ** chain_b,
326  u32 n_bytes_to_drop)
327 {
328  vlib_buffer_t *next = *chain_b;
329  u32 to_drop = n_bytes_to_drop;
330  ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
331  while (to_drop && (next->flags & VLIB_BUFFER_NEXT_PRESENT))
332  {
333  next = vlib_get_buffer (vm, next->next_buffer);
334  if (next->current_length > to_drop)
335  {
336  vlib_buffer_advance (next, to_drop);
337  to_drop = 0;
338  }
339  else
340  {
341  to_drop -= next->current_length;
342  next->current_length = 0;
343  }
344  }
345  *chain_b = next;
346 
347  if (to_drop == 0)
348  b->total_length_not_including_first_buffer -= n_bytes_to_drop;
349 }
350 
351 /**
352  * Enqueue buffer chain tail
353  */
354 always_inline int
356  u32 offset, u8 is_in_order)
357 {
358  vlib_buffer_t *chain_b;
359  u32 chain_bi, len, diff;
361  u8 *data;
362  u32 written = 0;
363  int rv = 0;
364 
365  if (is_in_order && offset)
366  {
367  diff = offset - b->current_length;
369  return 0;
370  chain_b = b;
371  session_enqueue_discard_chain_bytes (vm, b, &chain_b, diff);
372  chain_bi = vlib_get_buffer_index (vm, chain_b);
373  }
374  else
375  chain_bi = b->next_buffer;
376 
377  do
378  {
379  chain_b = vlib_get_buffer (vm, chain_bi);
380  data = vlib_buffer_get_current (chain_b);
381  len = chain_b->current_length;
382  if (!len)
383  continue;
384  if (is_in_order)
385  {
386  rv = svm_fifo_enqueue (s->rx_fifo, len, data);
387  if (rv == len)
388  {
389  written += rv;
390  }
391  else if (rv < len)
392  {
393  return (rv > 0) ? (written + rv) : written;
394  }
395  else if (rv > len)
396  {
397  written += rv;
398 
399  /* written more than what was left in chain */
401  return written;
402 
403  /* drop the bytes that have already been delivered */
404  session_enqueue_discard_chain_bytes (vm, b, &chain_b, rv - len);
405  }
406  }
407  else
408  {
409  rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset, len, data);
410  if (rv)
411  {
412  clib_warning ("failed to enqueue multi-buffer seg");
413  return -1;
414  }
415  offset += len;
416  }
417  }
418  while ((chain_bi = (chain_b->flags & VLIB_BUFFER_NEXT_PRESENT)
419  ? chain_b->next_buffer : 0));
420 
421  if (is_in_order)
422  return written;
423 
424  return 0;
425 }
426 
427 void
430 {
431  if (s->flags & SESSION_F_CUSTOM_FIFO_TUNING)
432  {
433  app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
434  app_worker_session_fifo_tuning (app_wrk, s, f, act, len);
435  if (CLIB_ASSERT_ENABLE)
436  {
437  segment_manager_t *sm;
438  sm = segment_manager_get (f->segment_manager);
439  ASSERT (f->size >= 4096);
440  ASSERT (f->size <= sm->max_fifo_size);
441  }
442  }
443 }
444 
445 /*
446  * Enqueue data for delivery to session peer. Does not notify peer of enqueue
447  * event but on request can queue notification events for later delivery by
448  * calling stream_server_flush_enqueue_events().
449  *
450  * @param tc Transport connection which is to be enqueued data
451  * @param b Buffer to be enqueued
452  * @param offset Offset at which to start enqueueing if out-of-order
453  * @param queue_event Flag to indicate if peer is to be notified or if event
454  * is to be queued. The former is useful when more data is
455  * enqueued and only one event is to be generated.
456  * @param is_in_order Flag to indicate if data is in order
457  * @return Number of bytes enqueued or a negative value if enqueueing failed.
458  */
459 int
461  vlib_buffer_t * b, u32 offset,
462  u8 queue_event, u8 is_in_order)
463 {
464  session_t *s;
465  int enqueued = 0, rv, in_order_off;
466 
467  s = session_get (tc->s_index, tc->thread_index);
468 
469  if (is_in_order)
470  {
471  enqueued = svm_fifo_enqueue (s->rx_fifo,
472  b->current_length,
474  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT)
475  && enqueued >= 0))
476  {
477  in_order_off = enqueued > b->current_length ? enqueued : 0;
478  rv = session_enqueue_chain_tail (s, b, in_order_off, 1);
479  if (rv > 0)
480  enqueued += rv;
481  }
482  }
483  else
484  {
485  rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset,
486  b->current_length,
488  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && !rv))
489  session_enqueue_chain_tail (s, b, offset + b->current_length, 0);
490  /* if something was enqueued, report even this as success for ooo
491  * segment handling */
492  return rv;
493  }
494 
495  if (queue_event)
496  {
497  /* Queue RX event on this fifo. Eventually these will need to be flushed
498  * by calling stream_server_flush_enqueue_events () */
499  session_worker_t *wrk;
500 
502  if (!(s->flags & SESSION_F_RX_EVT))
503  {
504  s->flags |= SESSION_F_RX_EVT;
505  vec_add1 (wrk->session_to_enqueue[tc->proto], s->session_index);
506  }
507 
509  }
510 
511  return enqueued;
512 }
513 
514 int
516  session_dgram_hdr_t * hdr,
517  vlib_buffer_t * b, u8 proto, u8 queue_event)
518 {
519  int enqueued = 0, rv, in_order_off;
520 
522  >= b->current_length + sizeof (*hdr));
523 
524  svm_fifo_enqueue (s->rx_fifo, sizeof (session_dgram_hdr_t), (u8 *) hdr);
525  enqueued = svm_fifo_enqueue (s->rx_fifo, b->current_length,
527  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && enqueued >= 0))
528  {
529  in_order_off = enqueued > b->current_length ? enqueued : 0;
530  rv = session_enqueue_chain_tail (s, b, in_order_off, 1);
531  if (rv > 0)
532  enqueued += rv;
533  }
534  if (queue_event)
535  {
536  /* Queue RX event on this fifo. Eventually these will need to be flushed
537  * by calling stream_server_flush_enqueue_events () */
538  session_worker_t *wrk;
539 
541  if (!(s->flags & SESSION_F_RX_EVT))
542  {
543  s->flags |= SESSION_F_RX_EVT;
544  vec_add1 (wrk->session_to_enqueue[proto], s->session_index);
545  }
546 
548  }
549  return enqueued;
550 }
551 
552 int
554  u32 offset, u32 max_bytes)
555 {
556  session_t *s = session_get (tc->s_index, tc->thread_index);
557  return svm_fifo_peek (s->tx_fifo, offset, max_bytes, buffer);
558 }
559 
560 u32
562 {
563  session_t *s = session_get (tc->s_index, tc->thread_index);
564  u32 rv;
565 
566  rv = svm_fifo_dequeue_drop (s->tx_fifo, max_bytes);
568 
569  if (svm_fifo_needs_deq_ntf (s->tx_fifo, max_bytes))
571 
572  return rv;
573 }
574 
575 static inline int
577  svm_fifo_t * f, session_evt_type_t evt_type)
578 {
579  app_worker_t *app_wrk;
580  application_t *app;
581  int i;
582 
583  app = application_get (app_index);
584  if (!app)
585  return -1;
586 
587  for (i = 0; i < f->n_subscribers; i++)
588  {
589  app_wrk = application_get_worker (app, f->subscribers[i]);
590  if (!app_wrk)
591  continue;
592  if (app_worker_lock_and_send_event (app_wrk, s, evt_type))
593  return -1;
594  }
595 
596  return 0;
597 }
598 
599 /**
600  * Notify session peer that new data has been enqueued.
601  *
602  * @param s Stream session for which the event is to be generated.
603  * @param lock Flag to indicate if call should lock message queue.
604  *
605  * @return 0 on success or negative number if failed to send notification.
606  */
607 static inline int
609 {
610  app_worker_t *app_wrk;
611  u32 session_index;
612  u8 n_subscribers;
613 
614  session_index = s->session_index;
615  n_subscribers = svm_fifo_n_subscribers (s->rx_fifo);
616 
617  app_wrk = app_worker_get_if_valid (s->app_wrk_index);
618  if (PREDICT_FALSE (!app_wrk))
619  {
620  SESSION_DBG ("invalid s->app_index = %d", s->app_wrk_index);
621  return 0;
622  }
623 
624  SESSION_EVT (SESSION_EVT_ENQ, s, svm_fifo_max_dequeue_prod (s->rx_fifo));
625 
626  s->flags &= ~SESSION_F_RX_EVT;
627 
628  /* Application didn't confirm accept yet */
629  if (PREDICT_FALSE (s->session_state == SESSION_STATE_ACCEPTING))
630  return 0;
631 
634  return -1;
635 
636  if (PREDICT_FALSE (n_subscribers))
637  {
638  s = session_get (session_index, vlib_get_thread_index ());
639  return session_notify_subscribers (app_wrk->app_index, s,
641  }
642 
643  return 0;
644 }
645 
646 int
648 {
650 }
651 
652 static void
654 {
655  u32 session_index = pointer_to_uword (arg);
656  session_t *s;
657 
658  s = session_get_if_valid (session_index, vlib_get_thread_index ());
659  if (!s)
660  return;
661 
663 }
664 
665 /**
666  * Like session_enqueue_notify, but can be called from a thread that does not
667  * own the session.
668  */
669 void
671 {
672  u32 thread_index = session_thread_from_handle (sh);
673  u32 session_index = session_index_from_handle (sh);
674 
675  /*
676  * Pass session index (u32) as opposed to handle (u64) in case pointers
677  * are not 64-bit.
678  */
679  session_send_rpc_evt_to_thread (thread_index,
681  uword_to_pointer (session_index, void *));
682 }
683 
684 int
686 {
687  app_worker_t *app_wrk;
688 
690 
691  app_wrk = app_worker_get_if_valid (s->app_wrk_index);
692  if (PREDICT_FALSE (!app_wrk))
693  return -1;
694 
697  return -1;
698 
699  if (PREDICT_FALSE (s->tx_fifo->n_subscribers))
700  return session_notify_subscribers (app_wrk->app_index, s,
702 
703  return 0;
704 }
705 
706 /**
707  * Flushes queue of sessions that are to be notified of new data
708  * enqueued events.
709  *
710  * @param thread_index Thread index for which the flush is to be performed.
711  * @return 0 on success or a positive number indicating the number of
712  * failures due to API queue being full.
713  */
714 int
716 {
717  session_worker_t *wrk = session_main_get_worker (thread_index);
718  session_t *s;
719  int i, errors = 0;
720  u32 *indices;
721 
722  indices = wrk->session_to_enqueue[transport_proto];
723 
724  for (i = 0; i < vec_len (indices); i++)
725  {
726  s = session_get_if_valid (indices[i], thread_index);
727  if (PREDICT_FALSE (!s))
728  {
729  errors++;
730  continue;
731  }
732 
734  0 /* TODO/not needed */ );
735 
737  errors++;
738  }
739 
740  vec_reset_length (indices);
741  wrk->session_to_enqueue[transport_proto] = indices;
742 
743  return errors;
744 }
745 
746 int
748 {
750  int i, errors = 0;
751  for (i = 0; i < 1 + vtm->n_threads; i++)
752  errors += session_main_flush_enqueue_events (transport_proto, i);
753  return errors;
754 }
755 
756 int
758  session_error_t err)
759 {
760  session_handle_t ho_handle, wrk_handle;
761  u32 opaque = 0, new_ti, new_si;
762  app_worker_t *app_wrk;
763  session_t *s = 0;
764 
765  /*
766  * Find connection handle and cleanup half-open table
767  */
768  ho_handle = session_lookup_half_open_handle (tc);
769  if (ho_handle == HALF_OPEN_LOOKUP_INVALID_VALUE)
770  {
771  SESSION_DBG ("half-open was removed!");
772  return -1;
773  }
775 
776  /* Get the app's index from the handle we stored when opening connection
777  * and the opaque (api_context for external apps) from transport session
778  * index */
779  app_wrk = app_worker_get_if_valid (session_handle_data (ho_handle));
780  if (!app_wrk)
781  return -1;
782 
783  wrk_handle = app_worker_lookup_half_open (app_wrk, tc->proto, ho_handle);
784  if (wrk_handle == SESSION_INVALID_HANDLE)
785  return -1;
786 
787  /* Make sure this is the same half-open index */
788  if (session_handle_index (wrk_handle) != session_handle_index (ho_handle))
789  return -1;
790 
791  opaque = session_handle_data (wrk_handle);
792 
793  if (err)
794  return app_worker_connect_notify (app_wrk, s, err, opaque);
795 
797  s->session_state = SESSION_STATE_CONNECTING;
798  s->app_wrk_index = app_wrk->wrk_index;
799  new_si = s->session_index;
800  new_ti = s->thread_index;
801 
802  if ((err = app_worker_init_connected (app_wrk, s)))
803  {
804  session_free (s);
805  app_worker_connect_notify (app_wrk, 0, err, opaque);
806  return -1;
807  }
808 
809  s = session_get (new_si, new_ti);
810  s->session_state = SESSION_STATE_READY;
812 
813  if (app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque))
814  {
815  s = session_get (new_si, new_ti);
817  return -1;
818  }
819 
820  return 0;
821 }
822 
823 static void
825 {
826  u32 session_index = pointer_to_uword (arg);
827  segment_manager_t *sm;
828  app_worker_t *app_wrk;
829  session_t *s;
830 
831  s = session_get_if_valid (session_index, vlib_get_thread_index ());
832  if (!s)
833  return;
834 
835  app_wrk = app_worker_get_if_valid (s->app_wrk_index);
836  if (!app_wrk)
837  return;
838 
839  /* Attach fifos to the right session and segment slice */
843 
844  /* Notify app that it has data on the new session */
846 }
847 
848 typedef struct _session_switch_pool_args
849 {
850  u32 session_index;
851  u32 thread_index;
852  u32 new_thread_index;
853  u32 new_session_index;
855 
856 /**
857  * Notify old thread of the session pool switch
858  */
859 static void
860 session_switch_pool (void *cb_args)
861 {
863  session_handle_t new_sh;
864  segment_manager_t *sm;
865  app_worker_t *app_wrk;
866  session_t *s;
867  void *rargs;
868 
869  ASSERT (args->thread_index == vlib_get_thread_index ());
870  s = session_get (args->session_index, args->thread_index);
871 
873  s->thread_index);
874 
875  new_sh = session_make_handle (args->new_session_index,
876  args->new_thread_index);
877 
878  app_wrk = app_worker_get_if_valid (s->app_wrk_index);
879  if (app_wrk)
880  {
881  /* Cleanup fifo segment slice state for fifos */
885 
886  /* Notify app, using old session, about the migration event */
887  app_worker_migrate_notify (app_wrk, s, new_sh);
888  }
889 
890  /* Trigger app read and fifo updates on the new thread */
891  rargs = uword_to_pointer (args->new_session_index, void *);
892  session_send_rpc_evt_to_thread (args->new_thread_index,
894 
895  session_free (s);
896  clib_mem_free (cb_args);
897 }
898 
899 /**
900  * Move dgram session to the right thread
901  */
902 int
904  u32 old_thread_index, session_t ** new_session)
905 {
906  session_t *new_s;
907  session_switch_pool_args_t *rpc_args;
908 
909  /*
910  * Clone half-open session to the right thread.
911  */
912  new_s = session_clone_safe (tc->s_index, old_thread_index);
913  new_s->connection_index = tc->c_index;
914  new_s->session_state = SESSION_STATE_READY;
915  new_s->flags |= SESSION_F_IS_MIGRATING;
916 
918 
919  /*
920  * Ask thread owning the old session to clean it up and make us the tx
921  * fifo owner
922  */
923  rpc_args = clib_mem_alloc (sizeof (*rpc_args));
924  rpc_args->new_session_index = new_s->session_index;
925  rpc_args->new_thread_index = new_s->thread_index;
926  rpc_args->session_index = tc->s_index;
927  rpc_args->thread_index = old_thread_index;
928  session_send_rpc_evt_to_thread (rpc_args->thread_index, session_switch_pool,
929  rpc_args);
930 
931  tc->s_index = new_s->session_index;
932  new_s->connection_index = tc->c_index;
933  *new_session = new_s;
934  return 0;
935 }
936 
937 /**
938  * Notification from transport that connection is being closed.
939  *
940  * A disconnect is sent to application but state is not removed. Once
941  * disconnect is acknowledged by application, session disconnect is called.
942  * Ultimately this leads to close being called on transport (passive close).
943  */
944 void
946 {
947  app_worker_t *app_wrk;
948  session_t *s;
949 
950  s = session_get (tc->s_index, tc->thread_index);
951  if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
952  return;
953  s->session_state = SESSION_STATE_TRANSPORT_CLOSING;
954  app_wrk = app_worker_get (s->app_wrk_index);
955  app_worker_close_notify (app_wrk, s);
956 }
957 
958 /**
959  * Notification from transport that connection is being deleted
960  *
961  * This removes the session if it is still valid. It should be called only on
962  * previously fully established sessions. For instance failed connects should
963  * call stream_session_connect_notify and indicate that the connect has
964  * failed.
965  */
966 void
968 {
969  session_t *s;
970 
971  /* App might've been removed already */
972  if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
973  return;
974 
975  switch (s->session_state)
976  {
977  case SESSION_STATE_CREATED:
978  /* Session was created but accept notification was not yet sent to the
979  * app. Cleanup everything. */
982  session_free (s);
983  break;
984  case SESSION_STATE_ACCEPTING:
985  case SESSION_STATE_TRANSPORT_CLOSING:
986  case SESSION_STATE_CLOSING:
987  case SESSION_STATE_TRANSPORT_CLOSED:
988  /* If transport finishes or times out before we get a reply
989  * from the app, mark transport as closed and wait for reply
990  * before removing the session. Cleanup session table in advance
991  * because transport will soon be closed and closed sessions
992  * are assumed to have been removed from the lookup table */
994  s->session_state = SESSION_STATE_TRANSPORT_DELETED;
997  break;
998  case SESSION_STATE_APP_CLOSED:
999  /* Cleanup lookup table as transport needs to still be valid.
1000  * Program transport close to ensure that all session events
1001  * have been cleaned up. Once transport close is called, the
1002  * session is just removed because both transport and app have
1003  * confirmed the close*/
1005  s->session_state = SESSION_STATE_TRANSPORT_DELETED;
1009  break;
1010  case SESSION_STATE_TRANSPORT_DELETED:
1011  break;
1012  case SESSION_STATE_CLOSED:
1014  session_delete (s);
1015  break;
1016  default:
1017  clib_warning ("session state %u", s->session_state);
1019  session_delete (s);
1020  break;
1021  }
1022 }
1023 
1024 /**
1025  * Notification from transport that it is closed
1026  *
1027  * Should be called by transport, prior to calling delete notify, once it
1028  * knows that no more data will be exchanged. This could serve as an
1029  * early acknowledgment of an active close especially if transport delete
1030  * can be delayed a long time, e.g., tcp time-wait.
1031  */
1032 void
1034 {
1035  app_worker_t *app_wrk;
1036  session_t *s;
1037 
1038  if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
1039  return;
1040 
1041  /* Transport thinks that app requested close but it actually didn't.
1042  * Can happen for tcp if fin and rst are received in close succession. */
1043  if (s->session_state == SESSION_STATE_READY)
1044  {
1047  s->session_state = SESSION_STATE_TRANSPORT_CLOSED;
1048  }
1049  /* If app close has not been received or has not yet resulted in
1050  * a transport close, only mark the session transport as closed */
1051  else if (s->session_state <= SESSION_STATE_CLOSING)
1052  {
1053  s->session_state = SESSION_STATE_TRANSPORT_CLOSED;
1054  }
1055  /* If app also closed, switch to closed */
1056  else if (s->session_state == SESSION_STATE_APP_CLOSED)
1057  s->session_state = SESSION_STATE_CLOSED;
1058 
1059  app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1060  if (app_wrk)
1062 }
1063 
1064 /**
1065  * Notify application that connection has been reset.
1066  */
1067 void
1069 {
1070  app_worker_t *app_wrk;
1071  session_t *s;
1072 
1073  s = session_get (tc->s_index, tc->thread_index);
1075  if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
1076  return;
1077  s->session_state = SESSION_STATE_TRANSPORT_CLOSING;
1078  app_wrk = app_worker_get (s->app_wrk_index);
1079  app_worker_reset_notify (app_wrk, s);
1080 }
1081 
1082 int
1084 {
1085  app_worker_t *app_wrk;
1086  session_t *s;
1087 
1088  s = session_get (tc->s_index, tc->thread_index);
1089  app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1090  if (!app_wrk)
1091  return -1;
1092  s->session_state = SESSION_STATE_ACCEPTING;
1093  if (app_worker_accept_notify (app_wrk, s))
1094  {
1095  /* On transport delete, no notifications should be sent. Unless, the
1096  * accept is retried and successful. */
1097  s->session_state = SESSION_STATE_CREATED;
1098  return -1;
1099  }
1100  return 0;
1101 }
1102 
1103 /**
1104  * Accept a stream session. Optionally ping the server by callback.
1105  */
1106 int
1108  u32 thread_index, u8 notify)
1109 {
1110  session_t *s;
1111  int rv;
1112 
1114  s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
1115  s->session_state = SESSION_STATE_CREATED;
1116 
1117  if ((rv = app_worker_init_accepted (s)))
1118  {
1119  session_free (s);
1120  return rv;
1121  }
1122 
1124 
1125  /* Shoulder-tap the server */
1126  if (notify)
1127  {
1128  app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
1129  if ((rv = app_worker_accept_notify (app_wrk, s)))
1130  {
1133  session_free (s);
1134  return rv;
1135  }
1136  }
1137 
1138  return 0;
1139 }
1140 
1141 int
1143  u32 thread_index)
1144 {
1145  app_worker_t *app_wrk;
1146  session_t *s;
1147  int rv;
1148 
1150  s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
1151 
1152  if ((rv = app_worker_init_accepted (s)))
1153  {
1154  session_free (s);
1155  return rv;
1156  }
1157 
1158  app_wrk = app_worker_get (s->app_wrk_index);
1159  if ((rv = app_worker_accept_notify (app_wrk, s)))
1160  {
1162  session_free (s);
1163  return rv;
1164  }
1165 
1166  s->session_state = SESSION_STATE_READY;
1168 
1169  return 0;
1170 }
1171 
1172 int
1173 session_open_cl (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
1174 {
1177  app_worker_t *app_wrk;
1178  session_handle_t sh;
1179  session_t *s;
1180  int rv;
1181 
1183  rv = transport_connect (rmt->transport_proto, tep);
1184  if (rv < 0)
1185  {
1186  SESSION_DBG ("Transport failed to open connection.");
1187  return rv;
1188  }
1189 
1190  tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
1191 
1192  /* For dgram type of service, allocate session and fifos now */
1193  app_wrk = app_worker_get (app_wrk_index);
1195  s->app_wrk_index = app_wrk->wrk_index;
1196  s->session_state = SESSION_STATE_OPENED;
1197  if (app_worker_init_connected (app_wrk, s))
1198  {
1199  session_free (s);
1200  return -1;
1201  }
1202 
1203  sh = session_handle (s);
1205  return app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque);
1206 }
1207 
1208 int
1209 session_open_vc (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
1210 {
1213  u64 handle, wrk_handle;
1214  int rv;
1215 
1217  rv = transport_connect (rmt->transport_proto, tep);
1218  if (rv < 0)
1219  {
1220  SESSION_DBG ("Transport failed to open connection.");
1221  return rv;
1222  }
1223 
1224  tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
1225 
1226  /* If transport offers a stream service, only allocate session once the
1227  * connection has been established.
1228  * Add connection to half-open table and save app and tc index. The
1229  * latter is needed to help establish the connection while the former
1230  * is needed when the connect notify comes and we have to notify the
1231  * external app
1232  */
1233  handle = session_make_handle (tc->c_index, app_wrk_index);
1234  session_lookup_add_half_open (tc, handle);
1235 
1236  /* Store the half-open handle in the connection. Transport will use it
1237  * when cleaning up @ref session_half_open_delete_notify
1238  */
1239  tc->s_ho_handle = handle;
1240 
1241  /* Track the half-open connections in case we want to forcefully
1242  * clean them up @ref session_cleanup_half_open
1243  */
1244  wrk_handle = session_make_handle (tc->c_index, opaque);
1245  app_worker_add_half_open (app_worker_get (app_wrk_index),
1246  rmt->transport_proto, handle, wrk_handle);
1247 
1248  return 0;
1249 }
1250 
1251 int
1252 session_open_app (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
1253 {
1256 
1257  sep->app_wrk_index = app_wrk_index;
1258  sep->opaque = opaque;
1259 
1260  return transport_connect (rmt->transport_proto, tep_cfg);
1261 }
1262 
1264 
1265 /* *INDENT-OFF* */
1270 };
1271 /* *INDENT-ON* */
1272 
1273 /**
1274  * Ask transport to open connection to remote transport endpoint.
1275  *
1276  * Stores handle for matching request with reply since the call can be
1277  * asynchronous. For instance, for TCP the 3-way handshake must complete
1278  * before reply comes. Session is only created once connection is established.
1279  *
1280  * @param app_index Index of the application requesting the connect
1281  * @param st Session type requested.
1282  * @param tep Remote transport endpoint
1283  * @param opaque Opaque data (typically, api_context) the application expects
1284  * on open completion.
1285  */
1286 int
1287 session_open (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
1288 {
1290  tst = transport_protocol_service_type (rmt->transport_proto);
1291  return session_open_srv_fns[tst] (app_wrk_index, rmt, opaque);
1292 }
1293 
1294 /**
1295  * Ask transport to listen on session endpoint.
1296  *
1297  * @param s Session for which listen will be called. Note that unlike
1298  * established sessions, listen sessions are not associated to a
1299  * thread.
1300  * @param sep Local endpoint to be listened on.
1301  */
1302 int
1304 {
1305  transport_endpoint_t *tep;
1306  int tc_index;
1307  u32 s_index;
1308 
1309  /* Transport bind/listen */
1310  tep = session_endpoint_to_transport (sep);
1311  s_index = ls->session_index;
1313  s_index, tep);
1314 
1315  if (tc_index < 0)
1316  return tc_index;
1317 
1318  /* Attach transport to session. Lookup tables are populated by the app
1319  * worker because local tables (for ct sessions) are not backed by a fib */
1320  ls = listen_session_get (s_index);
1321  ls->connection_index = tc_index;
1322 
1323  return 0;
1324 }
1325 
1326 /**
1327  * Ask transport to stop listening on local transport endpoint.
1328  *
1329  * @param s Session to stop listening on. It must be in state LISTENING.
1330  */
1331 int
1333 {
1336 
1337  if (s->session_state != SESSION_STATE_LISTENING)
1338  return SESSION_E_NOLISTEN;
1339 
1341 
1342  /* If no transport, assume everything was cleaned up already */
1343  if (!tc)
1344  return SESSION_E_NONE;
1345 
1346  if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1348 
1350  return 0;
1351 }
1352 
1353 /**
1354  * Initialize session closing procedure.
1355  *
1356  * Request is always sent to session node to ensure that all outstanding
1357  * requests are served before transport is notified.
1358  */
1359 void
1361 {
1362  if (!s)
1363  return;
1364 
1365  if (s->session_state >= SESSION_STATE_CLOSING)
1366  {
1367  /* Session will only be removed once both app and transport
1368  * acknowledge the close */
1369  if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED
1370  || s->session_state == SESSION_STATE_TRANSPORT_DELETED)
1372  return;
1373  }
1374 
1375  s->session_state = SESSION_STATE_CLOSING;
1377 }
1378 
1379 /**
1380  * Force a close without waiting for data to be flushed
1381  */
1382 void
1384 {
1385  if (s->session_state >= SESSION_STATE_CLOSING)
1386  return;
1387  /* Drop all outstanding tx data */
1389  s->session_state = SESSION_STATE_CLOSING;
1391 }
1392 
1393 /**
1394  * Notify transport the session can be disconnected. This should eventually
1395  * result in a delete notification that allows us to cleanup session state.
1396  * Called for both active/passive disconnects.
1397  *
1398  * Must be called from the session's thread.
1399  */
1400 void
1402 {
1403  if (s->session_state >= SESSION_STATE_APP_CLOSED)
1404  {
1405  if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
1406  s->session_state = SESSION_STATE_CLOSED;
1407  /* If transport is already deleted, just free the session */
1408  else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
1410  return;
1411  }
1412 
1413  /* If the tx queue wasn't drained, the transport can continue to try
1414  * sending the outstanding data (in closed state it cannot). It MUST however
1415  * at one point, either after sending everything or after a timeout, call
1416  * delete notify. This will finally lead to the complete cleanup of the
1417  * session.
1418  */
1419  s->session_state = SESSION_STATE_APP_CLOSED;
1420 
1422  s->thread_index);
1423 }
1424 
1425 /**
1426  * Force transport close
1427  */
1428 void
1430 {
1431  if (s->session_state >= SESSION_STATE_APP_CLOSED)
1432  {
1433  if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
1434  s->session_state = SESSION_STATE_CLOSED;
1435  else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
1437  return;
1438  }
1439 
1440  s->session_state = SESSION_STATE_APP_CLOSED;
1442  s->thread_index);
1443 }
1444 
1445 /**
1446  * Cleanup transport and session state.
1447  *
1448  * Notify transport of the cleanup and free the session. This should
1449  * be called only if transport reported some error and is already
1450  * closed.
1451  */
1452 void
1454 {
1455  /* Delete from main lookup table before we axe the the transport */
1457  if (s->session_state != SESSION_STATE_TRANSPORT_DELETED)
1459  s->thread_index);
1460  /* Since we called cleanup, no delete notification will come. So, make
1461  * sure the session is properly freed. */
1463  session_free (s);
1464 }
1465 
1466 /**
1467  * Allocate event queues in the shared-memory segment
1468  *
1469  * That can either be a newly created memfd segment, that will need to be
1470  * mapped by all stack users, or the binary api's svm region. The latter is
1471  * assumed to be already mapped. NOTE that this assumption DOES NOT hold if
1472  * api clients bootstrap shm api over sockets (i.e. use memfd segments) and
1473  * vpp uses api svm region for event queues.
1474  */
1475 void
1477 {
1478  u32 evt_q_length = 2048, evt_size = sizeof (session_event_t);
1479  ssvm_private_t *eqs = &smm->evt_qs_segment;
1480  uword eqs_size = 64 << 20;
1481  pid_t vpp_pid = getpid ();
1482  void *oldheap;
1483  int i;
1484 
1486  evt_q_length = smm->configured_event_queue_length;
1487 
1488  if (smm->evt_qs_use_memfd_seg)
1489  {
1490  if (smm->evt_qs_segment_size)
1491  eqs_size = smm->evt_qs_segment_size;
1492 
1493  eqs->ssvm_size = eqs_size;
1494  eqs->i_am_master = 1;
1495  eqs->my_pid = vpp_pid;
1496  eqs->name = format (0, "%s%c", "evt-qs-segment", 0);
1497  eqs->requested_va = smm->session_baseva;
1498 
1500  {
1501  clib_warning ("failed to initialize queue segment");
1502  return;
1503  }
1504  }
1505 
1506  if (smm->evt_qs_use_memfd_seg)
1507  oldheap = ssvm_push_heap (eqs->sh);
1508  else
1509  oldheap = vl_msg_push_heap ();
1510 
1511  for (i = 0; i < vec_len (smm->wrk); i++)
1512  {
1513  svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
1515  {evt_q_length, evt_size, 0}
1516  ,
1517  {evt_q_length >> 1, 256, 0}
1518  };
1519  cfg->consumer_pid = 0;
1520  cfg->n_rings = 2;
1521  cfg->q_nitems = evt_q_length;
1522  cfg->ring_cfgs = rc;
1523  smm->wrk[i].vpp_event_queue = svm_msg_q_alloc (cfg);
1524  if (smm->evt_qs_use_memfd_seg)
1525  {
1527  clib_warning ("eventfd returned");
1528  }
1529  }
1530 
1531  if (smm->evt_qs_use_memfd_seg)
1532  ssvm_pop_heap (oldheap);
1533  else
1534  vl_msg_pop_heap (oldheap);
1535 }
1536 
1539 {
1540  session_main_t *smm = &session_main;
1541  if (smm->evt_qs_use_memfd_seg)
1542  return &smm->evt_qs_segment;
1543  return 0;
1544 }
1545 
1546 u64
1548 {
1549  svm_fifo_t *f;
1550 
1551  if (!s->rx_fifo)
1552  return SESSION_INVALID_HANDLE;
1553 
1554  f = s->rx_fifo;
1555  return segment_manager_make_segment_handle (f->segment_manager,
1556  f->segment_index);
1557 }
1558 
1559 /* *INDENT-OFF* */
1564  session_tx_fifo_dequeue_and_snd
1565 };
1566 /* *INDENT-ON* */
1567 
1568 void
1570  const transport_proto_vft_t * vft, u8 is_ip4,
1571  u32 output_node)
1572 {
1573  session_main_t *smm = &session_main;
1574  session_type_t session_type;
1575  u32 next_index = ~0;
1576 
1577  session_type = session_type_from_proto_and_ip (transport_proto, is_ip4);
1578 
1579  vec_validate (smm->session_type_to_next, session_type);
1580  vec_validate (smm->session_tx_fns, session_type);
1581 
1582  /* *INDENT-OFF* */
1583  if (output_node != ~0)
1584  {
1585  foreach_vlib_main (({
1586  next_index = vlib_node_add_next (this_vlib_main,
1587  session_queue_node.index,
1588  output_node);
1589  }));
1590  }
1591  /* *INDENT-ON* */
1592 
1593  smm->session_type_to_next[session_type] = next_index;
1594  smm->session_tx_fns[session_type] =
1595  session_tx_fns[vft->transport_options.tx_type];
1596 }
1597 
1600 {
1601  session_main_t *smm = &session_main;
1602  session_worker_t *wrk;
1603  u32 thread;
1604 
1605  smm->last_transport_proto_type += 1;
1606 
1607  for (thread = 0; thread < vec_len (smm->wrk); thread++)
1608  {
1609  wrk = session_main_get_worker (thread);
1611  }
1612 
1613  return smm->last_transport_proto_type;
1614 }
1615 
1618 {
1619  if (s->session_state != SESSION_STATE_LISTENING)
1622  else
1624  s->connection_index);
1625 }
1626 
1627 void
1629 {
1630  if (s->session_state != SESSION_STATE_LISTENING)
1632  s->connection_index, s->thread_index, tep,
1633  is_lcl);
1634  else
1636  s->connection_index, tep, is_lcl);
1637 }
1638 
1641 {
1643  s->connection_index);
1644 }
1645 
1646 void
1648 {
1649  ASSERT (vlib_get_thread_index () == 0);
1652 }
1653 
1654 static clib_error_t *
1656 {
1657  segment_manager_main_init_args_t _sm_args = { 0 }, *sm_args = &_sm_args;
1658  session_main_t *smm = &session_main;
1660  u32 num_threads, preallocated_sessions_per_worker;
1661  session_worker_t *wrk;
1662  int i;
1663 
1664  num_threads = 1 /* main thread */ + vtm->n_threads;
1665 
1666  if (num_threads < 1)
1667  return clib_error_return (0, "n_thread_stacks not set");
1668 
1669  /* Allocate cache line aligned worker contexts */
1670  vec_validate_aligned (smm->wrk, num_threads - 1, CLIB_CACHE_LINE_BYTES);
1671 
1672  for (i = 0; i < num_threads; i++)
1673  {
1674  wrk = &smm->wrk[i];
1675  wrk->ctrl_head = clib_llist_make_head (wrk->event_elts, evt_list);
1676  wrk->new_head = clib_llist_make_head (wrk->event_elts, evt_list);
1677  wrk->old_head = clib_llist_make_head (wrk->event_elts, evt_list);
1678  wrk->vm = vlib_mains[i];
1679  wrk->last_vlib_time = vlib_time_now (vm);
1682 
1683  if (num_threads > 1)
1685  }
1686 
1687  /* Allocate vpp event queues segment and queue */
1689 
1690  /* Initialize fifo segment main baseva and timeout */
1691  sm_args->baseva = smm->session_baseva + smm->evt_qs_segment_size;
1692  sm_args->size = smm->session_va_space_size;
1693  segment_manager_main_init (sm_args);
1694 
1695  /* Preallocate sessions */
1696  if (smm->preallocated_sessions)
1697  {
1698  if (num_threads == 1)
1699  {
1701  }
1702  else
1703  {
1704  int j;
1705  preallocated_sessions_per_worker =
1706  (1.1 * (f64) smm->preallocated_sessions /
1707  (f64) (num_threads - 1));
1708 
1709  for (j = 1; j < num_threads; j++)
1710  {
1711  pool_init_fixed (smm->wrk[j].sessions,
1712  preallocated_sessions_per_worker);
1713  }
1714  }
1715  }
1716 
1719  transport_init ();
1720 
1721  smm->is_enabled = 1;
1722 
1723  /* Enable transports */
1724  transport_enable_disable (vm, 1);
1725  session_debug_init ();
1726 
1727  return 0;
1728 }
1729 
1730 void
1732 {
1733  u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
1735  u8 have_workers = vtm->n_threads != 0;
1736 
1737  /* *INDENT-OFF* */
1738  foreach_vlib_main (({
1739  if (have_workers && ii == 0)
1740  {
1741  vlib_node_set_state (this_vlib_main, session_queue_process_node.index,
1742  state);
1743  if (is_en)
1744  {
1745  vlib_node_t *n = vlib_get_node (this_vlib_main,
1747  vlib_start_process (this_vlib_main, n->runtime_index);
1748  }
1749  else
1750  {
1751  vlib_process_signal_event_mt (this_vlib_main,
1754  }
1755 
1756  continue;
1757  }
1758  vlib_node_set_state (this_vlib_main, session_queue_node.index,
1759  state);
1760  }));
1761  /* *INDENT-ON* */
1762 }
1763 
1764 clib_error_t *
1766 {
1767  clib_error_t *error = 0;
1768  if (is_en)
1769  {
1770  if (session_main.is_enabled)
1771  return 0;
1772 
1773  error = session_manager_main_enable (vm);
1775  }
1776  else
1777  {
1778  session_main.is_enabled = 0;
1780  }
1781 
1782  return error;
1783 }
1784 
1785 clib_error_t *
1787 {
1788  session_main_t *smm = &session_main;
1789 
1790  smm->is_enabled = 0;
1791  smm->session_enable_asap = 0;
1793 
1794 #if (HIGH_SEGMENT_BASEVA > (4ULL << 30))
1795  smm->session_va_space_size = 128ULL << 30;
1796  smm->evt_qs_segment_size = 64 << 20;
1797 #else
1798  smm->session_va_space_size = 128 << 20;
1799  smm->evt_qs_segment_size = 1 << 20;
1800 #endif
1801 
1802  smm->last_transport_proto_type = TRANSPORT_PROTO_QUIC;
1803 
1804  return 0;
1805 }
1806 
1807 static clib_error_t *
1809 {
1810  session_main_t *smm = &session_main;
1811  if (smm->session_enable_asap)
1812  {
1814  vnet_session_enable_disable (vm, 1 /* is_en */ );
1816  }
1817  return 0;
1818 }
1819 
1822 
1823 static clib_error_t *
1825 {
1826  session_main_t *smm = &session_main;
1827  u32 nitems;
1828  uword tmp;
1829 
1830  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1831  {
1832  if (unformat (input, "event-queue-length %d", &nitems))
1833  {
1834  if (nitems >= 2048)
1835  smm->configured_event_queue_length = nitems;
1836  else
1837  clib_warning ("event queue length %d too small, ignored", nitems);
1838  }
1839  else if (unformat (input, "preallocated-sessions %d",
1840  &smm->preallocated_sessions))
1841  ;
1842  else if (unformat (input, "v4-session-table-buckets %d",
1844  ;
1845  else if (unformat (input, "v4-halfopen-table-buckets %d",
1847  ;
1848  else if (unformat (input, "v6-session-table-buckets %d",
1850  ;
1851  else if (unformat (input, "v6-halfopen-table-buckets %d",
1853  ;
1854  else if (unformat (input, "v4-session-table-memory %U",
1855  unformat_memory_size, &tmp))
1856  {
1857  if (tmp >= 0x100000000)
1858  return clib_error_return (0, "memory size %llx (%lld) too large",
1859  tmp, tmp);
1861  }
1862  else if (unformat (input, "v4-halfopen-table-memory %U",
1863  unformat_memory_size, &tmp))
1864  {
1865  if (tmp >= 0x100000000)
1866  return clib_error_return (0, "memory size %llx (%lld) too large",
1867  tmp, tmp);
1869  }
1870  else if (unformat (input, "v6-session-table-memory %U",
1871  unformat_memory_size, &tmp))
1872  {
1873  if (tmp >= 0x100000000)
1874  return clib_error_return (0, "memory size %llx (%lld) too large",
1875  tmp, tmp);
1877  }
1878  else if (unformat (input, "v6-halfopen-table-memory %U",
1879  unformat_memory_size, &tmp))
1880  {
1881  if (tmp >= 0x100000000)
1882  return clib_error_return (0, "memory size %llx (%lld) too large",
1883  tmp, tmp);
1885  }
1886  else if (unformat (input, "local-endpoints-table-memory %U",
1887  unformat_memory_size, &tmp))
1888  {
1889  if (tmp >= 0x100000000)
1890  return clib_error_return (0, "memory size %llx (%lld) too large",
1891  tmp, tmp);
1892  smm->local_endpoints_table_memory = tmp;
1893  }
1894  else if (unformat (input, "local-endpoints-table-buckets %d",
1896  ;
1897  else if (unformat (input, "evt_qs_memfd_seg"))
1898  smm->evt_qs_use_memfd_seg = 1;
1899  else if (unformat (input, "evt_qs_seg_size %U", unformat_memory_size,
1900  &smm->evt_qs_segment_size))
1901  ;
1902  else if (unformat (input, "enable"))
1903  smm->session_enable_asap = 1;
1904  else
1905  return clib_error_return (0, "unknown input `%U'",
1906  format_unformat_error, input);
1907  }
1908  return 0;
1909 }
1910 
1912 
1913 /*
1914  * fd.io coding-style-patch-verification: ON
1915  *
1916  * Local Variables:
1917  * eval: (c-set-style "gnu")
1918  * End:
1919  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:507
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
#define session_endpoint_to_transport_cfg(_sep)
Definition: session_types.h:95
u32 preallocated_sessions
Preallocate session config parameter.
Definition: session.h:194
int app_worker_lock_and_send_event(app_worker_t *app, session_t *s, u8 evt_type)
Send event to application.
void transport_close(transport_proto_t tp, u32 conn_index, u8 thread_index)
Definition: transport.c:321
static session_open_service_fn session_open_srv_fns[TRANSPORT_N_SERVICES]
Definition: session.c:1266
u32 connection_index
Index of the transport connection associated to the session.
uword evt_qs_segment_size
Definition: session.h:176
int app_worker_init_accepted(session_t *s)
void session_transport_reset(session_t *s)
Force transport close.
Definition: session.c:1429
int session_lookup_del_connection(transport_connection_t *tc)
Delete transport connection from session table.
int app_worker_reset_notify(app_worker_t *app_wrk, session_t *s)
void * svm_msg_q_msg_data(svm_msg_q_t *mq, svm_msg_q_msg_t *msg)
Get data for message in queue.
static u8 svm_msg_q_ring_is_full(svm_msg_q_t *mq, u32 ring_index)
int session_open(u32 app_wrk_index, session_endpoint_t *rmt, u32 opaque)
Ask transport to open connection to remote transport endpoint.
Definition: session.c:1287
session_type_t session_type
Type built from transport and network protocol types.
uword requested_va
Definition: ssvm.h:86
svm_msg_q_t * vpp_event_queue
vpp event message queue for worker
Definition: session.h:80
void transport_get_listener_endpoint(transport_proto_t tp, u32 conn_index, transport_endpoint_t *tep, u8 is_lcl)
Definition: transport.c:389
static u32 svm_fifo_max_enqueue_prod(svm_fifo_t *f)
Maximum number of bytes that can be enqueued into fifo.
Definition: svm_fifo.h:522
static void clib_rwlock_writer_lock(clib_rwlock_t *p)
Definition: lock.h:173
uword ssvm_size
Definition: ssvm.h:85
transport_proto
Definition: session.api:22
svm_fifo_t * tx_fifo
int app_worker_connect_notify(app_worker_t *app_wrk, session_t *s, session_error_t err, u32 opaque)
session_main_t session_main
Definition: session.c:25
int session_tx_fifo_peek_bytes(transport_connection_t *tc, u8 *buffer, u32 offset, u32 max_bytes)
Definition: session.c:553
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
Definition: init.h:176
void session_queue_run_on_main_thread(vlib_main_t *vm)
Definition: session.c:1647
u32 session_index
Index in thread pool where session was allocated.
int session_stop_listen(session_t *s)
Ask transport to stop listening on local transport endpoint.
Definition: session.c:1332
static void session_switch_pool_reply(void *arg)
Definition: session.c:824
int app_worker_del_half_open(app_worker_t *app_wrk, transport_proto_t tp, session_handle_t ho_handle)
unsigned long u64
Definition: types.h:89
transport_connection_t * listen_session_get_transport(session_t *s)
Definition: session.c:1640
static svm_msg_q_t * session_main_get_vpp_event_queue(u32 thread_index)
Definition: session.h:633
transport_proto_t session_add_transport_proto(void)
Definition: session.c:1599
u32 configured_v4_halfopen_table_buckets
Definition: session.h:182
u8 session_enable_asap
Enable session manager at startup.
Definition: session.h:168
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
void session_transport_delete_notify(transport_connection_t *tc)
Notification from transport that connection is being deleted.
Definition: session.c:967
transport_connection_t * session_get_transport(session_t *s)
Definition: session.c:1617
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:291
svm_fifo_t * rx_fifo
Pointers to rx/tx buffers.
session_worker_t * wrk
Worker contexts.
Definition: session.h:143
static session_t * session_get_if_valid(u64 si, u32 thread_index)
Definition: session.h:308
void session_add_self_custom_tx_evt(transport_connection_t *tc, u8 has_prio)
Definition: session.c:122
u32 ** session_to_enqueue
Per-proto vector of sessions to enqueue.
Definition: session.h:92
#define pool_get_aligned_will_expand(P, YESNO, A)
See if pool_get will expand the pool or not.
Definition: pool.h:258
int session_enqueue_dgram_connection(session_t *s, session_dgram_hdr_t *hdr, vlib_buffer_t *b, u8 proto, u8 queue_event)
Definition: session.c:515
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static int session_enqueue_notify_inline(session_t *s)
Notify session peer that new data has been enqueued.
Definition: session.c:608
int session_main_flush_enqueue_events(u8 transport_proto, u32 thread_index)
Flushes queue of sessions that are to be notified of new data enqueued events.
Definition: session.c:715
session_evt_type_t
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:590
static transport_proto_t session_get_transport_proto(session_t *s)
void session_send_rpc_evt_to_thread(u32 thread_index, void *fp, void *rpc_args)
Definition: session.c:110
int svm_fifo_peek(svm_fifo_t *f, u32 offset, u32 len, u8 *dst)
Peek data from fifo.
Definition: svm_fifo.c:1038
void session_transport_reset_notify(transport_connection_t *tc)
Notify application that connection has been reset.
Definition: session.c:1068
int session_open_vc(u32 app_wrk_index, session_endpoint_t *rmt, u32 opaque)
Definition: session.c:1209
void svm_fifo_dequeue_drop_all(svm_fifo_t *f)
Drop all data from fifo.
Definition: svm_fifo.c:1099
ssvm_shared_header_t * sh
Definition: ssvm.h:84
u32 transport_start_listen(transport_proto_t tp, u32 session_index, transport_endpoint_t *tep)
Definition: transport.c:336
static session_t * session_get(u32 si, u32 thread_index)
Definition: session.h:301
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
session_evt_elt_t * event_elts
Pool of session event list elements.
Definition: session.h:101
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:518
u32 flags
Session flags.
int session_enqueue_stream_connection(transport_connection_t *tc, vlib_buffer_t *b, u32 offset, u8 queue_event, u8 is_in_order)
Definition: session.c:460
u64 session_lookup_half_open_handle(transport_connection_t *tc)
static session_t * session_clone_safe(u32 session_index, u32 thread_index)
Definition: session.h:398
u32 local_endpoints_table_memory
Transport table (preallocation) size parameters.
Definition: session.h:190
static u32 session_handle_data(session_handle_t ho_handle)
void session_node_enable_disable(u8 is_en)
Definition: session.c:1731
vlib_node_registration_t session_queue_node
(constructor) VLIB_REGISTER_NODE (session_queue_node)
vlib_main_t ** vlib_mains
Definition: buffer.c:332
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
app_worker_t * application_get_worker(application_t *app, u32 wrk_map_index)
Definition: application.c:663
session_fifo_rx_fn session_tx_fifo_peek_and_snd
clib_error_t * session_main_init(vlib_main_t *vm)
Definition: session.c:1786
session_t * sessions
Worker session pool.
Definition: session.h:77
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
static session_fifo_rx_fn * session_tx_fns[TRANSPORT_TX_N_FNS]
Definition: session.c:1560
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
static session_handle_t session_handle(session_t *s)
void session_get_endpoint(session_t *s, transport_endpoint_t *tep, u8 is_lcl)
Definition: session.c:1628
u8 session_type_t
void segment_manager_attach_fifo(segment_manager_t *sm, svm_fifo_t *f, session_t *s)
void session_transport_closing_notify(transport_connection_t *tc)
Notification from transport that connection is being closed.
Definition: session.c:945
u64 segment_manager_make_segment_handle(u32 segment_manager_index, u32 segment_index)
enum session_ft_action_ session_ft_action_t
int session_open_cl(u32 app_wrk_index, session_endpoint_t *rmt, u32 opaque)
Definition: session.c:1173
#define clib_llist_make_head(LP, name)
Initialize llist head.
Definition: llist.h:106
static session_worker_t * session_main_get_worker(u32 thread_index)
Definition: session.h:619
void session_free_w_fifos(session_t *s)
Definition: session.c:261
#define session_endpoint_to_transport(_sep)
Definition: session_types.h:94
void app_namespaces_init(void)
vlib_main_t * vm
Convenience pointer to this worker&#39;s vlib_main.
Definition: session.h:89
static clib_error_t * session_config_fn(vlib_main_t *vm, unformat_input_t *input)
Definition: session.c:1824
svm_msg_q_t * svm_msg_q_alloc(svm_msg_q_cfg_t *cfg)
Allocate message queue.
Definition: message_queue.c:41
int session_send_ctrl_evt_to_thread(session_t *s, session_evt_type_t evt_type)
Definition: session.c:93
void session_fifo_tuning(session_t *s, svm_fifo_t *f, session_ft_action_t act, u32 len)
Definition: session.c:428
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
void segment_manager_dealloc_fifos(svm_fifo_t *rx_fifo, svm_fifo_t *tx_fifo)
clib_llist_index_t new_head
Head of list of elements.
Definition: session.h:110
u32 * session_type_to_next
Per session type output nodes.
Definition: session.h:157
void session_cleanup_half_open(transport_proto_t tp, session_handle_t ho_handle)
Definition: session.c:286
static int session_enqueue_chain_tail(session_t *s, vlib_buffer_t *b, u32 offset, u8 is_in_order)
Enqueue buffer chain tail.
Definition: session.c:355
static void * ssvm_push_heap(ssvm_shared_header_t *sh)
Definition: ssvm.h:144
#define clib_error_return(e, args...)
Definition: error.h:99
uword session_baseva
Session ssvm segment configs.
Definition: session.h:174
pthread_t thread[MAX_CONNS]
Definition: main.c:142
void session_half_open_delete_notify(transport_proto_t tp, session_handle_t ho_handle)
Definition: session.c:292
unsigned int u32
Definition: types.h:88
int session_send_io_evt_to_thread(svm_fifo_t *f, session_evt_type_t evt_type)
Definition: session.c:79
int ssvm_master_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:415
u8 session_is_valid(u32 si, u8 thread_index)
Definition: session.c:224
static void ssvm_pop_heap(void *oldheap)
Definition: ssvm.h:152
void transport_reset(transport_proto_t tp, u32 conn_index, u8 thread_index)
Definition: transport.c:327
#define HALF_OPEN_LOOKUP_INVALID_VALUE
#define SESSION_INVALID_HANDLE
Definition: session_types.h:23
int session_lookup_del_half_open(transport_connection_t *tc)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:293
static clib_error_t * session_manager_main_enable(vlib_main_t *vm)
Definition: session.c:1655
struct _transport_proto_vft transport_proto_vft_t
int session_dequeue_notify(session_t *s)
Definition: session.c:685
struct _session_endpoint_cfg session_endpoint_cfg_t
u32 configured_v6_halfopen_table_memory
Definition: session.h:187
u32 configured_v6_session_table_buckets
Definition: session.h:184
static session_type_t session_type_from_proto_and_ip(transport_proto_t proto, u8 is_ip4)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
static void clib_rwlock_init(clib_rwlock_t *p)
Definition: lock.h:133
#define CLIB_US_TIME_FREQ
Definition: time.h:207
int svm_msg_q_alloc_consumer_eventfd(svm_msg_q_t *mq)
Allocate event fd for queue consumer.
session_event_t evt
Definition: session.h:64
transport_service_type_t transport_protocol_service_type(transport_proto_t tp)
Definition: transport.c:290
void vl_msg_pop_heap(void *oldheap)
Definition: cli.c:720
uword session_va_space_size
Definition: session.h:175
void session_send_rpc_evt_to_thread_force(u32 thread_index, void *fp, void *rpc_args)
Definition: session.c:102
u32 configured_v4_session_table_buckets
Session table size parameters.
Definition: session.h:180
void transport_get_endpoint(transport_proto_t tp, u32 conn_index, u32 thread_index, transport_endpoint_t *tep, u8 is_lcl)
Definition: transport.c:373
int app_worker_accept_notify(app_worker_t *app_wrk, session_t *s)
vl_api_ip_proto_t proto
Definition: acl_types.api:50
u32 app_index
Index of application that owns the listener.
struct _unformat_input_t unformat_input_t
u32 configured_v6_halfopen_table_buckets
Definition: session.h:186
u32 configured_event_queue_length
vpp fifo event queue configured length
Definition: session.h:171
u8 is_enabled
Session manager is enabled.
Definition: session.h:166
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:302
#define APP_INVALID_INDEX
Definition: application.h:172
int svm_fifo_enqueue(svm_fifo_t *f, u32 len, const u8 *src)
Enqueue data to fifo.
Definition: svm_fifo.c:836
segment_manager_t * app_worker_get_connect_segment_manager(app_worker_t *)
static clib_error_t * session_main_loop_init(vlib_main_t *vm)
Definition: session.c:1808
enum transport_service_type_ transport_service_type_t
#define PREDICT_FALSE(x)
Definition: clib.h:118
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:182
#define always_inline
Definition: ipsec.h:28
int session_lookup_del_session(session_t *s)
u32 wrk_index
Worker index in global worker pool.
Definition: application.h:37
app_worker_t * app_worker_get_if_valid(u32 wrk_index)
int app_worker_session_fifo_tuning(app_worker_t *app_wrk, session_t *s, svm_fifo_t *f, session_ft_action_t act, u32 len)
static u32 svm_fifo_max_dequeue_prod(svm_fifo_t *f)
Fifo max bytes to dequeue optimized for producer.
Definition: svm_fifo.h:416
session_fifo_rx_fn ** session_tx_fns
Per transport rx function that can either dequeue or peek.
Definition: session.h:152
u64 session_segment_handle(session_t *s)
Definition: session.c:1547
#define foreach_vlib_main(body)
Definition: threads.h:241
static session_evt_elt_t * session_evt_alloc_old(session_worker_t *wrk)
Definition: session.h:284
void transport_cleanup(transport_proto_t tp, u32 conn_index, u8 thread_index)
Definition: transport.c:302
vlib_main_t * vm
Definition: in2out_ed.c:1599
clib_error_t * vnet_session_enable_disable(vlib_main_t *vm, u8 is_en)
Definition: session.c:1765
u32 transport_stop_listen(transport_proto_t tp, u32 conn_index)
Definition: transport.c:343
struct _session_switch_pool_args session_switch_pool_args_t
u8 len
Definition: ip_types.api:92
clib_rwlock_t peekers_rw_locks
Peekers rw lock.
Definition: session.h:116
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P with alignment A.
Definition: pool.h:246
#define SESSION_DBG(_fmt, _args...)
static void clib_rwlock_writer_unlock(clib_rwlock_t *p)
Definition: lock.h:187
static u8 svm_fifo_set_event(svm_fifo_t *f)
Set fifo event flag.
Definition: svm_fifo.h:699
u32 n_rings
number of msg rings
Definition: message_queue.h:54
u8 evt_qs_use_memfd_seg
Definition: session.h:177
void transport_init(void)
Definition: transport.c:795
int app_worker_transport_closed_notify(app_worker_t *app_wrk, session_t *s)
session_fifo_rx_fn session_tx_fifo_dequeue_and_snd
ssvm_private_t evt_qs_segment
Event queues memfd segment initialized only if so configured.
Definition: session.h:146
transport_proto_t last_transport_proto_type
Definition: session.h:159
static u8 svm_fifo_needs_deq_ntf(svm_fifo_t *f, u32 n_last_deq)
Check if fifo needs dequeue notification.
Definition: svm_fifo.h:790
#define UNFORMAT_END_OF_INPUT
Definition: format.h:145
u32 runtime_index
Definition: node.h:285
session_handle_t listener_handle
Parent listener session index if the result of an accept.
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:218
int transport_connect(transport_proto_t tp, transport_endpoint_cfg_t *tep)
Definition: transport.c:315
static void svm_msg_q_unlock(svm_msg_q_t *mq)
Unlock message queue.
void segment_manager_detach_fifo(segment_manager_t *sm, svm_fifo_t *f)
static transport_connection_t * transport_get_connection(transport_proto_t tp, u32 conn_index, u8 thread_index)
Definition: transport.h:149
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
static void vlib_process_signal_event_mt(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Signal event to process from any thread.
Definition: node_funcs.h:958
static void session_cleanup_notify(session_t *s, session_cleanup_ntf_t ntf)
Definition: session.c:250
void session_free(session_t *s)
Definition: session.c:210
int app_worker_cleanup_notify(app_worker_t *app_wrk, session_t *s, session_cleanup_ntf_t ntf)
#define clib_warning(format, args...)
Definition: error.h:59
static int session_send_evt_to_thread(void *data, void *args, u32 thread_index, session_evt_type_t evt_type)
Definition: session.c:28
int session_dgram_connect_notify(transport_connection_t *tc, u32 old_thread_index, session_t **new_session)
Move dgram session to the right thread.
Definition: session.c:903
Don&#39;t register connection in lookup.
struct _transport_connection transport_connection_t
#define HIGH_SEGMENT_BASEVA
Definition: svm_common.h:87
int app_worker_init_connected(app_worker_t *app_wrk, session_t *s)
static session_evt_elt_t * session_evt_alloc_new(session_worker_t *wrk)
Definition: session.h:274
u32 my_pid
Definition: ssvm.h:87
void transport_enable_disable(vlib_main_t *vm, u8 is_en)
Definition: transport.c:784
static u8 vlib_thread_is_main_w_barrier(void)
Definition: threads.h:521
#define pool_init_fixed(pool, max_elts)
initialize a fixed-size, preallocated pool
Definition: pool.h:86
application_t * application_get(u32 app_index)
Definition: application.c:426
static u32 session_thread_from_handle(session_handle_t handle)
int session_listen(session_t *ls, session_endpoint_cfg_t *sep)
Ask transport to listen on session endpoint.
Definition: session.c:1303
static u32 session_index_from_handle(session_handle_t handle)
#define uword_to_pointer(u, type)
Definition: types.h:136
void session_vpp_event_queues_allocate(session_main_t *smm)
Allocate event queues in the shared-memory segment.
Definition: session.c:1476
#define ASSERT(truth)
static int session_notify_subscribers(u32 app_index, session_t *s, svm_fifo_t *f, session_evt_type_t evt_type)
Definition: session.c:576
void svm_msg_q_add_and_unlock(svm_msg_q_t *mq, svm_msg_q_msg_t *msg)
Producer enqueue one message to queue with mutex held.
int app_worker_add_half_open(app_worker_t *app_wrk, transport_proto_t tp, session_handle_t ho_handle, session_handle_t wrk_handle)
u8 data[128]
Definition: ipsec_types.api:89
void sesssion_reschedule_tx(transport_connection_t *tc)
Definition: session.c:150
int session_main_flush_all_enqueue_events(u8 transport_proto)
Definition: session.c:747
#define CLIB_ASSERT_ENABLE
void segment_manager_main_init(segment_manager_main_init_args_t *a)
static void clib_mem_free(void *p)
Definition: mem.h:215
enum _transport_proto transport_proto_t
static void svm_fifo_clear_deq_ntf(svm_fifo_t *f)
Clear the want notification flag and set has notification.
Definition: svm_fifo.h:757
int svm_fifo_enqueue_with_offset(svm_fifo_t *f, u32 offset, u32 len, u8 *src)
Enqueue a future segment.
Definition: svm_fifo.c:894
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:148
static session_handle_t session_make_handle(u32 session_index, u32 data)
clib_time_type_t last_vlib_time
vlib_time_now last time around the track
Definition: session.h:83
u32 configured_v4_halfopen_table_memory
Definition: session.h:183
int app_worker_migrate_notify(app_worker_t *app_wrk, session_t *s, session_handle_t new_sh)
u32 configured_v4_session_table_memory
Definition: session.h:181
int session_stream_accept_notify(transport_connection_t *tc)
Definition: session.c:1083
static void * clib_mem_alloc(uword size)
Definition: mem.h:157
void session_enqueue_notify_thread(session_handle_t sh)
Like session_enqueue_notify, but can be called from a thread that does not own the session...
Definition: session.c:670
static uword pointer_to_uword(const void *p)
Definition: types.h:131
svm_msg_q_ring_cfg_t * ring_cfgs
array of ring cfgs
Definition: message_queue.h:55
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u8 * name
Definition: ssvm.h:88
int() session_fifo_rx_fn(session_worker_t *wrk, vlib_node_runtime_t *node, session_evt_elt_t *e, int *n_tx_packets)
Definition: session.h:130
u8 thread_index
Index of the thread that allocated the session.
session_t * session_alloc(u32 thread_index)
Definition: session.c:184
void session_transport_cleanup(session_t *s)
Cleanup transport and session state.
Definition: session.c:1453
static void session_enqueue_discard_chain_bytes(vlib_main_t *vm, vlib_buffer_t *b, vlib_buffer_t **chain_b, u32 n_bytes_to_drop)
Discards bytes from buffer chain.
Definition: session.c:324
template key/value backing page structure
Definition: bihash_doc.h:44
static void session_switch_pool(void *cb_args)
Notify old thread of the session pool switch.
Definition: session.c:860
static transport_connection_t * transport_get_half_open(transport_proto_t tp, u32 conn_index)
Definition: transport.h:162
static u32 session_handle_index(session_handle_t ho_handle)
u64 app_worker_lookup_half_open(app_worker_t *app_wrk, transport_proto_t tp, session_handle_t ho_handle)
u32 local_endpoints_table_buckets
Definition: session.h:191
app_worker_t * app_worker_get(u32 wrk_index)
u32 q_nitems
msg queue size (not rings)
Definition: message_queue.h:53
u64 session_handle_t
void session_lookup_init(void)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
session_fifo_rx_fn session_tx_fifo_dequeue_internal
clib_llist_index_t old_head
Head of list of pending events.
Definition: session.h:113
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
volatile u8 session_state
State in session layer state machine.
int session_enqueue_notify(session_t *s)
Definition: session.c:647
void session_transport_closed_notify(transport_connection_t *tc)
Notification from transport that it is closed.
Definition: session.c:1033
void session_debug_init(void)
void session_close(session_t *s)
Initialize session closing procedure.
Definition: session.c:1360
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
int session_stream_connect_notify(transport_connection_t *tc, session_error_t err)
Definition: session.c:757
void session_reset(session_t *s)
Force a close without waiting for data to be flushed.
Definition: session.c:1383
ssvm_private_t * session_main_get_evt_q_segment(void)
Definition: session.c:1538
struct _segment_manager segment_manager_t
segment_manager_t * segment_manager_get(u32 index)
session_cleanup_ntf_t
int session_send_io_evt_to_thread_custom(void *data, u32 thread_index, session_evt_type_t evt_type)
Definition: session.c:86
unformat_function_t unformat_memory_size
Definition: format.h:296
static session_evt_elt_t * session_evt_alloc_ctrl(session_worker_t *wrk)
Definition: session.h:250
u32 app_index
Index of owning app.
Definition: application.h:43
static u8 svm_fifo_n_subscribers(svm_fifo_t *f)
Fifo number of subscribers getter.
Definition: svm_fifo.h:616
int session_lookup_add_connection(transport_connection_t *tc, u64 value)
Add transport connection to a session table.
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
Connection descheduled by the session layer.
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1540
u32 configured_v6_session_table_memory
Definition: session.h:185
clib_us_time_t last_vlib_us_time
vlib_time_now rounded to us precision and as u64
Definition: session.h:86
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
vlib_node_registration_t session_queue_process_node
(constructor) VLIB_REGISTER_NODE (session_queue_process_node)
vl_api_dhcp_client_state_t state
Definition: dhcp.api:201
static u32 vlib_num_workers()
Definition: threads.h:376
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
void * vl_msg_push_heap(void)
Definition: cli.c:713
int(* session_open_service_fn)(u32, session_endpoint_t *, u32)
Definition: session.c:1263
enum session_error_ session_error_t
u32 app_wrk_index
Index of the app worker that owns the session.
void transport_cleanup_half_open(transport_proto_t tp, u32 conn_index)
Definition: transport.c:308
u32 session_tx_fifo_dequeue_drop(transport_connection_t *tc, u32 max_bytes)
Definition: session.c:561
static void session_delete(session_t *s)
Cleans up session and lookup table.
Definition: session.c:274
u8 si
Definition: lisp_types.api:47
void session_transport_close(session_t *s)
Notify transport the session can be disconnected.
Definition: session.c:1401
clib_llist_index_t ctrl_head
Head of control events list.
Definition: session.h:107
int session_dgram_accept(transport_connection_t *tc, u32 listener_index, u32 thread_index)
Definition: session.c:1142
int app_worker_close_notify(app_worker_t *app_wrk, session_t *s)
struct _session_endpoint session_endpoint_t
int session_stream_accept(transport_connection_t *tc, u32 listener_index, u32 thread_index, u8 notify)
Accept a stream session.
Definition: session.c:1107
int consumer_pid
pid of msg consumer
Definition: message_queue.h:52
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static transport_connection_t * transport_get_listener(transport_proto_t tp, u32 conn_index)
Definition: transport.h:156
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
int svm_fifo_dequeue_drop(svm_fifo_t *f, u32 len)
Dequeue and drop bytes from fifo.
Definition: svm_fifo.c:1061
static int svm_msg_q_lock(svm_msg_q_t *mq)
Lock, or block trying, the message queue.
static void session_enqueue_notify_rpc(void *arg)
Definition: session.c:653
int session_open_app(u32 app_wrk_index, session_endpoint_t *rmt, u32 opaque)
Definition: session.c:1252
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1617
void session_register_transport(transport_proto_t transport_proto, const transport_proto_vft_t *vft, u8 is_ip4, u32 output_node)
Initialize session layer for given transport proto and ip version.
Definition: session.c:1569
int session_lookup_add_half_open(transport_connection_t *tc, u64 value)
struct _svm_fifo svm_fifo_t
u8 transport_protocol_is_cl(transport_proto_t tp)
Definition: transport.c:349
static u8 svm_msg_q_is_full(svm_msg_q_t *mq)
Check if message queue is full.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
svm_msg_q_msg_t svm_msg_q_alloc_msg_w_ring(svm_msg_q_t *mq, u32 ring_index)
Allocate message buffer on ring.
#define SESSION_EVT(_evt, _args...)
int i_am_master
Definition: ssvm.h:90
session_t * session_alloc_for_connection(transport_connection_t *tc)
Definition: session.c:300
static void session_program_transport_ctrl_evt(session_t *s, session_evt_type_t evt)
Definition: session.c:163
static session_t * listen_session_get(u32 ls_index)
Definition: session.h:594
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:171
static u8 transport_connection_is_descheduled(transport_connection_t *tc)
Definition: transport.h:202