FD.io VPP  v20.01-48-g3e0dafb74
Vector Packet Processing
segment_manager.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
17 #include <vnet/session/session.h>
19 
20 typedef struct segment_manager_main_
21 {
22  segment_manager_t *segment_managers; /**< Pool of segment managers */
23  clib_valloc_main_t va_allocator; /**< Virtual address allocator */
24  u32 seg_name_counter; /**< Counter for segment names */
25 
26  /*
27  * Configuration
28  */
29  u32 default_fifo_size; /**< default rx/tx fifo size */
30  u32 default_segment_size; /**< default fifo segment size */
31  u32 default_app_mq_size; /**< default app msg q size */
33 
35 
36 #define segment_manager_foreach_segment_w_lock(VAR, SM, BODY) \
37 do { \
38  clib_rwlock_reader_lock (&(SM)->segments_rwlock); \
39  pool_foreach((VAR), ((SM)->segments), (BODY)); \
40  clib_rwlock_reader_unlock (&(SM)->segments_rwlock); \
41 } while (0)
42 
45 {
46  app_worker_t *app_wrk = app_worker_get (sm->app_wrk_index);
48 }
49 
52 {
53  props->add_segment_size = sm_main.default_segment_size;
54  props->rx_fifo_size = sm_main.default_fifo_size;
55  props->tx_fifo_size = sm_main.default_fifo_size;
56  props->evt_q_size = sm_main.default_app_mq_size;
57  props->n_slices = vlib_num_workers () + 1;
58  return props;
59 }
60 
61 static u8
63 {
64  return (sm->app_wrk_index == SEGMENT_MANAGER_INVALID_APP_INDEX);
65 }
66 
67 void
69 {
70  sm->app_wrk_index = SEGMENT_MANAGER_INVALID_APP_INDEX;
71 }
72 
75 {
76  return (seg - sm->segments);
77 }
78 
79 /**
80  * Adds segment to segment manager's pool
81  *
82  * If needed a writer's lock is acquired before allocating a new segment
83  * to avoid affecting any of the segments pool readers.
84  */
85 int
87 {
88  uword baseva = (uword) ~ 0ULL, alloc_size, page_size;
89  u32 rnd_margin = 128 << 10, fs_index = ~0;
92  fifo_segment_t *fs;
93  u8 *seg_name;
94  int rv;
95 
96  props = segment_manager_properties_get (sm);
97 
98  /* Not configured for addition of new segments and not first */
99  if (!props->add_segment && !segment_size)
100  {
101  clib_warning ("cannot allocate new segment");
102  return VNET_API_ERROR_INVALID_VALUE;
103  }
104 
105  /*
106  * Allocate fifo segment and grab lock if needed
107  */
108  if (vlib_num_workers ())
109  clib_rwlock_writer_lock (&sm->segments_rwlock);
110 
111  pool_get_zero (sm->segments, fs);
112 
113  /*
114  * Allocate ssvm segment
115  */
116  segment_size = segment_size ? segment_size : props->add_segment_size;
117  page_size = clib_mem_get_page_size ();
118  /* Protect against segment size u32 wrap */
119  segment_size = clib_max (segment_size + page_size - 1, segment_size);
120  segment_size = segment_size & ~(page_size - 1);
121 
122  if (props->segment_type != SSVM_SEGMENT_PRIVATE)
123  {
124  seg_name = format (0, "%d-%d%c", getpid (), smm->seg_name_counter++, 0);
125  alloc_size = (uword) segment_size + rnd_margin;
126  baseva = clib_valloc_alloc (&smm->va_allocator, alloc_size, 0);
127  if (!baseva)
128  {
129  clib_warning ("out of space for segments");
130  pool_put (sm->segments, fs);
131  goto done;
132  }
133  }
134  else
135  seg_name = format (0, "%s%c", "process-private", 0);
136 
137  fs->ssvm.ssvm_size = segment_size;
138  fs->ssvm.name = seg_name;
139  fs->ssvm.requested_va = baseva;
140 
141  if ((rv = ssvm_master_init (&fs->ssvm, props->segment_type)))
142  {
143  clib_warning ("svm_master_init ('%v', %u) failed", seg_name,
144  segment_size);
145 
146  if (props->segment_type != SSVM_SEGMENT_PRIVATE)
147  clib_valloc_free (&smm->va_allocator, baseva);
148  pool_put (sm->segments, fs);
149  goto done;
150  }
151 
152  /*
153  * Initialize fifo segment
154  */
155  fs->n_slices = props->n_slices;
156  fifo_segment_init (fs);
157 
158  /*
159  * Save segment index before dropping lock, if any held
160  */
161  fs_index = fs - sm->segments;
162 
163 done:
164 
165  if (vlib_num_workers ())
166  clib_rwlock_writer_unlock (&sm->segments_rwlock);
167 
168  return fs_index;
169 }
170 
171 /**
172  * Remove segment without lock
173  */
174 void
176 {
178 
179  if (ssvm_type (&fs->ssvm) != SSVM_SEGMENT_PRIVATE)
180  {
182 
183  if (sm->app_wrk_index != SEGMENT_MANAGER_INVALID_APP_INDEX)
184  {
185  app_worker_t *app_wrk;
186  u64 segment_handle;
187  app_wrk = app_worker_get (sm->app_wrk_index);
188  segment_handle = segment_manager_segment_handle (sm, fs);
189  app_worker_del_segment_notify (app_wrk, segment_handle);
190  }
191  }
192 
193  ssvm_delete (&fs->ssvm);
194 
195  if (CLIB_DEBUG)
196  clib_memset (fs, 0xfb, sizeof (*fs));
197  pool_put (sm->segments, fs);
198 }
199 
200 /**
201  * Removes segment after acquiring writer lock
202  */
203 static inline void
205 {
206  fifo_segment_t *fs;
207  u8 is_prealloc;
208 
209  clib_rwlock_writer_lock (&sm->segments_rwlock);
210  fs = segment_manager_get_segment (sm, fs_index);
212  if (is_prealloc && !segment_manager_app_detached (sm))
213  {
214  clib_rwlock_writer_unlock (&sm->segments_rwlock);
215  return;
216  }
217 
219  clib_rwlock_writer_unlock (&sm->segments_rwlock);
220 }
221 
222 /**
223  * Reads a segment from the segment manager's pool without lock
224  */
227 {
228  return pool_elt_at_index (sm->segments, segment_index);
229 }
230 
231 u64
233  fifo_segment_t * segment)
234 {
235  u32 segment_index = segment_manager_segment_index (sm, segment);
236  return (((u64) segment_manager_index (sm) << 32) | segment_index);
237 }
238 
239 static void
240 segment_manager_parse_segment_handle (u64 segment_handle, u32 * sm_index,
241  u32 * segment_index)
242 {
243  *sm_index = segment_handle >> 32;
244  *segment_index = segment_handle & 0xFFFFFFFF;
245 }
246 
247 u64
249  u32 segment_index)
250 {
251  return (((u64) segment_manager_index << 32) | segment_index);
252 }
253 
256 {
257  u32 sm_index, segment_index;
258  segment_manager_t *sm;
259 
260  segment_manager_parse_segment_handle (segment_handle, &sm_index,
261  &segment_index);
262  sm = segment_manager_get (sm_index);
263  if (!sm || pool_is_free_index (sm->segments, segment_index))
264  return 0;
265  return pool_elt_at_index (sm->segments, segment_index);
266 }
267 
268 /**
269  * Reads a segment from the segment manager's pool and acquires reader lock
270  *
271  * Caller must drop the reader's lock by calling
272  * @ref segment_manager_segment_reader_unlock once it finishes working with
273  * the segment.
274  */
277 {
278  clib_rwlock_reader_lock (&sm->segments_rwlock);
279  return pool_elt_at_index (sm->segments, segment_index);
280 }
281 
282 void
284 {
285  clib_rwlock_reader_unlock (&sm->segments_rwlock);
286 }
287 
288 void
290 {
291  clib_rwlock_writer_unlock (&sm->segments_rwlock);
292 }
293 
296 {
298  segment_manager_t *sm;
299 
300  pool_get_zero (smm->segment_managers, sm);
301  clib_rwlock_init (&sm->segments_rwlock);
302  return sm;
303 }
304 
305 /**
306  * Initializes segment manager based on options provided.
307  * Returns error if ssvm segment(s) allocation fails.
308  */
309 int
311  u32 prealloc_fifo_pairs)
312 {
313  u32 rx_fifo_size, tx_fifo_size, pair_size;
314  u32 rx_rounded_data_size, tx_rounded_data_size;
315  u64 approx_total_size, max_seg_size = ((u64) 1 << 32) - (128 << 10);
317  fifo_segment_t *segment;
318  u32 approx_segment_count;
319  int seg_index, i;
320 
321  props = segment_manager_properties_get (sm);
322  first_seg_size = clib_max (first_seg_size, sm_main.default_segment_size);
323 
324  if (prealloc_fifo_pairs)
325  {
326  /* Figure out how many segments should be preallocated */
327  rx_rounded_data_size = (1 << (max_log2 (props->rx_fifo_size)));
328  tx_rounded_data_size = (1 << (max_log2 (props->tx_fifo_size)));
329 
330  rx_fifo_size = sizeof (svm_fifo_t) + rx_rounded_data_size;
331  tx_fifo_size = sizeof (svm_fifo_t) + tx_rounded_data_size;
332  pair_size = rx_fifo_size + tx_fifo_size;
333 
334  approx_total_size = (u64) prealloc_fifo_pairs *pair_size;
335  if (first_seg_size > approx_total_size)
336  max_seg_size = first_seg_size;
337  approx_segment_count = (approx_total_size + (max_seg_size - 1))
338  / max_seg_size;
339 
340  /* Allocate the segments */
341  for (i = 0; i < approx_segment_count + 1; i++)
342  {
343  seg_index = segment_manager_add_segment (sm, max_seg_size);
344  if (seg_index < 0)
345  {
346  clib_warning ("Failed to preallocate segment %d", i);
347  return seg_index;
348  }
349 
350  segment = segment_manager_get_segment (sm, seg_index);
351  if (i == 0)
352  sm->event_queue = segment_manager_alloc_queue (segment, props);
353 
355  props->rx_fifo_size,
356  props->tx_fifo_size,
357  &prealloc_fifo_pairs);
359  if (prealloc_fifo_pairs == 0)
360  break;
361  }
362  }
363  else
364  {
365  seg_index = segment_manager_add_segment (sm, first_seg_size);
366  if (seg_index < 0)
367  {
368  clib_warning ("Failed to allocate segment");
369  return seg_index;
370  }
371  segment = segment_manager_get_segment (sm, seg_index);
372  sm->event_queue = segment_manager_alloc_queue (segment, props);
373  }
374 
375  return 0;
376 }
377 
378 /**
379  * Cleanup segment manager.
380  */
381 void
383 {
385  fifo_segment_t *fifo_segment;
386 
389 
390  /* If we have empty preallocated segments that haven't been removed, remove
391  * them now. Apart from that, the first segment in the first segment manager
392  * is not removed when all fifos are removed. It can only be removed when
393  * the manager is explicitly deleted/detached by the app. */
394  clib_rwlock_writer_lock (&sm->segments_rwlock);
395 
396  /* *INDENT-OFF* */
397  pool_foreach (fifo_segment, sm->segments, ({
398  segment_manager_del_segment (sm, fifo_segment);
399  }));
400  /* *INDENT-ON* */
401 
402  clib_rwlock_writer_unlock (&sm->segments_rwlock);
403 
404  clib_rwlock_free (&sm->segments_rwlock);
405  if (CLIB_DEBUG)
406  clib_memset (sm, 0xfe, sizeof (*sm));
407  pool_put (smm->segment_managers, sm);
408 }
409 
410 void
412 {
414  if (segment_manager_has_fifos (sm))
416  else
417  {
418  ASSERT (!sm->first_is_protected || segment_manager_app_detached (sm));
420  }
421 }
422 
425 {
426  return pool_elt_at_index (sm_main.segment_managers, index);
427 }
428 
431 {
432  if (pool_is_free_index (sm_main.segment_managers, index))
433  return 0;
434  return pool_elt_at_index (sm_main.segment_managers, index);
435 }
436 
437 u32
439 {
440  return sm - sm_main.segment_managers;
441 }
442 
443 u8
445 {
446  fifo_segment_t *seg;
447  u8 first = 1;
448 
449  /* *INDENT-OFF* */
451  if (CLIB_DEBUG && !first && !fifo_segment_has_fifos (seg)
453  {
454  clib_warning ("segment %d has no fifos!",
456  first = 0;
457  }
458  if (fifo_segment_has_fifos (seg))
459  {
461  return 1;
462  }
463  }));
464  /* *INDENT-ON* */
465 
466  return 0;
467 }
468 
469 /**
470  * Initiate disconnects for all sessions 'owned' by a segment manager
471  */
472 void
474 {
475  session_handle_t *handles = 0, *handle;
476  fifo_segment_t *fs;
477  session_t *session;
478  int slice_index;
479  svm_fifo_t *f;
480 
481  ASSERT (pool_elts (sm->segments) != 0);
482 
483  /* Across all fifo segments used by the server */
484  /* *INDENT-OFF* */
486  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
487  {
488  f = fifo_segment_get_slice_fifo_list (fs, slice_index);
489 
490  /*
491  * Remove any residual sessions from the session lookup table
492  * Don't bother deleting the individual fifos, we're going to
493  * throw away the fifo segment in a minute.
494  */
495  while (f)
496  {
497  session = session_get_if_valid (f->master_session_index,
498  f->master_thread_index);
499  if (session)
500  vec_add1 (handles, session_handle (session));
501  f = f->next;
502  }
503  }
504 
505  /* Instead of removing the segment, test when cleaning up disconnected
506  * sessions if the segment can be removed.
507  */
508  }));
509  /* *INDENT-ON* */
510 
511  vec_foreach (handle, handles)
513 }
514 
515 int
517  u32 thread_index,
518  u32 rx_fifo_size, u32 tx_fifo_size,
519  svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo)
520 {
521  rx_fifo_size = clib_max (rx_fifo_size, sm_main.default_fifo_size);
522  *rx_fifo = fifo_segment_alloc_fifo_w_slice (fifo_segment, thread_index,
523  rx_fifo_size,
525 
526  tx_fifo_size = clib_max (tx_fifo_size, sm_main.default_fifo_size);
527  *tx_fifo = fifo_segment_alloc_fifo_w_slice (fifo_segment, thread_index,
528  tx_fifo_size,
530 
531  if (*rx_fifo == 0)
532  {
533  /* This would be very odd, but handle it... */
534  if (*tx_fifo != 0)
535  {
536  fifo_segment_free_fifo (fifo_segment, *tx_fifo);
537  *tx_fifo = 0;
538  }
539  return -1;
540  }
541  if (*tx_fifo == 0)
542  {
543  if (*rx_fifo != 0)
544  {
545  fifo_segment_free_fifo (fifo_segment, *rx_fifo);
546  *rx_fifo = 0;
547  }
548  return -1;
549  }
550 
551  return 0;
552 }
553 
554 int
556  u32 thread_index,
557  svm_fifo_t ** rx_fifo,
558  svm_fifo_t ** tx_fifo)
559 {
560  int alloc_fail = 1, rv = 0, new_fs_index;
562  fifo_segment_t *fs = 0;
563  u32 sm_index, fs_index;
564  u8 added_a_segment = 0;
565  u64 fs_handle;
566 
567  props = segment_manager_properties_get (sm);
568 
569  /*
570  * Find the first free segment to allocate the fifos in
571  */
572 
573  /* *INDENT-OFF* */
575  alloc_fail = segment_manager_try_alloc_fifos (fs,
576  thread_index,
577  props->rx_fifo_size,
578  props->tx_fifo_size,
579  rx_fifo, tx_fifo);
580  /* Exit with lock held, drop it after notifying app */
581  if (!alloc_fail)
582  goto alloc_success;
583  }));
584  /* *INDENT-ON* */
585 
586 alloc_check:
587 
588  if (!alloc_fail)
589  {
590 
591  alloc_success:
592 
593  ASSERT (rx_fifo && tx_fifo);
594  sm_index = segment_manager_index (sm);
595  fs_index = segment_manager_segment_index (sm, fs);
596  (*tx_fifo)->segment_manager = sm_index;
597  (*rx_fifo)->segment_manager = sm_index;
598  (*tx_fifo)->segment_index = fs_index;
599  (*rx_fifo)->segment_index = fs_index;
600 
601  if (added_a_segment)
602  {
603  app_worker_t *app_wrk;
604  fs_handle = segment_manager_segment_handle (sm, fs);
605  app_wrk = app_worker_get (sm->app_wrk_index);
606  rv = app_worker_add_segment_notify (app_wrk, fs_handle);
607  }
608  /* Drop the lock after app is notified */
610  return rv;
611  }
612 
613  /*
614  * Allocation failed, see if we can add a new segment
615  */
616  if (props->add_segment)
617  {
618  if (added_a_segment)
619  {
620  clib_warning ("Added a segment, still can't allocate a fifo");
622  return SESSION_ERROR_NEW_SEG_NO_SPACE;
623  }
624  if ((new_fs_index = segment_manager_add_segment (sm, 0)) < 0)
625  {
626  clib_warning ("Failed to add new segment");
627  return SESSION_ERROR_SEG_CREATE;
628  }
629  fs = segment_manager_get_segment_w_lock (sm, new_fs_index);
630  alloc_fail = segment_manager_try_alloc_fifos (fs, thread_index,
631  props->rx_fifo_size,
632  props->tx_fifo_size,
633  rx_fifo, tx_fifo);
634  added_a_segment = 1;
635  goto alloc_check;
636  }
637  else
638  {
639  clib_warning ("Can't add new seg and no space to allocate fifos!");
640  return SESSION_ERROR_NO_SPACE;
641  }
642 }
643 
644 void
646 {
647  segment_manager_t *sm;
648  fifo_segment_t *fs;
649  u32 segment_index;
650 
651  if (!rx_fifo || !tx_fifo)
652  return;
653 
654  /* It's possible to have no segment manager if the session was removed
655  * as result of a detach. */
656  if (!(sm = segment_manager_get_if_valid (rx_fifo->segment_manager)))
657  return;
658 
659  segment_index = rx_fifo->segment_index;
660  fs = segment_manager_get_segment_w_lock (sm, segment_index);
661  fifo_segment_free_fifo (fs, rx_fifo);
662  fifo_segment_free_fifo (fs, tx_fifo);
663 
664  /*
665  * Try to remove svm segment if it has no fifos. This can be done only if
666  * the segment is not the first in the segment manager or if it is first
667  * and it is not protected. Moreover, if the segment is first and the app
668  * has detached from the segment manager, remove the segment manager.
669  */
670  if (!fifo_segment_has_fifos (fs))
671  {
673 
674  /* Remove segment if it holds no fifos or first but not protected */
675  if (segment_index != 0 || !sm->first_is_protected)
676  segment_manager_lock_and_del_segment (sm, segment_index);
677 
678  /* Remove segment manager if no sessions and detached from app */
680  && !segment_manager_has_fifos (sm))
681  {
683  }
684  }
685  else
687 }
688 
689 int
691 {
692  fifo_segment_t *fs;
693  int rv;
694 
695  fs = segment_manager_get_segment_w_lock (sm, f->segment_index);
696  rv = fifo_segment_grow_fifo (fs, f, size);
698 
699  return rv;
700 }
701 
702 int
704 {
705  fifo_segment_t *fs;
706  int rv;
707 
708  fs = segment_manager_get_segment_w_lock (sm, f->segment_index);
711 
712  return rv;
713 }
714 
715 int
717  u8 is_producer)
718 {
719  int rv;
720 
721  rv = svm_fifo_reduce_size (f, size, is_producer);
722 
723  /* Nothing to collect at this point */
724  if (!is_producer)
725  return rv;
726 
727  if (f->flags & SVM_FIFO_F_COLLECT_CHUNKS)
729 
730  return rv;
731 }
732 
733 u32
735 {
736  u32 fifo_evt_size, notif_q_size, q_hdrs;
737  u32 msg_q_sz, fifo_evt_ring_sz, session_ntf_ring_sz;
738 
739  fifo_evt_size = 1 << max_log2 (sizeof (session_event_t));
740  notif_q_size = clib_max (16, q_len >> 4);
741 
742  msg_q_sz = q_len * sizeof (svm_msg_q_msg_t);
743  fifo_evt_ring_sz = q_len * fifo_evt_size;
744  session_ntf_ring_sz = notif_q_size * 256;
745  q_hdrs = sizeof (svm_queue_t) + sizeof (svm_msg_q_t);
746 
747  return (msg_q_sz + fifo_evt_ring_sz + session_ntf_ring_sz + q_hdrs);
748 }
749 
750 /**
751  * Allocates shm queue in the first segment
752  *
753  * Must be called with lock held
754  */
755 svm_msg_q_t *
757  segment_manager_props_t * props)
758 {
759  u32 fifo_evt_size, session_evt_size = 256, notif_q_size;
760  svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
761  svm_msg_q_t *q;
762  void *oldheap;
763 
764  fifo_evt_size = sizeof (session_event_t);
765  notif_q_size = clib_max (16, props->evt_q_size >> 4);
766  /* *INDENT-OFF* */
768  {props->evt_q_size, fifo_evt_size, 0},
769  {notif_q_size, session_evt_size, 0}
770  };
771  /* *INDENT-ON* */
772  cfg->consumer_pid = 0;
773  cfg->n_rings = 2;
774  cfg->q_nitems = props->evt_q_size;
775  cfg->ring_cfgs = rc;
776 
777  oldheap = ssvm_push_heap (segment->ssvm.sh);
778  q = svm_msg_q_alloc (cfg);
780  ssvm_pop_heap (oldheap);
781 
782  if (props->use_mq_eventfd)
783  {
785  clib_warning ("failed to alloc eventfd");
786  }
787  return q;
788 }
789 
790 svm_msg_q_t *
792 {
793  return sm->event_queue;
794 }
795 
796 /**
797  * Frees shm queue allocated in the first segment
798  */
799 void
801 {
802  fifo_segment_t *segment;
804  void *oldheap;
805 
806  ASSERT (!pool_is_free_index (sm->segments, 0));
807 
808  segment = segment_manager_get_segment_w_lock (sm, 0);
809  sh = segment->ssvm.sh;
810 
811  oldheap = ssvm_push_heap (sh);
812  svm_queue_free (q);
813  ssvm_pop_heap (oldheap);
815 }
816 
817 /*
818  * Init segment vm address allocator
819  */
820 void
822 {
824  clib_valloc_chunk_t _ip, *ip = &_ip;
825 
826  ip->baseva = a->baseva;
827  ip->size = a->size;
828 
829  clib_valloc_init (&sm->va_allocator, ip, 1 /* lock */ );
830 
831  sm->default_fifo_size = 1 << 12;
832  sm->default_segment_size = 1 << 20;
833  sm->default_app_mq_size = 128;
834 }
835 
836 static clib_error_t *
838  vlib_cli_command_t * cmd)
839 {
841  u8 show_segments = 0, verbose = 0;
842  segment_manager_t *sm;
843  fifo_segment_t *seg;
844 
846  {
847  if (unformat (input, "segments"))
848  show_segments = 1;
849  else if (unformat (input, "verbose"))
850  verbose = 1;
851  else
852  return clib_error_return (0, "unknown input `%U'",
853  format_unformat_error, input);
854  }
855  vlib_cli_output (vm, "%d segment managers allocated",
856  pool_elts (smm->segment_managers));
857  if (verbose && pool_elts (smm->segment_managers))
858  {
859  vlib_cli_output (vm, "%-10s%=15s%=12s", "Index", "App Index",
860  "Segments");
861 
862  /* *INDENT-OFF* */
863  pool_foreach (sm, smm->segment_managers, ({
864  vlib_cli_output (vm, "%-10d%=15d%=12d", segment_manager_index (sm),
865  sm->app_wrk_index, pool_elts (sm->segments));
866  }));
867  /* *INDENT-ON* */
868 
869  }
870  if (show_segments)
871  {
872  vlib_cli_output (vm, "%U", format_fifo_segment, 0, verbose);
873 
874  /* *INDENT-OFF* */
875  pool_foreach (sm, smm->segment_managers, ({
876  segment_manager_foreach_segment_w_lock (seg, sm, ({
877  vlib_cli_output (vm, "%U", format_fifo_segment, seg, verbose);
878  }));
879  }));
880  /* *INDENT-ON* */
881 
882  }
883  return 0;
884 }
885 
886 /* *INDENT-OFF* */
887 VLIB_CLI_COMMAND (segment_manager_show_command, static) =
888 {
889  .path = "show segment-manager",
890  .short_help = "show segment-manager [segments][verbose]",
891  .function = segment_manager_show_fn,
892 };
893 /* *INDENT-ON* */
894 
895 void
897 {
899  app_worker_t *app_wrk;
900  fifo_segment_t *fs;
901  const u8 *app_name;
902  int slice_index;
903  u8 *s = 0, *str;
904  svm_fifo_t *f;
905 
906  if (!sm)
907  {
908  if (verbose)
909  vlib_cli_output (vm, "%-40s%-20s%-15s%-10s", "Connection", "App",
910  "API Client", "SegManager");
911  else
912  vlib_cli_output (vm, "%-40s%-20s", "Connection", "App");
913  return;
914  }
915 
916  app_wrk = app_worker_get (sm->app_wrk_index);
917  app_name = application_name_from_index (app_wrk->app_index);
918 
919  clib_rwlock_reader_lock (&sm->segments_rwlock);
920 
921  /* *INDENT-OFF* */
922  pool_foreach (fs, sm->segments, ({
923  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
924  {
925  f = fifo_segment_get_slice_fifo_list (fs, slice_index);
926  while (f)
927  {
928  u32 session_index, thread_index;
929  session_t *session;
930 
931  session_index = f->master_session_index;
932  thread_index = f->master_thread_index;
933 
934  session = session_get (session_index, thread_index);
935  str = format (0, "%U", format_session, session, verbose);
936 
937  if (verbose)
938  s = format (s, "%-40s%-20s%-15u%-10u", str, app_name,
939  app_wrk->api_client_index, app_wrk->connects_seg_manager);
940  else
941  s = format (s, "%-40s%-20s", str, app_name);
942 
943  vlib_cli_output (vm, "%v", s);
944  vec_reset_length (s);
945  vec_free (str);
946 
947  f = f->next;
948  }
949  vec_free (s);
950  }
951  }));
952  /* *INDENT-ON* */
953 
954  clib_rwlock_reader_unlock (&sm->segments_rwlock);
955 }
956 
957 /*
958  * fd.io coding-style-patch-verification: ON
959  *
960  * Local Variables:
961  * eval: (c-set-style "gnu")
962  * End:
963  */
u32 segment_manager_index(segment_manager_t *sm)
static void clib_rwlock_reader_lock(clib_rwlock_t *p)
Definition: lock.h:150
void segment_manager_segment_reader_unlock(segment_manager_t *sm)
u64 segment_manager_segment_handle(segment_manager_t *sm, fifo_segment_t *segment)
svm_msg_q_t * segment_manager_alloc_queue(fifo_segment_t *segment, segment_manager_props_t *props)
Allocates shm queue in the first segment.
uword requested_va
Definition: ssvm.h:88
a
Definition: bitmap.h:538
static void clib_rwlock_writer_lock(clib_rwlock_t *p)
Definition: lock.h:173
int segment_manager_add_segment(segment_manager_t *sm, uword segment_size)
Adds segment to segment manager&#39;s pool.
uword ssvm_size
Definition: ssvm.h:85
svm_fifo_t * fifo_segment_get_slice_fifo_list(fifo_segment_t *fs, u32 slice_index)
Definition: fifo_segment.c:979
fifo_segment_t * segment_manager_get_segment(segment_manager_t *sm, u32 segment_index)
Reads a segment from the segment manager&#39;s pool without lock.
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:240
unsigned long u64
Definition: types.h:89
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
struct segment_manager_main_ segment_manager_main_t
svm_fifo_t * fifo_segment_alloc_fifo_w_slice(fifo_segment_t *fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype)
Allocate fifo in fifo segment.
Definition: fifo_segment.c:487
static void clib_rwlock_free(clib_rwlock_t *p)
Definition: lock.h:140
static session_t * session_get_if_valid(u64 si, u32 thread_index)
Definition: session.h:302
fifo_segment_t * segment_manager_get_segment_w_handle(u64 segment_handle)
void segment_manager_format_sessions(segment_manager_t *sm, int verbose)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
clib_valloc_main_t va_allocator
Virtual address allocator.
static segment_manager_main_t sm_main
int i
void segment_manager_free(segment_manager_t *sm)
Cleanup segment manager.
ssvm_shared_header_t * sh
Definition: ssvm.h:84
int app_worker_add_segment_notify(app_worker_t *app_wrk, u64 segment_handle)
Send an API message to the external app, to map new segment.
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
segment_manager_t * segment_managers
Pool of segment managers.
int segment_manager_init(segment_manager_t *sm, uword first_seg_size, u32 prealloc_fifo_pairs)
Initializes segment manager based on options provided.
u32 segment_manager_evt_q_expected_size(u32 q_len)
unsigned char u8
Definition: types.h:56
static void segment_manager_parse_segment_handle(u64 segment_handle, u32 *sm_index, u32 *segment_index)
static u32 segment_manager_segment_index(segment_manager_t *sm, fifo_segment_t *seg)
void ssvm_delete(ssvm_private_t *ssvm)
Definition: ssvm.c:437
static session_handle_t session_handle(session_t *s)
struct _svm_fifo svm_fifo_t
#define fifo_segment_flags(_fs)
Definition: fifo_segment.h:84
u64 segment_manager_make_segment_handle(u32 segment_manager_index, u32 segment_index)
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:498
svm_msg_q_t * svm_msg_q_alloc(svm_msg_q_cfg_t *cfg)
Allocate message queue.
Definition: message_queue.c:41
static u8 segment_manager_app_detached(segment_manager_t *sm)
void segment_manager_dealloc_fifos(svm_fifo_t *rx_fifo, svm_fifo_t *tx_fifo)
static void * ssvm_push_heap(ssvm_shared_header_t *sh)
Definition: ssvm.h:143
#define clib_error_return(e, args...)
Definition: error.h:99
u32 default_fifo_size
default rx/tx fifo size
unsigned int u32
Definition: types.h:88
static void segment_manager_lock_and_del_segment(segment_manager_t *sm, u32 fs_index)
Removes segment after acquiring writer lock.
int ssvm_master_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:425
static void ssvm_pop_heap(void *oldheap)
Definition: ssvm.h:151
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
void fifo_segment_preallocate_fifo_pairs(fifo_segment_t *fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 *n_fifo_pairs)
Pre-allocates fifo pairs in fifo segment.
Definition: fifo_segment.c:698
u32 seg_name_counter
Counter for segment names.
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:519
static void clib_rwlock_init(clib_rwlock_t *p)
Definition: lock.h:133
static void clib_rwlock_reader_unlock(clib_rwlock_t *p)
Definition: lock.h:165
void segment_manager_app_detach(segment_manager_t *sm)
u32 default_app_mq_size
default app msg q size
static session_t * session_get_from_handle(session_handle_t handle)
Definition: session.h:315
void segment_manager_del_segment(segment_manager_t *sm, fifo_segment_t *fs)
Remove segment without lock.
void segment_manager_dealloc_queue(segment_manager_t *sm, svm_queue_t *q)
Frees shm queue allocated in the first segment.
struct _unformat_input_t unformat_input_t
uword clib_mem_get_page_size(void)
Definition: mem.c:51
const u8 * application_name_from_index(u32 app_index)
Returns app name for app-index.
Definition: application.c:360
u64 size
Definition: vhost_user.h:140
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:287
void segment_manager_init_free(segment_manager_t *sm)
Initiate segment manager cleanup.
struct _segment_manager_props segment_manager_props_t
int fifo_segment_grow_fifo(fifo_segment_t *fs, svm_fifo_t *f, u32 chunk_size)
Grow fifo size by adding an additional chunk of memory.
Definition: fifo_segment.c:758
u8 segment_manager_has_fifos(segment_manager_t *sm)
#define always_inline
Definition: ipsec.h:28
uword size
size in bytes of this chunk
Definition: valloc.h:33
vlib_main_t * vm
Definition: in2out_ed.c:1810
int segment_manager_grow_fifo(segment_manager_t *sm, svm_fifo_t *f, u32 size)
Grows fifo owned by segment manager.
int segment_manager_collect_fifo_chunks(segment_manager_t *sm, svm_fifo_t *f)
Collect fifo chunks that are no longer used.
uword baseva
base VA for this chunk
Definition: valloc.h:32
#define SEGMENT_MANAGER_INVALID_APP_INDEX
static void clib_rwlock_writer_unlock(clib_rwlock_t *p)
Definition: lock.h:187
u32 n_rings
number of msg rings
Definition: message_queue.h:54
svm_msg_q_t * segment_manager_event_queue(segment_manager_t *sm)
ssvm_private_t ssvm
ssvm segment data
Definition: fifo_segment.h:63
#define UNFORMAT_END_OF_INPUT
Definition: format.h:145
segment_manager_t * segment_manager_alloc(void)
static segment_manager_props_t * segment_manager_properties_get(segment_manager_t *sm)
#define clib_warning(format, args...)
Definition: error.h:59
int svm_fifo_reduce_size(svm_fifo_t *f, u32 len, u8 try_shrink)
Request to reduce fifo size by amount of bytes.
Definition: svm_fifo.c:807
#define segment_manager_foreach_segment_w_lock(VAR, SM, BODY)
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:284
int segment_manager_alloc_session_fifos(segment_manager_t *sm, u32 thread_index, svm_fifo_t **rx_fifo, svm_fifo_t **tx_fifo)
void svm_queue_free(svm_queue_t *q)
Definition: queue.c:89
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:152
u8 n_slices
number of fifo segment slices
Definition: fifo_segment.h:65
segment_manager_props_t * application_get_segment_manager_properties(u32 app_index)
Definition: application.c:1305
uword clib_valloc_alloc(clib_valloc_main_t *vam, uword size, int os_out_of_memory_on_failure)
Allocate virtual space.
Definition: valloc.c:151
#define ASSERT(truth)
segment_manager_props_t * segment_manager_props_init(segment_manager_props_t *props)
void fifo_segment_free_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated in fifo segment.
Definition: fifo_segment.c:538
fifo_segment_t * segment_manager_get_segment_w_lock(segment_manager_t *sm, u32 segment_index)
Reads a segment from the segment manager&#39;s pool and acquires reader lock.
void segment_manager_main_init(segment_manager_main_init_args_t *a)
svm_msg_q_ring_cfg_t * ring_cfgs
array of ring cfgs
Definition: message_queue.h:55
#define clib_max(x, y)
Definition: clib.h:288
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u8 * name
Definition: ssvm.h:87
int app_worker_del_segment_notify(app_worker_t *app_wrk, u64 segment_handle)
void segment_manager_segment_writer_unlock(segment_manager_t *sm)
vl_api_address_t ip
Definition: l2.api:490
app_worker_t * app_worker_get(u32 wrk_index)
u32 q_nitems
msg queue size (not rings)
Definition: message_queue.h:53
u64 session_handle_t
static uword max_log2(uword x)
Definition: clib.h:191
void session_close(session_t *s)
Initialize session closing procedure.
Definition: session.c:1216
u64 uword
Definition: types.h:112
void segment_manager_del_sessions(segment_manager_t *sm)
Initiate disconnects for all sessions &#39;owned&#39; by a segment manager.
uword clib_valloc_free(clib_valloc_main_t *vam, uword baseva)
Free virtual space.
Definition: valloc.c:228
struct _segment_manager segment_manager_t
u32 default_segment_size
default fifo segment size
segment_manager_t * segment_manager_get(u32 index)
struct _svm_queue svm_queue_t
void fifo_segment_update_free_bytes(fifo_segment_t *fs)
Update fifo segment free bytes estimate.
Definition: fifo_segment.c:934
u8 fifo_segment_has_fifos(fifo_segment_t *fs)
Definition: fifo_segment.c:963
u32 app_index
Index of owning app.
Definition: application.h:43
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
int fifo_segment_collect_fifo_chunks(fifo_segment_t *fs, svm_fifo_t *f)
Collect unused chunks for fifo.
Definition: fifo_segment.c:804
void clib_valloc_init(clib_valloc_main_t *vam, clib_valloc_chunk_t *template, int need_lock)
Initialize a virtual memory allocation arena.
Definition: valloc.c:129
int segment_manager_try_alloc_fifos(fifo_segment_t *fifo_segment, u32 thread_index, u32 rx_fifo_size, u32 tx_fifo_size, svm_fifo_t **rx_fifo, svm_fifo_t **tx_fifo)
static u32 vlib_num_workers()
Definition: threads.h:372
#define vec_foreach(var, vec)
Vector iterator.
static clib_error_t * segment_manager_show_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
int consumer_pid
pid of msg consumer
Definition: message_queue.h:52
int svm_msg_q_alloc_producer_eventfd(svm_msg_q_t *mq)
Allocate event fd for queue consumer.
format_function_t format_fifo_segment
Definition: fifo_segment.h:227
int fifo_segment_init(fifo_segment_t *fs)
Initialize fifo segment shared header.
Definition: fifo_segment.c:89
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:689
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
segment_manager_t * segment_manager_get_if_valid(u32 index)
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:171
int segment_manager_shrink_fifo(segment_manager_t *sm, svm_fifo_t *f, u32 size, u8 is_producer)
Request to shrink fifo owned by segment manager.
ssvm_segment_type_t ssvm_type(const ssvm_private_t *ssvm)
Definition: ssvm.c:443
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128