FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
fifo_segment.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <svm/fifo_segment.h>
17 #include <vppinfra/mem.h>
18 
19 static inline void *
21 {
22  uword cur_pos, cur_pos_align, new_pos;
23 
24  cur_pos = clib_atomic_load_relax_n (&fsh->byte_index);
25  cur_pos_align = round_pow2_u64 (cur_pos, align);
26  size = round_pow2_u64 (size, align);
27  new_pos = cur_pos_align + size;
28 
29  if (new_pos >= fsh->max_byte_index)
30  return 0;
31 
32  while (!clib_atomic_cmp_and_swap_acq_relax (&fsh->byte_index, &cur_pos,
33  &new_pos, 1 /* weak */))
34  {
35  cur_pos_align = round_pow2_u64 (cur_pos, align);
36  new_pos = cur_pos_align + size;
37  if (new_pos >= fsh->max_byte_index)
38  return 0;
39  }
40  return uword_to_pointer ((u8 *) fsh + cur_pos_align, void *);
41 }
42 
43 static inline void *
45 {
46  return fsh_alloc_aligned (fsh, size, 8);
47 }
48 
49 static inline fifo_segment_slice_t *
51 {
52  return &fsh->slices[slice_index];
53 }
54 
55 static inline fifo_slice_private_t *
57 {
58  ASSERT (slice_index < fs->n_slices);
59  return &fs->slices[slice_index];
60 }
61 
63 #define _(sym,str) str,
65 #undef _
66 };
67 
68 static inline uword
70 {
71  uword cur_pos = clib_atomic_load_relax_n (&fsh->byte_index);
72  ASSERT (fsh->max_byte_index > cur_pos);
73  return fsh->max_byte_index - cur_pos;
74 }
75 
76 static inline void
78 {
80 }
81 
82 static inline void
84 {
86 }
87 
88 static inline uword
90 {
92  return n_cached;
93 }
94 
95 static inline void
97 {
99 }
100 
101 static inline u32
103 {
105 }
106 
107 static inline uword
109 {
111  uword total_vm = 0;
112  int i;
113 
114  for (i = 0; i < fsh->n_slices; i++)
115  {
116  fss = fsh_slice_get (fsh, i);
117  total_vm += clib_atomic_load_relax_n (&fss->virtual_mem);
118  }
119  return total_vm;
120 }
121 
122 void
124  int n_bytes)
125 {
126  fifo_segment_slice_t *fss = fsh_slice_get (fsh, slice_index);
127  fss->virtual_mem += n_bytes;
128 }
129 
130 static inline int
132 {
133  return (fl_index < FS_CHUNK_VEC_LEN);
134 }
135 
136 #define FS_CL_HEAD_MASK 0xFFFFFFFFFFFF
137 #define FS_CL_HEAD_TMASK 0xFFFF000000000000
138 #define FS_CL_HEAD_TINC (1ULL << 48)
139 
140 static svm_fifo_chunk_t *
142  fifo_segment_slice_t *fss, u32 fl_index)
143 {
144  fs_sptr_t headsp = clib_atomic_load_relax_n (&fss->free_chunks[fl_index]);
145  return fs_chunk_ptr (fsh, headsp & FS_CL_HEAD_MASK);
146 }
147 
148 static void
150  fifo_segment_slice_t *fss, u32 fl_index,
152 {
153  fs_sptr_t old_head, new_head, csp;
154 
155  csp = fs_chunk_sptr (fsh, c);
156  ASSERT (csp <= FS_CL_HEAD_MASK);
157  old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]);
158 
159  do
160  {
161  c->next = old_head & FS_CL_HEAD_MASK;
162  new_head = csp + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK);
163  }
164  while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head,
165  &new_head, 0 /* weak */, __ATOMIC_RELEASE,
166  __ATOMIC_ACQUIRE));
167 }
168 
169 static void
171  fifo_segment_slice_t *fss, u32 fl_index,
172  svm_fifo_chunk_t *head, svm_fifo_chunk_t *tail)
173 {
174  fs_sptr_t old_head, new_head, headsp;
175 
176  headsp = fs_chunk_sptr (fsh, head);
177  ASSERT (headsp <= FS_CL_HEAD_MASK);
178  old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]);
179 
180  do
181  {
182  tail->next = old_head & FS_CL_HEAD_MASK;
183  new_head = headsp + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK);
184  }
185  while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head,
186  &new_head, 0 /* weak */, __ATOMIC_RELEASE,
187  __ATOMIC_ACQUIRE));
188 }
189 
190 static svm_fifo_chunk_t *
192  u32 fl_index)
193 {
194  fs_sptr_t old_head, new_head;
196 
197  ASSERT (fss_chunk_fl_index_is_valid (fss, fl_index));
198 
199  old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]);
200 
201  /* Lock-free stacks are affected by ABA if a side allocates a chunk and
202  * shortly thereafter frees it. To circumvent that, reuse the upper bits
203  * of the head of the list shared pointer, i.e., offset to where the chunk
204  * is, as a tag. The tag is incremented with each push/pop operation and
205  * therefore collisions can only happen if an element is popped and pushed
206  * exactly after a complete wrap of the tag (16 bits). It's unlikely either
207  * of the sides will be descheduled for that long */
208  do
209  {
210  if (!(old_head & FS_CL_HEAD_MASK))
211  return 0;
212  c = fs_chunk_ptr (fsh, old_head & FS_CL_HEAD_MASK);
213  new_head = c->next + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK);
214  }
215  while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head,
216  &new_head, 0 /* weak */, __ATOMIC_RELEASE,
217  __ATOMIC_ACQUIRE));
218 
219  return c;
220 }
221 
222 static void
224  svm_fifo_shared_t *sf)
225 {
226  sf->next = fss->free_fifos;
227  fss->free_fifos = fs_sptr (fsh, sf);
228 }
229 
230 static void
234 {
235  tail->next = fss->free_fifos;
236  fss->free_fifos = fs_sptr (fsh, head);
237 }
238 
241 {
242  svm_fifo_shared_t *sf;
243  sf = fs_ptr (fsh, fss->free_fifos);
244  fss->free_fifos = sf->next;
245  return sf;
246 }
247 
248 static inline void
250 {
251  if (pfss->active_fifos)
252  {
253  pfss->active_fifos->prev = f;
254  f->next = pfss->active_fifos;
255  }
256  pfss->active_fifos = f;
257 }
258 
259 static inline void
261 {
263  {
264  if (f->prev)
265  f->prev->next = f->next;
266  else
267  pfss->active_fifos = f->next;
268  if (f->next)
269  f->next->prev = f->prev;
270  }
271 }
272 
273 static inline uword
275 {
277 }
278 
279 static inline void
281 {
283 }
284 
285 static inline void
287 {
289 }
290 
291 /**
292  * Initialize fifo segment shared header
293  */
294 int
296 {
297  u32 align = 8, offset = 2 * 4096, slices_sz, i;
298  uword max_fifo, seg_start, seg_sz;
301  void *seg_data;
302 
303  /* TODO remove ssvm heap entirely */
304  sh = fs->ssvm.sh;
305 
306  seg_data = (u8 *) sh + offset;
307  seg_sz = sh->ssvm_size - offset;
308 
309  fs->n_slices = clib_max (fs->n_slices, 1);
310  slices_sz = sizeof (fifo_segment_slice_t) * fs->n_slices;
311 
312  seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align);
313  fsh = uword_to_pointer (seg_start, void *);
314  CLIB_MEM_UNPOISON (fsh, seg_sz);
315  memset (fsh, 0, sizeof (*fsh) + slices_sz);
316 
317  fsh->byte_index = sizeof (*fsh) + slices_sz;
318  fsh->max_byte_index = seg_sz;
319  fsh->n_slices = fs->n_slices;
320  max_fifo = clib_min ((seg_sz - slices_sz) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE);
321  fsh->max_log2_fifo_size = min_log2 (max_fifo);
322  fsh->n_cached_bytes = 0;
323  fsh->n_reserved_bytes = fsh->byte_index;
324  fsh->start_byte_index = fsh->byte_index;
325  ASSERT (fsh->max_byte_index <= sh->ssvm_size - offset);
326 
327  fs->max_byte_index = fsh->max_byte_index;
328  fs->h = fsh;
329  sh->opaque[0] = (void *) ((u8 *) fsh - (u8 *) fs->ssvm.sh);
330 
331  /* Allow random offsets */
332  fs->ssvm.sh->ssvm_va = 0;
333 
334  vec_validate (fs->slices, fs->n_slices - 1);
335  for (i = 0; i < fs->n_slices; i++)
336  fs->slices[i].fifos =
338 
339  sh->ready = 1;
340  return (0);
341 }
342 
343 /**
344  * Create a fifo segment and initialize as master
345  */
346 int
348 {
349  fifo_segment_t *fs;
350  uword baseva;
351  int rv;
352 
353  /* Allocate a fresh segment */
354  pool_get_zero (sm->segments, fs);
355 
356  baseva = a->segment_type == SSVM_SEGMENT_PRIVATE ? ~0ULL : sm->next_baseva;
357  fs->ssvm.ssvm_size = a->segment_size;
358  fs->ssvm.is_server = 1;
359  fs->ssvm.my_pid = getpid ();
360  fs->ssvm.name = format (0, "%s%c", a->segment_name, 0);
361  fs->ssvm.requested_va = baseva;
362 
363  if ((rv = ssvm_server_init (&fs->ssvm, a->segment_type)))
364  {
365  pool_put (sm->segments, fs);
366  return (rv);
367  }
368 
369  /* Note: requested_va updated due to seg base addr randomization */
370  sm->next_baseva = fs->ssvm.sh->ssvm_va + fs->ssvm.ssvm_size;
371 
372  fifo_segment_init (fs);
373  vec_add1 (a->new_segment_indices, fs - sm->segments);
374  return (0);
375 }
376 
377 /**
378  * Attach as slave to a fifo segment
379  */
380 int
382 {
384  fifo_segment_t *fs;
385  int rv;
386 
387  pool_get_zero (sm->segments, fs);
388 
389  fs->ssvm.ssvm_size = a->segment_size;
390  fs->ssvm.my_pid = getpid ();
391  fs->ssvm.name = format (0, "%s%c", a->segment_name, 0);
392  fs->ssvm.requested_va = 0;
393  if (a->segment_type == SSVM_SEGMENT_MEMFD)
394  fs->ssvm.fd = a->memfd_fd;
395  else
397 
398  if ((rv = ssvm_client_init (&fs->ssvm, a->segment_type)))
399  {
400  pool_put (sm->segments, fs);
401  return (rv);
402  }
403 
404  /* Probably a segment without fifos */
405  if (!fs->ssvm.sh->opaque[0])
406  goto done;
407 
408  fsh = fs->h = (void *) fs->ssvm.sh + (uword) fs->ssvm.sh->opaque[0];
409  fs->max_byte_index = fsh->max_byte_index;
410  vec_validate (fs->slices, 0);
411  fs->slices[0].fifos =
413 
414 done:
415  vec_add1 (a->new_segment_indices, fs - sm->segments);
416  return (0);
417 }
418 
419 void
421 {
423  ssvm_delete (&s->ssvm);
424  clib_memset (s, 0xfe, sizeof (*s));
425  pool_put (sm->segments, s);
426 }
427 
428 u32
430 {
431  return s - sm->segments;
432 }
433 
436 {
437  return pool_elt_at_index (sm->segments, segment_index);
438 }
439 
442 {
443  if (pool_is_free_index (sm->segments, segment_index))
444  return 0;
445  return pool_elt_at_index (sm->segments, segment_index);
446 }
447 
448 void
449 fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size)
450 {
451  *address = (char *) seg->ssvm.sh->ssvm_va;
452  *size = seg->ssvm.ssvm_size;
453 }
454 
455 void
457  u32 timeout_in_seconds)
458 {
459  sm->next_baseva = baseva;
460  sm->timeout_in_seconds = timeout_in_seconds;
461 }
462 
463 static inline u32
465 {
467  return 0;
469  FS_CHUNK_VEC_LEN - 1);
470 }
471 
472 static inline u32
474 {
475  return 1 << (fl_index + FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE);
476 }
477 
478 static inline int
480 {
481  /*
482  * 4K minimum. It's not likely that anything good will happen
483  * with a smaller FIFO.
484  */
485  return size >= FIFO_SEGMENT_MIN_FIFO_SIZE &&
486  size <= (1ULL << fsh->max_log2_fifo_size);
487 }
488 
491  fifo_segment_slice_t * fss, u32 data_bytes)
492 {
493  u32 fl_index, fl_size, n_alloc = 0, req_bytes = data_bytes;
494  svm_fifo_chunk_t *c, *first = 0, *next;
495 
496  fl_index = fs_freelist_for_size (req_bytes);
497  if (fl_index > 0)
498  fl_index -= 1;
499 
500  fl_size = fs_freelist_index_to_size (fl_index);
501 
502  while (req_bytes)
503  {
504  c = fss_chunk_free_list_pop (fsh, fss, fl_index);
505  if (c)
506  {
507  c->next = fs_chunk_sptr (fsh, first);
508  first = c;
509  n_alloc += fl_size;
510  req_bytes -= clib_min (fl_size, req_bytes);
511  }
512  else
513  {
514  /* Failed to allocate with smaller chunks */
515  if (fl_index == 0)
516  {
517  /* Free all chunks if any allocated */
518  c = first;
519  while (c)
520  {
521  fl_index = fs_freelist_for_size (c->length);
522  next = fs_chunk_ptr (fsh, c->next);
523  fss_chunk_free_list_push (fsh, fss, fl_index, c);
524  c = next;
525  }
526  n_alloc = 0;
527  first = 0;
528  /* As last attempt, try allocating a chunk larger than
529  * the requested size, if possible */
530  fl_index = fs_freelist_for_size (data_bytes) + 1;
531  if (!fss_chunk_fl_index_is_valid (fss, fl_index))
532  return 0;
533  first = fss_chunk_free_list_pop (fsh, fss, fl_index);
534  if (first)
535  {
536  first->next = 0;
537  n_alloc = fs_freelist_index_to_size (fl_index);
538  goto done;
539  }
540  return 0;
541  }
542  fl_index -= 1;
543  fl_size = fl_size >> 1;
544  }
545  }
546 
547 done:
548  fss_fl_chunk_bytes_sub (fss, n_alloc);
549  fsh_cached_bytes_sub (fsh, n_alloc);
550  return first;
551 }
552 
553 static int
555  fifo_segment_slice_t * fss, u32 batch_size)
556 {
557  svm_fifo_shared_t *f, *head = 0, *tail;
558  uword size;
559  u8 *fmem;
560  int i;
561 
562  ASSERT (batch_size != 0);
563 
564  size = (uword) sizeof (*f) * batch_size;
565 
567  if (fmem == 0)
568  return -1;
569 
570  /* Carve fifo hdr space */
571  tail = f = (svm_fifo_shared_t *) fmem;
572  for (i = 0; i < batch_size; i++)
573  {
574  clib_memset (f, 0, sizeof (*f));
575  f->next = fs_sptr (fsh, head);
576  head = f;
577  fmem += sizeof (*f);
578  f = (svm_fifo_shared_t *) fmem;
579  }
580 
581  fss_fifo_free_list_push_list (fsh, fss, head, tail);
582 
583  return 0;
584 }
585 
586 static int
588  fifo_segment_slice_t * fss,
589  u32 fl_index, u32 batch_size)
590 {
591  svm_fifo_chunk_t *c, *head = 0, *tail;
592  uword size, total_chunk_bytes;
593  u32 rounded_data_size;
594  u8 *cmem;
595  int i;
596 
597  ASSERT (batch_size != 0);
598 
599  rounded_data_size = fs_freelist_index_to_size (fl_index);
600  total_chunk_bytes = (uword) batch_size *rounded_data_size;
601  size = (uword) (sizeof (*c) + rounded_data_size) * batch_size;
602 
603  cmem = fsh_alloc_aligned (fsh, size, 8 /* chunk hdr is 24B */);
604  if (cmem == 0)
605  return -1;
606 
607  /* Carve fifo + chunk space */
608  tail = c = (svm_fifo_chunk_t *) cmem;
609  for (i = 0; i < batch_size; i++)
610  {
611  c->start_byte = 0;
612  c->length = rounded_data_size;
613  c->next = fs_chunk_sptr (fsh, head);
614  head = c;
615  cmem += sizeof (*c) + rounded_data_size;
616  c = (svm_fifo_chunk_t *) cmem;
617  }
618 
619  fss_chunk_free_list_push_list (fsh, fss, fl_index, head, tail);
620  fss->num_chunks[fl_index] += batch_size;
621  fss_fl_chunk_bytes_add (fss, total_chunk_bytes);
622  fsh_cached_bytes_add (fsh, total_chunk_bytes);
623 
624  return 0;
625 }
626 
627 static int
629  fifo_segment_slice_t * fss,
630  u32 fl_index, u32 batch_size)
631 {
632  if (fsh_try_alloc_fifo_hdr_batch (fsh, fss, batch_size))
633  return 0;
634  return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size);
635 }
636 
637 static svm_fifo_shared_t *
639 {
640  svm_fifo_shared_t *sf;
641 
642  if (!fss->free_fifos)
643  {
644  if (fsh_try_alloc_fifo_hdr_batch (fsh, fss,
646  return 0;
647  }
648 
649  sf = fss_fifo_free_list_pop (fsh, fss);
650  clib_memset (sf, 0, sizeof (*sf));
651 
652  return sf;
653 }
654 
655 static svm_fifo_chunk_t *
657  fifo_segment_slice_t * fss, u32 data_bytes)
658 {
660  u32 fl_index;
661 
662  fl_index = fs_freelist_for_size (data_bytes);
663 
664 free_list:
665  c = fss_chunk_free_list_pop (fsh, fss, fl_index);
666  if (c)
667  {
668  c->next = 0;
671  }
672  else
673  {
674  u32 chunk_size, batch = FIFO_SEGMENT_ALLOC_BATCH_SIZE;
675  uword n_free;
676 
677  chunk_size = fs_freelist_index_to_size (fl_index);
678  n_free = fsh_n_free_bytes (fsh);
679 
680  if (chunk_size <= n_free)
681  {
682  batch = chunk_size * batch <= n_free ? batch : 1;
683  if (!fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch))
684  goto free_list;
685  }
686  /* Failed to allocate larger chunk, try to allocate multi-chunk
687  * that is close to what was actually requested */
688  if (data_bytes <= fss_fl_chunk_bytes (fss))
689  {
690  c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes);
691  if (c)
692  goto done;
694  if (!batch || fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
695  goto done;
696  }
697  if (data_bytes <= fss_fl_chunk_bytes (fss) + n_free)
698  {
699  u32 min_size = FIFO_SEGMENT_MIN_FIFO_SIZE;
700  if (n_free < min_size)
701  goto done;
702  batch = (data_bytes - fss_fl_chunk_bytes (fss)) / min_size;
703  batch = clib_min (batch + 1, n_free / min_size);
704  if (fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
705  goto done;
706  c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes);
707  }
708  }
709 
710 done:
711 
712  return c;
713 }
714 
715 /**
716  * Try to allocate new fifo
717  *
718  * Tries the following steps in order:
719  * - grab fifo and chunk from freelists
720  * - batch fifo and chunk allocation
721  * - single fifo allocation
722  * - grab multiple fifo chunks from freelists
723  */
724 static svm_fifo_shared_t *
725 fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
726 {
728  u32 fl_index, min_size;
730  svm_fifo_shared_t *sf = 0;
731 
732  fss = fsh_slice_get (fsh, slice_index);
733  min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096);
734  fl_index = fs_freelist_for_size (min_size);
735 
736  if (!fss_chunk_fl_index_is_valid (fss, fl_index))
737  return 0;
738 
739  sf = fsh_try_alloc_fifo_hdr (fsh, fss);
740  if (!sf)
741  return 0;
742 
743  c = fsh_try_alloc_chunk (fsh, fss, min_size);
744  if (!c)
745  {
746  fss_fifo_free_list_push (fsh, fss, sf);
747  return 0;
748  }
749 
750  sf->start_chunk = fs_chunk_sptr (fsh, c);
751  while (c->next)
752  c = fs_chunk_ptr (fsh, c->next);
753  sf->end_chunk = fs_chunk_sptr (fsh, c);
754  sf->size = data_bytes;
755  sf->slice_index = slice_index;
756 
757  return sf;
758 }
759 
761 fsh_alloc_chunk (fifo_segment_header_t * fsh, u32 slice_index, u32 chunk_size)
762 {
765 
766  fss = fsh_slice_get (fsh, slice_index);
767  c = fsh_try_alloc_chunk (fsh, fss, chunk_size);
768 
769  return c;
770 }
771 
772 static void
775 {
776  u32 n_collect = 0, fl_index;
778 
779  while (c)
780  {
781  CLIB_MEM_UNPOISON (c, sizeof (*c));
782  next = fs_chunk_ptr (fsh, c->next);
783  fl_index = fs_freelist_for_size (c->length);
784  fss_chunk_free_list_push (fsh, fss, fl_index, c);
785  n_collect += fs_freelist_index_to_size (fl_index);
786  c = next;
787  }
788 
789  fss_fl_chunk_bytes_add (fss, n_collect);
790  fsh_cached_bytes_add (fsh, n_collect);
791 }
792 
793 void
796 {
798  fss = fsh_slice_get (fsh, slice_index);
799  fsh_slice_collect_chunks (fsh, fss, c);
800 }
801 
802 svm_fifo_t *
803 fs_fifo_alloc (fifo_segment_t *fs, u32 slice_index)
804 {
805  fifo_slice_private_t *pfss = &fs->slices[slice_index];
806  svm_fifo_t *f;
807 
808  f = clib_mem_bulk_alloc (pfss->fifos);
809  clib_memset (f, 0, sizeof (*f));
810  return f;
811 }
812 
813 void
815 {
816  fifo_slice_private_t *pfss;
817 
818  if (CLIB_DEBUG)
819  clib_memset (f, 0xfc, sizeof (*f));
820 
821  pfss = &fs->slices[slice_index];
822  clib_mem_bulk_free (pfss->fifos, f);
823 }
824 
825 void
827 {
828  int slice_index;
829  svm_msg_q_t *mq = 0;
830 
831  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
832  clib_mem_bulk_destroy (fs->slices[slice_index].fifos);
833 
834  vec_free (fs->slices);
835 
836  vec_foreach (fs->mqs, mq)
837  svm_msg_q_cleanup (mq);
838 
839  vec_free (fs->mqs);
840 }
841 
842 /**
843  * Allocate fifo in fifo segment
844  */
845 svm_fifo_t *
847  u32 data_bytes, fifo_segment_ftype_t ftype)
848 {
849  fifo_segment_header_t *fsh = fs->h;
850  fifo_slice_private_t *pfss;
852  svm_fifo_shared_t *sf;
853  svm_fifo_t *f = 0;
854 
855  ASSERT (slice_index < fs->n_slices);
856 
857  if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size))
858  return 0;
859 
860  sf = fs_try_alloc_fifo (fsh, slice_index, data_bytes);
861  if (!sf)
862  goto done;
863 
864  f = fs_fifo_alloc (fs, slice_index);
865  f->fs_hdr = fsh;
866  f->shr = sf;
867 
868  svm_fifo_init (f, data_bytes);
869 
870  fss = fsh_slice_get (fsh, slice_index);
871  pfss = fs_slice_private_get (fs, slice_index);
872 
873  /* If rx fifo type add to active fifos list. When cleaning up segment,
874  * we need a list of active sessions that should be disconnected. Since
875  * both rx and tx fifos keep pointers to the session, it's enough to track
876  * only one. */
877  if (ftype == FIFO_SEGMENT_RX_FIFO)
878  {
881  }
882 
883  fsh_active_fifos_update (fsh, 1);
884  fss->virtual_mem += svm_fifo_size (f);
885 
886 done:
887  return (f);
888 }
889 
890 svm_fifo_t *
892 {
893  svm_fifo_t *f = fs_fifo_alloc (fs, 0);
894  svm_fifo_shared_t *sf;
895 
896  sf = (svm_fifo_shared_t *) ((u8 *) fs->h + offset);
897  f->fs_hdr = fs->h;
898  f->shr = sf;
899 
900  f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
901  f->segment_index = SVM_FIFO_INVALID_INDEX;
902  f->refcnt = 1;
903  return f;
904 }
905 
906 svm_fifo_t *
908 {
909  svm_fifo_t *nf = fs_fifo_alloc (fs, 0);
910  clib_memcpy (nf, f, sizeof (*f));
911  return nf;
912 }
913 
914 /**
915  * Free fifo allocated in fifo segment
916  */
917 void
919 {
920  fifo_segment_header_t *fsh = fs->h;
921  fifo_slice_private_t *pfss;
923  svm_fifo_shared_t *sf;
924 
925  ASSERT (f->refcnt > 0);
926 
927  if (--f->refcnt > 0)
928  return;
929 
930  /*
931  * Cleanup shared state
932  */
933 
934  sf = f->shr;
935  fss = fsh_slice_get (fsh, sf->slice_index);
936  pfss = fs_slice_private_get (fs, sf->slice_index);
937 
938  /* Free fifo chunks */
939  fsh_slice_collect_chunks (fsh, fss, fs_chunk_ptr (fsh, f->shr->start_chunk));
940 
941  sf->start_chunk = sf->end_chunk = 0;
942  sf->head_chunk = sf->tail_chunk = 0;
943 
944  /* Add to free list */
945  fss_fifo_free_list_push (fsh, fss, sf);
946 
947  fss->virtual_mem -= svm_fifo_size (f);
948 
949  /*
950  * Cleanup private state
951  */
952 
953  /* Remove from active list. Only rx fifos are tracked */
955  {
958  }
959 
962 
963  if (CLIB_DEBUG)
964  {
965  sf->master_session_index = ~0;
966  f->master_thread_index = ~0;
967  }
968 
969  f->ooo_enq = f->ooo_deq = 0;
970  f->prev = 0;
971 
972  fs_fifo_free (fs, f, f->shr->slice_index);
973 
974  fsh_active_fifos_update (fsh, -1);
975 }
976 
977 void
979 {
980  fs_fifo_free (fs, f, 0 /* clients attach fifos in slice 0 */);
981 }
982 
983 void
985 {
986  fifo_slice_private_t *pfss;
988  svm_fifo_t *of = *f;
989  u32 slice_index;
990 
991  slice_index = of->master_thread_index;
992  fss = fsh_slice_get (fs->h, slice_index);
993  pfss = fs_slice_private_get (fs, slice_index);
994  fss->virtual_mem -= svm_fifo_size (of);
995  if (of->flags & SVM_FIFO_F_LL_TRACKED)
996  pfss_fifo_del_active_list (pfss, of);
997 
998  /* Collect chunks that were provided in return for those detached */
999  fsh_slice_collect_chunks (fs->h, fss, of->chunks_at_attach);
1000  of->chunks_at_attach = 0;
1001 
1002  /* Collect hdr that was provided in return for the detached */
1003  fss_fifo_free_list_push (fs->h, fss, of->hdr_at_attach);
1004  of->hdr_at_attach = 0;
1005 
1006  clib_mem_bulk_free (pfss->fifos, *f);
1007  *f = 0;
1008 }
1009 
1010 void
1012 {
1013  svm_fifo_chunk_t *c, *nc, *pc = 0;
1014  fifo_slice_private_t *pfss;
1015  fifo_segment_slice_t *fss;
1016  svm_fifo_t *nf, *of;
1017 
1018  nf = fs_fifo_alloc (fs, slice_index);
1019  clib_memcpy_fast (nf, *f, sizeof (*nf));
1020 
1021  fss = fsh_slice_get (fs->h, slice_index);
1022  pfss = fs_slice_private_get (fs, slice_index);
1023  fss->virtual_mem += svm_fifo_size (nf);
1024  nf->next = nf->prev = 0;
1025  if (nf->flags & SVM_FIFO_F_LL_TRACKED)
1026  pfss_fifo_add_active_list (pfss, nf);
1027 
1028  /* Allocate shared hdr and chunks to be collected at detach in return
1029  * for those that are being attached now */
1030  of = *f;
1031  of->hdr_at_attach = fsh_try_alloc_fifo_hdr (fs->h, fss);
1032 
1033  c = fs_chunk_ptr (fs->h, nf->shr->start_chunk);
1034  of->chunks_at_attach = pc = fsh_try_alloc_chunk (fs->h, fss, c->length);
1035 
1036  while ((c = fs_chunk_ptr (fs->h, c->next)))
1037  {
1038  nc = fsh_try_alloc_chunk (fs->h, fss, c->length);
1039  pc->next = fs_chunk_sptr (fs->h, nc);
1040  pc = nc;
1041  }
1042 
1043  nf->shr->slice_index = slice_index;
1044  *f = nf;
1045 }
1046 
1047 uword
1049 {
1050  return (u8 *) f->shr - (u8 *) f->fs_hdr;
1051 }
1052 
1055  u32 chunk_size)
1056 {
1057  fifo_segment_header_t *fsh = fs->h;
1058  fifo_segment_slice_t *fss;
1059 
1060  fss = fsh_slice_get (fsh, slice_index);
1061  return fsh_try_alloc_chunk (fsh, fss, chunk_size);
1062 }
1063 
1064 void
1067 {
1068  fsh_collect_chunks (fs->h, slice_index, c);
1069 }
1070 
1071 uword
1073 {
1074  return (u8 *) c - (u8 *) fs->h;
1075 }
1076 
1077 svm_msg_q_t *
1079  svm_msg_q_cfg_t *cfg)
1080 {
1081  fifo_segment_header_t *fsh = fs->h;
1082  svm_msg_q_shared_t *smq;
1083  svm_msg_q_t *mq;
1084  void *base;
1085  u32 size;
1086 
1087  if (!fs->mqs)
1088  {
1089  u32 n_mqs = clib_max (fs->h->n_mqs, 1);
1090  vec_validate (fs->mqs, n_mqs - 1);
1091  }
1092 
1093  size = svm_msg_q_size_to_alloc (cfg);
1094  base = fsh_alloc_aligned (fsh, size, 8);
1095  fsh->n_reserved_bytes += size;
1096 
1097  smq = svm_msg_q_init (base, cfg);
1098  mq = vec_elt_at_index (fs->mqs, mq_index);
1099  svm_msg_q_attach (mq, smq);
1100 
1101  return mq;
1102 }
1103 
1104 svm_msg_q_t *
1106 {
1107  svm_msg_q_t *mq;
1108 
1109  if (!fs->mqs)
1110  {
1111  u32 n_mqs = clib_max (fs->h->n_mqs, 1);
1112  vec_validate (fs->mqs, n_mqs - 1);
1113  }
1114 
1115  mq = vec_elt_at_index (fs->mqs, mq_index);
1116 
1117  if (!mq->q.shr)
1118  {
1119  svm_msg_q_shared_t *smq;
1120  smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset);
1121  svm_msg_q_attach (mq, smq);
1122  }
1123 
1124  ASSERT (fifo_segment_msg_q_offset (fs, mq_index) == offset);
1125 
1126  return mq;
1127 }
1128 
1129 void
1131 {
1132  svm_msg_q_shared_t *smq;
1133  u32 n_mqs, size, i;
1134  uword offset = 0, n_alloced;
1135  svm_msg_q_t *mq;
1136 
1137  n_mqs = fs->h->n_mqs;
1138  if (n_fds && n_mqs != n_fds)
1139  {
1140  clib_warning ("expected %u fds got %u", n_mqs, n_fds);
1141  return;
1142  }
1143 
1144  vec_validate (fs->mqs, n_mqs - 1);
1145  n_alloced = fs->h->n_reserved_bytes - fs->h->start_byte_index;
1146  ASSERT (n_alloced % n_mqs == 0);
1147  size = n_alloced / n_mqs;
1148 
1149  offset = fs->h->start_byte_index;
1150  for (i = 0; i < n_mqs; i++)
1151  {
1152  mq = vec_elt_at_index (fs->mqs, i);
1153  smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset);
1154  svm_msg_q_attach (mq, smq);
1155  if (n_fds)
1156  svm_msg_q_set_eventfd (mq, fds[i]);
1157  offset += size;
1158  }
1159 }
1160 
1161 uword
1163 {
1164  svm_msg_q_t *mq = vec_elt_at_index (fs->mqs, mq_index);
1165 
1166  if (mq->q.shr == 0)
1167  return ~0ULL;
1168 
1169  return (uword) ((u8 *) mq->q.shr - (u8 *) fs->h) -
1170  sizeof (svm_msg_q_shared_t);
1171 }
1172 
1173 int
1175  u32 batch_size)
1176 {
1177  fifo_segment_header_t *fsh = fs->h;
1178  fifo_segment_slice_t *fss;
1179 
1180  fss = fsh_slice_get (fsh, slice_index);
1181  return fsh_try_alloc_fifo_hdr_batch (fsh, fss, batch_size);
1182 }
1183 
1184 int
1186  u32 chunk_size, u32 batch_size)
1187 {
1188  fifo_segment_header_t *fsh = fs->h;
1189  fifo_segment_slice_t *fss;
1190  u32 fl_index;
1191 
1192  if (!fs_chunk_size_is_valid (fsh, chunk_size))
1193  {
1194  clib_warning ("chunk size out of range %d", chunk_size);
1195  return -1;
1196  }
1197 
1198  fl_index = fs_freelist_for_size (chunk_size);
1199  fss = fsh_slice_get (fsh, slice_index);
1200 
1201  return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size);
1202 }
1203 
1204 /**
1205  * Pre-allocates fifo pairs in fifo segment
1206  */
1207 void
1209  u32 rx_fifo_size, u32 tx_fifo_size,
1210  u32 * n_fifo_pairs)
1211 {
1212  u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc;
1213  u32 hdrs, pairs_per_slice, alloc_now;
1214  fifo_segment_header_t *fsh = fs->h;
1215  int rx_fl_index, tx_fl_index, i;
1216  fifo_segment_slice_t *fss;
1217  uword space_available;
1218 
1219  /* Parameter check */
1220  if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
1221  return;
1222 
1223  if (!fs_chunk_size_is_valid (fsh, rx_fifo_size))
1224  {
1225  clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
1226  return;
1227  }
1228 
1229  if (!fs_chunk_size_is_valid (fsh, tx_fifo_size))
1230  {
1231  clib_warning ("tx fifo_size out of range %d", tx_fifo_size);
1232  return;
1233  }
1234 
1235  rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size)));
1236  rx_fl_index = fs_freelist_for_size (rx_fifo_size);
1237  tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size)));
1238  tx_fl_index = fs_freelist_for_size (tx_fifo_size);
1239 
1240  hdrs = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t);
1241 
1242  /* Calculate space requirements */
1243  pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size;
1244  space_available = fsh_n_free_bytes (fsh);
1245  pairs_to_alloc = space_available / pair_size;
1246  pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs);
1247  pairs_per_slice = pairs_to_alloc / fs->n_slices;
1248  pairs_per_slice += pairs_to_alloc % fs->n_slices ? 1 : 0;
1249 
1250  if (!pairs_per_slice)
1251  return;
1252 
1253  for (i = 0; i < fs->n_slices; i++)
1254  {
1255  alloc_now = clib_min (pairs_per_slice, *n_fifo_pairs);
1256  if (0 == alloc_now)
1257  break;
1258 
1259  fss = fsh_slice_get (fsh, i);
1260  if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, alloc_now))
1261  clib_warning ("rx prealloc failed: pairs %u", alloc_now);
1262  if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, alloc_now))
1263  clib_warning ("tx prealloc failed: pairs %u", alloc_now);
1264 
1265  /* Account for the pairs allocated */
1266  *n_fifo_pairs -= alloc_now;
1267  }
1268 }
1269 
1270 /**
1271  * Get number of active fifos
1272  */
1273 u32
1275 {
1276  return fsh_n_active_fifos (fs->h);
1277 }
1278 
1279 static u32
1281 {
1283  u32 count = 0;
1284 
1285  f = fs_ptr (fsh, fss->free_fifos);
1286  if (f == 0)
1287  return 0;
1288 
1289  while (f)
1290  {
1291  f = fs_ptr (fsh, f->next);
1292  count++;
1293  }
1294  return count;
1295 }
1296 
1297 u32
1299 {
1300  fifo_segment_header_t *fsh = fs->h;
1301  fifo_segment_slice_t *fss;
1302  int slice_index;
1303  u32 count = 0;
1304 
1305  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
1306  {
1307  fss = fsh_slice_get (fsh, slice_index);
1308  count += fs_slice_num_free_fifos (fsh, fss);
1309  }
1310  return count;
1311 }
1312 
1313 static u32
1316 {
1317  u32 count = 0, rounded_size, fl_index;
1319  int i;
1320 
1321  /* Count all free chunks? */
1322  if (size == ~0)
1323  {
1324  for (i = 0; i < FS_CHUNK_VEC_LEN; i++)
1325  {
1326  c = fss_chunk_free_list_head (fsh, fss, i);
1327  if (c == 0)
1328  continue;
1329 
1330  while (c)
1331  {
1332  c = fs_chunk_ptr (fsh, c->next);
1333  count++;
1334  }
1335  }
1336  return count;
1337  }
1338 
1339  rounded_size = (1 << (max_log2 (size)));
1340  fl_index = fs_freelist_for_size (rounded_size);
1341 
1342  if (fl_index >= FS_CHUNK_VEC_LEN)
1343  return 0;
1344 
1345  c = fss_chunk_free_list_head (fsh, fss, fl_index);
1346  if (c == 0)
1347  return 0;
1348 
1349  while (c)
1350  {
1351  c = fs_chunk_ptr (fsh, c->next);
1352  count++;
1353  }
1354  return count;
1355 }
1356 
1357 u32
1359 {
1360  fifo_segment_header_t *fsh = fs->h;
1361  fifo_segment_slice_t *fss;
1362  int slice_index;
1363  u32 count = 0;
1364 
1365  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
1366  {
1367  fss = fsh_slice_get (fsh, slice_index);
1368  count += fs_slice_num_free_chunks (fsh, fss, size);
1369  }
1370  return count;
1371 }
1372 
1373 uword
1375 {
1376  return fs->h->max_byte_index - fs->h->n_reserved_bytes;
1377 }
1378 
1379 u8
1381 {
1382  return (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT) ? 1 : 0;
1383 }
1384 
1385 void
1387 {
1389 }
1390 
1391 void *
1393 {
1394  void *rv = fsh_alloc (fs->h, size);
1395  /* Mark externally allocated bytes as reserved. This helps
1396  * @ref fifo_segment_size report bytes used only for fifos */
1397  fs->h->n_reserved_bytes += size;
1398  return rv;
1399 }
1400 
1401 uword
1403 {
1404  return fsh_n_free_bytes (fs->h);
1405 }
1406 
1407 uword
1409 {
1410  return fsh_n_cached_bytes (fs->h);
1411 }
1412 
1413 uword
1415 {
1416  return fsh_n_free_bytes (fs->h) + fsh_n_cached_bytes (fs->h);
1417 }
1418 
1419 uword
1421 {
1422  fifo_segment_header_t *fsh = fs->h;
1423  fifo_segment_slice_t *fss;
1424  uword n_bytes = 0;
1425  int slice_index;
1426 
1427  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
1428  {
1429  fss = fsh_slice_get (fsh, slice_index);
1430  n_bytes += fss_fl_chunk_bytes (fss);
1431  }
1432 
1433  return n_bytes;
1434 }
1435 
1436 u8
1438 {
1439  return (fsh_n_active_fifos (fs->h) != 0);
1440 }
1441 
1442 svm_fifo_t *
1444 {
1445  fifo_slice_private_t *pfss;
1446 
1447  pfss = fs_slice_private_get (fs, slice_index);
1448  return pfss->active_fifos;
1449 }
1450 
1451 u8
1453 {
1454  uword size, in_use;
1455 
1456  size = fifo_segment_size (fs);
1457  in_use =
1459  return (in_use * 100) / size;
1460 }
1461 
1464 {
1465  if (!fsh->high_watermark || !fsh->low_watermark)
1466  return MEMORY_PRESSURE_NO_PRESSURE;
1467 
1468  /* once the no-memory is detected, the status continues
1469  * until memory usage gets below the high watermark
1470  */
1471  if (fsh_has_reached_mem_limit (fsh))
1472  {
1473  if (usage >= fsh->high_watermark)
1474  return MEMORY_PRESSURE_NO_MEMORY;
1475  else
1476  fsh_reset_mem_limit (fsh);
1477  }
1478 
1479  if (usage >= fsh->high_watermark)
1480  return MEMORY_PRESSURE_HIGH_PRESSURE;
1481 
1482  else if (usage >= fsh->low_watermark)
1483  return MEMORY_PRESSURE_LOW_PRESSURE;
1484 
1485  return MEMORY_PRESSURE_NO_PRESSURE;
1486 }
1487 
1490 {
1491  fifo_segment_header_t *fsh = fs->h;
1493 
1494  return fifo_segment_determine_status (fsh, usage);
1495 }
1496 
1497 u8 *
1498 format_fifo_segment_type (u8 * s, va_list * args)
1499 {
1500  fifo_segment_t *sp;
1501  sp = va_arg (*args, fifo_segment_t *);
1502  ssvm_segment_type_t st = ssvm_type (&sp->ssvm);
1503 
1504  if (st == SSVM_SEGMENT_PRIVATE)
1505  s = format (s, "%s", "private");
1506  else if (st == SSVM_SEGMENT_MEMFD)
1507  s = format (s, "%s", "memfd");
1508  else if (st == SSVM_SEGMENT_SHM)
1509  s = format (s, "%s", "shm");
1510  else
1511  s = format (s, "%s", "unknown");
1512  return s;
1513 }
1514 
1515 /**
1516  * Segment format function
1517  */
1518 u8 *
1519 format_fifo_segment (u8 * s, va_list * args)
1520 {
1521  u32 count, indent, active_fifos, free_fifos;
1522  fifo_segment_t *fs = va_arg (*args, fifo_segment_t *);
1523  int verbose __attribute__ ((unused)) = va_arg (*args, int);
1524  uword est_chunk_bytes, est_free_seg_bytes, free_chunks;
1525  uword chunk_bytes = 0, free_seg_bytes, chunk_size;
1526  uword tracked_cached_bytes;
1527  uword fifo_hdr = 0, reserved;
1528  fifo_segment_header_t *fsh;
1529  fifo_segment_slice_t *fss;
1531  u32 slice_index;
1532  char *address;
1533  size_t size;
1534  int i;
1535  uword allocated, in_use, virt;
1536  f64 usage;
1538 
1539  indent = format_get_indent (s) + 2;
1540 
1541  if (fs == 0)
1542  {
1543  s = format (s, "%-20s%10s%15s%15s%15s%15s", "Name", "Type",
1544  "HeapSize (M)", "ActiveFifos", "FreeFifos", "Address");
1545  return s;
1546  }
1547 
1548  fifo_segment_info (fs, &address, &size);
1549  active_fifos = fifo_segment_num_fifos (fs);
1550  free_fifos = fifo_segment_num_free_fifos (fs);
1551 
1552  s = format (s, "%-20v%10U%15llu%15u%15u%15llx", ssvm_name (&fs->ssvm),
1553  format_fifo_segment_type, fs, size >> 20ULL, active_fifos,
1554  free_fifos, address);
1555 
1556  if (!verbose)
1557  return s;
1558 
1559  fsh = fs->h;
1560 
1561  free_chunks = fifo_segment_num_free_chunks (fs, ~0);
1562  if (free_chunks)
1563  s =
1564  format (s, "\n\n%UFree/Allocated chunks by size:\n", format_white_space,
1565  indent + 2);
1566  else
1567  s = format (s, "\n");
1568 
1569  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
1570  {
1571  fss = fsh_slice_get (fsh, slice_index);
1572  for (i = 0; i < FS_CHUNK_VEC_LEN; i++)
1573  {
1574  c = fss_chunk_free_list_head (fsh, fss, i);
1575  if (c == 0 && fss->num_chunks[i] == 0)
1576  continue;
1577  count = 0;
1578  while (c)
1579  {
1580  c = fs_chunk_ptr (fsh, c->next);
1581  count++;
1582  }
1583 
1584  chunk_size = fs_freelist_index_to_size (i);
1585  s = format (s, "%U%-5u kB: %u/%u\n", format_white_space, indent + 2,
1586  chunk_size >> 10, count, fss->num_chunks[i]);
1587 
1588  chunk_bytes += count * chunk_size;
1589  }
1590  }
1591 
1592  fifo_hdr = free_fifos * sizeof (svm_fifo_t);
1593  est_chunk_bytes = fifo_segment_fl_chunk_bytes (fs);
1594  est_free_seg_bytes = fifo_segment_free_bytes (fs);
1595  free_seg_bytes = fifo_segment_free_bytes (fs);
1596  tracked_cached_bytes = fifo_segment_cached_bytes (fs);
1597  allocated = fifo_segment_size (fs);
1598  in_use = fifo_segment_size (fs) - est_free_seg_bytes - tracked_cached_bytes;
1599  usage = (100.0 * in_use) / allocated;
1600  mem_st = fifo_segment_get_mem_status (fs);
1601  virt = fsh_virtual_mem (fsh);
1602  reserved = fsh->n_reserved_bytes;
1603 
1604  s = format (s, "\n%Useg free bytes: %U (%lu) estimated: %U (%lu) reserved:"
1605  " %U (%lu)\n", format_white_space, indent + 2,
1606  format_memory_size, free_seg_bytes, free_seg_bytes,
1607  format_memory_size, est_free_seg_bytes, est_free_seg_bytes,
1608  format_memory_size, reserved, reserved);
1609  s = format (s, "%Uchunk free bytes: %U (%lu) estimated: %U (%lu) tracked:"
1610  " %U (%lu)\n", format_white_space, indent + 2,
1611  format_memory_size, chunk_bytes, chunk_bytes,
1612  format_memory_size, est_chunk_bytes, est_chunk_bytes,
1613  format_memory_size, tracked_cached_bytes, tracked_cached_bytes);
1614  s = format (s, "%Ufifo active: %u hdr free: %u bytes: %U (%u) \n",
1615  format_white_space, indent + 2, fsh->n_active_fifos, free_fifos,
1616  format_memory_size, fifo_hdr, fifo_hdr);
1617  s = format (s, "%Usegment usage: %.2f%% (%U / %U) virt: %U status: %s\n",
1619  in_use, format_memory_size, allocated, format_memory_size, virt,
1621  s = format (s, "\n");
1622 
1623  return s;
1624 }
1625 
1626 /*
1627  * fd.io coding-style-patch-verification: ON
1628  *
1629  * Local Variables:
1630  * eval: (c-set-style "gnu")
1631  * End:
1632  */
ssvm_private_t::requested_va
uword requested_va
Definition: ssvm.h:85
svm_msg_q_shared_
Definition: message_queue.h:65
fsh_alloc_chunk
svm_fifo_chunk_t * fsh_alloc_chunk(fifo_segment_header_t *fsh, u32 slice_index, u32 chunk_size)
Allocate chunks in fifo segment.
Definition: fifo_segment.c:761
fifo_segment_header_::max_byte_index
uword max_byte_index
Definition: fifo_types.h:151
fsh_try_alloc_fifo_hdr_batch
static int fsh_try_alloc_fifo_hdr_batch(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 batch_size)
Definition: fifo_segment.c:554
svm_fifo_shr_::master_session_index
u32 master_session_index
session layer session index
Definition: fifo_types.h:71
FS_CL_HEAD_TINC
#define FS_CL_HEAD_TINC
Definition: fifo_segment.c:138
fs_sptr
static fs_sptr_t fs_sptr(fifo_segment_header_t *fsh, void *p)
Definition: fifo_types.h:167
svm_fifo_size
static u32 svm_fifo_size(svm_fifo_t *f)
Definition: svm_fifo.h:754
fifo_segment_chunk_offset
uword fifo_segment_chunk_offset(fifo_segment_t *fs, svm_fifo_chunk_t *c)
Definition: fifo_segment.c:1072
fifo_segment_cached_bytes
uword fifo_segment_cached_bytes(fifo_segment_t *fs)
Fifo segment number of cached bytes.
Definition: fifo_segment.c:1408
fifo_segment_main_t::timeout_in_seconds
u32 timeout_in_seconds
Time to wait during attach.
Definition: fifo_segment.h:81
svm_fifo_shr_::size
u32 size
size of the fifo in bytes
Definition: fifo_types.h:70
fifo_segment_delete
void fifo_segment_delete(fifo_segment_main_t *sm, fifo_segment_t *s)
Definition: fifo_segment.c:420
fss_fifo_free_list_push_list
static void fss_fifo_free_list_push_list(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, svm_fifo_shared_t *head, svm_fifo_shared_t *tail)
Definition: fifo_segment.c:231
pfss_fifo_del_active_list
static void pfss_fifo_del_active_list(fifo_slice_private_t *pfss, svm_fifo_t *f)
Definition: fifo_segment.c:260
fsh_cached_bytes_sub
static void fsh_cached_bytes_sub(fifo_segment_header_t *fsh, uword size)
Definition: fifo_segment.c:83
fifo_segment_alloc_fifo_w_slice
svm_fifo_t * fifo_segment_alloc_fifo_w_slice(fifo_segment_t *fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype)
Allocate fifo in fifo segment.
Definition: fifo_segment.c:846
clib_memcpy
#define clib_memcpy(d, s, n)
Definition: string.h:197
fifo_segment_get_mem_usage
u8 fifo_segment_get_mem_usage(fifo_segment_t *fs)
Definition: fifo_segment.c:1452
fs_fifo_alloc
svm_fifo_t * fs_fifo_alloc(fifo_segment_t *fs, u32 slice_index)
Definition: fifo_segment.c:803
fsh_active_fifos_update
static void fsh_active_fifos_update(fifo_segment_header_t *fsh, int inc)
Definition: fifo_segment.c:96
fifo_segment_header_::slices
fifo_segment_slice_t slices[0]
Definition: fifo_types.h:154
ssvm_shared_header_t::ssvm_size
uword ssvm_size
Definition: ssvm.h:69
clib_max
#define clib_max(x, y)
Definition: clib.h:335
fifo_segment_attach_fifo
void fifo_segment_attach_fifo(fifo_segment_t *fs, svm_fifo_t **f, u32 slice_index)
Definition: fifo_segment.c:1011
fss_chunk_free_list_push_list
static void fss_chunk_free_list_push_list(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index, svm_fifo_chunk_t *head, svm_fifo_chunk_t *tail)
Definition: fifo_segment.c:170
fs_try_alloc_multi_chunk
svm_fifo_chunk_t * fs_try_alloc_multi_chunk(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 data_bytes)
Definition: fifo_segment.c:490
fs_ptr
static void * fs_ptr(fifo_segment_header_t *fsh, fs_sptr_t sp)
Definition: fifo_types.h:161
fifo_segment_get_segment
fifo_segment_t * fifo_segment_get_segment(fifo_segment_main_t *sm, u32 segment_index)
Definition: fifo_segment.c:435
f
vlib_frame_t * f
Definition: interface_output.c:1080
pointer_to_uword
static uword pointer_to_uword(const void *p)
Definition: types.h:131
pool_elt_at_index
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:553
fs_chunk_sptr
static fs_sptr_t fs_chunk_sptr(fifo_segment_header_t *fsh, svm_fifo_chunk_t *c)
Definition: fifo_types.h:179
ssvm_private_t::ssvm_size
uword ssvm_size
Definition: ssvm.h:84
svm_msg_q_init
svm_msg_q_shared_t * svm_msg_q_init(void *base, svm_msg_q_cfg_t *cfg)
Definition: message_queue.c:72
fss_chunk_free_list_pop
static svm_fifo_chunk_t * fss_chunk_free_list_pop(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index)
Definition: fifo_segment.c:191
fifo_segment_msg_q_offset
uword fifo_segment_msg_q_offset(fifo_segment_t *fs, u32 mq_index)
Message queue offset on segment.
Definition: fifo_segment.c:1162
fifo_segment_msg_q_attach
svm_msg_q_t * fifo_segment_msg_q_attach(fifo_segment_t *fs, uword offset, u32 mq_index)
Attach message queue at fifo segment offset.
Definition: fifo_segment.c:1105
fifo_segment_slice_::virtual_mem
uword virtual_mem
Slice sum of all fifo sizes.
Definition: fifo_types.h:126
fifo_segment_alloc_fifo_w_offset
svm_fifo_t * fifo_segment_alloc_fifo_w_offset(fifo_segment_t *fs, uword offset)
Definition: fifo_segment.c:891
usage
static void usage(void)
Definition: health_check.c:14
fifo_segment_header_::byte_index
uword byte_index
Definition: fifo_types.h:150
next
u16 * next
Definition: nat44_ei_out2in.c:718
svm_fifo_shr_::slice_index
u8 slice_index
segment slice for fifo
Definition: fifo_types.h:73
fifo_segment_header_::n_mqs
u8 n_mqs
Num mqs for mqs segment.
Definition: fifo_types.h:148
fifo_segment_t
Definition: fifo_segment.h:67
fifo_segment_header_::n_active_fifos
u32 n_active_fifos
Number of active fifos.
Definition: fifo_types.h:140
fifo_segment_info
void fifo_segment_info(fifo_segment_t *seg, char **address, size_t *size)
Definition: fifo_segment.c:449
fifo_segment_detach_fifo
void fifo_segment_detach_fifo(fifo_segment_t *fs, svm_fifo_t **f)
Definition: fifo_segment.c:984
fs_freelist_index_to_size
static u32 fs_freelist_index_to_size(u32 fl_index)
Definition: fifo_segment.c:473
svm_fifo_shr_::tail_chunk
fs_sptr_t tail_chunk
tracks chunk where tail lands
Definition: fifo_types.h:84
svm_fifo_chunk_
Definition: fifo_types.h:38
fifo_segment_num_free_fifos
u32 fifo_segment_num_free_fifos(fifo_segment_t *fs)
Definition: fifo_segment.c:1298
first
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
ssvm_shared_header_t::opaque
void * opaque[SSVM_N_OPAQUE]
Definition: ssvm.h:73
FS_CL_HEAD_MASK
#define FS_CL_HEAD_MASK
Definition: fifo_segment.c:136
fifo_segment_num_free_chunks
u32 fifo_segment_num_free_chunks(fifo_segment_t *fs, u32 size)
Find number of free chunks of given size.
Definition: fifo_segment.c:1358
clib_mem_bulk_free
void clib_mem_bulk_free(clib_mem_bulk_handle_t h, void *p)
Definition: mem_bulk.c:179
pool_put
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:305
fifo_segment_main_t::next_baseva
uword next_baseva
Where to put the next one.
Definition: fifo_segment.h:80
fifo_segment_header_::n_cached_bytes
uword n_cached_bytes
Cached bytes.
Definition: fifo_types.h:139
svm_fifo_shr_
Definition: fifo_types.h:63
fifo_segment_prealloc_fifo_chunks
int fifo_segment_prealloc_fifo_chunks(fifo_segment_t *fs, u32 slice_index, u32 chunk_size, u32 batch_size)
Try to preallocate fifo chunks on segment.
Definition: fifo_segment.c:1185
ssvm_private_t::sh
ssvm_shared_header_t * sh
Definition: ssvm.h:83
ssvm_shared_header_t::ssvm_va
uword ssvm_va
Definition: ssvm.h:67
format_fifo_segment_type
u8 * format_fifo_segment_type(u8 *s, va_list *args)
Definition: fifo_segment.c:1498
fs_slice_private_get
static fifo_slice_private_t * fs_slice_private_get(fifo_segment_t *fs, u32 slice_index)
Definition: fifo_segment.c:56
fs_try_alloc_fifo_batch
static int fs_try_alloc_fifo_batch(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index, u32 batch_size)
Definition: fifo_segment.c:628
svm_fifo_shr_::head_chunk
fs_sptr_t head_chunk
tracks chunk where head lands
Definition: fifo_types.h:77
fifo_segment_slice_::n_fl_chunk_bytes
uword n_fl_chunk_bytes
Chunk bytes on freelist.
Definition: fifo_types.h:125
clib_memcpy_fast
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
Definition: string.h:92
fsh_alloc_aligned
static void * fsh_alloc_aligned(fifo_segment_header_t *fsh, uword size, uword align)
Definition: fifo_segment.c:20
SSVM_SEGMENT_MEMFD
@ SSVM_SEGMENT_MEMFD
Definition: ssvm.h:50
ssvm_segment_type_t
enum ssvm_segment_type_ ssvm_segment_type_t
fifo_segment_collect_chunk
void fifo_segment_collect_chunk(fifo_segment_t *fs, u32 slice_index, svm_fifo_chunk_t *c)
Definition: fifo_segment.c:1065
fsh_collect_chunks
void fsh_collect_chunks(fifo_segment_header_t *fsh, u32 slice_index, svm_fifo_chunk_t *c)
Return chunks to fifo segment.
Definition: fifo_segment.c:794
fss_fl_chunk_bytes_sub
static void fss_fl_chunk_bytes_sub(fifo_segment_slice_t *fss, uword size)
Definition: fifo_segment.c:286
fifo_segment_header_::n_reserved_bytes
u32 n_reserved_bytes
Bytes not to be allocated.
Definition: fifo_types.h:141
svm_fifo_shr_::next
fs_sptr_t next
next in freelist
Definition: fifo_types.h:74
foreach_segment_mem_status
#define foreach_segment_mem_status
Definition: fifo_segment.h:43
svm_fifo_t
struct _svm_fifo svm_fifo_t
max_log2
static uword max_log2(uword x)
Definition: clib.h:223
fsh_slice_get
static fifo_segment_slice_t * fsh_slice_get(fifo_segment_header_t *fsh, u32 slice_index)
Definition: fifo_segment.c:50
pool_is_free_index
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:302
fifo_segment_header_::low_watermark
u8 low_watermark
Memory pressure watermark low.
Definition: fifo_types.h:146
fifo_segment_header_::max_log2_fifo_size
u32 max_log2_fifo_size
Max log2(chunk size) for fs.
Definition: fifo_types.h:142
svm_fifo_shr_::end_chunk
fs_sptr_t end_chunk
end chunk in fifo chunk list
Definition: fifo_types.h:67
fsh_virtual_mem_update
void fsh_virtual_mem_update(fifo_segment_header_t *fsh, u32 slice_index, int n_bytes)
Definition: fifo_segment.c:123
fifo_segment_get_mem_status
fifo_segment_mem_status_t fifo_segment_get_mem_status(fifo_segment_t *fs)
Definition: fifo_segment.c:1489
svm_fifo_free_ooo_data
void svm_fifo_free_ooo_data(svm_fifo_t *f)
Cleanup fifo ooo data.
Definition: svm_fifo.c:115
fifo_segment_cleanup
void fifo_segment_cleanup(fifo_segment_t *fs)
Definition: fifo_segment.c:826
fifo_segment_create
int fifo_segment_create(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Create a fifo segment and initialize as master.
Definition: fifo_segment.c:347
count
u8 count
Definition: dhcp.api:208
svm_msg_q_cleanup
void svm_msg_q_cleanup(svm_msg_q_t *mq)
Cleanup mq's private data.
Definition: message_queue.c:170
fs_freelist_for_size
static u32 fs_freelist_for_size(u32 size)
Definition: fifo_segment.c:464
fs_chunk_size_is_valid
static int fs_chunk_size_is_valid(fifo_segment_header_t *fsh, u32 size)
Definition: fifo_segment.c:479
fsh_try_alloc_chunk_batch
static int fsh_try_alloc_chunk_batch(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index, u32 batch_size)
Definition: fifo_segment.c:587
vec_add1
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:606
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
ssvm_delete
void ssvm_delete(ssvm_private_t *ssvm)
Definition: ssvm.c:445
fifo_segment_msg_q_alloc
svm_msg_q_t * fifo_segment_msg_q_alloc(fifo_segment_t *fs, u32 mq_index, svm_msg_q_cfg_t *cfg)
Allocate message queue on segment.
Definition: fifo_segment.c:1078
offset
struct clib_bihash_value offset
template key/value backing page structure
fifo_segment_get_segment_if_valid
fifo_segment_t * fifo_segment_get_segment_if_valid(fifo_segment_main_t *sm, u32 segment_index)
Definition: fifo_segment.c:441
svm_msg_q_queue_::shr
svm_msg_q_shared_queue_t * shr
pointer to shared queue
Definition: message_queue.h:43
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
fifo_segment_header_
Definition: fifo_types.h:137
fsh_n_cached_bytes
static uword fsh_n_cached_bytes(fifo_segment_header_t *fsh)
Definition: fifo_segment.c:89
c
svmdb_client_t * c
Definition: vpp_get_metrics.c:48
fs_sptr_t
uword fs_sptr_t
Definition: fifo_types.h:36
fifo_segment_preallocate_fifo_pairs
void fifo_segment_preallocate_fifo_pairs(fifo_segment_t *fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 *n_fifo_pairs)
Pre-allocates fifo pairs in fifo segment.
Definition: fifo_segment.c:1208
FIFO_SEGMENT_F_MEM_LIMIT
@ FIFO_SEGMENT_F_MEM_LIMIT
Definition: fifo_segment.h:40
fifo_segment_attach
int fifo_segment_attach(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Attach as slave to a fifo segment.
Definition: fifo_segment.c:381
fifo_segment_free_client_fifo
void fifo_segment_free_client_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated by external applications.
Definition: fifo_segment.c:978
uword
u64 uword
Definition: types.h:112
fifo_segment_fl_chunk_bytes
uword fifo_segment_fl_chunk_bytes(fifo_segment_t *fs)
Number of bytes on chunk free lists.
Definition: fifo_segment.c:1420
fss_chunk_free_list_head
static svm_fifo_chunk_t * fss_chunk_free_list_head(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index)
Definition: fifo_segment.c:141
round_pow2_u64
static u64 round_pow2_u64(u64 x, u64 pow2)
Definition: clib.h:285
FIFO_SEGMENT_MIN_FIFO_SIZE
#define FIFO_SEGMENT_MIN_FIFO_SIZE
4kB min fifo size
Definition: fifo_segment.h:32
fsh_cached_bytes_add
static void fsh_cached_bytes_add(fifo_segment_header_t *fsh, uword size)
Definition: fifo_segment.c:77
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
fsh_reset_mem_limit
void fsh_reset_mem_limit(fifo_segment_header_t *fsh)
Fifo segment reset mem limit flag.
Definition: fifo_segment.c:1386
f64
double f64
Definition: types.h:142
format_fifo_segment
u8 * format_fifo_segment(u8 *s, va_list *args)
Segment format function.
Definition: fifo_segment.c:1519
fifo_segment_alloc
void * fifo_segment_alloc(fifo_segment_t *fs, uword size)
Fifo segment reset mem limit flag.
Definition: fifo_segment.c:1392
OOO_SEGMENT_INVALID_INDEX
#define OOO_SEGMENT_INVALID_INDEX
Definition: svm_fifo.h:28
vec_validate
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment)
Definition: vec.h:523
ssvm_private_t::attach_timeout
int attach_timeout
shm segments attach timeout (sec)
Definition: ssvm.h:94
address
manual_print typedef address
Definition: ip_types.api:96
svm_msg_q_attach
void svm_msg_q_attach(svm_msg_q_t *mq, void *smq_base)
Definition: message_queue.c:144
fs_try_alloc_fifo
static svm_fifo_shared_t * fs_try_alloc_fifo(fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
Try to allocate new fifo.
Definition: fifo_segment.c:725
clib_mem_bulk_init
clib_mem_bulk_handle_t clib_mem_bulk_init(u32 elt_sz, u32 align, u32 min_elts_per_chunk)
Definition: mem_bulk.c:54
heap_elt_t::next
i32 next
Definition: heap.h:78
svm_fifo_chunk_::next
fs_sptr_t next
pointer to next chunk in linked-lists
Definition: fifo_types.h:42
clib_min
#define clib_min(x, y)
Definition: clib.h:342
clib_atomic_fetch_add_rel
#define clib_atomic_fetch_add_rel(a, b)
Definition: atomics.h:60
CLIB_CACHE_LINE_BYTES
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
fifo_segment_duplicate_fifo
svm_fifo_t * fifo_segment_duplicate_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Definition: fifo_segment.c:907
svm_fifo_init
void svm_fifo_init(svm_fifo_t *f, u32 size)
Initialize fifo.
Definition: svm_fifo.c:372
format_memory_size
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:209
svm_msg_q_
Definition: message_queue.h:72
SSVM_SEGMENT_PRIVATE
@ SSVM_SEGMENT_PRIVATE
Definition: ssvm.h:51
fifo_segment_t::n_slices
u8 n_slices
number of fifo segment slices
Definition: fifo_segment.h:72
FIFO_SEGMENT_RX_FIFO
@ FIFO_SEGMENT_RX_FIFO
Definition: fifo_segment.h:26
fifo_segment_alloc_chunk_w_slice
svm_fifo_chunk_t * fifo_segment_alloc_chunk_w_slice(fifo_segment_t *fs, u32 slice_index, u32 chunk_size)
Definition: fifo_segment.c:1054
pfss_fifo_add_active_list
static void pfss_fifo_add_active_list(fifo_slice_private_t *pfss, svm_fifo_t *f)
Definition: fifo_segment.c:249
svm_msg_q_size_to_alloc
uword svm_msg_q_size_to_alloc(svm_msg_q_cfg_t *cfg)
Definition: message_queue.c:104
clib_atomic_fetch_sub_rel
#define clib_atomic_fetch_sub_rel(a, b)
Definition: atomics.h:61
svm_fifo_shr_::start_chunk
fs_sptr_t start_chunk
first chunk in fifo chunk list
Definition: fifo_types.h:66
clib_atomic_fetch_add_relax
#define clib_atomic_fetch_add_relax(a, b)
Definition: atomics.h:63
fifo_segment_t::ssvm
ssvm_private_t ssvm
ssvm segment data
Definition: fifo_segment.h:69
vec_free
#define vec_free(V)
Free vector's memory (no header).
Definition: vec.h:395
fifo_segment_slice_::num_chunks
u32 num_chunks[FS_CHUNK_VEC_LEN]
Allocated chunks by chunk size.
Definition: fifo_types.h:127
fifo_segment_num_fifos
u32 fifo_segment_num_fifos(fifo_segment_t *fs)
Get number of active fifos.
Definition: fifo_segment.c:1274
fss_fl_chunk_bytes
static uword fss_fl_chunk_bytes(fifo_segment_slice_t *fss)
Definition: fifo_segment.c:274
svm_msg_q_set_eventfd
void svm_msg_q_set_eventfd(svm_msg_q_t *mq, int fd)
Set event fd for queue.
Definition: message_queue.c:485
fifo_segment_slice_::free_fifos
fs_sptr_t free_fifos
Freelists of fifo shared hdrs
Definition: fifo_types.h:124
size
u32 size
Definition: vhost_user.h:125
SSVM_SEGMENT_SHM
@ SSVM_SEGMENT_SHM
Definition: ssvm.h:49
clib_bihash_value
template key/value backing page structure
Definition: bihash_doc.h:44
fifo_segment_slice_t
struct fifo_segment_slice_ fifo_segment_slice_t
fifo_segment_free_fifo
void fifo_segment_free_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated in fifo segment.
Definition: fifo_segment.c:918
fifo_segment_main_init
void fifo_segment_main_init(fifo_segment_main_t *sm, u64 baseva, u32 timeout_in_seconds)
Definition: fifo_segment.c:456
fifo_slice_private_
Definition: fifo_types.h:130
FS_CHUNK_VEC_LEN
#define FS_CHUNK_VEC_LEN
number of chunk sizes
Definition: fifo_types.h:26
fsh_n_active_fifos
static u32 fsh_n_active_fifos(fifo_segment_header_t *fsh)
Definition: fifo_segment.c:102
u64
unsigned long u64
Definition: types.h:89
fss_fifo_free_list_push
static void fss_fifo_free_list_push(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, svm_fifo_shared_t *sf)
Definition: fifo_segment.c:223
format
description fragment has unexpected format
Definition: map.api:433
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
n_free
u32 n_free
Definition: interface_output.c:1078
ssvm_private_t::my_pid
u32 my_pid
Definition: ssvm.h:86
format_get_indent
static u32 format_get_indent(u8 *s)
Definition: format.h:72
ssvm_server_init
int ssvm_server_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:433
fsh_virtual_mem
static uword fsh_virtual_mem(fifo_segment_header_t *fsh)
Definition: fifo_segment.c:108
clib_mem_bulk_alloc
void * clib_mem_bulk_alloc(clib_mem_bulk_handle_t h)
Definition: mem_bulk.c:141
fss_fl_chunk_bytes_add
static void fss_fl_chunk_bytes_add(fifo_segment_slice_t *fss, uword size)
Definition: fifo_segment.c:280
vlib_frame_t::flags
u16 flags
Definition: node.h:378
svm_fifo_free_chunk_lookup
void svm_fifo_free_chunk_lookup(svm_fifo_t *f)
Cleanup fifo chunk lookup rb tree.
Definition: svm_fifo.c:761
FS_CL_HEAD_TMASK
#define FS_CL_HEAD_TMASK
Definition: fifo_segment.c:137
clib_atomic_fetch_sub_relax
#define clib_atomic_fetch_sub_relax(a, b)
Definition: atomics.h:64
SVM_FIFO_INVALID_INDEX
#define SVM_FIFO_INVALID_INDEX
Definition: svm_fifo.h:30
u32
unsigned int u32
Definition: types.h:88
ssvm_type
ssvm_segment_type_t ssvm_type(const ssvm_private_t *ssvm)
Definition: ssvm.c:451
fsh_has_reached_mem_limit
u8 fsh_has_reached_mem_limit(fifo_segment_header_t *fsh)
Fifo segment has reached mem limit.
Definition: fifo_segment.c:1380
clib_atomic_load_relax_n
#define clib_atomic_load_relax_n(a)
Definition: atomics.h:50
fifo_segment_index
u32 fifo_segment_index(fifo_segment_main_t *sm, fifo_segment_t *s)
Definition: fifo_segment.c:429
CLIB_MEM_UNPOISON
#define CLIB_MEM_UNPOISON(a, s)
Definition: sanitizer.h:47
n_bytes
u32 n_bytes
Definition: interface_output.c:401
ssvm_private_t::is_server
int is_server
Definition: ssvm.h:89
fifo_segment_t::max_byte_index
uword max_byte_index
Definition: fifo_segment.h:71
vec_foreach
#define vec_foreach(var, vec)
Vector iterator.
Definition: vec_bootstrap.h:213
fifo_segment_main_t
Definition: fifo_segment.h:77
fss_fifo_free_list_pop
svm_fifo_shared_t * fss_fifo_free_list_pop(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
Definition: fifo_segment.c:240
fifo_segment_slice_
Definition: fifo_types.h:120
fifo_segment_available_bytes
uword fifo_segment_available_bytes(fifo_segment_t *fs)
Definition: fifo_segment.c:1414
fifo_segment_init
int fifo_segment_init(fifo_segment_t *fs)
Initialize fifo segment shared header.
Definition: fifo_segment.c:295
svm_msg_q_cfg_
Definition: message_queue.h:85
FIFO_SEGMENT_MAX_FIFO_SIZE
#define FIFO_SEGMENT_MAX_FIFO_SIZE
2GB max fifo size
Definition: fifo_segment.h:33
clib_memset
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
fs_chunk_ptr
static svm_fifo_chunk_t * fs_chunk_ptr(fifo_segment_header_t *fsh, fs_sptr_t cp)
Definition: fifo_types.h:173
fsh_try_alloc_chunk
static svm_fifo_chunk_t * fsh_try_alloc_chunk(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 data_bytes)
Definition: fifo_segment.c:656
fifo_segment_header_::start_byte_index
uword start_byte_index
Definition: fifo_types.h:152
ssvm_shared_header_t::ready
volatile u32 ready
Definition: ssvm.h:76
fifo_segment_size
uword fifo_segment_size(fifo_segment_t *fs)
Fifo segment allocated size.
Definition: fifo_segment.c:1374
fifo_segment_determine_status
fifo_segment_mem_status_t fifo_segment_determine_status(fifo_segment_header_t *fsh, u8 usage)
Definition: fifo_segment.c:1463
fifo_segment_t::mqs
svm_msg_q_t * mqs
private vec of attached mqs
Definition: fifo_segment.h:74
pool_get_zero
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:258
u8
unsigned char u8
Definition: types.h:56
ssvm_private_t::name
u8 * name
Definition: ssvm.h:87
fifo_segment_header_::n_slices
u8 n_slices
Number of slices.
Definition: fifo_types.h:144
a
a
Definition: bitmap.h:544
fifo_segment_free_bytes
uword fifo_segment_free_bytes(fifo_segment_t *fs)
Fifo segment estimate of number of free bytes.
Definition: fifo_segment.c:1402
ssvm_private_t::fd
int fd
memfd segments
Definition: ssvm.h:93
fsh_slice_collect_chunks
static void fsh_slice_collect_chunks(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, svm_fifo_chunk_t *c)
Definition: fifo_segment.c:773
fss_chunk_fl_index_is_valid
static int fss_chunk_fl_index_is_valid(fifo_segment_slice_t *fss, u32 fl_index)
Definition: fifo_segment.c:131
uword_to_pointer
#define uword_to_pointer(u, type)
Definition: types.h:136
clib_atomic_cmp_and_swap_acq_relax
#define clib_atomic_cmp_and_swap_acq_relax(addr, exp, new, weak)
Definition: atomics.h:41
fifo_segment_slice_::free_chunks
fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]
Free chunks by size.
Definition: fifo_types.h:123
clib_warning
#define clib_warning(format, args...)
Definition: error.h:59
fifo_segment_main_t::segments
fifo_segment_t * segments
pool of fifo segments
Definition: fifo_segment.h:79
fifo_segment_t::slices
fifo_slice_private_t * slices
private slice information
Definition: fifo_segment.h:73
rv
int __clib_unused rv
Definition: application.c:491
mem.h
FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE
#define FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE
4kB min fifo size
Definition: fifo_segment.h:31
fs_fifo_free
void fs_fifo_free(fifo_segment_t *fs, svm_fifo_t *f, u32 slice_index)
Definition: fifo_segment.c:814
clib_atomic_load_acq_n
#define clib_atomic_load_acq_n(a)
Definition: atomics.h:51
fifo_segment_ftype_t
fifo_segment_ftype_t
Definition: fifo_segment.h:23
fifo_slice_private_::active_fifos
svm_fifo_t * active_fifos
Linked list of active RX fifos.
Definition: fifo_types.h:134
fifo_segment_fifo_offset
uword fifo_segment_fifo_offset(svm_fifo_t *f)
Definition: fifo_segment.c:1048
ssvm_name
u8 * ssvm_name(const ssvm_private_t *ssvm)
Definition: ssvm.c:457
fifo_segment_prealloc_fifo_hdrs
int fifo_segment_prealloc_fifo_hdrs(fifo_segment_t *fs, u32 slice_index, u32 batch_size)
Try to preallocate fifo headers.
Definition: fifo_segment.c:1174
min_log2
static uword min_log2(uword x)
Definition: clib.h:176
fifo_segment_create_args_t
Definition: fifo_segment.h:84
fifo_segment_t::h
fifo_segment_header_t * h
fifo segment data
Definition: fifo_segment.h:70
clib_mem_bulk_destroy
void clib_mem_bulk_destroy(clib_mem_bulk_handle_t h)
Definition: mem_bulk.c:83
fsh_n_free_bytes
static uword fsh_n_free_bytes(fifo_segment_header_t *fsh)
Definition: fifo_segment.c:69
fifo_segment_msg_qs_discover
void fifo_segment_msg_qs_discover(fifo_segment_t *fs, int *fds, u32 n_fds)
Discover mqs on mq only segment.
Definition: fifo_segment.c:1130
fifo_segment_has_fifos
u8 fifo_segment_has_fifos(fifo_segment_t *fs)
Definition: fifo_segment.c:1437
fss_chunk_free_list_push
static void fss_chunk_free_list_push(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index, svm_fifo_chunk_t *c)
Definition: fifo_segment.c:149
fifo_segment_header_::flags
u8 flags
Segment flags.
Definition: fifo_types.h:143
fifo_segment_get_slice_fifo_list
svm_fifo_t * fifo_segment_get_slice_fifo_list(fifo_segment_t *fs, u32 slice_index)
Definition: fifo_segment.c:1443
fifo_slice_private_::fifos
clib_mem_bulk_handle_t fifos
Bulk fifo allocator.
Definition: fifo_types.h:132
fifo_segment.h
fs_slice_num_free_fifos
static u32 fs_slice_num_free_fifos(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
Definition: fifo_segment.c:1280
ssvm_client_init
int ssvm_client_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:439
SVM_FIFO_F_LL_TRACKED
@ SVM_FIFO_F_LL_TRACKED
Definition: svm_fifo.h:42
fifo_segment_mem_status_strings
static char * fifo_segment_mem_status_strings[]
Definition: fifo_segment.c:62
svm_msg_q_::q
svm_msg_q_queue_t q
queue for exchanging messages
Definition: message_queue.h:74
FIFO_SEGMENT_ALLOC_BATCH_SIZE
#define FIFO_SEGMENT_ALLOC_BATCH_SIZE
Definition: fifo_segment.h:34
fs_slice_num_free_chunks
static u32 fs_slice_num_free_chunks(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 size)
Definition: fifo_segment.c:1314
format_white_space
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
fifo_segment_header_::pct_first_alloc
u8 pct_first_alloc
Pct of fifo size to alloc.
Definition: fifo_types.h:147
fsh_alloc
static void * fsh_alloc(fifo_segment_header_t *fsh, uword size)
Definition: fifo_segment.c:44
ssvm_shared_header_t
Definition: ssvm.h:55
fsh_try_alloc_fifo_hdr
static svm_fifo_shared_t * fsh_try_alloc_fifo_hdr(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
Definition: fifo_segment.c:638
fifo_segment_header_::high_watermark
u8 high_watermark
Memory pressure watermark high.
Definition: fifo_types.h:145
fifo_segment_mem_status_t
fifo_segment_mem_status_t
Definition: fifo_segment.h:49