FD.io VPP  v20.01-48-g3e0dafb74
Vector Packet Processing
fifo_segment.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <svm/fifo_segment.h>
17 
18 /**
19  * Fifo segment free space
20  *
21  * Queries the underlying memory manager, dlmalloc, for free space. Since this
22  * ends up walking the internal data structures, it should not be called
23  * indiscriminately.
24  *
25  * @param fs fifo segment
26  * @return number of free bytes
27  */
28 static uword
30 {
31  struct dlmallinfo dlminfo;
32 
33  dlminfo = mspace_mallinfo (fsh->ssvm_sh->heap);
34  return dlminfo.fordblks;
35 }
36 
37 static inline void
39 {
41 }
42 
43 static inline uword
45 {
47  return n_free > fsh->n_reserved_bytes ? n_free - fsh->n_reserved_bytes : 0;
48 }
49 
50 static inline void
52 {
54 }
55 
56 static void
58 {
59  uword thresh;
60 
62  return;
63 
64  thresh = clib_max (0.01 * fsh->ssvm_sh->ssvm_size,
65  2 * fsh->n_reserved_bytes);
66  if (fsh->n_free_bytes > thresh)
67  return;
68 
71 }
72 
73 static inline fifo_segment_slice_t *
75 {
76  return &fsh->slices[slice_index];
77 }
78 
79 static inline void
81 {
83 }
84 
85 /**
86  * Initialize fifo segment shared header
87  */
88 int
90 {
94  u32 max_chunk_sz, max_chunks;
95  uword max_fifo;
96  void *oldheap;
97  int i;
98 
99  sh = fs->ssvm.sh;
100  oldheap = ssvm_push_heap (sh);
101 
102  /*
103  * Manually align the fifo segment header to sizeof(uword) = 8 bytes.
104  * Long story made short: the "process-private" fifo segment
105  * is allocated from the main heap, not mmapped. dlmalloc
106  * only guarantees 4-byte alignment, and on aarch64
107  * the fsh can end up 4-byte but not 8-byte aligned.
108  * That eventually causes the atomic op in fifo_segment_update_free_bytes
109  * to backfire.
110  */
111  fsh = clib_mem_alloc_aligned (sizeof (*fsh), sizeof (uword));
112  clib_memset (fsh, 0, sizeof (*fsh));
113  fs->h = sh->opaque[0] = fsh;
114  fs->n_slices = clib_max (fs->n_slices, 1);
115 
116  fsh->ssvm_sh = fs->ssvm.sh;
117  fsh->n_slices = fs->n_slices;
118  max_fifo = clib_min ((fsh_free_space (fsh) - 4096) / 2,
120  fsh->max_log2_chunk_size = max_log2 (max_fifo);
121 
122  fsh->slices = clib_mem_alloc (sizeof (*fss) * fs->n_slices);
123  clib_memset (fsh->slices, 0, sizeof (*fss) * fs->n_slices);
125 
126  for (i = 0; i < fs->n_slices; i++)
127  {
128  fss = fsh_slice_get (fsh, i);
129  vec_validate_init_empty (fss->free_chunks, max_chunk_sz, 0);
130  }
131 
132  ssvm_pop_heap (oldheap);
133 
134  fsh->n_free_bytes = fsh_free_space (fsh);
135  max_chunks = fsh->n_free_bytes / FIFO_SEGMENT_MIN_FIFO_SIZE;
136  fsh->n_reserved_bytes = (max_chunks / 4) * sizeof (rb_node_t);
137  sh->ready = 1;
138  return (0);
139 }
140 
141 /**
142  * Create a fifo segment and initialize as master
143  */
144 int
146 {
147  fifo_segment_t *fs;
148  uword baseva;
149  int rv;
150 
151  /* Allocate a fresh segment */
152  pool_get_zero (sm->segments, fs);
153 
154  baseva = a->segment_type == SSVM_SEGMENT_PRIVATE ? ~0ULL : sm->next_baseva;
155  fs->ssvm.ssvm_size = a->segment_size;
156  fs->ssvm.i_am_master = 1;
157  fs->ssvm.my_pid = getpid ();
158  fs->ssvm.name = format (0, "%s%c", a->segment_name, 0);
159  fs->ssvm.requested_va = baseva;
160 
161  if ((rv = ssvm_master_init (&fs->ssvm, a->segment_type)))
162  {
163  pool_put (sm->segments, fs);
164  return (rv);
165  }
166 
167  /* Note: requested_va updated due to seg base addr randomization */
168  sm->next_baseva = fs->ssvm.sh->ssvm_va + fs->ssvm.ssvm_size;
169 
170  fifo_segment_init (fs);
171  vec_add1 (a->new_segment_indices, fs - sm->segments);
172  return (0);
173 }
174 
175 /**
176  * Attach as slave to a fifo segment
177  */
178 int
180 {
181  fifo_segment_t *fs;
182  int rv;
183 
184  pool_get_zero (sm->segments, fs);
185 
186  fs->ssvm.ssvm_size = a->segment_size;
187  fs->ssvm.my_pid = getpid ();
188  fs->ssvm.name = format (0, "%s%c", a->segment_name, 0);
189  fs->ssvm.requested_va = sm->next_baseva;
191  fs->ssvm.fd = a->memfd_fd;
192  else
194 
195  if ((rv = ssvm_slave_init (&fs->ssvm, a->segment_type)))
196  {
197  _vec_len (fs) = vec_len (fs) - 1;
198  return (rv);
199  }
200 
201  /* Fish the segment header */
202  fs->h = fs->ssvm.sh->opaque[0];
203 
204  vec_add1 (a->new_segment_indices, fs - sm->segments);
205  return (0);
206 }
207 
208 void
210 {
211  ssvm_delete (&s->ssvm);
212  clib_memset (s, 0xfe, sizeof (*s));
213  pool_put (sm->segments, s);
214 }
215 
216 u32
218 {
219  return s - sm->segments;
220 }
221 
224 {
225  return pool_elt_at_index (sm->segments, segment_index);
226 }
227 
228 void
229 fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size)
230 {
231  *address = (char *) seg->ssvm.sh->ssvm_va;
232  *size = seg->ssvm.ssvm_size;
233 }
234 
235 void
237  u32 timeout_in_seconds)
238 {
239  sm->next_baseva = baseva;
240  sm->timeout_in_seconds = timeout_in_seconds;
241 }
242 
243 static inline u32
245 {
247 }
248 
249 static inline u32
251 {
252  return 1 << (fl_index + FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE);
253 }
254 
255 static inline int
257 {
258  /*
259  * 4K minimum. It's not likely that anything good will happen
260  * with a smaller FIFO.
261  */
262  return size >= FIFO_SEGMENT_MIN_FIFO_SIZE
263  && size <= (1 << fsh->max_log2_chunk_size);
264 }
265 
266 static svm_fifo_t *
268  u32 fl_index, u32 data_bytes)
269 {
271  svm_fifo_t *f;
272 
273  f = fss->free_fifos;
274  c = fss->free_chunks[fl_index];
275 
276  if (!f || !c)
277  return 0;
278 
279  fss->free_fifos = f->next;
280  fss->free_chunks[fl_index] = c->next;
281  c->next = c;
282  c->start_byte = 0;
283  c->length = data_bytes;
284  memset (f, 0, sizeof (*f));
285  f->start_chunk = c;
286  f->end_chunk = c;
287 
288  fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
289  return f;
290 }
291 
292 static svm_fifo_t *
294  fifo_segment_slice_t * fss,
295  u32 data_bytes)
296 {
297  svm_fifo_chunk_t *c, *first = 0, *last = 0;
298  u32 fl_index, fl_size, n_alloc = 0;
299  svm_fifo_t *f;
300 
301  f = fss->free_fifos;
302  if (!f)
303  {
304  void *oldheap = ssvm_push_heap (fsh->ssvm_sh);
306  ssvm_pop_heap (oldheap);
307  if (!f)
308  return 0;
309  memset (f, 0, sizeof (*f));
310  fsh_free_bytes_sub (fsh, sizeof (*f));
311  }
312  else
313  {
314  fss->free_fifos = f->next;
315  }
316 
317  fl_index = fs_freelist_for_size (data_bytes);
318  if (fl_index > 0)
319  fl_index -= 1;
320 
321  fl_size = fs_freelist_index_to_size (fl_index);
322 
323  while (data_bytes)
324  {
325  c = fss->free_chunks[fl_index];
326  if (c)
327  {
328  fss->free_chunks[fl_index] = c->next;
329  if (!last)
330  last = c;
331  c->next = first;
332  first = c;
333  n_alloc += fl_size;
334  c->length = clib_min (fl_size, data_bytes);
335  data_bytes -= c->length;
336  }
337  else
338  {
339  /* Failed to allocate with smaller chunks */
340  if (fl_index == 0)
341  {
342  /* free all chunks if any allocated */
343  c = first;
344  while (c)
345  {
346  fl_index = fs_freelist_for_size (c->length);
347  fl_size = fs_freelist_index_to_size (fl_index);
348  c->next = fss->free_chunks[fl_index];
349  fss->free_chunks[fl_index] = c;
350  fss->n_fl_chunk_bytes += fl_size;
351  data_bytes += fl_size;
352  }
353  first = last = 0;
354  fl_index = fs_freelist_for_size (data_bytes);
355  if (fss->free_chunks[fl_index + 1])
356  {
357  fl_index += 1;
358  fl_size = fs_freelist_index_to_size (fl_index);
359  continue;
360  }
361 
362  f->next = fss->free_fifos;
363  fss->free_fifos = f;
364  return 0;
365  }
366  fl_index -= 1;
367  fl_size = fl_size >> 1;
368  }
369  }
370 
371  f->start_chunk = first;
372  f->end_chunk = last;
373  last->next = first;
374  fss->n_fl_chunk_bytes -= n_alloc;
375  return f;
376 }
377 
378 static int
380  fifo_segment_slice_t * fss,
381  u32 fl_index, u32 batch_size)
382 {
383  u32 size, hdrs, rounded_data_size;
385  svm_fifo_t *f;
386  void *oldheap;
387  u8 *fmem;
388  int i;
389 
390  rounded_data_size = fs_freelist_index_to_size (fl_index);
391  hdrs = sizeof (*f) + sizeof (*c);
392  size = (hdrs + rounded_data_size) * batch_size;
393 
394  oldheap = ssvm_push_heap (fsh->ssvm_sh);
396  0 /* align_offset */ ,
397  0 /* os_out_of_memory */ );
398  ssvm_pop_heap (oldheap);
399 
400  /* Out of space.. */
401  if (fmem == 0)
402  return -1;
403 
404  /* Carve fifo + chunk space */
405  for (i = 0; i < batch_size; i++)
406  {
407  f = (svm_fifo_t *) fmem;
408  memset (f, 0, sizeof (*f));
409  f->next = fss->free_fifos;
410  fss->free_fifos = f;
411  c = (svm_fifo_chunk_t *) (fmem + sizeof (*f));
412  c->start_byte = 0;
413  c->length = rounded_data_size;
414  c->next = fss->free_chunks[fl_index];
415  fss->free_chunks[fl_index] = c;
416  fmem += hdrs + rounded_data_size;
417  }
418 
419  fss->n_fl_chunk_bytes += batch_size * rounded_data_size;
420  fsh_free_bytes_sub (fsh, size);
421 
422  return 0;
423 }
424 
425 /**
426  * Try to allocate new fifo
427  *
428  * Tries the following steps in order:
429  * - grab fifo and chunk from freelists
430  * - batch fifo and chunk allocation
431  * - single fifo allocation
432  * - grab multiple fifo chunks from freelists
433  */
434 static svm_fifo_t *
436  u32 data_bytes)
437 {
438  u32 fifo_sz, fl_index;
439  svm_fifo_t *f = 0;
440  uword n_free_bytes;
441 
442  fl_index = fs_freelist_for_size (data_bytes);
443  fifo_sz = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t);
444  fifo_sz += 1 << max_log2 (data_bytes);
445 
446  if (fss->free_fifos && fss->free_chunks[fl_index])
447  {
448  f = fs_try_alloc_fifo_freelist (fss, fl_index, data_bytes);
449  if (f)
450  goto done;
451  }
452 
453  fsh_check_mem (fsh);
454  n_free_bytes = fsh_n_free_bytes (fsh);
455  if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < n_free_bytes)
456  {
457  if (fs_try_alloc_fifo_batch (fsh, fss, fl_index,
459  goto done;
460 
461  f = fs_try_alloc_fifo_freelist (fss, fl_index, data_bytes);
462  goto done;
463  }
464  if (fifo_sz <= n_free_bytes)
465  {
466  void *oldheap = ssvm_push_heap (fsh->ssvm_sh);
467  f = svm_fifo_create (data_bytes);
468  ssvm_pop_heap (oldheap);
469  if (f)
470  {
471  fsh_free_bytes_sub (fsh, fifo_sz);
472  goto done;
473  }
474  }
475  if (data_bytes <= fss->n_fl_chunk_bytes)
476  f = fs_try_alloc_fifo_freelist_multi_chunk (fsh, fss, data_bytes);
477 
478 done:
479 
480  return f;
481 }
482 
483 /**
484  * Allocate fifo in fifo segment
485  */
486 svm_fifo_t *
488  u32 data_bytes, fifo_segment_ftype_t ftype)
489 {
490  fifo_segment_header_t *fsh = fs->h;
492  svm_fifo_t *f = 0;
493 
494  ASSERT (slice_index < fs->n_slices);
495 
496  fss = fsh_slice_get (fsh, slice_index);
497  f = fs_try_alloc_fifo (fsh, fss, data_bytes);
498  if (!f)
499  goto done;
500 
501  f->slice_index = slice_index;
502 
503  /* (re)initialize the fifo, as in svm_fifo_create */
504  svm_fifo_init (f, data_bytes);
505 
506  /* Initialize chunks and rbtree for multi-chunk fifos */
507  if (f->start_chunk->next != f->start_chunk)
508  {
509  void *oldheap = ssvm_push_heap (fsh->ssvm_sh);
511  ssvm_pop_heap (oldheap);
512  }
513 
514  /* If rx fifo type add to active fifos list. When cleaning up segment,
515  * we need a list of active sessions that should be disconnected. Since
516  * both rx and tx fifos keep pointers to the session, it's enough to track
517  * only one. */
518  if (ftype == FIFO_SEGMENT_RX_FIFO)
519  {
520  if (fss->fifos)
521  {
522  fss->fifos->prev = f;
523  f->next = fss->fifos;
524  }
525  fss->fifos = f;
526  f->flags |= SVM_FIFO_F_LL_TRACKED;
527  }
528  fsh_active_fifos_update (fsh, 1);
529 
530 done:
531  return (f);
532 }
533 
534 /**
535  * Free fifo allocated in fifo segment
536  */
537 void
539 {
540  fifo_segment_header_t *fsh = fs->h;
541  svm_fifo_chunk_t *cur, *next;
543  void *oldheap;
544  int fl_index;
545 
546  ASSERT (f->refcnt > 0);
547 
548  if (--f->refcnt > 0)
549  return;
550 
551  fss = fsh_slice_get (fsh, f->slice_index);
552 
553  /* Remove from active list. Only rx fifos are tracked */
554  if (f->flags & SVM_FIFO_F_LL_TRACKED)
555  {
556  if (f->prev)
557  f->prev->next = f->next;
558  else
559  fss->fifos = f->next;
560  if (f->next)
561  f->next->prev = f->prev;
562  f->flags &= ~SVM_FIFO_F_LL_TRACKED;
563  }
564 
565  /* Add to free list */
566  f->next = fss->free_fifos;
567  f->prev = 0;
568  fss->free_fifos = f;
569 
570  /* Free fifo chunks */
571  cur = f->start_chunk;
572  do
573  {
574  next = cur->next;
575  fl_index = fs_freelist_for_size (cur->length);
576  ASSERT (fl_index < vec_len (fss->free_chunks));
577  cur->next = fss->free_chunks[fl_index];
578  fss->free_chunks[fl_index] = cur;
579  fss->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index);
580  cur = next;
581  }
582  while (cur != f->start_chunk);
583 
584  f->start_chunk = f->end_chunk = f->new_chunks = 0;
585  f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0;
586 
587  oldheap = ssvm_push_heap (fsh->ssvm_sh);
589  ssvm_pop_heap (oldheap);
590 
591  /* not allocated on segment heap */
593 
594  if (CLIB_DEBUG)
595  {
596  f->master_session_index = ~0;
597  f->master_thread_index = ~0;
598  }
599 
600  fsh_active_fifos_update (fsh, -1);
601 }
602 
603 int
605  u32 batch_size)
606 {
607  fifo_segment_header_t *fsh = fs->h;
609  svm_fifo_t *f;
610  void *oldheap;
611  u32 size;
612  u8 *fmem;
613  int i;
614 
615  fss = fsh_slice_get (fsh, slice_index);
616  size = (sizeof (*f)) * batch_size;
617 
618  oldheap = ssvm_push_heap (fsh->ssvm_sh);
620  0 /* align_offset */ ,
621  0 /* os_out_of_memory */ );
622  ssvm_pop_heap (oldheap);
623 
624  /* Out of space.. */
625  if (fmem == 0)
626  return -1;
627 
628  /* Carve fifo + chunk space */
629  for (i = 0; i < batch_size; i++)
630  {
631  f = (svm_fifo_t *) fmem;
632  memset (f, 0, sizeof (*f));
633  f->next = fss->free_fifos;
634  fss->free_fifos = f;
635  fmem += sizeof (*f);
636  }
637 
638  fsh_free_bytes_sub (fsh, size);
639 
640  return 0;
641 }
642 
643 int
645  u32 chunk_size, u32 batch_size)
646 {
647  u32 size, rounded_data_size, fl_index;
648  fifo_segment_header_t *fsh = fs->h;
651  void *oldheap;
652  u8 *cmem;
653  int i;
654 
655  if (!fs_chunk_size_is_valid (fsh, chunk_size))
656  {
657  clib_warning ("chunk size out of range %d", chunk_size);
658  return -1;
659  }
660 
661  fl_index = fs_freelist_for_size (chunk_size);
662  rounded_data_size = fs_freelist_index_to_size (fl_index);
663  size = (sizeof (*c) + rounded_data_size) * batch_size;
664 
665  oldheap = ssvm_push_heap (fsh->ssvm_sh);
667  0 /* align_offset */ ,
668  0 /* os_out_of_memory */ );
669  ssvm_pop_heap (oldheap);
670 
671  /* Out of space.. */
672  if (cmem == 0)
673  return -1;
674 
675  fss = fsh_slice_get (fsh, slice_index);
676 
677  /* Carve fifo + chunk space */
678  for (i = 0; i < batch_size; i++)
679  {
680  c = (svm_fifo_chunk_t *) cmem;
681  c->start_byte = 0;
682  c->length = rounded_data_size;
683  c->next = fss->free_chunks[fl_index];
684  fss->free_chunks[fl_index] = c;
685  cmem += sizeof (*c) + rounded_data_size;
686  }
687 
688  fss->n_fl_chunk_bytes += batch_size * rounded_data_size;
689  fsh_free_bytes_sub (fsh, size);
690 
691  return 0;
692 }
693 
694 /**
695  * Pre-allocates fifo pairs in fifo segment
696  */
697 void
699  u32 rx_fifo_size, u32 tx_fifo_size,
700  u32 * n_fifo_pairs)
701 {
702  u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc;
703  u32 hdrs, pairs_per_slice, alloc_now;
704  fifo_segment_header_t *fsh = fs->h;
705  int rx_fl_index, tx_fl_index, i;
707  uword space_available;
708 
709  /* Parameter check */
710  if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
711  return;
712 
713  if (!fs_chunk_size_is_valid (fsh, rx_fifo_size))
714  {
715  clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
716  return;
717  }
718 
719  if (!fs_chunk_size_is_valid (fsh, tx_fifo_size))
720  {
721  clib_warning ("tx fifo_size out of range %d", tx_fifo_size);
722  return;
723  }
724 
725  rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size)));
726  rx_fl_index = fs_freelist_for_size (rx_fifo_size);
727  tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size)));
728  tx_fl_index = fs_freelist_for_size (tx_fifo_size);
729 
730  hdrs = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t);
731 
732  /* Calculate space requirements */
733  pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size;
734  space_available = fsh_free_space (fsh);
735  pairs_to_alloc = space_available / pair_size;
736  pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs);
737  pairs_per_slice = pairs_to_alloc / fs->n_slices;
738  pairs_per_slice += pairs_to_alloc % fs->n_slices ? 1 : 0;
739 
740  if (!pairs_per_slice)
741  return;
742 
743  for (i = 0; i < fs->n_slices; i++)
744  {
745  fss = fsh_slice_get (fsh, i);
746  alloc_now = clib_min (pairs_per_slice, *n_fifo_pairs);
747  if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, alloc_now))
748  clib_warning ("rx prealloc failed: pairs %u", alloc_now);
749  if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, alloc_now))
750  clib_warning ("tx prealloc failed: pairs %u", alloc_now);
751 
752  /* Account for the pairs allocated */
753  *n_fifo_pairs -= alloc_now;
754  }
755 }
756 
757 int
759 {
760  fifo_segment_header_t *fsh = fs->h;
763  void *oldheap;
764  int fl_index;
765 
766  fl_index = fs_freelist_for_size (chunk_size);
767  fss = fsh_slice_get (fsh, f->slice_index);
768 
769  oldheap = ssvm_push_heap (fsh->ssvm_sh);
770 
771  c = fss->free_chunks[fl_index];
772 
773  if (!c)
774  {
775  fsh_check_mem (fsh);
776  if (fsh_n_free_bytes (fsh) < chunk_size)
777  {
778  ssvm_pop_heap (oldheap);
779  return -1;
780  }
781 
782  c = svm_fifo_chunk_alloc (chunk_size);
783  if (!c)
784  {
785  ssvm_pop_heap (oldheap);
786  return -1;
787  }
788  fsh_free_bytes_sub (fsh, chunk_size + sizeof (*c));
789  }
790  else
791  {
792  fss->free_chunks[fl_index] = c->next;
793  c->next = 0;
794  fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
795  }
796 
797  svm_fifo_add_chunk (f, c);
798 
799  ssvm_pop_heap (oldheap);
800  return 0;
801 }
802 
803 int
805 {
806  fifo_segment_header_t *fsh = fs->h;
807  svm_fifo_chunk_t *cur, *next;
809  void *oldheap;
810  int fl_index;
811 
812  oldheap = ssvm_push_heap (fsh->ssvm_sh);
813  cur = svm_fifo_collect_chunks (f);
814 
815  fss = fsh_slice_get (fsh, f->slice_index);
816 
817  while (cur)
818  {
819  next = cur->next;
820  fl_index = fs_freelist_for_size (cur->length);
821  cur->next = fss->free_chunks[fl_index];
822  fss->free_chunks[fl_index] = cur;
823  cur = next;
824  }
825 
826  ssvm_pop_heap (oldheap);
827 
828  return 0;
829 }
830 
831 /**
832  * Get number of active fifos
833  */
834 u32
836 {
838 }
839 
840 static u32
842 {
843  svm_fifo_t *f;
844  u32 count = 0;
845 
846  f = fss->free_fifos;
847  if (f == 0)
848  return 0;
849 
850  while (f)
851  {
852  f = f->next;
853  count++;
854  }
855  return count;
856 }
857 
858 u32
860 {
861  fifo_segment_header_t *fsh = fs->h;
863  int slice_index;
864  u32 count = 0;
865 
866  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
867  {
868  fss = fsh_slice_get (fsh, slice_index);
869  count += fs_slice_num_free_fifos (fss);
870  }
871  return count;
872 }
873 
874 static u32
876 {
877  u32 count = 0, rounded_size, fl_index;
879  int i;
880 
881  /* Count all free chunks? */
882  if (size == ~0)
883  {
884  for (i = 0; i < vec_len (fss->free_chunks); i++)
885  {
886  c = fss->free_chunks[i];
887  if (c == 0)
888  continue;
889 
890  while (c)
891  {
892  c = c->next;
893  count++;
894  }
895  }
896  return count;
897  }
898 
899  rounded_size = (1 << (max_log2 (size)));
900  fl_index = fs_freelist_for_size (rounded_size);
901 
902  if (fl_index >= vec_len (fss->free_chunks))
903  return 0;
904 
905  c = fss->free_chunks[fl_index];
906  if (c == 0)
907  return 0;
908 
909  while (c)
910  {
911  c = c->next;
912  count++;
913  }
914  return count;
915 }
916 
917 u32
919 {
920  fifo_segment_header_t *fsh = fs->h;
922  int slice_index;
923  u32 count = 0;
924 
925  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
926  {
927  fss = fsh_slice_get (fsh, slice_index);
928  count += fs_slice_num_free_chunks (fss, size);
929  }
930  return count;
931 }
932 
933 void
935 {
936  fsh_update_free_bytes (fs->h);
937 }
938 
939 uword
941 {
942  return fsh_n_free_bytes (fs->h);
943 }
944 
945 uword
947 {
948  fifo_segment_header_t *fsh = fs->h;
950  uword n_bytes = 0;
951  int slice_index;
952 
953  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
954  {
955  fss = fsh_slice_get (fsh, slice_index);
956  n_bytes += fss->n_fl_chunk_bytes;
957  }
958 
959  return n_bytes;
960 }
961 
962 u8
964 {
965  fifo_segment_header_t *fsh = fs->h;
967  int slice_index;
968 
969  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
970  {
971  fss = fsh_slice_get (fsh, slice_index);
972  if (fss->fifos)
973  return 1;
974  }
975  return 0;
976 }
977 
978 svm_fifo_t *
980 {
981  fifo_segment_header_t *fsh = fs->h;
983 
984  fss = fsh_slice_get (fsh, slice_index);
985  return fss->fifos;
986 }
987 
988 u8 *
989 format_fifo_segment_type (u8 * s, va_list * args)
990 {
991  fifo_segment_t *sp;
992  sp = va_arg (*args, fifo_segment_t *);
993  ssvm_segment_type_t st = ssvm_type (&sp->ssvm);
994 
995  if (st == SSVM_SEGMENT_PRIVATE)
996  s = format (s, "%s", "private-heap");
997  else if (st == SSVM_SEGMENT_MEMFD)
998  s = format (s, "%s", "memfd");
999  else if (st == SSVM_SEGMENT_SHM)
1000  s = format (s, "%s", "shm");
1001  else
1002  s = format (s, "%s", "unknown");
1003  return s;
1004 }
1005 
1006 /**
1007  * Segment format function
1008  */
1009 u8 *
1010 format_fifo_segment (u8 * s, va_list * args)
1011 {
1012  u32 count, indent, active_fifos, free_fifos, fifo_hdr = 0;
1013  fifo_segment_t *fs = va_arg (*args, fifo_segment_t *);
1014  int verbose __attribute__ ((unused)) = va_arg (*args, int);
1015  uword est_chunk_bytes, est_free_seg_bytes, free_chunks;
1016  uword chunk_bytes = 0, free_seg_bytes, chunk_size;
1017  fifo_segment_header_t *fsh;
1018  fifo_segment_slice_t *fss;
1020  u32 slice_index;
1021  char *address;
1022  size_t size;
1023  int i;
1024 
1025  indent = format_get_indent (s) + 2;
1026 #if USE_DLMALLOC == 0
1027  s = format (s, "%U segment heap: %U\n", format_white_space, indent,
1028  format_mheap, fsh->ssvm_sh->heap, verbose);
1029  s = format (s, "%U segment has %u active fifos\n",
1031 #endif
1032 
1033  if (fs == 0)
1034  {
1035  s = format (s, "%-15s%15s%15s%15s%15s%15s", "Name", "Type",
1036  "HeapSize (M)", "ActiveFifos", "FreeFifos", "Address");
1037  return s;
1038  }
1039 
1040  fifo_segment_info (fs, &address, &size);
1041  active_fifos = fifo_segment_num_fifos (fs);
1042  free_fifos = fifo_segment_num_free_fifos (fs);
1043 
1044  s = format (s, "%-15v%15U%15llu%15u%15u%15llx", ssvm_name (&fs->ssvm),
1045  format_fifo_segment_type, fs, size >> 20ULL, active_fifos,
1046  free_fifos, address);
1047 
1048  if (!verbose)
1049  return s;
1050 
1051  fsh = fs->h;
1052 
1053  free_chunks = fifo_segment_num_free_chunks (fs, ~0);
1054  if (free_chunks)
1055  s = format (s, "\n\n%UFree chunks by size:\n", format_white_space,
1056  indent + 2);
1057  else
1058  s = format (s, "\n");
1059 
1060  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
1061  {
1062  fss = fsh_slice_get (fsh, slice_index);
1063  for (i = 0; i < vec_len (fss->free_chunks); i++)
1064  {
1065  c = fss->free_chunks[i];
1066  if (c == 0)
1067  continue;
1068  count = 0;
1069  while (c)
1070  {
1071  c = c->next;
1072  count++;
1073  }
1074 
1075  chunk_size = fs_freelist_index_to_size (i);
1076  s = format (s, "%U%-5u kB: %u\n", format_white_space, indent + 2,
1077  chunk_size >> 10, count);
1078 
1079  chunk_bytes += count * chunk_size;
1080  }
1081  }
1082 
1083  fifo_hdr = free_fifos * sizeof (svm_fifo_t);
1084  est_chunk_bytes = fifo_segment_fl_chunk_bytes (fs);
1085  est_free_seg_bytes = fifo_segment_free_bytes (fs);
1087  free_seg_bytes = fifo_segment_free_bytes (fs);
1088 
1089  s = format (s, "\n%Useg free bytes: %U (%lu) estimated: %U (%lu)\n",
1091  free_seg_bytes, free_seg_bytes, format_memory_size,
1092  est_free_seg_bytes, est_free_seg_bytes);
1093  s = format (s, "%Uchunk free bytes: %U (%lu) estimated: %U (%lu)\n",
1094  format_white_space, indent + 2, format_memory_size, chunk_bytes,
1095  chunk_bytes, format_memory_size, est_chunk_bytes,
1096  est_chunk_bytes);
1097  s = format (s, "%Ufifo hdr free bytes: %U (%u) reserved %U (%lu)\n",
1098  format_white_space, indent + 2, format_memory_size, fifo_hdr,
1099  fifo_hdr, format_memory_size, fsh->n_reserved_bytes,
1100  fsh->n_reserved_bytes);
1101  s = format (s, "\n");
1102 
1103  return s;
1104 }
1105 
1106 /*
1107  * fd.io coding-style-patch-verification: ON
1108  *
1109  * Local Variables:
1110  * eval: (c-set-style "gnu")
1111  * End:
1112  */
u32 length
length of chunk in bytes
Definition: svm_fifo.h:61
void fifo_segment_info(fifo_segment_t *seg, char **address, size_t *size)
Definition: fifo_segment.c:229
u8 count
Definition: dhcp.api:208
#define FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE
4kB min fifo size
Definition: fifo_segment.h:29
u8 n_slices
Number of slices.
Definition: fifo_segment.h:58
fifo_segment_header_t * h
fifo segment data
Definition: fifo_segment.h:64
#define clib_min(x, y)
Definition: clib.h:295
svm_fifo_chunk_t * svm_fifo_collect_chunks(svm_fifo_t *f)
Removes chunks that are after fifo end byte.
Definition: svm_fifo.c:715
static void fsh_check_mem(fifo_segment_header_t *fsh)
Definition: fifo_segment.c:57
uword requested_va
Definition: ssvm.h:88
i32 next
Definition: heap.h:78
static uword fsh_n_free_bytes(fifo_segment_header_t *fsh)
Definition: fifo_segment.c:44
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
Definition: mem.h:113
a
Definition: bitmap.h:538
uword ssvm_size
Definition: ssvm.h:85
fifo_segment_slice_t * slices
Definition: fifo_segment.h:51
svm_fifo_t * fifo_segment_get_slice_fifo_list(fifo_segment_t *fs, u32 slice_index)
Definition: fifo_segment.c:979
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:240
u8 flags
Segment flags.
Definition: fifo_segment.h:57
void svm_fifo_free_chunk_lookup(svm_fifo_t *f)
Cleanup fifo chunk lookup rb tree.
Definition: svm_fifo.c:847
volatile u32 ready
Definition: ssvm.h:77
unsigned long u64
Definition: types.h:89
void fifo_segment_delete(fifo_segment_main_t *sm, fifo_segment_t *s)
Definition: fifo_segment.c:209
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
svm_fifo_t * fifo_segment_alloc_fifo_w_slice(fifo_segment_t *fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype)
Allocate fifo in fifo segment.
Definition: fifo_segment.c:487
u32 n_active_fifos
Number of active fifos.
Definition: fifo_segment.h:54
void * opaque[SSVM_N_OPAQUE]
Definition: ssvm.h:74
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
void svm_fifo_init(svm_fifo_t *f, u32 size)
Initialize fifo.
Definition: svm_fifo.c:392
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
ssvm_shared_header_t * sh
Definition: ssvm.h:84
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
DLMALLOC_EXPORT struct dlmallinfo mspace_mallinfo(mspace msp)
static u32 fs_slice_num_free_fifos(fifo_segment_slice_t *fss)
Definition: fifo_segment.c:841
static u32 fs_freelist_index_to_size(u32 fl_index)
Definition: fifo_segment.c:250
static int fs_try_alloc_fifo_batch(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 fl_index, u32 batch_size)
Definition: fifo_segment.c:379
unsigned char u8
Definition: types.h:56
u32 fifo_segment_num_free_fifos(fifo_segment_t *fs)
Definition: fifo_segment.c:859
void ssvm_delete(ssvm_private_t *ssvm)
Definition: ssvm.c:437
static void fsh_update_free_bytes(fifo_segment_header_t *fsh)
Definition: fifo_segment.c:51
struct _svm_fifo svm_fifo_t
uword next_baseva
Where to put the next one.
Definition: fifo_segment.h:71
void svm_fifo_init_chunks(svm_fifo_t *f)
Initialize fifo chunks and rbtree.
Definition: svm_fifo.c:408
u32 fifo_segment_num_free_chunks(fifo_segment_t *fs, u32 size)
Find number of free chunks of given size.
Definition: fifo_segment.c:918
fifo_segment_t * fifo_segment_get_segment(fifo_segment_main_t *sm, u32 segment_index)
Definition: fifo_segment.c:223
static fifo_segment_slice_t * fsh_slice_get(fifo_segment_header_t *fsh, u32 slice_index)
Definition: fifo_segment.c:74
u8 * ssvm_name(const ssvm_private_t *ssvm)
Definition: ssvm.c:449
enum ssvm_segment_type_ ssvm_segment_type_t
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
svm_fifo_chunk_t * svm_fifo_chunk_alloc(u32 size)
Creates a fifo chunk in the current heap.
Definition: svm_fifo.c:471
static u32 fs_freelist_for_size(u32 size)
Definition: fifo_segment.c:244
static int fs_chunk_size_is_valid(fifo_segment_header_t *fsh, u32 size)
Definition: fifo_segment.c:256
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:209
void svm_fifo_free_ooo_data(svm_fifo_t *f)
Cleanup fifo ooo data.
Definition: svm_fifo.c:132
static svm_fifo_t * fs_try_alloc_fifo_freelist_multi_chunk(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 data_bytes)
Definition: fifo_segment.c:293
uword n_fl_chunk_bytes
Chunk bytes on freelist.
Definition: fifo_segment.h:46
static void * ssvm_push_heap(ssvm_shared_header_t *sh)
Definition: ssvm.h:143
unsigned int u32
Definition: types.h:88
int attach_timeout
shm segments attach timeout (sec)
Definition: ssvm.h:94
int ssvm_master_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:425
u32 max_log2_chunk_size
Max log2(chunk size) for fs.
Definition: fifo_segment.h:56
static void ssvm_pop_heap(void *oldheap)
Definition: ssvm.h:151
char * segment_name
segment name
Definition: fifo_segment.h:80
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
int fifo_segment_attach(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Attach as slave to a fifo segment.
Definition: fifo_segment.c:179
void fifo_segment_preallocate_fifo_pairs(fifo_segment_t *fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 *n_fifo_pairs)
Pre-allocates fifo pairs in fifo segment.
Definition: fifo_segment.c:698
int fifo_segment_prealloc_fifo_chunks(fifo_segment_t *fs, u32 slice_index, u32 chunk_size, u32 batch_size)
Try to preallocate fifo chunks on segment.
Definition: fifo_segment.c:644
u8 * format_mheap(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:357
u8 * format_fifo_segment_type(u8 *s, va_list *args)
Definition: fifo_segment.c:989
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:519
struct svm_fifo_chunk_ * next
pointer to next chunk in linked-lists
Definition: svm_fifo.h:62
static uword fsh_free_space(fifo_segment_header_t *fsh)
Fifo segment free space.
Definition: fifo_segment.c:29
svm_fifo_t * free_fifos
Freelists by fifo size.
Definition: fifo_segment.h:44
u8 * format_fifo_segment(u8 *s, va_list *args)
Segment format function.
u64 size
Definition: vhost_user.h:140
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:287
int fifo_segment_grow_fifo(fifo_segment_t *fs, svm_fifo_t *f, u32 chunk_size)
Grow fifo size by adding an additional chunk of memory.
Definition: fifo_segment.c:758
ssvm_shared_header_t * ssvm_sh
Fixed array of slices.
Definition: fifo_segment.h:52
#define clib_atomic_fetch_add_rel(a, b)
Definition: atomics.h:54
int fifo_segment_create(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Create a fifo segment and initialize as master.
Definition: fifo_segment.c:145
#define FIFO_SEGMENT_MIN_FIFO_SIZE
4kB min fifo size
Definition: fifo_segment.h:30
svm_fifo_t * fifos
Linked list of active RX fifos.
Definition: fifo_segment.h:43
u32 segment_size
size of the segment
Definition: fifo_segment.h:78
ssvm_private_t ssvm
ssvm segment data
Definition: fifo_segment.h:63
svmdb_client_t * c
void fifo_segment_main_init(fifo_segment_main_t *sm, u64 baseva, u32 timeout_in_seconds)
Definition: fifo_segment.c:236
#define clib_warning(format, args...)
Definition: error.h:59
void svm_fifo_add_chunk(svm_fifo_t *f, svm_fifo_chunk_t *c)
Grow fifo size by adding chunk to chunk list.
Definition: svm_fifo.c:579
uword fifo_segment_fl_chunk_bytes(fifo_segment_t *fs)
Number of bytes on chunk free lists.
Definition: fifo_segment.c:946
int ssvm_slave_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:431
u32 my_pid
Definition: ssvm.h:86
u32 start_byte
chunk start byte
Definition: svm_fifo.h:60
int memfd_fd
fd for memfd segments
Definition: fifo_segment.h:79
int fd
memfd segments
Definition: ssvm.h:93
u8 n_slices
number of fifo segment slices
Definition: fifo_segment.h:65
#define clib_atomic_load_relax_n(a)
Definition: atomics.h:47
u32 fifo_segment_index(fifo_segment_main_t *sm, fifo_segment_t *s)
Definition: fifo_segment.c:217
#define ASSERT(truth)
manual_print typedef address
Definition: ip_types.api:84
void fifo_segment_free_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated in fifo segment.
Definition: fifo_segment.c:538
fifo_segment_t * segments
pool of fifo segments
Definition: fifo_segment.h:70
fifo_segment_ftype_t
Definition: fifo_segment.h:21
#define FIFO_SEGMENT_MAX_FIFO_SIZE
2GB max fifo size
Definition: fifo_segment.h:31
static u32 fs_slice_num_free_chunks(fifo_segment_slice_t *fss, u32 size)
Definition: fifo_segment.c:875
static svm_fifo_t * fs_try_alloc_fifo(fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, u32 data_bytes)
Try to allocate new fifo.
Definition: fifo_segment.c:435
static void * clib_mem_alloc(uword size)
Definition: mem.h:153
#define clib_atomic_store_rel_n(a, b)
Definition: atomics.h:49
#define clib_max(x, y)
Definition: clib.h:288
u8 * name
Definition: ssvm.h:87
#define clib_atomic_fetch_sub_rel(a, b)
Definition: atomics.h:55
ssvm_segment_type_t segment_type
type of segment requested
Definition: fifo_segment.h:77
uword n_free_bytes
Segment free bytes.
Definition: fifo_segment.h:53
u32 fifo_segment_num_fifos(fifo_segment_t *fs)
Get number of active fifos.
Definition: fifo_segment.c:835
uword ssvm_size
Definition: ssvm.h:70
static void fsh_free_bytes_sub(fifo_segment_header_t *fsh, int size)
Definition: fifo_segment.c:38
static svm_fifo_t * fs_try_alloc_fifo_freelist(fifo_segment_slice_t *fss, u32 fl_index, u32 data_bytes)
Definition: fifo_segment.c:267
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static uword max_log2(uword x)
Definition: clib.h:191
u64 uword
Definition: types.h:112
u32 n_reserved_bytes
Bytes not to be allocated.
Definition: fifo_segment.h:55
int fifo_segment_prealloc_fifo_hdrs(fifo_segment_t *fs, u32 slice_index, u32 batch_size)
Try to preallocate fifo headers.
Definition: fifo_segment.c:604
void fifo_segment_update_free_bytes(fifo_segment_t *fs)
Update fifo segment free bytes estimate.
Definition: fifo_segment.c:934
u8 fifo_segment_has_fifos(fifo_segment_t *fs)
Definition: fifo_segment.c:963
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:161
int fifo_segment_collect_fifo_chunks(fifo_segment_t *fs, svm_fifo_t *f)
Collect unused chunks for fifo.
Definition: fifo_segment.c:804
#define FIFO_SEGMENT_ALLOC_BATCH_SIZE
Definition: fifo_segment.h:32
u32 timeout_in_seconds
Time to wait during attach.
Definition: fifo_segment.h:72
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:487
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
int fifo_segment_init(fifo_segment_t *fs)
Initialize fifo segment shared header.
Definition: fifo_segment.c:89
static void fsh_active_fifos_update(fifo_segment_header_t *fsh, int inc)
Definition: fifo_segment.c:80
int i_am_master
Definition: ssvm.h:89
uword fifo_segment_free_bytes(fifo_segment_t *fs)
Fifo segment estimate of number of free bytes.
Definition: fifo_segment.c:940
svm_fifo_chunk_t ** free_chunks
Freelists by chunk size.
Definition: fifo_segment.h:45
u32 * new_segment_indices
return vec of new seg indices
Definition: fifo_segment.h:81
ssvm_segment_type_t ssvm_type(const ssvm_private_t *ssvm)
Definition: ssvm.c:443
svm_fifo_t * svm_fifo_create(u32 data_size_in_bytes)
Creates a fifo in the current heap.
Definition: svm_fifo.c:436