FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
mem_bulk.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vppinfra/clib.h>
17 #include <vppinfra/mem.h>
18 #include <vppinfra/time.h>
19 #include <vppinfra/format.h>
20 #include <vppinfra/clib_error.h>
21 
22 /* while usage of dlmalloc APIs is genrally discouraged, in this particular
23  * case there is significant benefit of calling them directly due to
24  * smaller memory consuption (no wwp and headroom space) */
25 #include <vppinfra/dlmalloc.h>
26 
27 #define CLIB_MEM_BULK_DEFAULT_MIN_ELTS_PER_CHUNK 32
28 
30 {
35 
36 typedef struct
37 {
43  void *mspace;
46 
47 static inline uword
49 {
50  return (uword) b->elts_per_chunk * b->elt_sz + b->chunk_hdr_sz;
51 }
52 
53 __clib_export clib_mem_bulk_handle_t
54 clib_mem_bulk_init (u32 elt_sz, u32 align, u32 min_elts_per_chunk)
55 {
58  uword sz;
59 
60  if ((b = mspace_memalign (heap->mspace, 16, sizeof (clib_mem_bulk_t))) == 0)
61  return 0;
62 
63  if (align < 16)
64  align = 16;
65 
66  if (min_elts_per_chunk == 0)
67  min_elts_per_chunk = CLIB_MEM_BULK_DEFAULT_MIN_ELTS_PER_CHUNK;
68 
70  clib_memset (b, 0, sizeof (clib_mem_bulk_t));
71  b->mspace = heap->mspace;
72  b->align = align;
73  b->elt_sz = round_pow2 (elt_sz, align);
74  b->chunk_hdr_sz = round_pow2 (sizeof (clib_mem_bulk_chunk_hdr_t), align);
75  b->elts_per_chunk = min_elts_per_chunk;
76  sz = bulk_chunk_size (b);
77  b->chunk_align = max_pow2 (sz);
78  b->elts_per_chunk += (b->chunk_align - sz) / b->elt_sz;
79  return b;
80 }
81 
82 __clib_export void
84 {
85  clib_mem_bulk_t *b = h;
87  void *ms = b->mspace;
88 
89  c = b->full_chunks;
90 
91 again:
92  while (c)
93  {
94  next = c->next;
96  mspace_free (ms, c);
97  c = next;
98  }
99 
100  if (b->avail_chunks)
101  {
102  c = b->avail_chunks;
103  b->avail_chunks = 0;
104  goto again;
105  }
106 
107  CLIB_MEM_POISON (b, sizeof (clib_mem_bulk_t));
108  mspace_free (ms, b);
109 }
110 
111 static inline void *
113 {
114  return (u8 *) c + b->chunk_hdr_sz + index * b->elt_sz;
115 }
116 
117 static inline void
120 {
121  c->next = *first;
122  c->prev = 0;
123  if (c->next)
124  c->next->prev = c;
125  *first = c;
126 }
127 
128 static inline void
131 {
132  if (c->next)
133  c->next->prev = c->prev;
134  if (c->prev)
135  c->prev->next = c->next;
136  else
137  *first = c->next;
138 }
139 
140 __clib_export void *
142 {
143  clib_mem_bulk_t *b = h;
144  clib_mem_bulk_chunk_hdr_t *c = b->avail_chunks;
145  u32 elt_idx;
146 
147  if (b->avail_chunks == 0)
148  {
149  u32 i, sz = bulk_chunk_size (b);
150  c = mspace_memalign (b->mspace, b->chunk_align, sz);
151  CLIB_MEM_UNPOISON (c, sz);
152  clib_memset (c, 0, sizeof (clib_mem_bulk_chunk_hdr_t));
153  b->avail_chunks = c;
154  c->n_free = b->elts_per_chunk;
155 
156  /* populate freelist */
157  for (i = 0; i < b->elts_per_chunk - 1; i++)
158  *((u32 *) get_chunk_elt_ptr (b, c, i)) = i + 1;
159  *((u32 *) get_chunk_elt_ptr (b, c, i)) = ~0;
160  }
161 
162  ASSERT (c->freelist != ~0);
163  elt_idx = c->freelist;
164  c->freelist = *((u32 *) get_chunk_elt_ptr (b, c, elt_idx));
165  c->n_free--;
166 
167  if (c->n_free == 0)
168  {
169  /* chunk is full */
170  ASSERT (c->freelist == ~0);
171  remove_from_chunk_list (&b->avail_chunks, c);
172  add_to_chunk_list (&b->full_chunks, c);
173  }
174 
175  return get_chunk_elt_ptr (b, c, elt_idx);
176 }
177 
178 __clib_export void
180 {
181  clib_mem_bulk_t *b = h;
182  uword offset = (uword) p & (b->chunk_align - 1);
183  clib_mem_bulk_chunk_hdr_t *c = (void *) ((u8 *) p - offset);
184  u32 elt_idx = (offset - b->chunk_hdr_sz) / b->elt_sz;
185 
186  ASSERT (elt_idx < b->elts_per_chunk);
187  ASSERT (get_chunk_elt_ptr (b, c, elt_idx) == p);
188 
189  c->n_free++;
190 
191  if (c->n_free == b->elts_per_chunk)
192  {
193  /* chunk is empty - give it back */
194  remove_from_chunk_list (&b->avail_chunks, c);
196  mspace_free (b->mspace, c);
197  return;
198  }
199 
200  if (c->n_free == 1)
201  {
202  /* move chunk to avail chunks */
203  remove_from_chunk_list (&b->full_chunks, c);
204  add_to_chunk_list (&b->avail_chunks, c);
205  }
206 
207  /* add elt to freelist */
208  *(u32 *) p = c->freelist;
209  c->freelist = elt_idx;
210 }
211 
212 __clib_export u8 *
213 format_clib_mem_bulk (u8 *s, va_list *args)
214 {
215  clib_mem_bulk_t *b = va_arg (*args, clib_mem_bulk_handle_t);
217  uword n_chunks = 0, n_free_elts = 0, n_elts, chunk_sz;
218 
219  c = b->full_chunks;
220  while (c)
221  {
222  n_chunks++;
223  c = c->next;
224  }
225 
226  c = b->avail_chunks;
227  while (c)
228  {
229  n_chunks++;
230  n_free_elts += c->n_free;
231  c = c->next;
232  }
233 
234  n_elts = n_chunks * b->elts_per_chunk;
235  chunk_sz = b->chunk_hdr_sz + (uword) b->elts_per_chunk * b->elt_sz;
236 
237  s = format (s, "%u bytes/elt, align %u, chunk-align %u, ", b->elt_sz,
238  b->align, b->chunk_align);
239  s = format (s, "%u elts-per-chunk, chunk size %lu bytes", b->elts_per_chunk,
240  chunk_sz);
241 
242  if (n_chunks == 0)
243  return format (s, "\nempty");
244 
245  s = format (s, "\n%lu chunks allocated, ", n_chunks);
246  s = format (s, "%lu / %lu free elts (%.1f%%), ", n_free_elts, n_elts,
247  (f64) n_free_elts * 100 / n_elts);
248  s = format (s, "%lu bytes of memory consumed", n_chunks * chunk_sz);
249 
250  return s;
251 }
clib_mem_bulk_chunk_hdr
Definition: mem_bulk.c:29
clib_mem_bulk_alloc
__clib_export void * clib_mem_bulk_alloc(clib_mem_bulk_handle_t h)
Definition: mem_bulk.c:141
clib_mem_get_heap
static clib_mem_heap_t * clib_mem_get_heap(void)
Definition: mem.h:359
clib_mem_bulk_init
__clib_export clib_mem_bulk_handle_t clib_mem_bulk_init(u32 elt_sz, u32 align, u32 min_elts_per_chunk)
Definition: mem_bulk.c:54
clib_mem_bulk_handle_t
void * clib_mem_bulk_handle_t
Definition: mem.h:560
CLIB_MEM_POISON
#define CLIB_MEM_POISON(a, s)
Definition: sanitizer.h:46
clib.h
clib_mem_heap_t::mspace
void * mspace
Definition: mem.h:113
mspace_memalign
DLMALLOC_EXPORT void * mspace_memalign(mspace msp, size_t alignment, size_t bytes)
next
u16 * next
Definition: nat44_ei_out2in.c:718
get_chunk_elt_ptr
static void * get_chunk_elt_ptr(clib_mem_bulk_t *b, clib_mem_bulk_chunk_hdr_t *c, u32 index)
Definition: mem_bulk.c:112
first
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
clib_mem_bulk_chunk_hdr::prev
struct clib_mem_bulk_chunk_hdr * prev
Definition: mem_bulk.c:33
clib_mem_bulk_free
__clib_export void clib_mem_bulk_free(clib_mem_bulk_handle_t h, void *p)
Definition: mem_bulk.c:179
h
h
Definition: flowhash_template.h:372
round_pow2
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:279
add_to_chunk_list
static void add_to_chunk_list(clib_mem_bulk_chunk_hdr_t **first, clib_mem_bulk_chunk_hdr_t *c)
Definition: mem_bulk.c:118
clib_mem_bulk_t::elts_per_chunk
u32 elts_per_chunk
Definition: mem_bulk.c:40
clib_mem_bulk_t::chunk_hdr_sz
u32 chunk_hdr_sz
Definition: mem_bulk.c:39
offset
struct clib_bihash_value offset
template key/value backing page structure
c
svmdb_client_t * c
Definition: vpp_get_metrics.c:48
uword
u64 uword
Definition: types.h:112
time.h
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
f64
double f64
Definition: types.h:142
clib_mem_bulk_chunk_hdr_t
struct clib_mem_bulk_chunk_hdr clib_mem_bulk_chunk_hdr_t
format.h
remove_from_chunk_list
static void remove_from_chunk_list(clib_mem_bulk_chunk_hdr_t **first, clib_mem_bulk_chunk_hdr_t *c)
Definition: mem_bulk.c:129
dlmalloc.h
clib_mem_bulk_t::chunk_align
u32 chunk_align
Definition: mem_bulk.c:42
clib_mem_bulk_t
Definition: mem_bulk.c:36
clib_mem_bulk_t::elt_sz
u32 elt_sz
Definition: mem_bulk.c:38
index
u32 index
Definition: flow_types.api:221
clib_bihash_value
template key/value backing page structure
Definition: bihash_doc.h:44
clib_mem_bulk_t::mspace
void * mspace
Definition: mem_bulk.c:43
format
description fragment has unexpected format
Definition: map.api:433
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
clib_mem_bulk_t::align
u32 align
Definition: mem_bulk.c:41
u32
unsigned int u32
Definition: types.h:88
CLIB_MEM_UNPOISON
#define CLIB_MEM_UNPOISON(a, s)
Definition: sanitizer.h:47
clib_mem_bulk_chunk_hdr::n_free
u32 n_free
Definition: mem_bulk.c:32
clib_mem_bulk_chunk_hdr::freelist
u32 freelist
Definition: mem_bulk.c:31
clib_memset
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
clib_mem_bulk_t::full_chunks
clib_mem_bulk_chunk_hdr_t * full_chunks
Definition: mem_bulk.c:44
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
u8
unsigned char u8
Definition: types.h:56
clib_error.h
format_clib_mem_bulk
__clib_export u8 * format_clib_mem_bulk(u8 *s, va_list *args)
Definition: mem_bulk.c:213
clib_mem_bulk_chunk_hdr::next
struct clib_mem_bulk_chunk_hdr * next
Definition: mem_bulk.c:33
max_pow2
static uword max_pow2(uword x)
Definition: clib.h:258
mem.h
clib_mem_bulk_destroy
__clib_export void clib_mem_bulk_destroy(clib_mem_bulk_handle_t h)
Definition: mem_bulk.c:83
clib_mem_heap_t
Definition: mem.h:107
mspace_free
DLMALLOC_EXPORT void mspace_free(mspace msp, void *mem)
CLIB_MEM_BULK_DEFAULT_MIN_ELTS_PER_CHUNK
#define CLIB_MEM_BULK_DEFAULT_MIN_ELTS_PER_CHUNK
Definition: mem_bulk.c:27
bulk_chunk_size
static uword bulk_chunk_size(clib_mem_bulk_t *b)
Definition: mem_bulk.c:48