FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
mem_dlmalloc.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vppinfra/format.h>
17 #include <vppinfra/dlmalloc.h>
18 #include <vppinfra/os.h>
19 #include <vppinfra/lock.h>
20 #include <vppinfra/hash.h>
21 #include <vppinfra/elf_clib.h>
22 
24 
25 typedef struct
26 {
27  /* Address of callers: outer first, inner last. */
28  uword callers[12];
29 
30  /* Count of allocations with this traceback. */
31 #if CLIB_VEC64 > 0
32  u64 n_allocations;
33 #else
35 #endif
36 
37  /* Count of bytes allocated with this traceback. */
39 
40  /* Offset of this item */
43 
44 typedef struct
45 {
48 
50 
51  /* Indices of free traces. */
53 
54  /* Hash table mapping callers to trace index. */
56 
57  /* Hash table mapping mheap offset to trace index. */
60 
62 
63 void
65 {
67  mheap_trace_t *t;
68  uword i, n_callers, trace_index, *p;
70  uword save_enabled;
71 
72  if (tm->enabled == 0)
73  return;
74 
75  /* Spurious Coverity warnings be gone. */
76  memset (&trace, 0, sizeof (trace));
77 
78  /* Skip our frame and mspace_get_aligned's frame */
79  n_callers = clib_backtrace (trace.callers, ARRAY_LEN (trace.callers), 2);
80  if (n_callers == 0)
81  return;
82 
83  /* $$$ This looks like dreck to remove... */
84  if (0)
85  for (i = n_callers; i < ARRAY_LEN (trace.callers); i++)
86  trace.callers[i] = 0;
87 
88  clib_spinlock_lock (&tm->lock);
89 
90  /* Turn off tracing to avoid embarrassment... */
91  save_enabled = tm->enabled;
92  tm->enabled = 0;
93 
94  if (!tm->trace_by_callers)
95  tm->trace_by_callers =
96  hash_create_shmem (0, sizeof (trace.callers), sizeof (uword));
97 
98  p = hash_get_mem (tm->trace_by_callers, &trace.callers);
99  if (p)
100  {
101  trace_index = p[0];
102  t = tm->traces + trace_index;
103  }
104  else
105  {
106  i = vec_len (tm->trace_free_list);
107  if (i > 0)
108  {
109  trace_index = tm->trace_free_list[i - 1];
110  _vec_len (tm->trace_free_list) = i - 1;
111  }
112  else
113  {
114  mheap_trace_t *old_start = tm->traces;
115  mheap_trace_t *old_end = vec_end (tm->traces);
116 
117  vec_add2 (tm->traces, t, 1);
118 
119  if (tm->traces != old_start)
120  {
121  hash_pair_t *p;
122  mheap_trace_t *q;
123  /* *INDENT-OFF* */
125  ({
126  q = uword_to_pointer (p->key, mheap_trace_t *);
127  ASSERT (q >= old_start && q < old_end);
128  p->key = pointer_to_uword (tm->traces + (q - old_start));
129  }));
130  /* *INDENT-ON* */
131  }
132  trace_index = t - tm->traces;
133  }
134 
135  t = tm->traces + trace_index;
136  t[0] = trace;
137  t->n_allocations = 0;
138  t->n_bytes = 0;
139  hash_set_mem (tm->trace_by_callers, t->callers, trace_index);
140  }
141 
142  t->n_allocations += 1;
143  t->n_bytes += size;
144  t->offset = offset; /* keep a sample to autopsy */
145  hash_set (tm->trace_index_by_offset, offset, t - tm->traces);
146  tm->enabled = save_enabled;
147  clib_spinlock_unlock (&tm->lock);
148 }
149 
150 void
152 {
153  mheap_trace_t *t;
154  uword trace_index, *p;
156  uword save_enabled;
157 
158  if (tm->enabled == 0)
159  return;
160 
161  clib_spinlock_lock (&tm->lock);
162 
163  /* Turn off tracing for a moment */
164  save_enabled = tm->enabled;
165  tm->enabled = 0;
166 
167  p = hash_get (tm->trace_index_by_offset, offset);
168  if (!p)
169  {
170  tm->enabled = save_enabled;
171  clib_spinlock_unlock (&tm->lock);
172  return;
173  }
174 
175  trace_index = p[0];
176  hash_unset (tm->trace_index_by_offset, offset);
177  ASSERT (trace_index < vec_len (tm->traces));
178 
179  t = tm->traces + trace_index;
180  ASSERT (t->n_allocations > 0);
181  ASSERT (t->n_bytes >= size);
182  t->n_allocations -= 1;
183  t->n_bytes -= size;
184  if (t->n_allocations == 0)
185  {
187  vec_add1 (tm->trace_free_list, trace_index);
188  memset (t, 0, sizeof (t[0]));
189  }
190  tm->enabled = save_enabled;
191  clib_spinlock_unlock (&tm->lock);
192 }
193 
194 always_inline void
196 {
197  vec_free (tm->traces);
201 }
202 
203 /* Initialize CLIB heap based on memory/size given by user.
204  Set memory to 0 and CLIB will try to allocate its own heap. */
205 void *
207 {
208  u8 *heap;
209 
210  if (memory)
211  {
212  heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ );
213  mspace_disable_expand (heap);
214  }
215  else
216  heap = create_mspace (memory_size, 1 /* locked */ );
217 
218  clib_mem_set_heap (heap);
219 
220  if (mheap_trace_main.lock == 0)
221  clib_spinlock_init (&mheap_trace_main.lock);
222 
223  return heap;
224 }
225 
226 void *
228 {
229  return clib_mem_init (memory, memory_size);
230 }
231 
232 #ifdef CLIB_LINUX_KERNEL
233 #include <asm/page.h>
234 
235 uword
237 {
238  return PAGE_SIZE;
239 }
240 #endif
241 
242 #ifdef CLIB_UNIX
243 uword
245 {
246  return getpagesize ();
247 }
248 #endif
249 
250 /* Make a guess for standalone. */
251 #ifdef CLIB_STANDALONE
252 uword
254 {
255  return 4096;
256 }
257 #endif
258 
259 u8 *
260 format_clib_mem_usage (u8 * s, va_list * va)
261 {
262  int verbose = va_arg (*va, int);
263  return format (s, "$$$$ heap at %llx verbose %d", clib_mem_get_heap (),
264  verbose);
265 }
266 
267 /*
268  * Magic decoder ring for mallinfo stats (ala dlmalloc):
269  *
270  * size_t arena; / * Non-mmapped space allocated (bytes) * /
271  * size_t ordblks; / * Number of free chunks * /
272  * size_t smblks; / * Number of free fastbin blocks * /
273  * size_t hblks; / * Number of mmapped regions * /
274  * size_t hblkhd; / * Space allocated in mmapped regions (bytes) * /
275  * size_t usmblks; / * Maximum total allocated space (bytes) * /
276  * size_t fsmblks; / * Space in freed fastbin blocks (bytes) * /
277  * size_t uordblks; / * Total allocated space (bytes) * /
278  * size_t fordblks; / * Total free space (bytes) * /
279  * size_t keepcost; / * Top-most, releasable space (bytes) * /
280  *
281  */
282 
283 u8 *
284 format_msize (u8 * s, va_list * va)
285 {
286  uword a = va_arg (*va, uword);
287 
288  if (a >= 1ULL << 30)
289  s = format (s, "%.2fG", (((f64) a) / ((f64) (1ULL << 30))));
290  else if (a >= 1ULL << 20)
291  s = format (s, "%.2fM", (((f64) a) / ((f64) (1ULL << 20))));
292  else if (a >= 1ULL << 10)
293  s = format (s, "%.2fK", (((f64) a) / ((f64) (1ULL << 10))));
294  else
295  s = format (s, "%lld", a);
296  return s;
297 }
298 
299 static int
300 mheap_trace_sort (const void *_t1, const void *_t2)
301 {
302  const mheap_trace_t *t1 = _t1;
303  const mheap_trace_t *t2 = _t2;
304  word cmp;
305 
306  cmp = (word) t2->n_bytes - (word) t1->n_bytes;
307  if (!cmp)
308  cmp = (word) t2->n_allocations - (word) t1->n_allocations;
309  return cmp;
310 }
311 
312 u8 *
313 format_mheap_trace (u8 * s, va_list * va)
314 {
315  mheap_trace_main_t *tm = va_arg (*va, mheap_trace_main_t *);
316  int verbose = va_arg (*va, int);
317  int have_traces = 0;
318  int i;
319 
320  clib_spinlock_lock (&tm->lock);
321  if (vec_len (tm->traces) > 0)
322  {
323  have_traces = 1;
324 
325  /* Make a copy of traces since we'll be sorting them. */
326  mheap_trace_t *t, *traces_copy;
327  u32 indent, total_objects_traced;
328 
329  traces_copy = vec_dup (tm->traces);
330 
331  qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]),
333 
334  total_objects_traced = 0;
335  s = format (s, "\n");
336  vec_foreach (t, traces_copy)
337  {
338  /* Skip over free elements. */
339  if (t->n_allocations == 0)
340  continue;
341 
342  total_objects_traced += t->n_allocations;
343 
344  /* When not verbose only report allocations of more than 1k. */
345  if (!verbose && t->n_bytes < 1024)
346  continue;
347 
348  if (t == traces_copy)
349  s = format (s, "%=9s%=9s %=10s Traceback\n", "Bytes", "Count",
350  "Sample");
351  s = format (s, "%9d%9d %p", t->n_bytes, t->n_allocations, t->offset);
352  indent = format_get_indent (s);
353  for (i = 0; i < ARRAY_LEN (t->callers) && t->callers[i]; i++)
354  {
355  if (i > 0)
356  s = format (s, "%U", format_white_space, indent);
357 #if defined(CLIB_UNIX) && !defined(__APPLE__)
358  /* $$$$ does this actually work? */
359  s =
361  t->callers[i]);
362 #else
363  s = format (s, " %p\n", t->callers[i]);
364 #endif
365  }
366  }
367 
368  s = format (s, "%d total traced objects\n", total_objects_traced);
369 
370  vec_free (traces_copy);
371  }
372  clib_spinlock_unlock (&tm->lock);
373  if (have_traces == 0)
374  s = format (s, "no traced allocations\n");
375 
376  return s;
377 }
378 
379 
380 u8 *
381 format_mheap (u8 * s, va_list * va)
382 {
383  void *heap = va_arg (*va, u8 *);
384  int verbose = va_arg (*va, int);
385  struct dlmallinfo mi;
387 
388  mi = mspace_mallinfo (heap);
389 
390  s = format (s, "total: %U, used: %U, free: %U, trimmable: %U",
391  format_msize, mi.arena,
394  if (verbose > 0)
395  {
396  s = format (s, "\n free chunks %llu free fastbin blks %llu",
397  mi.ordblks, mi.smblks);
398  s =
399  format (s, "\n max total allocated %U", format_msize, mi.usmblks);
400  }
401 
402  s = format (s, "\n%U", format_mheap_trace, tm, verbose);
403  return s;
404 }
405 
406 void
408 {
409  clib_warning ("unimp");
410 }
411 
412 /* Call serial number for debugger breakpoints. */
414 
415 void
417 {
418  clib_warning ("unimp");
419 }
420 
421 void
422 mheap_trace (void *v, int enable)
423 {
424  (void) mspace_enable_disable_trace (v, enable);
425 
426  if (enable == 0)
427  mheap_trace_main_free (&mheap_trace_main);
428 }
429 
430 void
431 clib_mem_trace (int enable)
432 {
434 
435  tm->enabled = enable;
436  mheap_trace (clib_mem_get_heap (), enable);
437 }
438 
439 uword
441 {
442  uword rv;
444 
445  rv = tm->enabled;
446  tm->enabled = enable;
447  return rv;
448 }
449 
450 /*
451  * These API functions seem like layering violations, but
452  * by introducing them we greatly reduce the number
453  * of code changes required to use dlmalloc spaces
454  */
455 void *
457 {
458  void *rv;
459  if (memory == 0)
460  return create_mspace (size, locked);
461  else
462  {
463  rv = create_mspace_with_base (memory, size, locked);
464  if (rv)
466  return rv;
467  }
468 }
469 
470 /*
471  * fd.io coding-style-patch-verification: ON
472  *
473  * Local Variables:
474  * eval: (c-set-style "gnu")
475  * End:
476  */
void * clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]
Definition: mem_dlmalloc.c:23
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:862
#define hash_set(h, key, value)
Definition: hash.h:255
vhost_user_memory_t memory
Definition: vhost_user.h:122
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
#define hash_unset(h, key)
Definition: hash.h:261
a
Definition: bitmap.h:538
void * clib_mem_init(void *memory, uword memory_size)
Definition: mem_dlmalloc.c:206
unsigned long u64
Definition: types.h:89
uword clib_backtrace(uword *callers, uword max_callers, uword n_frames_to_skip)
Definition: backtrace.c:226
uword callers[12]
Definition: mem_dlmalloc.c:28
u8 * format_msize(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:284
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
void mheap_trace(void *v, int enable)
Definition: mem_dlmalloc.c:422
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
#define hash_set_mem(h, key, value)
Definition: hash.h:275
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
mheap_trace_t * traces
Definition: mem_dlmalloc.c:49
DLMALLOC_EXPORT struct dlmallinfo mspace_mallinfo(mspace msp)
#define CLIB_MAX_MHEAPS
Definition: mem.h:58
unsigned char u8
Definition: types.h:56
double f64
Definition: types.h:142
DLMALLOC_EXPORT mspace create_mspace_with_base(void *base, size_t capacity, int locked)
MALLINFO_FIELD_TYPE uordblks
Definition: dlmalloc.h:786
memset(h->entries, 0, sizeof(h->entries[0])*entries)
i64 word
Definition: types.h:111
MALLINFO_FIELD_TYPE arena
Definition: dlmalloc.h:779
#define always_inline
Definition: clib.h:94
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
unsigned int u32
Definition: types.h:88
#define vec_end(v)
End (last data address) of vector.
MALLINFO_FIELD_TYPE ordblks
Definition: dlmalloc.h:780
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
u8 * format_mheap(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:381
#define hash_get(h, key)
Definition: hash.h:249
uword size
#define hash_unset_mem(h, key)
Definition: hash.h:291
mheap_trace_main_t mheap_trace_main
Definition: mem_dlmalloc.c:61
MALLINFO_FIELD_TYPE usmblks
Definition: dlmalloc.h:784
void clib_mem_usage(clib_mem_usage_t *u)
Definition: mem_dlmalloc.c:407
DLMALLOC_EXPORT void mspace_disable_expand(mspace msp)
#define v
Definition: acl.c:496
u64 memory_size
Definition: vhost_user.h:115
#define hash_free(h)
Definition: hash.h:310
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:373
void clib_mem_validate(void)
Definition: mem_dlmalloc.c:416
static int mheap_trace_sort(const void *_t1, const void *_t2)
Definition: mem_dlmalloc.c:300
uword * trace_index_by_offset
Definition: mem_dlmalloc.c:58
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:261
MALLINFO_FIELD_TYPE keepcost
Definition: dlmalloc.h:788
#define clib_warning(format, args...)
Definition: error.h:59
DLMALLOC_EXPORT int mspace_enable_disable_trace(mspace msp, int enable)
#define ARRAY_LEN(x)
Definition: clib.h:61
uword * trace_by_callers
Definition: mem_dlmalloc.c:55
u8 * format_mheap_trace(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:313
static void * clib_mem_get_heap(void)
Definition: mem.h:255
static void mheap_trace_main_free(mheap_trace_main_t *tm)
Definition: mem_dlmalloc.c:195
MALLINFO_FIELD_TYPE smblks
Definition: dlmalloc.h:781
#define hash_create_shmem(elts, key_bytes, value_bytes)
Definition: hash.h:684
#define ASSERT(truth)
uword clib_mem_trace_enable_disable(uword enable)
Definition: mem_dlmalloc.c:440
void mheap_get_trace(uword offset, uword size)
Definition: mem_dlmalloc.c:64
void * mheap_alloc_with_lock(void *memory, uword size, int locked)
Definition: mem_dlmalloc.c:456
uword clib_mem_validate_serial
Definition: mem_dlmalloc.c:413
uword clib_mem_get_page_size(void)
Definition: mem_dlmalloc.c:244
template key/value backing page structure
Definition: bihash_doc.h:44
void qsort(void *base, uword n, uword size, int(*compar)(const void *, const void *))
Definition: qsort.c:56
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define hash_foreach_pair(p, v, body)
Iterate over hash pairs.
Definition: hash.h:373
u64 uword
Definition: types.h:112
void mheap_put_trace(uword offset, uword size)
Definition: mem_dlmalloc.c:151
format_function_t format_clib_elf_symbol_with_address
Definition: elf_clib.h:134
#define hash_get_mem(h, key)
Definition: hash.h:269
struct clib_bihash_value offset
template key/value backing page structure
#define vec_foreach(var, vec)
Vector iterator.
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:431
void * clib_mem_init_thread_safe(void *memory, uword memory_size)
Definition: mem_dlmalloc.c:227
MALLINFO_FIELD_TYPE fordblks
Definition: dlmalloc.h:787
clib_spinlock_t lock
Definition: mem_dlmalloc.c:46
u8 * format_clib_mem_usage(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:260