66 while (__sync_lock_test_and_set (&h->
lock, 1))
94 uword n_user_data_words;
95 word small_bin, large_bin;
105 ASSERT (n_user_data_words > 0);
189 nf->
free_elt.prev_uoffset = uoffset;
236 #define MHEAP_VM_MAP (1 << 0) 237 #define MHEAP_VM_UNMAP (1 << 1) 238 #define MHEAP_VM_NOMAP (0 << 1) 239 #define MHEAP_VM_ROUND (1 << 2) 240 #define MHEAP_VM_ROUND_UP MHEAP_VM_ROUND 241 #define MHEAP_VM_ROUND_DOWN (0 << 2) 248 return (addr + mheap_page_size - 1) & ~(mheap_page_size - 1);
254 return addr & ~(mheap_page_size - 1);
266 end_addr = start_addr +
size;
277 if (end_page > start_page)
279 mapped_bytes = end_page - start_page;
298 return mheap_vm (v, flags, start_addr, end_addr - start_addr);
307 #if !defined (CLIB_HAVE_VEC128) || defined (__ALTIVEC__) || defined (__i386__) 314 #define _(i) ((uword) u8x16_compare_byte_mask (u8x16_is_equal (b, c->bins.as_u8x16[i])) << (uword) ((i)*16)) 377 uword * n_user_data_bytes_arg,
385 word o0, o1, f0, f1, search_n_user_data_bytes;
386 word lo_free_usize, hi_free_usize;
391 search_n_user_data_bytes = *n_user_data_bytes_arg;
394 o0 = o1 = f0 = f1 = 0;
405 ASSERT (this_object_n_user_data_bytes >= search_n_user_data_bytes);
409 if (this_object_n_user_data_bytes < search_n_user_data_bytes)
414 f1 = f0 + this_object_n_user_data_bytes;
417 o0 = ((f1 - search_n_user_data_bytes) & ~(align - 1)) - align_offset;
431 o1 = o0 + search_n_user_data_bytes;
434 if (o0 >= f0 && o1 <= f1)
454 search_n_user_data_bytes += f1 - o1;
467 uword f0_page_start, f0_page_end;
468 uword o0_page_start, o0_page_end;
477 if (o0_page_start < f0_page_start)
478 o0_page_start = f0_page_start;
479 if (o0_page_end > f0_page_end)
480 o0_page_end = f0_page_end;
482 if (o0_page_end > o0_page_start)
484 o0_page_end - o0_page_start);
491 if (lo_free_usize > 0)
493 ASSERT (lo_free_usize >= (
word) MHEAP_MIN_USER_DATA_BYTES);
500 if (hi_free_usize > 0)
508 *n_user_data_bytes_arg = search_n_user_data_bytes;
518 uword * n_user_bytes_arg,
522 uword bin, n_user_bytes,
i, bi;
524 n_user_bytes = *n_user_bytes_arg;
531 && align_offset == 0)
570 uword n_user_data_bytes,
575 uword f0, f1, o0, o1;
580 if (_vec_len (v) == 0)
601 o1 = o0 + n_user_data_bytes;
624 if (f1_page > f0_page)
644 uword n_user_data_bytes,
657 align_offset &= (align - 1);
758 uword n_user_data_bytes, bin;
760 uword trace_uoffset, trace_n_user_data_bytes;
780 trace_uoffset = uoffset;
781 trace_n_user_data_bytes = n_user_data_bytes;
814 uword f0, f1, n_combine;
817 f1 = f0 + n_user_data_bytes;
875 if (!mheap_page_size)
914 size = memory + memory_size -
v;
922 memset (h, 0,
sizeof (h[0]));
935 if (!(h->
flags & MHEAP_FLAG_DISABLE_VM))
954 #ifdef CLIB_HAVE_VEC128 962 _mheap_free (
void *v)
977 uword elt_size),
void *arg)
980 u8 *stack_heap, *clib_mem_mheap_save;
981 u8 tmp_heap_memory[16 * 1024];
988 clib_mem_mheap_save = 0;
997 stack_heap =
mheap_alloc (tmp_heap_memory,
sizeof (tmp_heap_memory));
998 clib_mem_mheap_save =
v;
1013 if (clib_mem_mheap_save)
1039 uword used = 0, free = 0, free_vm_unmapped = 0;
1084 return format (s,
"%wd", n_bytes);
1086 return format (s,
"%wdk", n_bytes / 1024);
1127 "alloc. from small object cache: %Ld hits %Ld attempts (%.2f%%) replacements %d",
1136 "\n%Ualloc. from free-list: %Ld attempts, %Ld hits (%.2f%%), %Ld considered (per-attempt %.2f)",
1147 s =
format (s,
"\n%Ualloc. from vector-expand: %Ld",
1150 s =
format (s,
"\n%Uallocs: %Ld %.2f clocks/call",
1154 s =
format (s,
"\n%Ufrees: %Ld %.2f clocks/call",
1164 void *v = va_arg (*va,
u8 *);
1165 int verbose = va_arg (*va,
int);
1183 "%d objects, %U of %U used, %U free, %U reclaimed, %U overhead",
1200 memset (hist, 0,
sizeof (hist));
1216 s =
format (s,
"\n%U%=12s%=12s%=16s",
1218 "Size",
"Count",
"Fraction");
1224 s =
format (s,
"\n%U%12d%12wd%16.4f",
1228 (
f64) hist[i] / (
f64) n_hist);
1240 u32 indent, total_objects_traced;
1243 qsort (traces_copy,
vec_len (traces_copy),
sizeof (traces_copy[0]),
1246 total_objects_traced = 0;
1257 if (!verbose && t->
n_bytes < 1024)
1260 if (t == traces_copy)
1261 s =
format (s,
"%=9s%=9s %=10s Traceback\n",
"Bytes",
"Count",
1280 s =
format (s,
"%d total traced objects\n", total_objects_traced);
1289 s =
format (s,
"\n first corrupt object: %p, size %wd\n %U",
1307 s =
format (s,
"%8d: ", i);
1312 s =
format (s,
"(%8d) ", o);
1314 s =
format (s,
" %8d ", o);
1316 if ((i % 8) == 7 || (i + 1) >= h->
n_elts)
1344 uword elt_count, elt_size;
1345 uword free_count_from_free_lists, free_size_from_free_lists;
1346 uword small_elt_free_count, small_elt_free_size;
1348 #define CHECK(x) if (! (x)) { mheap_validate_breakpoint (); os_panic (); } 1356 free_size_from_free_lists = free_count_from_free_lists = 0;
1395 free_count_from_free_lists += 1;
1396 free_size_from_free_lists += s;
1411 small_elt_free_count = small_elt_free_size = 0;
1429 small_elt_free_count += 1;
1430 small_elt_free_size += s;
1436 uword elt_free_size, elt_free_count;
1438 elt_count = elt_size = elt_free_size = elt_free_count = 0;
1466 CHECK (free_count_from_free_lists == elt_free_count);
1467 CHECK (free_size_from_free_lists == elt_free_size);
1468 CHECK (elt_count == h->
n_elts + elt_free_count + small_elt_free_count);
1496 uword i, n_callers, trace_index, *p;
1500 memset (&trace, 0,
sizeof (trace));
1521 t = tm->
traces + trace_index;
1538 if (tm->
traces != old_start)
1545 q = uword_to_pointer (p->key, mheap_trace_t *);
1546 ASSERT (q >= old_start && q < old_end);
1547 p->key = pointer_to_uword (tm->traces + (q - old_start));
1551 trace_index = t - tm->
traces;
1554 t = tm->
traces + trace_index;
1573 uword trace_index, *p;
1585 t = tm->
traces + trace_index;
1594 memset (t, 0,
sizeof (t[0]));
#define MHEAP_LOG2_N_SMALL_OBJECT_BINS
static_always_inline uword mheap_page_truncate(uword addr)
struct mheap_elt_t::@14::@16 free_elt
#define hash_set(h, key, value)
sll srl srl sll sra u16x4 i
static void remove_free_elt2(void *v, mheap_elt_t *e)
#define MHEAP_ELT_OVERHEAD_BYTES
#define hash_unset(h, key)
union mheap_small_object_cache_t::@18 bins
static_always_inline uword mheap_vm(void *v, uword flags, clib_address_t start_addr, uword size)
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
static u8 * format_mheap_byte_count(u8 *s, va_list *va)
uword offsets[BITS(uword)]
uword vm_alloc_offset_from_header
uword bytes_free_reclaimed
static uword mheap_elt_size_to_user_n_words(uword n_bytes)
void * mheap_alloc(void *memory, uword size)
uword clib_backtrace(uword *callers, uword max_callers, uword n_frames_to_skip)
struct mheap_stats_t::@17 free_list
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static u64 clib_cpu_time_now(void)
u64 n_small_object_cache_hits
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
u64 n_small_object_cache_attempts
static mheap_t * mheap_header(u8 *v)
#define hash_set_mem(h, key, value)
#define STRUCT_OFFSET_OF(t, f)
#define MHEAP_N_SMALL_OBJECT_BINS
static void mheap_put_trace(void *v, uword offset, uword size)
#define MHEAP_FLAG_THREAD_SAFE
static void new_free_elt(void *v, uword uoffset, uword n_user_data_bytes)
#define vec_bytes(v)
Number of data bytes in vector.
static uword mheap_max_size(void *v)
static void mheap_maybe_unlock(void *v)
static uword mheap_small_object_cache_mask(mheap_small_object_cache_t *c, uword bin)
u8 * format_mheap(u8 *s, va_list *va)
static uword min_log2(uword x)
#define foreach_set_bit(var, mask, body)
vhost_user_memory_t memory
static uword mheap_get_small_object(mheap_t *h, uword bin)
mheap_trace_main_t trace_main
#define static_always_inline
static uword pow2_mask(uword x)
#define MHEAP_FLAG_DISABLE_VM
void mheap_usage(void *v, clib_mem_usage_t *usage)
static uword mheap_elt_data_bytes(mheap_elt_t *e)
static mheap_elt_t * mheap_prev_elt(mheap_elt_t *e)
#define vec_end(v)
End (last data address) of vector.
static uword mheap_bytes_overhead(void *v)
static uword pointer_to_uword(const void *p)
static void mheap_trace_main_free(mheap_trace_main_t *tm)
uword non_empty_free_elt_heads[(MHEAP_N_BINS+BITS(uword)-1)/BITS(uword)]
#define hash_create_mem(elts, key_bytes, value_bytes)
#define hash_unset_mem(h, key)
static void mheap_maybe_lock(void *v)
static uword mheap_elts(void *v)
#define vec_dup(V)
Return copy of vector (no header, no alignment)
static uword mheap_page_size
static_always_inline uword mheap_vm_elt(void *v, uword flags, uword offset)
static u32 * elt_data(void *v, heap_elt_t *e)
#define MHEAP_N_USER_DATA_INVALID
#define uword_to_pointer(u, type)
static uword mheap_put_small_object(mheap_t *h, uword bin, uword offset)
static void mheap_validate_breakpoint()
void * mheap_get_aligned(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
static void free_last_elt(void *v, mheap_elt_t *e)
#define vec_free(V)
Free vector's memory (no header).
static uword mheap_elt_uoffset(void *v, mheap_elt_t *e)
static void * clib_mem_set_heap(void *heap)
static never_inline void * mheap_get_extend_vector(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
mheap_small_object_cache_t small_object_cache
static uword max_pow2(uword x)
static uword round_pow2(uword x, uword pow2)
static void set_free_elt(void *v, uword uoffset, uword n_user_data_bytes)
static void * clib_mem_get_heap(void)
void * mheap_alloc_with_flags(void *memory, uword memory_size, uword flags)
static uword mheap_get_search_free_list(void *v, uword *n_user_bytes_arg, uword align, uword align_offset)
void mheap_trace(void *v, int enable)
static void * mheap_elt_data(void *v, mheap_elt_t *e)
#define MHEAP_FLAG_VALIDATE
static_always_inline uword mheap_page_round(uword addr)
static uword mheap_get_search_free_bin(void *v, uword bin, uword *n_user_data_bytes_arg, uword align, uword align_offset)
static mheap_elt_t * mheap_first_corrupt(void *v)
#define MHEAP_FLAG_SMALL_OBJECT_CACHE
static mheap_elt_t * mheap_elt_at_uoffset(void *v, uword uo)
uword mheap_bytes(void *v)
template key/value backing page structure
#define MHEAP_HAVE_SMALL_OBJECT_CACHE
static void clib_mem_vm_free(void *addr, uword size)
void qsort(void *base, uword n, uword size, int(*compar)(const void *, const void *))
void mheap_put(void *v, uword uoffset)
static int mheap_trace_sort(const void *t1, const void *t2)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define hash_foreach_pair(p, v, body)
Iterate over hash pairs.
static uword max_log2(uword x)
#define MHEAP_VM_ROUND_UP
static uword mheap_elt_size_to_user_n_bytes(uword n_bytes)
static void mheap_get_trace(void *v, uword offset, uword size)
static_always_inline uword os_get_thread_index(void)
static u8 * format_mheap_stats(u8 *s, va_list *va)
static mheap_elt_t * mheap_next_elt(mheap_elt_t *e)
format_function_t format_clib_elf_symbol_with_address
#define hash_get_mem(h, key)
struct clib_bihash_value offset
template key/value backing page structure
static void * clib_mem_vm_unmap(void *addr, uword size)
#define STRUCT_SIZE_OF(t, f)
void mheap_validate(void *v)
static void remove_free_elt(void *v, mheap_elt_t *e, uword bin)
#define vec_foreach(var, vec)
Vector iterator.
void mheap_foreach(void *v, uword(*func)(void *arg, void *v, void *elt_data, uword elt_size), void *arg)
static void set_first_free_elt_offset(mheap_t *h, uword bin, uword uoffset)
#define CLIB_MEMORY_BARRIER()
#define MHEAP_USER_DATA_WORD_BYTES
static uword user_data_size_to_bin_index(uword n_user_data_bytes)
static u8 * mheap_vector(mheap_t *h)
uword clib_mem_get_page_size(void)
uword * trace_index_by_offset
static void * clib_mem_vm_alloc(uword size)
static void mheap_usage_no_lock(void *v, clib_mem_usage_t *usage)
#define MHEAP_MIN_USER_DATA_BYTES
static void mheap_elt_set_size(void *v, uword uoffset, uword n_user_data_bytes, uword is_free)
u32 first_free_elt_uoffset_by_bin[MHEAP_N_BINS]
static void * clib_mem_vm_map(void *addr, uword size)